github-mirrors/logstash
1
0
Fork 0
mirror of https://github.com/elastic/logstash.git synced 2025-04-23 22:27:21 -04:00
Code Issues Projects Releases Packages Wiki Activity

Cleanup plugins to move to plugin system

Browse source 
Plugins have moved to https://github.com/logstash-plugins/... into
individual repositories. One plugin per repo.

* Add plugin install task and add required plugins for testing
* Adding a plugin task to install plugins
* Added required plugins to a prep task for the test part.

Fixes #1984

Conflicts:
	lib/logstash/filters/multiline.rb
	lib/logstash/outputs/hipchat.rb
	logstash.gemspec
This commit is contained in:
Richard Pijnenburg 2014-10-30 09:57:11 +00:00 • committed by Jordan Sissel
parent b37ef67b2f
commit fec0be423a
197 changed files with 29 additions and 25134 deletions
Download patch file Download diff file Expand all files Collapse all files

485
lib/logstash/codecs/collectd.rb
View file

@ -1,485 +0,0 @@
# encoding utf-8
require "date"
require "logstash/codecs/base"
require "logstash/namespace"
require "logstash/errors"
require "tempfile"
require "time"
# Read events from the connectd binary protocol over the network via udp.
# See https://collectd.org/wiki/index.php/Binary_protocol
#
# Configuration in your Logstash configuration file can be as simple as:
# input {
# udp {
# port => 28526
# buffer_size => 1452
# codec => collectd { }
# }
# }
#
# A sample collectd.conf to send to Logstash might be:
#
# Hostname "host.example.com"
# LoadPlugin interface
# LoadPlugin load
# LoadPlugin memory
# LoadPlugin network
# <Plugin interface>
# Interface "eth0"
# IgnoreSelected false
# </Plugin>
# <Plugin network>
# <Server "10.0.0.1" "25826">
# </Server>
# </Plugin>
#
# Be sure to replace "10.0.0.1" with the IP of your Logstash instance.
#
class ProtocolError < LogStash::Error; end
class HeaderError < LogStash::Error; end
class EncryptionError < LogStash::Error; end
class NaNError < LogStash::Error; end
class LogStash::Codecs::Collectd < LogStash::Codecs::Base
config_name "collectd"
milestone 1
AUTHFILEREGEX = /([^:]+): (.+)/
PLUGIN_TYPE = 2
COLLECTD_TYPE = 4
SIGNATURE_TYPE = 512
ENCRYPTION_TYPE = 528
TYPEMAP = {
0 => "host",
1 => "@timestamp",
PLUGIN_TYPE => "plugin",
3 => "plugin_instance",
COLLECTD_TYPE => "collectd_type",
5 => "type_instance",
6 => "values",
7 => "interval",
8 => "@timestamp",
9 => "interval",
256 => "message",
257 => "severity",
SIGNATURE_TYPE => "signature",
ENCRYPTION_TYPE => "encryption"
}
PLUGIN_TYPE_FIELDS = {
'host' => true,
'@timestamp' => true,
}
COLLECTD_TYPE_FIELDS = {
'host' => true,
'@timestamp' => true,
'plugin' => true,
'plugin_instance' => true,
'type_instance' => true,
}
INTERVAL_VALUES_FIELDS = {
"interval" => true,
"values" => true,
}
INTERVAL_BASE_FIELDS = {
'host' => true,
'collectd_type' => true,
'plugin' => true,
'plugin_instance' => true,
'@timestamp' => true,
'type_instance' => true,
}
INTERVAL_TYPES = {
7 => true,
9 => true,
}
SECURITY_NONE = "None"
SECURITY_SIGN = "Sign"
SECURITY_ENCR = "Encrypt"
# File path(s) to collectd types.db to use.
# The last matching pattern wins if you have identical pattern names in multiple files.
# If no types.db is provided the included types.db will be used (currently 5.4.0).
config :typesdb, :validate => :array
# Prune interval records. Defaults to true.
config :prune_intervals, :validate => :boolean, :default => true
# Security Level. Default is "None". This setting mirrors the setting from the
# collectd [Network plugin](https://collectd.org/wiki/index.php/Plugin:Network)
config :security_level, :validate => [SECURITY_NONE, SECURITY_SIGN, SECURITY_ENCR],
:default => "None"
# What to do when a value in the event is NaN (Not a Number)
# - change_value (default): Change the NaN to the value of the nan_value option and add nan_tag as a tag
# - warn: Change the NaN to the value of the nan_value option, print a warning to the log and add nan_tag as a tag
# - drop: Drop the event containing the NaN (this only drops the single event, not the whole packet)
config :nan_handling, :validate => ['change_value','warn','drop'], :default => 'change_value'
# Only relevant when nan_handeling is set to 'change_value'
# Change NaN to this configured value
config :nan_value, :validate => :number, :default => 0
# The tag to add to the event if a NaN value was found
# Set this to an empty string ('') if you don't want to tag
config :nan_tag, :validate => :string, :default => '_collectdNaN'
# Path to the authentication file. This file should have the same format as
# the [AuthFile](http://collectd.org/documentation/manpages/collectd.conf.5.shtml#authfile_filename)
# in collectd. You only need to set this option if the security_level is set to
# "Sign" or "Encrypt"
config :authfile, :validate => :string
public
def register
@logger.info("Starting Collectd codec...")
init_lambdas!
if @typesdb.nil?
@typesdb = LogStash::Environment.vendor_path("collectd/types.db")
if !File.exists?(@typesdb)
raise "You must specify 'typesdb => ...' in your collectd input (I looked for '#{@typesdb}')"
end
@logger.info("Using types.db", :typesdb => @typesdb.to_s)
end
@types = get_types(@typesdb)
if ([SECURITY_SIGN, SECURITY_ENCR].include?(@security_level))
if @authfile.nil?
raise "Security level is set to #{@security_level}, but no authfile was configured"
else
# Load OpenSSL and instantiate Digest and Crypto functions
require 'openssl'
@sha256 = OpenSSL::Digest::Digest.new('sha256')
@sha1 = OpenSSL::Digest::Digest.new('sha1')
@cipher = OpenSSL::Cipher.new('AES-256-OFB')
@auth = {}
parse_authfile
end
end
end # def register
public
def get_types(paths)
types = {}
# Get the typesdb
paths = Array(paths) # Make sure a single path is still forced into an array type
paths.each do |path|
@logger.info("Getting Collectd typesdb info", :typesdb => path.to_s)
File.open(path, 'r').each_line do |line|
typename, *line = line.strip.split
@logger.debug("typename", :typename => typename.to_s)
next if typename.nil? || typename[0,1] == '#'
types[typename] = line.collect { |l| l.strip.split(":")[0] }
end
end
@logger.debug("Collectd Types", :types => types.to_s)
return types
end # def get_types
def init_lambdas!
# Lambdas for hash + closure methodology
# This replaces when statements for fixed values and is much faster
string_decoder = lambda { |body| body.pack("C*")[0..-2] }
numeric_decoder = lambda { |body| body.slice!(0..7).pack("C*").unpack("E")[0] }
counter_decoder = lambda { |body| body.slice!(0..7).pack("C*").unpack("Q>")[0] }
gauge_decoder = lambda { |body| body.slice!(0..7).pack("C*").unpack("E")[0] }
derive_decoder = lambda { |body| body.slice!(0..7).pack("C*").unpack("q>")[0] }
# For Low-Resolution time
time_decoder = lambda do |body|
byte1, byte2 = body.pack("C*").unpack("NN")
Time.at(( ((byte1 << 32) + byte2))).utc
end
# Hi-Resolution time
hirestime_decoder = lambda do |body|
byte1, byte2 = body.pack("C*").unpack("NN")
Time.at(( ((byte1 << 32) + byte2) * (2**-30) )).utc
end
# Hi resolution intervals
hiresinterval_decoder = lambda do |body|
byte1, byte2 = body.pack("C*").unpack("NN")
Time.at(( ((byte1 << 32) + byte2) * (2**-30) )).to_i
end
# Value type decoder
value_type_decoder = lambda do |body|
body.slice!(0..1) # Prune the header
if body.length % 9 == 0 # Should be 9 fields
count = 0
retval = []
# Iterate through and take a slice each time
types = body.slice!(0..((body.length/9)-1))
while body.length > 0
# Use another hash + closure here...
v = @values_decoder[types[count]].call(body)
if types[count] == 1 && v.nan?
case @nan_handling
when 'drop'; drop = true
else
v = @nan_value
add_nan_tag = true
@nan_handling == 'warn' && @logger.warn("NaN replaced by #{@nan_value}")
end
end
retval << v
count += 1
end
else
@logger.error("Incorrect number of data fields for collectd record", :body => body.to_s)
end
return retval, drop, add_nan_tag
end
# Signature
signature_decoder = lambda do |body|
if body.length < 32
@logger.warning("SHA256 signature too small (got #{body.length} bytes instead of 32)")
elsif body.length < 33
@logger.warning("Received signature without username")
else
retval = []
# Byte 32 till the end contains the username as chars (=unsigned ints)
retval << body[32..-1].pack('C*')
# Byte 0 till 31 contain the signature
retval << body[0..31].pack('C*')
end
return retval
end
# Encryption
encryption_decoder = lambda do |body|
retval = []
user_length = (body.slice!(0) << 8) + body.slice!(0)
retval << body.slice!(0..user_length-1).pack('C*') # Username
retval << body.slice!(0..15).pack('C*') # IV
retval << body.pack('C*')
return retval
end
@id_decoder = {
0 => string_decoder,
1 => time_decoder,
2 => string_decoder,
3 => string_decoder,
4 => string_decoder,
5 => string_decoder,
6 => value_type_decoder,
7 => numeric_decoder,
8 => hirestime_decoder,
9 => hiresinterval_decoder,
256 => string_decoder,
257 => numeric_decoder,
512 => signature_decoder,
528 => encryption_decoder
}
# TYPE VALUES:
# 0: COUNTER
# 1: GAUGE
# 2: DERIVE
# 3: ABSOLUTE
@values_decoder = {
0 => counter_decoder,
1 => gauge_decoder,
2 => derive_decoder,
3 => counter_decoder
}
end # def init_lambdas!
public
def get_values(id, body)
drop = false
add_tag = false
if id == 6
retval, drop, add_nan_tag = @id_decoder[id].call(body)
# Use hash + closure/lambda to speed operations
else
retval = @id_decoder[id].call(body)
end
return retval, drop, add_nan_tag
end
private
def parse_authfile
# We keep the authfile parsed in memory so we don't have to open the file
# for every event.
@logger.debug("Parsing authfile #{@authfile}")
if !File.exist?(@authfile)
raise LogStash::ConfigurationError, "The file #{@authfile} was not found"
end
@auth.clear
@authmtime = File.stat(@authfile).mtime
File.readlines(@authfile).each do |line|
#line.chomp!
k,v = line.scan(AUTHFILEREGEX).flatten
if k && v
@logger.debug("Added authfile entry '#{k}' with key '#{v}'")
@auth[k] = v
else
@logger.info("Ignoring malformed authfile line '#{line.chomp}'")
end
end
end # def parse_authfile
private
def get_key(user)
return if @authmtime.nil? or @authfile.nil?
# Validate that our auth data is still up-to-date
parse_authfile if @authmtime < File.stat(@authfile).mtime
key = @auth[user]
@logger.warn("User #{user} is not found in the authfile #{@authfile}") if key.nil?
return key
end # def get_key
private
def verify_signature(user, signature, payload)
# The user doesn't care about the security
return true if @security_level == SECURITY_NONE
# We probably got and array of ints, pack it!
payload = payload.pack('C*') if payload.is_a?(Array)
key = get_key(user)
return false if key.nil?
return OpenSSL::HMAC.digest(@sha256, key, user+payload) == signature
end # def verify_signature
private
def decrypt_packet(user, iv, content)
# Content has to have at least a SHA1 hash (20 bytes), a header (4 bytes) and
# one byte of data
return [] if content.length < 26
content = content.pack('C*') if content.is_a?(Array)
key = get_key(user)
if key.nil?
@logger.debug("Key was nil")
return []
end
# Set the correct state of the cipher instance
@cipher.decrypt
@cipher.padding = 0
@cipher.iv = iv
@cipher.key = @sha256.digest(key);
# Decrypt the content
plaintext = @cipher.update(content) + @cipher.final
# Reset the state, as adding a new key to an already instantiated state
# results in an exception
@cipher.reset
# The plaintext contains a SHA1 hash as checksum in the first 160 bits
# (20 octets) of the rest of the data
hash = plaintext.slice!(0..19)
if @sha1.digest(plaintext) != hash
@logger.warn("Unable to decrypt packet, checksum mismatch")
return []
end
return plaintext.unpack('C*')
end # def decrypt_packet
public
def decode(payload)
payload = payload.bytes.to_a
collectd = {}
was_encrypted = false
while payload.length > 0 do
typenum = (payload.slice!(0) << 8) + payload.slice!(0)
# Get the length of the data in this part, but take into account that
# the header is 4 bytes
length = ((payload.slice!(0) << 8) + payload.slice!(0)) - 4
# Validate that the part length is correct
raise(HeaderError) if length > payload.length
body = payload.slice!(0..length-1)
field = TYPEMAP[typenum]
if field.nil?
@logger.warn("Unknown typenumber: #{typenum}")
next
end
values, drop, add_nan_tag = get_values(typenum, body)
case typenum
when SIGNATURE_TYPE
raise(EncryptionError) unless verify_signature(values[0], values[1], payload)
next
when ENCRYPTION_TYPE
payload = decrypt_packet(values[0], values[1], values[2])
raise(EncryptionError) if payload.empty?
was_encrypted = true
next
when PLUGIN_TYPE
# We've reached a new plugin, delete everything except for the the host
# field, because there's only one per packet and the timestamp field,
# because that one goes in front of the plugin
collectd.each_key do |k|
collectd.delete(k) unless PLUGIN_TYPE_FIELDS.has_key?(k)
end
when COLLECTD_TYPE
# We've reached a new type within the plugin section, delete all fields
# that could have something to do with the previous type (if any)
collectd.each_key do |k|
collectd.delete(k) unless COLLECTD_TYPE_FIELDS.has_key?(k)
end
end
raise(EncryptionError) if !was_encrypted and @security_level == SECURITY_ENCR
# Fill in the fields.
if values.is_a?(Array)
if values.length > 1 # Only do this iteration on multi-value arrays
values.each_with_index do |value, x|
begin
type = collectd['collectd_type']
key = @types[type]
key_x = key[x]
# assign
collectd[key_x] = value
rescue
@logger.error("Invalid value for type=#{type.inspect}, key=#{@types[type].inspect}, index=#{x}")
end
end
else # Otherwise it's a single value
collectd['value'] = values[0] # So name it 'value' accordingly
end
elsif field != nil # Not an array, make sure it's non-empty
collectd[field] = values # Append values to collectd under key field
end
if INTERVAL_VALUES_FIELDS.has_key?(field)
if ((@prune_intervals && !INTERVAL_TYPES.has_key?(typenum)) || !@prune_intervals)
# Prune these *specific* keys if they exist and are empty.
# This is better than looping over all keys every time.
collectd.delete('type_instance') if collectd['type_instance'] == ""
collectd.delete('plugin_instance') if collectd['plugin_instance'] == ""
if add_nan_tag
collectd['tags'] ||= []
collectd['tags'] << @nan_tag
end
# This ugly little shallow-copy hack keeps the new event from getting munged by the cleanup
# With pass-by-reference we get hosed (if we pass collectd, then clean it up rapidly, values can disappear)
if !drop # Drop the event if it's flagged true
yield LogStash::Event.new(collectd.dup)
else
raise(NaNError)
end
end
# Clean up the event
collectd.each_key do |k|
collectd.delete(k) if !INTERVAL_BASE_FIELDS.has_key?(k)
end
end
end # while payload.length > 0 do
rescue EncryptionError, ProtocolError, HeaderError, NaNError
# basically do nothing, we just want out
end # def decode
end # class LogStash::Codecs::Collectd

18
lib/logstash/codecs/dots.rb
View file

@ -1,18 +0,0 @@
# encoding: utf-8
require "logstash/codecs/base"
class LogStash::Codecs::Dots < LogStash::Codecs::Base
config_name "dots"
milestone 1
public
def decode(data)
raise "Not implemented"
end # def decode
public
def encode(data)
@on_event.call(".")
end # def encode
end # class LogStash::Codecs::Dots

34
lib/logstash/codecs/edn.rb
View file

@ -1,34 +0,0 @@
require "logstash/codecs/base"
require "logstash/codecs/line"
require "logstash/util"
class LogStash::Codecs::EDN < LogStash::Codecs::Base
config_name "edn"
milestone 1
def register
require "edn"
end
public
def decode(data)
begin
yield LogStash::Event.new(EDN.read(data))
rescue => e
@logger.warn("EDN parse failure. Falling back to plain-text", :error => e, :data => data)
yield LogStash::Event.new("message" => data)
end
end
public
def encode(event)
# use normalize to make sure returned Hash is pure Ruby
# #to_edn which relies on pure Ruby object recognition
data = LogStash::Util.normalize(event.to_hash)
# timestamp is serialized as a iso8601 string
# merge to avoid modifying data which could have side effects if multiple outputs
@on_event.call(data.merge(LogStash::Event::TIMESTAMP => event.timestamp.to_iso8601).to_edn)
end
end

42
lib/logstash/codecs/edn_lines.rb
View file

@ -1,42 +0,0 @@
require "logstash/codecs/base"
require "logstash/codecs/line"
require "logstash/util"
class LogStash::Codecs::EDNLines < LogStash::Codecs::Base
config_name "edn_lines"
milestone 1
def register
require "edn"
end
public
def initialize(params={})
super(params)
@lines = LogStash::Codecs::Line.new
end
public
def decode(data)
@lines.decode(data) do |event|
begin
yield LogStash::Event.new(EDN.read(event["message"]))
rescue => e
@logger.warn("EDN parse failure. Falling back to plain-text", :error => e, :data => data)
yield LogStash::Event.new("message" => data)
end
end
end
public
def encode(event)
# use normalize to make sure returned Hash is pure Ruby for
# #to_edn which relies on pure Ruby object recognition
data = LogStash::Util.normalize(event.to_hash)
# timestamp is serialized as a iso8601 string
# merge to avoid modifying data which could have side effects if multiple outputs
@on_event.call(data.merge(LogStash::Event::TIMESTAMP => event.timestamp.to_iso8601).to_edn + NL)
end
end

63
lib/logstash/codecs/fluent.rb
View file

@ -1,63 +0,0 @@
# encoding: utf-8
require "logstash/codecs/base"
require "logstash/util/charset"
require "logstash/timestamp"
require "logstash/util"
# This codec handles fluentd's msgpack schema.
#
# For example, you can receive logs from fluent-logger-ruby with:
#
# input {
# tcp {
# codec => fluent
# port => 4000
# }
# }
#
# And from your ruby code in your own application:
#
# logger = Fluent::Logger::FluentLogger.new(nil, :host => "example.log", :port => 4000)
# logger.post("some_tag", { "your" => "data", "here" => "yay!" })
#
# Notes:
#
# * the fluent uses a second-precision time for events, so you will never see
# subsecond precision on events processed by this codec.
#
class LogStash::Codecs::Fluent < LogStash::Codecs::Base
config_name "fluent"
milestone 1
public
def register
require "msgpack"
@decoder = MessagePack::Unpacker.new
end
public
def decode(data)
@decoder.feed(data)
@decoder.each do |tag, epochtime, map|
event = LogStash::Event.new(map.merge(
LogStash::Event::TIMESTAMP => LogStash::Timestamp.at(epochtime),
"tags" => tag
))
yield event
end
end # def decode
public
def encode(event)
tag = event["tags"] || "log"
epochtime = event.timestamp.to_i
# use normalize to make sure returned Hash is pure Ruby for
# MessagePack#pack which relies on pure Ruby object recognition
data = LogStash::Util.normalize(event.to_hash)
# timestamp is serialized as a iso8601 string
# merge to avoid modifying data which could have side effects if multiple outputs
@on_event.call(MessagePack.pack([tag, epochtime, data.merge(LogStash::Event::TIMESTAMP => event.timestamp.to_iso8601)]))
end # def encode
end # class LogStash::Codecs::Fluent

103
lib/logstash/codecs/graphite.rb
View file

@ -1,103 +0,0 @@
# encoding: utf-8
require "logstash/codecs/base"
require "logstash/codecs/line"
require "logstash/timestamp"
# This codec will encode and decode Graphite formated lines.
class LogStash::Codecs::Graphite < LogStash::Codecs::Base
config_name "graphite"
milestone 2
EXCLUDE_ALWAYS = [ "@timestamp", "@version" ]
DEFAULT_METRICS_FORMAT = "*"
METRIC_PLACEHOLDER = "*"
# The metric(s) to use. This supports dynamic strings like %{host}
# for metric names and also for values. This is a hash field with key
# of the metric name, value of the metric value. Example:
#
# [ "%{host}/uptime", "%{uptime_1m}" ]
#
# The value will be coerced to a floating point value. Values which cannot be
# coerced will zero (0)
config :metrics, :validate => :hash, :default => {}
# Indicate that the event @fields should be treated as metrics and will be sent as is to graphite
config :fields_are_metrics, :validate => :boolean, :default => false
# Include only regex matched metric names
config :include_metrics, :validate => :array, :default => [ ".*" ]
# Exclude regex matched metric names, by default exclude unresolved %{field} strings
config :exclude_metrics, :validate => :array, :default => [ "%\{[^}]+\}" ]
# Defines format of the metric string. The placeholder '*' will be
# replaced with the name of the actual metric.
#
# metrics_format => "foo.bar.*.sum"
#
# NOTE: If no metrics_format is defined the name of the metric will be used as fallback.
config :metrics_format, :validate => :string, :default => DEFAULT_METRICS_FORMAT
public
def initialize(params={})
super(params)
@lines = LogStash::Codecs::Line.new
end
public
def decode(data)
@lines.decode(data) do |event|
name, value, time = event["message"].split(" ")
yield LogStash::Event.new(name => value.to_f, LogStash::Event::TIMESTAMP => LogStash::Timestamp.at(time.to_i))
end # @lines.decode
end # def decode
private
def construct_metric_name(metric)
if @metrics_format
return @metrics_format.gsub(METRIC_PLACEHOLDER, metric)
end
return metric
end
public
def encode(event)
# Graphite message format: metric value timestamp\n
messages = []
timestamp = event.sprintf("%{+%s}")
if @fields_are_metrics
@logger.debug("got metrics event", :metrics => event.to_hash)
event.to_hash.each do |metric,value|
next if EXCLUDE_ALWAYS.include?(metric)
next unless @include_metrics.empty? || @include_metrics.any? { |regexp| metric.match(regexp) }
next if @exclude_metrics.any? {|regexp| metric.match(regexp)}
messages << "#{construct_metric_name(metric)} #{event.sprintf(value.to_s).to_f} #{timestamp}"
end # data.to_hash.each
else
@metrics.each do |metric, value|
@logger.debug("processing", :metric => metric, :value => value)
metric = event.sprintf(metric)
next unless @include_metrics.any? {|regexp| metric.match(regexp)}
next if @exclude_metrics.any? {|regexp| metric.match(regexp)}
messages << "#{construct_metric_name(event.sprintf(metric))} #{event.sprintf(value).to_f} #{timestamp}"
end # @metrics.each
end # if @fields_are_metrics
if messages.empty?
@logger.debug("Message is empty, not emiting anything.", :messages => messages)
else
message = messages.join(NL) + NL
@logger.debug("Emiting carbon messages", :messages => messages)
@on_event.call(message)
end # if messages.empty?
end # def encode
end # class LogStash::Codecs::Graphite

48
lib/logstash/codecs/json.rb
View file

@ -1,48 +0,0 @@
# encoding: utf-8
require "logstash/codecs/base"
require "logstash/util/charset"
require "logstash/json"
# This codec may be used to decode (via inputs) and encode (via outputs)
# full JSON messages. If you are streaming JSON messages delimited
# by '\n' then see the `json_lines` codec.
# Encoding will result in a single JSON string.
class LogStash::Codecs::JSON < LogStash::Codecs::Base
config_name "json"
milestone 3
# The character encoding used in this codec. Examples include "UTF-8" and
# "CP1252".
#
# JSON requires valid UTF-8 strings, but in some cases, software that
# emits JSON does so in another encoding (nxlog, for example). In
# weird cases like this, you can set the `charset` setting to the
# actual encoding of the text and Logstash will convert it for you.
#
# For nxlog users, you'll want to set this to "CP1252".
config :charset, :validate => ::Encoding.name_list, :default => "UTF-8"
public
def register
@converter = LogStash::Util::Charset.new(@charset)
@converter.logger = @logger
end
public
def decode(data)
data = @converter.convert(data)
begin
yield LogStash::Event.new(LogStash::Json.load(data))
rescue LogStash::Json::ParserError => e
@logger.info("JSON parse failure. Falling back to plain-text", :error => e, :data => data)
yield LogStash::Event.new("message" => data)
end
end # def decode
public
def encode(event)
@on_event.call(event.to_json)
end # def encode
end # class LogStash::Codecs::JSON

53
lib/logstash/codecs/json_lines.rb
View file

@ -1,53 +0,0 @@
# encoding: utf-8
require "logstash/codecs/base"
require "logstash/codecs/line"
require "logstash/json"
# This codec will decode streamed JSON that is newline delimited.
# For decoding line-oriented JSON payload in the redis or file inputs,
# for example, use the json codec instead.
# Encoding will emit a single JSON string ending in a '\n'
class LogStash::Codecs::JSONLines < LogStash::Codecs::Base
config_name "json_lines"
milestone 3
# The character encoding used in this codec. Examples include "UTF-8" and
# "CP1252"
#
# JSON requires valid UTF-8 strings, but in some cases, software that
# emits JSON does so in another encoding (nxlog, for example). In
# weird cases like this, you can set the charset setting to the
# actual encoding of the text and logstash will convert it for you.
#
# For nxlog users, you'll want to set this to "CP1252"
config :charset, :validate => ::Encoding.name_list, :default => "UTF-8"
public
def initialize(params={})
super(params)
@lines = LogStash::Codecs::Line.new
@lines.charset = @charset
end
public
def decode(data)
@lines.decode(data) do |event|
begin
yield LogStash::Event.new(LogStash::Json.load(event["message"]))
rescue LogStash::Json::ParserError => e
@logger.info("JSON parse failure. Falling back to plain-text", :error => e, :data => data)
yield LogStash::Event.new("message" => event["message"])
end
end
end # def decode
public
def encode(event)
# Tack on a \n for now because previously most of logstash's JSON
# outputs emitted one per line, and whitespace is OK in json.
@on_event.call(event.to_json + NL)
end # def encode
end # class LogStash::Codecs::JSON

28
lib/logstash/codecs/json_spooler.rb
View file

@ -1,28 +0,0 @@
# encoding: utf-8
require "logstash/codecs/base"
require "logstash/codecs/spool"
require "logstash/json"
# This is the base class for logstash codecs.
class LogStash::Codecs::JsonSpooler < LogStash::Codecs::Spool
config_name "json_spooler"
milestone 0
public
def register
@logger.error("the json_spooler codec is deprecated and will be removed in a future release")
end
public
def decode(data)
super(LogStash::Json.load(data.force_encoding(Encoding::UTF_8))) do |event|
yield event
end
end # def decode
public
def encode(event)
super(event)
end # def encode
end # class LogStash::Codecs::Json

58
lib/logstash/codecs/line.rb
View file

@ -1,58 +0,0 @@
# encoding: utf-8
require "logstash/codecs/base"
require "logstash/util/charset"
# Line-oriented text data.
#
# Decoding behavior: Only whole line events will be emitted.
#
# Encoding behavior: Each event will be emitted with a trailing newline.
class LogStash::Codecs::Line < LogStash::Codecs::Base
config_name "line"
milestone 3
# Set the desired text format for encoding.
config :format, :validate => :string
# The character encoding used in this input. Examples include "UTF-8"
# and "cp1252"
#
# This setting is useful if your log files are in Latin-1 (aka cp1252)
# or in another character set other than UTF-8.
#
# This only affects "plain" format logs since json is UTF-8 already.
config :charset, :validate => ::Encoding.name_list, :default => "UTF-8"
public
def register
require "logstash/util/buftok"
@buffer = FileWatch::BufferedTokenizer.new
@converter = LogStash::Util::Charset.new(@charset)
@converter.logger = @logger
end
public
def decode(data)
@buffer.extract(data).each do |line|
yield LogStash::Event.new("message" => @converter.convert(line))
end
end # def decode
public
def flush(&block)
remainder = @buffer.flush
if !remainder.empty?
block.call(LogStash::Event.new("message" => @converter.convert(remainder)))
end
end
public
def encode(event)
if event.is_a? LogStash::Event and @format
@on_event.call(event.sprintf(@format) + NL)
else
@on_event.call(event.to_s + NL)
end
end # def encode
end # class LogStash::Codecs::Plain

48
lib/logstash/codecs/msgpack.rb
View file

@ -1,48 +0,0 @@
# encoding: utf-8
require "logstash/codecs/base"
require "logstash/timestamp"
require "logstash/util"
class LogStash::Codecs::Msgpack < LogStash::Codecs::Base
config_name "msgpack"
milestone 1
config :format, :validate => :string, :default => nil
public
def register
require "msgpack"
end
public
def decode(data)
begin
# Msgpack does not care about UTF-8
event = LogStash::Event.new(MessagePack.unpack(data))
event["tags"] ||= []
if @format
event["message"] ||= event.sprintf(@format)
end
rescue => e
# Treat as plain text and try to do the best we can with it?
@logger.warn("Trouble parsing msgpack input, falling back to plain text",
:input => data, :exception => e)
event["message"] = data
event["tags"] ||= []
event["tags"] << "_msgpackparsefailure"
end
yield event
end # def decode
public
def encode(event)
# use normalize to make sure returned Hash is pure Ruby for
# MessagePack#pack which relies on pure Ruby object recognition
data = LogStash::Util.normalize(event.to_hash)
# timestamp is serialized as a iso8601 string
# merge to avoid modifying data which could have side effects if multiple outputs
@on_event.call(MessagePack.pack(data.merge(LogStash::Event::TIMESTAMP => event.timestamp.to_iso8601)))
end # def encode
end # class LogStash::Codecs::Msgpack

194
lib/logstash/codecs/multiline.rb
View file

@ -1,194 +0,0 @@
# encoding: utf-8
require "logstash/codecs/base"
require "logstash/util/charset"
require "logstash/timestamp"
# The multiline codec will collapse multiline messages and merge them into a
# single event.
#
# The original goal of this codec was to allow joining of multiline messages
# from files into a single event. For example, joining Java exception and
# stacktrace messages into a single event.
#
# The config looks like this:
#
# input {
# stdin {
# codec => multiline {
# pattern => "pattern, a regexp"
# negate => "true" or "false"
# what => "previous" or "next"
# }
# }
# }
#
# The `pattern` should match what you believe to be an indicator that the field
# is part of a multi-line event.
#
# The `what` must be "previous" or "next" and indicates the relation
# to the multi-line event.
#
# The `negate` can be "true" or "false" (defaults to "false"). If "true", a
# message not matching the pattern will constitute a match of the multiline
# filter and the `what` will be applied. (vice-versa is also true)
#
# For example, Java stack traces are multiline and usually have the message
# starting at the far-left, with each subsequent line indented. Do this:
#
# input {
# stdin {
# codec => multiline {
# pattern => "^\s"
# what => "previous"
# }
# }
# }
#
# This says that any line starting with whitespace belongs to the previous line.
#
# Another example is to merge lines not starting with a date up to the previous
# line..
#
# input {
# file {
# path => "/var/log/someapp.log"
# codec => multiline {
# # Grok pattern names are valid! :)
# pattern => "^%{TIMESTAMP_ISO8601} "
# negate => true
# what => previous
# }
# }
# }
#
# This says that any line not starting with a timestamp should be merged with the previous line.
#
# One more common example is C line continuations (backslash). Here's how to do that:
#
# filter {
# multiline {
# type => "somefiletype"
# pattern => "\\$"
# what => "next"
# }
# }
#
# This says that any line ending with a backslash should be combined with the
# following line.
#
class LogStash::Codecs::Multiline < LogStash::Codecs::Base
config_name "multiline"
milestone 3
# The regular expression to match.
config :pattern, :validate => :string, :required => true
# If the pattern matched, does event belong to the next or previous event?
config :what, :validate => ["previous", "next"], :required => true
# Negate the regexp pattern ('if not matched').
config :negate, :validate => :boolean, :default => false
# Logstash ships by default with a bunch of patterns, so you don't
# necessarily need to define this yourself unless you are adding additional
# patterns.
#
# Pattern files are plain text with format:
#
# NAME PATTERN
#
# For example:
#
# NUMBER \d+
config :patterns_dir, :validate => :array, :default => []
# The character encoding used in this input. Examples include "UTF-8"
# and "cp1252"
#
# This setting is useful if your log files are in Latin-1 (aka cp1252)
# or in another character set other than UTF-8.
#
# This only affects "plain" format logs since JSON is UTF-8 already.
config :charset, :validate => ::Encoding.name_list, :default => "UTF-8"
# Tag multiline events with a given tag. This tag will only be added
# to events that actually have multiple lines in them.
config :multiline_tag, :validate => :string, :default => "multiline"
public
def register
require "grok-pure" # rubygem 'jls-grok'
# Detect if we are running from a jarfile, pick the right path.
patterns_path = []
patterns_path += ["#{File.dirname(__FILE__)}/../../../patterns/*"]
@grok = Grok.new
@patterns_dir = patterns_path.to_a + @patterns_dir
@patterns_dir.each do |path|
if File.directory?(path)
path = File.join(path, "*")
end
Dir.glob(path).each do |file|
@logger.info("Grok loading patterns from file", :path => file)
@grok.add_patterns_from_file(file)
end
end
@grok.compile(@pattern)
@logger.debug("Registered multiline plugin", :type => @type, :config => @config)
@buffer = []
@handler = method("do_#{@what}".to_sym)
@converter = LogStash::Util::Charset.new(@charset)
@converter.logger = @logger
end # def register
public
def decode(text, &block)
text = @converter.convert(text)
match = @grok.match(text)
@logger.debug("Multiline", :pattern => @pattern, :text => text,
:match => !match.nil?, :negate => @negate)
# Add negate option
match = (match and !@negate) || (!match and @negate)
@handler.call(text, match, &block)
end # def decode
def buffer(text)
@time = LogStash::Timestamp.now if @buffer.empty?
@buffer << text
end
def flush(&block)
if @buffer.any?
event = LogStash::Event.new(LogStash::Event::TIMESTAMP => @time, "message" => @buffer.join(NL))
# Tag multiline events
event.tag @multiline_tag if @multiline_tag && @buffer.size > 1
yield event
@buffer = []
end
end
def do_next(text, matched, &block)
buffer(text)
flush(&block) if !matched
end
def do_previous(text, matched, &block)
flush(&block) if !matched
buffer(text)
end
public
def encode(event)
# Nothing to do.
@on_event.call(event)
end # def encode
end # class LogStash::Codecs::Plain

263
lib/logstash/codecs/netflow.rb
View file

@ -1,263 +0,0 @@
# encoding: utf-8
require "logstash/filters/base"
require "logstash/namespace"
require "logstash/timestamp"
# The "netflow" codec is for decoding Netflow v5/v9 flows.
class LogStash::Codecs::Netflow < LogStash::Codecs::Base
config_name "netflow"
milestone 1
# Netflow v9 template cache TTL (minutes)
config :cache_ttl, :validate => :number, :default => 4000
# Specify into what field you want the Netflow data.
config :target, :validate => :string, :default => "netflow"
# Specify which Netflow versions you will accept.
config :versions, :validate => :array, :default => [5, 9]
# Override YAML file containing Netflow field definitions
#
# Each Netflow field is defined like so:
#
# ---
# id:
# - default length in bytes
# - :name
# id:
# - :uintN or :ip4_addr or :ip6_addr or :mac_addr or :string
# - :name
# id:
# - :skip
#
# See <https://github.com/logstash/logstash/tree/v%VERSION%/lib/logstash/codecs/netflow/netflow.yaml> for the base set.
config :definitions, :validate => :path
public
def initialize(params={})
super(params)
@threadsafe = false
end
public
def register
require "logstash/codecs/netflow/util"
@templates = Vash.new()
# Path to default Netflow v9 field definitions
filename = LogStash::Environment.plugin_path("codecs/netflow/netflow.yaml")
begin
@fields = YAML.load_file(filename)
rescue Exception => e
raise "#{self.class.name}: Bad syntax in definitions file #{filename}"
end
# Allow the user to augment/override/rename the supported Netflow fields
if @definitions
raise "#{self.class.name}: definitions file #{@definitions} does not exists" unless File.exists?(@definitions)
begin
@fields.merge!(YAML.load_file(@definitions))
rescue Exception => e
raise "#{self.class.name}: Bad syntax in definitions file #{@definitions}"
end
end
end # def register
public
def decode(payload, &block)
header = Header.read(payload)
unless @versions.include?(header.version)
@logger.warn("Ignoring Netflow version v#{header.version}")
return
end
if header.version == 5
flowset = Netflow5PDU.read(payload)
elsif header.version == 9
flowset = Netflow9PDU.read(payload)
else
@logger.warn("Unsupported Netflow version v#{header.version}")
return
end
flowset.records.each do |record|
if flowset.version == 5
event = LogStash::Event.new
# FIXME Probably not doing this right WRT JRuby?
#
# The flowset header gives us the UTC epoch seconds along with
# residual nanoseconds so we can set @timestamp to that easily
event.timestamp = LogStash::Timestamp.at(flowset.unix_sec, flowset.unix_nsec / 1000)
event[@target] = {}
# Copy some of the pertinent fields in the header to the event
['version', 'flow_seq_num', 'engine_type', 'engine_id', 'sampling_algorithm', 'sampling_interval', 'flow_records'].each do |f|
event[@target][f] = flowset[f]
end
# Create fields in the event from each field in the flow record
record.each_pair do |k,v|
case k.to_s
when /_switched$/
# The flow record sets the first and last times to the device
# uptime in milliseconds. Given the actual uptime is provided
# in the flowset header along with the epoch seconds we can
# convert these into absolute times
millis = flowset.uptime - v
seconds = flowset.unix_sec - (millis / 1000)
micros = (flowset.unix_nsec / 1000) - (millis % 1000)
if micros < 0
seconds--
micros += 1000000
end
# FIXME Again, probably doing this wrong WRT JRuby?
event[@target][k.to_s] = Time.at(seconds, micros).utc.strftime("%Y-%m-%dT%H:%M:%S.%3NZ")
else
event[@target][k.to_s] = v
end
end
yield event
elsif flowset.version == 9
case record.flowset_id
when 0
# Template flowset
record.flowset_data.templates.each do |template|
catch (:field) do
fields = []
template.fields.each do |field|
entry = netflow_field_for(field.field_type, field.field_length)
if ! entry
throw :field
end
fields += entry
end
# We get this far, we have a list of fields
#key = "#{flowset.source_id}|#{event["source"]}|#{template.template_id}"
key = "#{flowset.source_id}|#{template.template_id}"
@templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
# Purge any expired templates
@templates.cleanup!
end
end
when 1
# Options template flowset
record.flowset_data.templates.each do |template|
catch (:field) do
fields = []
template.option_fields.each do |field|
entry = netflow_field_for(field.field_type, field.field_length)
if ! entry
throw :field
end
fields += entry
end
# We get this far, we have a list of fields
#key = "#{flowset.source_id}|#{event["source"]}|#{template.template_id}"
key = "#{flowset.source_id}|#{template.template_id}"
@templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
# Purge any expired templates
@templates.cleanup!
end
end
when 256..65535
# Data flowset
#key = "#{flowset.source_id}|#{event["source"]}|#{record.flowset_id}"
key = "#{flowset.source_id}|#{record.flowset_id}"
template = @templates[key]
if ! template
#@logger.warn("No matching template for flow id #{record.flowset_id} from #{event["source"]}")
@logger.warn("No matching template for flow id #{record.flowset_id}")
next
end
length = record.flowset_length - 4
# Template shouldn't be longer than the record and there should
# be at most 3 padding bytes
if template.num_bytes > length or ! (length % template.num_bytes).between?(0, 3)
@logger.warn("Template length doesn't fit cleanly into flowset", :template_id => record.flowset_id, :template_length => template.num_bytes, :record_length => length)
next
end
array = BinData::Array.new(:type => template, :initial_length => length / template.num_bytes)
records = array.read(record.flowset_data)
records.each do |r|
event = LogStash::Event.new(
LogStash::Event::TIMESTAMP => LogStash::Timestamp.at(flowset.unix_sec),
@target => {}
)
# Fewer fields in the v9 header
['version', 'flow_seq_num'].each do |f|
event[@target][f] = flowset[f]
end
event[@target]['flowset_id'] = record.flowset_id
r.each_pair do |k,v|
case k.to_s
when /_switched$/
millis = flowset.uptime - v
seconds = flowset.unix_sec - (millis / 1000)
# v9 did away with the nanosecs field
micros = 1000000 - (millis % 1000)
event[@target][k.to_s] = Time.at(seconds, micros).utc.strftime("%Y-%m-%dT%H:%M:%S.%3NZ")
else
event[@target][k.to_s] = v
end
end
yield event
end
else
@logger.warn("Unsupported flowset id #{record.flowset_id}")
end
end
end
end # def filter
private
def uint_field(length, default)
# If length is 4, return :uint32, etc. and use default if length is 0
("uint" + (((length > 0) ? length : default) * 8).to_s).to_sym
end # def uint_field
private
def netflow_field_for(type, length)
if @fields.include?(type)
field = @fields[type]
if field.is_a?(Array)
if field[0].is_a?(Integer)
field[0] = uint_field(length, field[0])
end
# Small bit of fixup for skip or string field types where the length
# is dynamic
case field[0]
when :skip
field += [nil, {:length => length}]
when :string
field += [{:length => length, :trim_padding => true}]
end
@logger.debug("Definition complete", :field => field)
[field]
else
@logger.warn("Definition should be an array", :field => field)
nil
end
else
@logger.warn("Unsupported field", :type => type, :length => length)
nil
end
end # def netflow_field_for
end # class LogStash::Filters::Netflow

215
lib/logstash/codecs/netflow/netflow.yaml
View file

@ -1,215 +0,0 @@
---
1:
- 4
- :in_bytes
2:
- 4
- :in_pkts
3:
- 4
- :flows
4:
- :uint8
- :protocol
5:
- :uint8
- :src_tos
6:
- :uint8
- :tcp_flags
7:
- :uint16
- :l4_src_port
8:
- :ip4_addr
- :ipv4_src_addr
9:
- :uint8
- :src_mask
10:
- 2
- :input_snmp
11:
- :uint16
- :l4_dst_port
12:
- :ip4_addr
- :ipv4_dst_addr
13:
- :uint8
- :dst_mask
14:
- 2
- :output_snmp
15:
- :ip4_addr
- :ipv4_next_hop
16:
- 2
- :src_as
17:
- 2
- :dst_as
18:
- :ip4_addr
- :bgp_ipv4_next_hop
19:
- 4
- :mul_dst_pkts
20:
- 4
- :mul_dst_bytes
21:
- :uint32
- :last_switched
22:
- :uint32
- :first_switched
23:
- 4
- :out_bytes
24:
- 4
- :out_pkts
25:
- :uint16
- :min_pkt_length
26:
- :uint16
- :max_pkt_length
27:
- :ip6_addr
- :ipv6_src_addr
28:
- :ip6_addr
- :ipv6_dst_addr
29:
- :uint8
- :ipv6_src_mask
30:
- :uint8
- :ipv6_dst_mask
31:
- :uint24
- :ipv6_flow_label
32:
- :uint16
- :icmp_type
33:
- :uint8
- :mul_igmp_type
34:
- :uint32
- :sampling_interval
35:
- :uint8
- :sampling_algorithm
36:
- :uint16
- :flow_active_timeout
37:
- :uint16
- :flow_inactive_timeout
38:
- :uint8
- :engine_type
39:
- :uint8
- :engine_id
40:
- 4
- :total_bytes_exp
41:
- 4
- :total_pkts_exp
42:
- 4
- :total_flows_exp
43:
- :skip
44:
- :ip4_addr
- :ipv4_src_prefix
45:
- :ip4_addr
- :ipv4_dst_prefix
46:
- :uint8
- :mpls_top_label_type
47:
- :uint32
- :mpls_top_label_ip_addr
48:
- 4
- :flow_sampler_id
49:
- :uint8
- :flow_sampler_mode
50:
- :uint32
- :flow_sampler_random_interval
51:
- :skip
52:
- :uint8
- :min_ttl
53:
- :uint8
- :max_ttl
54:
- :uint16
- :ipv4_ident
55:
- :uint8
- :dst_tos
56:
- :mac_addr
- :in_src_max
57:
- :mac_addr
- :out_dst_max
58:
- :uint16
- :src_vlan
59:
- :uint16
- :dst_vlan
60:
- :uint8
- :ip_protocol_version
61:
- :uint8
- :direction
62:
- :ip6_addr
- :ipv6_next_hop
63:
- :ip6_addr
- :bgp_ipv6_next_hop
64:
- :uint32
- :ipv6_option_headers
64:
- :skip
65:
- :skip
66:
- :skip
67:
- :skip
68:
- :skip
69:
- :skip
80:
- :mac_addr
- :in_dst_mac
81:
- :mac_addr
- :out_src_mac
82:
- :string
- :if_name
83:
- :string
- :if_desc

212
lib/logstash/codecs/netflow/util.rb
View file

@ -1,212 +0,0 @@
# encoding: utf-8
require "bindata"
require "ipaddr"
class IP4Addr < BinData::Primitive
endian :big
uint32 :storage
def set(val)
ip = IPAddr.new(val)
if ! ip.ipv4?
raise ArgumentError, "invalid IPv4 address '#{val}'"
end
self.storage = ip.to_i
end
def get
IPAddr.new_ntoh([self.storage].pack('N')).to_s
end
end
class IP6Addr < BinData::Primitive
endian :big
uint128 :storage
def set(val)
ip = IPAddr.new(val)
if ! ip.ipv6?
raise ArgumentError, "invalid IPv6 address `#{val}'"
end
self.storage = ip.to_i
end
def get
IPAddr.new_ntoh((0..7).map { |i|
(self.storage >> (112 - 16 * i)) & 0xffff
}.pack('n8')).to_s
end
end
class MacAddr < BinData::Primitive
array :bytes, :type => :uint8, :initial_length => 6
def set(val)
ints = val.split(/:/).collect { |int| int.to_i(16) }
self.bytes = ints
end
def get
self.bytes.collect { |byte| byte.to_s(16) }.join(":")
end
end
class Header < BinData::Record
endian :big
uint16 :version
end
class Netflow5PDU < BinData::Record
endian :big
uint16 :version
uint16 :flow_records
uint32 :uptime
uint32 :unix_sec
uint32 :unix_nsec
uint32 :flow_seq_num
uint8 :engine_type
uint8 :engine_id
bit2 :sampling_algorithm
bit14 :sampling_interval
array :records, :initial_length => :flow_records do
ip4_addr :ipv4_src_addr
ip4_addr :ipv4_dst_addr
ip4_addr :ipv4_next_hop
uint16 :input_snmp
uint16 :output_snmp
uint32 :in_pkts
uint32 :in_bytes
uint32 :first_switched
uint32 :last_switched
uint16 :l4_src_port
uint16 :l4_dst_port
skip :length => 1
uint8 :tcp_flags # Split up the TCP flags maybe?
uint8 :protocol
uint8 :src_tos
uint16 :src_as
uint16 :dst_as
uint8 :src_mask
uint8 :dst_mask
skip :length => 2
end
end
class TemplateFlowset < BinData::Record
endian :big
array :templates, :read_until => lambda { array.num_bytes == flowset_length - 4 } do
uint16 :template_id
uint16 :field_count
array :fields, :initial_length => :field_count do
uint16 :field_type
uint16 :field_length
end
end
end
class OptionFlowset < BinData::Record
endian :big
array :templates, :read_until => lambda { flowset_length - 4 - array.num_bytes <= 2 } do
uint16 :template_id
uint16 :scope_length
uint16 :option_length
array :scope_fields, :initial_length => lambda { scope_length / 4 } do
uint16 :field_type
uint16 :field_length
end
array :option_fields, :initial_length => lambda { option_length / 4 } do
uint16 :field_type
uint16 :field_length
end
end
skip :length => lambda { templates.length.odd? ? 2 : 0 }
end
class Netflow9PDU < BinData::Record
endian :big
uint16 :version
uint16 :flow_records
uint32 :uptime
uint32 :unix_sec
uint32 :flow_seq_num
uint32 :source_id
array :records, :read_until => :eof do
uint16 :flowset_id
uint16 :flowset_length
choice :flowset_data, :selection => :flowset_id do
template_flowset 0
option_flowset 1
string :default, :read_length => lambda { flowset_length - 4 }
end
end
end
# https://gist.github.com/joshaven/184837
class Vash < Hash
def initialize(constructor = {})
@register ||= {}
if constructor.is_a?(Hash)
super()
merge(constructor)
else
super(constructor)
end
end
alias_method :regular_writer, :[]= unless method_defined?(:regular_writer)
alias_method :regular_reader, :[] unless method_defined?(:regular_reader)
def [](key)
sterilize(key)
clear(key) if expired?(key)
regular_reader(key)
end
def []=(key, *args)
if args.length == 2
value, ttl = args[1], args[0]
elsif args.length == 1
value, ttl = args[0], 60
else
raise ArgumentError, "Wrong number of arguments, expected 2 or 3, received: #{args.length+1}\n"+
"Example Usage: volatile_hash[:key]=value OR volatile_hash[:key, ttl]=value"
end
sterilize(key)
ttl(key, ttl)
regular_writer(key, value)
end
def merge(hsh)
hsh.map {|key,value| self[sterile(key)] = hsh[key]}
self
end
def cleanup!
now = Time.now.to_i
@register.map {|k,v| clear(k) if v < now}
end
def clear(key)
sterilize(key)
@register.delete key
self.delete key
end
private
def expired?(key)
Time.now.to_i > @register[key].to_i
end
def ttl(key, secs=60)
@register[key] = Time.now.to_i + secs.to_i
end
def sterile(key)
String === key ? key.chomp('!').chomp('=') : key.to_s.chomp('!').chomp('=').to_sym
end
def sterilize(key)
key = sterile(key)
end
end

19
lib/logstash/codecs/noop.rb
View file

@ -1,19 +0,0 @@
# encoding: utf-8
require "logstash/codecs/base"
class LogStash::Codecs::Noop < LogStash::Codecs::Base
config_name "noop"
milestone 1
public
def decode(data)
yield data
end # def decode
public
def encode(event)
@on_event.call event
end # def encode
end # class LogStash::Codecs::Noop

57
lib/logstash/codecs/oldlogstashjson.rb
View file

@ -1,57 +0,0 @@
# encoding: utf-8
require "logstash/codecs/base"
require "logstash/json"
class LogStash::Codecs::OldLogStashJSON < LogStash::Codecs::Base
config_name "oldlogstashjson"
milestone 2
# Map from v0 name to v1 name.
# Note: @source is gone and has no similar field.
V0_TO_V1 = {"@timestamp" => "@timestamp", "@message" => "message",
"@tags" => "tags", "@type" => "type",
"@source_host" => "host", "@source_path" => "path"}
public
def decode(data)
begin
obj = LogStash::Json.load(data.force_encoding(Encoding::UTF_8))
rescue LogStash::Json::ParserError => e
@logger.info("JSON parse failure. Falling back to plain-text", :error => e, :data => data)
yield LogStash::Event.new("message" => data)
return
end
h = {}
# Convert the old logstash schema to the new one.
V0_TO_V1.each do |key, val|
h[val] = obj[key] if obj.include?(key)
end
h.merge!(obj["@fields"]) if obj["@fields"].is_a?(Hash)
yield LogStash::Event.new(h)
end # def decode
public
def encode(event)
h = {}
# Convert the new logstash schema to the old one.
V0_TO_V1.each do |key, val|
h[key] = event[val] if event.include?(val)
end
event.to_hash.each do |field, val|
# TODO: might be better to V1_TO_V0 = V0_TO_V1.invert during
# initialization than V0_TO_V1.has_value? within loop
next if field == "@version" or V0_TO_V1.has_value?(field)
h["@fields"] ||= {}
h["@fields"][field] = val
end
# Tack on a \n because JSON outputs 1.1.x had them.
@on_event.call(LogStash::Json.dump(h) + NL)
end # def encode
end # class LogStash::Codecs::OldLogStashJSON

48
lib/logstash/codecs/plain.rb
View file

@ -1,48 +0,0 @@
# encoding: utf-8
require "logstash/codecs/base"
require "logstash/util/charset"
# The "plain" codec is for plain text with no delimiting between events.
#
# This is mainly useful on inputs and outputs that already have a defined
# framing in their transport protocol (such as zeromq, rabbitmq, redis, etc)
class LogStash::Codecs::Plain < LogStash::Codecs::Base
config_name "plain"
milestone 3
# Set the message you which to emit for each event. This supports sprintf
# strings.
#
# This setting only affects outputs (encoding of events).
config :format, :validate => :string
# The character encoding used in this input. Examples include "UTF-8"
# and "cp1252"
#
# This setting is useful if your log files are in Latin-1 (aka cp1252)
# or in another character set other than UTF-8.
#
# This only affects "plain" format logs since json is UTF-8 already.
config :charset, :validate => ::Encoding.name_list, :default => "UTF-8"
public
def register
@converter = LogStash::Util::Charset.new(@charset)
@converter.logger = @logger
end
public
def decode(data)
yield LogStash::Event.new("message" => @converter.convert(data))
end # def decode
public
def encode(event)
if event.is_a?(LogStash::Event) and @format
@on_event.call(event.sprintf(@format))
else
@on_event.call(event.to_s)
end
end # def encode
end # class LogStash::Codecs::Plain

41
lib/logstash/codecs/rubydebug.rb
View file

@ -1,41 +0,0 @@
# encoding: utf-8
require "logstash/codecs/base"
# The rubydebug codec will output your Logstash event data using
# the Ruby Awesome Print library.
#
class LogStash::Codecs::RubyDebug < LogStash::Codecs::Base
config_name "rubydebug"
milestone 3
# Should the event's metadata be included?
config :metadata, :validate => :boolean, :default => false
def register
require "ap"
if @metadata
@encoder = method(:encode_with_metadata)
else
@encoder = method(:encode_default)
end
end
public
def decode(data)
raise "Not implemented"
end # def decode
public
def encode(event)
@encoder.call(event)
end
def encode_default(event)
@on_event.call(event.to_hash.awesome_inspect + NL)
end # def encode_default
def encode_with_metadata(event)
@on_event.call(event.to_hash_with_metadata.awesome_inspect + NL)
end # def encode_with_metadata
end # class LogStash::Codecs::Dots

38
lib/logstash/codecs/spool.rb
View file

@ -1,38 +0,0 @@
# encoding: utf-8
require "logstash/codecs/base"
class LogStash::Codecs::Spool < LogStash::Codecs::Base
config_name 'spool'
milestone 1
config :spool_size, :validate => :number, :default => 50
attr_reader :buffer
public
def decode(data)
data.each do |event|
yield event
end
end # def decode
public
def encode(event)
@buffer ||= []
#buffer size is hard coded for now until a
#better way to pass args into codecs is implemented
if @buffer.length >= @spool_size
@on_event.call @buffer
@buffer = []
else
@buffer << event
end
end # def encode
public
def teardown
if !@buffer.nil? and @buffer.length > 0
@on_event.call @buffer
end
@buffer = []
end
end # class LogStash::Codecs::Spool

95
lib/logstash/filters/anonymize.rb
View file

@ -1,95 +0,0 @@
# encoding: utf-8
require "logstash/filters/base"
require "logstash/namespace"
# Anonymize fields using by replacing values with a consistent hash.
class LogStash::Filters::Anonymize < LogStash::Filters::Base
config_name "anonymize"
milestone 1
# The fields to be anonymized
config :fields, :validate => :array, :required => true
# Hashing key
# When using MURMUR3 the key is ignored but must still be set.
# When using IPV4_NETWORK key is the subnet prefix lentgh
config :key, :validate => :string, :required => true
# digest/hash type
config :algorithm, :validate => ['SHA1', 'SHA256', 'SHA384', 'SHA512', 'MD5', "MURMUR3", "IPV4_NETWORK"], :required => true, :default => 'SHA1'
public
def register
# require any library and set the anonymize function
case @algorithm
when "IPV4_NETWORK"
require 'ipaddr'
class << self; alias_method :anonymize, :anonymize_ipv4_network; end
when "MURMUR3"
require "murmurhash3"
class << self; alias_method :anonymize, :anonymize_murmur3; end
else
require 'openssl'
class << self; alias_method :anonymize, :anonymize_openssl; end
end
end # def register
public
def filter(event)
return unless filter?(event)
@fields.each do |field|
next unless event.include?(field)
if event[field].is_a?(Array)
event[field] = event[field].collect { |v| anonymize(v) }
else
event[field] = anonymize(event[field])
end
end
end # def filter
private
def anonymize_ipv4_network(ip_string)
# in JRuby 1.7.11 outputs as US-ASCII
IPAddr.new(ip_string).mask(@key.to_i).to_s.force_encoding(Encoding::UTF_8)
end
def anonymize_openssl(data)
digest = algorithm()
# in JRuby 1.7.11 outputs as ASCII-8BIT
OpenSSL::HMAC.hexdigest(digest, @key, data).force_encoding(Encoding::UTF_8)
end
def anonymize_murmur3(value)
case value
when Fixnum
MurmurHash3::V32.int_hash(value)
when String
MurmurHash3::V32.str_hash(value)
end
end
def algorithm
case @algorithm
#when 'SHA'
#return OpenSSL::Digest::SHA.new
when 'SHA1'
return OpenSSL::Digest::SHA1.new
#when 'SHA224'
#return OpenSSL::Digest::SHA224.new
when 'SHA256'
return OpenSSL::Digest::SHA256.new
when 'SHA384'
return OpenSSL::Digest::SHA384.new
when 'SHA512'
return OpenSSL::Digest::SHA512.new
#when 'MD4'
#return OpenSSL::Digest::MD4.new
when 'MD5'
return OpenSSL::Digest::MD5.new
else
@logger.error("Unknown algorithm")
end
end
end # class LogStash::Filters::Anonymize

53
lib/logstash/filters/checksum.rb
View file

@ -1,53 +0,0 @@
# encoding: utf-8
require "logstash/filters/base"
require "logstash/namespace"
require "yaml"
# This filter let's you create a checksum based on various parts
# of the logstash event.
# This can be useful for deduplication of messages or simply to provide
# a custom unique identifier.
#
# This is VERY experimental and is largely a proof-of-concept
class LogStash::Filters::Checksum < LogStash::Filters::Base
config_name "checksum"
milestone 1
ALGORITHMS = ["md5", "sha", "sha1", "sha256", "sha384",]
# A list of keys to use in creating the string to checksum
# Keys will be sorted before building the string
# keys and values will then be concatenated with pipe delimeters
# and checksummed
config :keys, :validate => :array, :default => ["message", "@timestamp", "type"]
config :algorithm, :validate => ALGORITHMS, :default => "sha256"
public
def register
require 'openssl'
@to_checksum = ""
end
public
def filter(event)
return unless filter?(event)
@logger.debug("Running checksum filter", :event => event)
@keys.sort.each do |k|
@logger.debug("Adding key to string", :current_key => k)
@to_checksum << "|#{k}|#{event[k]}"
end
@to_checksum << "|"
@logger.debug("Final string built", :to_checksum => @to_checksum)
# in JRuby 1.7.11 outputs as ASCII-8BIT
digested_string = OpenSSL::Digest.hexdigest(@algorithm, @to_checksum).force_encoding(Encoding::UTF_8)
@logger.debug("Digested string", :digested_string => digested_string)
event['logstash_checksum'] = digested_string
end
end # class LogStash::Filters::Checksum

35
lib/logstash/filters/clone.rb
View file

@ -1,35 +0,0 @@
# encoding: utf-8
require "logstash/filters/base"
require "logstash/namespace"
# The clone filter is for duplicating events.
# A clone will be made for each type in the clone list.
# The original event is left unchanged.
class LogStash::Filters::Clone < LogStash::Filters::Base
config_name "clone"
milestone 2
# A new clone will be created with the given type for each type in this list.
config :clones, :validate => :array, :default => []
public
def register
# Nothing to do
end
public
def filter(event)
return unless filter?(event)
@clones.each do |type|
clone = event.clone
clone["type"] = type
filter_matched(clone)
@logger.debug("Cloned event", :clone => clone, :event => event)
# Push this new event onto the stack at the LogStash::FilterWorker
yield clone
end
end
end # class LogStash::Filters::Clone

97
lib/logstash/filters/csv.rb
View file

@ -1,97 +0,0 @@
# encoding: utf-8
require "logstash/filters/base"
require "logstash/namespace"
require "csv"
# The CSV filter takes an event field containing CSV data, parses it,
# and stores it as individual fields (can optionally specify the names).
# This filter can also parse data with any separator, not just commas.
class LogStash::Filters::CSV < LogStash::Filters::Base
config_name "csv"
milestone 2
# The CSV data in the value of the `source` field will be expanded into a
# data structure.
config :source, :validate => :string, :default => "message"
# Define a list of column names (in the order they appear in the CSV,
# as if it were a header line). If `columns` is not configured, or there
# are not enough columns specified, the default column names are
# "column1", "column2", etc. In the case that there are more columns
# in the data than specified in this column list, extra columns will be auto-numbered:
# (e.g. "user_defined_1", "user_defined_2", "column3", "column4", etc.)
config :columns, :validate => :array, :default => []
# Define the column separator value. If this is not specified, the default
# is a comma ','.
# Optional.
config :separator, :validate => :string, :default => ","
# Define the character used to quote CSV fields. If this is not specified
# the default is a double quote '"'.
# Optional.
config :quote_char, :validate => :string, :default => '"'
# Define target field for placing the data.
# Defaults to writing to the root of the event.
config :target, :validate => :string
public
def register
# Nothing to do here
end # def register
public
def filter(event)
return unless filter?(event)
@logger.debug("Running csv filter", :event => event)
matches = 0
if event[@source]
if event[@source].is_a?(String)
event[@source] = [event[@source]]
end
if event[@source].length > 1
@logger.warn("csv filter only works on fields of length 1",
:source => @source, :value => event[@source],
:event => event)
return
end
raw = event[@source].first
begin
values = CSV.parse_line(raw, :col_sep => @separator, :quote_char => @quote_char)
if @target.nil?
# Default is to write to the root of the event.
dest = event
else
dest = event[@target] ||= {}
end
values.each_index do |i|
field_name = @columns[i] || "column#{i+1}"
dest[field_name] = values[i]
end
filter_matched(event)
rescue => e
event.tag "_csvparsefailure"
@logger.warn("Trouble parsing csv", :source => @source, :raw => raw,
:exception => e)
return
end # begin
end # if event
@logger.debug("Event after csv filter", :event => event)
end # def filter
end # class LogStash::Filters::Csv

236
lib/logstash/filters/date.rb
View file

@ -1,236 +0,0 @@
# encoding: utf-8
require "logstash/filters/base"
require "logstash/namespace"
require "logstash/timestamp"
# The date filter is used for parsing dates from fields, and then using that
# date or timestamp as the logstash timestamp for the event.
#
# For example, syslog events usually have timestamps like this:
#
# "Apr 17 09:32:01"
#
# You would use the date format "MMM dd HH:mm:ss" to parse this.
#
# The date filter is especially important for sorting events and for
# backfilling old data. If you don't get the date correct in your
# event, then searching for them later will likely sort out of order.
#
# In the absence of this filter, logstash will choose a timestamp based on the
# first time it sees the event (at input time), if the timestamp is not already
# set in the event. For example, with file input, the timestamp is set to the
# time of each read.
class LogStash::Filters::Date < LogStash::Filters::Base
if RUBY_ENGINE == "jruby"
JavaException = java.lang.Exception
UTC = org.joda.time.DateTimeZone.forID("UTC")
end
config_name "date"
milestone 3
# Specify a time zone canonical ID to be used for date parsing.
# The valid IDs are listed on the [Joda.org available time zones page](http://joda-time.sourceforge.net/timezones.html).
# This is useful in case the time zone cannot be extracted from the value,
# and is not the platform default.
# If this is not specified the platform default will be used.
# Canonical ID is good as it takes care of daylight saving time for you
# For example, `America/Los_Angeles` or `Europe/France` are valid IDs.
config :timezone, :validate => :string
# Specify a locale to be used for date parsing using either IETF-BCP47 or POSIX language tag.
# Simple examples are `en`,`en-US` for BCP47 or `en_US` for POSIX.
# If not specified, the platform default will be used.
#
# The locale is mostly necessary to be set for parsing month names (pattern with MMM) and
# weekday names (pattern with EEE).
#
config :locale, :validate => :string
# The date formats allowed are anything allowed by Joda-Time (java time
# library). You can see the docs for this format here:
#
# [joda.time.format.DateTimeFormat](http://joda-time.sourceforge.net/apidocs/org/joda/time/format/DateTimeFormat.html)
#
# An array with field name first, and format patterns following, `[ field,
# formats... ]`
#
# If your time field has multiple possible formats, you can do this:
#
# match => [ "logdate", "MMM dd YYY HH:mm:ss",
# "MMM d YYY HH:mm:ss", "ISO8601" ]
#
# The above will match a syslog (rfc3164) or iso8601 timestamp.
#
# There are a few special exceptions. The following format literals exist
# to help you save time and ensure correctness of date parsing.
#
# * "ISO8601" - should parse any valid ISO8601 timestamp, such as
# 2011-04-19T03:44:01.103Z
# * "UNIX" - will parse unix time in seconds since epoch
# * "UNIX_MS" - will parse unix time in milliseconds since epoch
# * "TAI64N" - will parse tai64n time values
#
# For example, if you have a field 'logdate', with a value that looks like
# 'Aug 13 2010 00:03:44', you would use this configuration:
#
# filter {
# date {
# match => [ "logdate", "MMM dd YYYY HH:mm:ss" ]
# }
# }
#
# If your field is nested in your structure, you can use the nested
# syntax [foo][bar] to match its value. For more information, please refer to
# http://logstash.net/docs/latest/configuration#fieldreferences
config :match, :validate => :array, :default => []
# Store the matching timestamp into the given target field. If not provided,
# default to updating the @timestamp field of the event.
config :target, :validate => :string, :default => "@timestamp"
# LOGSTASH-34
DATEPATTERNS = %w{ y d H m s S }
public
def initialize(config = {})
super
@parsers = Hash.new { |h,k| h[k] = [] }
end # def initialize
public
def register
require "java"
if @match.length < 2
raise LogStash::ConfigurationError, I18n.t("logstash.agent.configuration.invalid_plugin_register",
:plugin => "filter", :type => "date",
:error => "The match setting should contains first a field name and at least one date format, current value is #{@match}")
end
locale = nil
if @locale
if @locale.include? '_'
@logger.warn("Date filter now use BCP47 format for locale, replacing underscore with dash")
@locale.gsub!('_','-')
end
locale = java.util.Locale.forLanguageTag(@locale)
end
setupMatcher(@config["match"].shift, locale, @config["match"] )
end
def setupMatcher(field, locale, value)
value.each do |format|
parsers = []
case format
when "ISO8601"
iso_parser = org.joda.time.format.ISODateTimeFormat.dateTimeParser
if @timezone
iso_parser = iso_parser.withZone(org.joda.time.DateTimeZone.forID(@timezone))
else
iso_parser = iso_parser.withOffsetParsed
end
parsers << lambda { |date| iso_parser.parseMillis(date) }
#Fall back solution of almost ISO8601 date-time
almostISOparsers = [
org.joda.time.format.DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSSZ").getParser(),
org.joda.time.format.DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS").getParser()
].to_java(org.joda.time.format.DateTimeParser)
joda_parser = org.joda.time.format.DateTimeFormatterBuilder.new.append( nil, almostISOparsers ).toFormatter()
if @timezone
joda_parser = joda_parser.withZone(org.joda.time.DateTimeZone.forID(@timezone))
else
joda_parser = joda_parser.withOffsetParsed
end
parsers << lambda { |date| joda_parser.parseMillis(date) }
when "UNIX" # unix epoch
parsers << lambda do |date|
raise "Invalid UNIX epoch value '#{date}'" unless /^\d+(?:\.\d+)?$/ === date || date.is_a?(Numeric)
(date.to_f * 1000).to_i
end
when "UNIX_MS" # unix epoch in ms
parsers << lambda do |date|
raise "Invalid UNIX epoch value '#{date}'" unless /^\d+$/ === date || date.is_a?(Numeric)
date.to_i
end
when "TAI64N" # TAI64 with nanoseconds, -10000 accounts for leap seconds
parsers << lambda do |date|
# Skip leading "@" if it is present (common in tai64n times)
date = date[1..-1] if date[0, 1] == "@"
return (date[1..15].hex * 1000 - 10000)+(date[16..23].hex/1000000)
end
else
joda_parser = org.joda.time.format.DateTimeFormat.forPattern(format).withDefaultYear(Time.new.year)
if @timezone
joda_parser = joda_parser.withZone(org.joda.time.DateTimeZone.forID(@timezone))
else
joda_parser = joda_parser.withOffsetParsed
end
if (locale != nil)
joda_parser = joda_parser.withLocale(locale)
end
parsers << lambda { |date| joda_parser.parseMillis(date) }
end
@logger.debug("Adding type with date config", :type => @type,
:field => field, :format => format)
@parsers[field] << {
:parser => parsers,
:format => format
}
end
end
# def register
public
def filter(event)
@logger.debug? && @logger.debug("Date filter: received event", :type => event["type"])
return unless filter?(event)
@parsers.each do |field, fieldparsers|
@logger.debug? && @logger.debug("Date filter looking for field",
:type => event["type"], :field => field)
next unless event.include?(field)
fieldvalues = event[field]
fieldvalues = [fieldvalues] if !fieldvalues.is_a?(Array)
fieldvalues.each do |value|
next if value.nil?
begin
epochmillis = nil
success = false
last_exception = RuntimeError.new "Unknown"
fieldparsers.each do |parserconfig|
parserconfig[:parser].each do |parser|
begin
epochmillis = parser.call(value)
success = true
break # success
rescue StandardError, JavaException => e
last_exception = e
end
end # parserconfig[:parser].each
break if success
end # fieldparsers.each
raise last_exception unless success
# Convert joda DateTime to a ruby Time
event[@target] = LogStash::Timestamp.at(epochmillis / 1000, (epochmillis % 1000) * 1000)
@logger.debug? && @logger.debug("Date parsing done", :value => value, :timestamp => event[@target])
filter_matched(event)
rescue StandardError, JavaException => e
@logger.warn("Failed parsing date from field", :field => field,
:value => value, :exception => e)
# Raising here will bubble all the way up and cause an exit.
# TODO(sissel): Maybe we shouldn't raise?
# TODO(sissel): What do we do on a failure? Tag it like grok does?
#raise e
end # begin
end # fieldvalue.each
end # @parsers.each
return event
end # def filter
end # class LogStash::Filters::Date

209
lib/logstash/filters/dns.rb
View file

@ -1,209 +0,0 @@
# encoding: utf-8
# DNS Filter
#
# This filter will resolve any IP addresses from a field of your choosing.
#
require "logstash/filters/base"
require "logstash/namespace"
# The DNS filter performs a lookup (either an A record/CNAME record lookup
# or a reverse lookup at the PTR record) on records specified under the
# "reverse" and "resolve" arrays.
#
# The config should look like this:
#
# filter {
# dns {
# type => 'type'
# reverse => [ "source_host", "field_with_address" ]
# resolve => [ "field_with_fqdn" ]
# action => "replace"
# }
# }
#
# Caveats: at the moment, there's no way to tune the timeout with the 'resolv'
# core library. It does seem to be fixed in here:
#
# http://redmine.ruby-lang.org/issues/5100
#
# but isn't currently in JRuby.
class LogStash::Filters::DNS < LogStash::Filters::Base
config_name "dns"
milestone 2
# Reverse resolve one or more fields.
config :reverse, :validate => :array
# Forward resolve one or more fields.
config :resolve, :validate => :array
# Determine what action to do: append or replace the values in the fields
# specified under "reverse" and "resolve."
config :action, :validate => [ "append", "replace" ], :default => "append"
# Use custom nameserver.
config :nameserver, :validate => :string
# TODO(sissel): make 'action' required? This was always the intent, but it
# due to a typo it was never enforced. Thus the default behavior in past
# versions was 'append' by accident.
# resolv calls will be wrapped in a timeout instance
config :timeout, :validate => :number, :default => 2
public
def register
require "resolv"
require "timeout"
if @nameserver.nil?
@resolv = Resolv.new
else
@resolv = Resolv.new(resolvers=[::Resolv::Hosts.new, ::Resolv::DNS.new(:nameserver => [@nameserver], :search => [], :ndots => 1)])
end
@ip_validator = Resolv::AddressRegex
end # def register
public
def filter(event)
return unless filter?(event)
new_event = event.clone
if @resolve
begin
status = Timeout::timeout(@timeout) {
resolve(new_event)
}
return if status.nil?
rescue Timeout::Error
@logger.debug("DNS: resolve action timed out")
return
end
end
if @reverse
begin
status = Timeout::timeout(@timeout) {
reverse(new_event)
}
return if status.nil?
rescue Timeout::Error
@logger.debug("DNS: reverse action timed out")
return
end
end
filter_matched(new_event)
yield new_event
event.cancel
end
private
def resolve(event)
@resolve.each do |field|
is_array = false
raw = event[field]
if raw.is_a?(Array)
is_array = true
if raw.length > 1
@logger.warn("DNS: skipping resolve, can't deal with multiple values", :field => field, :value => raw)
return
end
raw = raw.first
end
begin
# in JRuby 1.7.11 outputs as US-ASCII
address = @resolv.getaddress(raw).force_encoding(Encoding::UTF_8)
rescue Resolv::ResolvError
@logger.debug("DNS: couldn't resolve the hostname.",
:field => field, :value => raw)
return
rescue Resolv::ResolvTimeout
@logger.debug("DNS: timeout on resolving the hostname.",
:field => field, :value => raw)
return
rescue SocketError => e
@logger.debug("DNS: Encountered SocketError.",
:field => field, :value => raw)
return
rescue NoMethodError => e
# see JRUBY-5647
@logger.debug("DNS: couldn't resolve the hostname.",
:field => field, :value => raw,
:extra => "NameError instead of ResolvError")
return
end
if @action == "replace"
if is_array
event[field] = [address]
else
event[field] = address
end
else
if !is_array
event[field] = [event[field], address]
else
event[field] << address
end
end
end
end
private
def reverse(event)
@reverse.each do |field|
raw = event[field]
is_array = false
if raw.is_a?(Array)
is_array = true
if raw.length > 1
@logger.warn("DNS: skipping reverse, can't deal with multiple values", :field => field, :value => raw)
return
end
raw = raw.first
end
if ! @ip_validator.match(raw)
@logger.debug("DNS: not an address",
:field => field, :value => event[field])
return
end
begin
# in JRuby 1.7.11 outputs as US-ASCII
hostname = @resolv.getname(raw).force_encoding(Encoding::UTF_8)
rescue Resolv::ResolvError
@logger.debug("DNS: couldn't resolve the address.",
:field => field, :value => raw)
return
rescue Resolv::ResolvTimeout
@logger.debug("DNS: timeout on resolving address.",
:field => field, :value => raw)
return
rescue SocketError => e
@logger.debug("DNS: Encountered SocketError.",
:field => field, :value => raw)
return
end
if @action == "replace"
if is_array
event[field] = [hostname]
else
event[field] = hostname
end
else
if !is_array
event[field] = [event[field], hostname]
else
event[field] << hostname
end
end
end
end
end # class LogStash::Filters::DNS

32
lib/logstash/filters/drop.rb
View file

@ -1,32 +0,0 @@
# encoding: utf-8
require "logstash/filters/base"
require "logstash/namespace"
# Drop filter.
#
# Drops everything that gets to this filter.
#
# This is best used in combination with conditionals, for example:
#
# filter {
# if [loglevel] == "debug" {
# drop { }
# }
# }
#
# The above will only pass events to the drop filter if the loglevel field is
# "debug". This will cause all events matching to be dropped.
class LogStash::Filters::Drop < LogStash::Filters::Base
config_name "drop"
milestone 3
public
def register
# nothing to do.
end
public
def filter(event)
event.cancel
end # def filter
end # class LogStash::Filters::Drop

122
lib/logstash/filters/fingerprint.rb
View file

@ -1,122 +0,0 @@
# encoding: utf-8
require "logstash/filters/base"
require "logstash/namespace"
# Fingerprint fields using by replacing values with a consistent hash.
class LogStash::Filters::Fingerprint < LogStash::Filters::Base
config_name "fingerprint"
milestone 1
# Source field(s)
config :source, :validate => :array, :default => 'message'
# Target field.
# will overwrite current value of a field if it exists.
config :target, :validate => :string, :default => 'fingerprint'
# When used with IPV4_NETWORK method fill in the subnet prefix length
# Not required for MURMUR3 or UUID methods
# With other methods fill in the HMAC key
config :key, :validate => :string
# Fingerprint method
config :method, :validate => ['SHA1', 'SHA256', 'SHA384', 'SHA512', 'MD5', "MURMUR3", "IPV4_NETWORK", "UUID", "PUNCTUATION"], :required => true, :default => 'SHA1'
# When set to true, we concatenate the values of all fields into 1 string like the old checksum filter.
config :concatenate_sources, :validate => :boolean, :default => false
def register
# require any library and set the anonymize function
case @method
when "IPV4_NETWORK"
require 'ipaddr'
@logger.error("Key value is empty. please fill in a subnet prefix length") if @key.nil?
class << self; alias_method :anonymize, :anonymize_ipv4_network; end
when "MURMUR3"
require "murmurhash3"
class << self; alias_method :anonymize, :anonymize_murmur3; end
when "UUID"
require "securerandom"
when "PUNCTUATION"
# nothing required
else
require 'openssl'
@logger.error("Key value is empty. Please fill in an encryption key") if @key.nil?
class << self; alias_method :anonymize, :anonymize_openssl; end
end
end # def register
public
def filter(event)
return unless filter?(event)
case @method
when "UUID"
event[@target] = SecureRandom.uuid
when "PUNCTUATION"
@source.sort.each do |field|
next unless event.include?(field)
event[@target] = event[field].tr('A-Za-z0-9 \t','')
end
else
if @concatenate_sources
to_string = ''
@source.sort.each do |k|
@logger.debug("Adding key to string")
to_string << "|#{k}|#{event[k]}"
end
to_string << "|"
@logger.debug("String built", :to_checksum => to_string)
event[@target] = anonymize(to_string)
else
@source.each do |field|
next unless event.include?(field)
if event[field].is_a?(Array)
event[@target] = event[field].collect { |v| anonymize(v) }
else
event[@target] = anonymize(event[field])
end
end # @source.each
end # concatenate_sources
end # casse @method
end # def filter
private
def anonymize_ipv4_network(ip_string)
# in JRuby 1.7.11 outputs as US-ASCII
IPAddr.new(ip_string).mask(@key.to_i).to_s.force_encoding(Encoding::UTF_8)
end
def anonymize_openssl(data)
digest = encryption_algorithm()
# in JRuby 1.7.11 outputs as ASCII-8BIT
OpenSSL::HMAC.hexdigest(digest, @key, data.to_s).force_encoding(Encoding::UTF_8)
end
def anonymize_murmur3(value)
case value
when Fixnum
MurmurHash3::V32.int_hash(value)
else
MurmurHash3::V32.str_hash(value.to_s)
end
end
def encryption_algorithm
case @method
when 'SHA1'
return OpenSSL::Digest::SHA1.new
when 'SHA256'
return OpenSSL::Digest::SHA256.new
when 'SHA384'
return OpenSSL::Digest::SHA384.new
when 'SHA512'
return OpenSSL::Digest::SHA512.new
when 'MD5'
return OpenSSL::Digest::MD5.new
else
@logger.error("Unknown algorithm")
end
end
end # class LogStash::Filters::Anonymize

147
lib/logstash/filters/geoip.rb
View file

@ -1,147 +0,0 @@
# encoding: utf-8
require "logstash/filters/base"
require "logstash/namespace"
require "tempfile"
# The GeoIP filter adds information about the geographical location of IP addresses,
# based on data from the Maxmind database.
#
# Starting with version 1.3.0 of Logstash, a [geoip][location] field is created if
# the GeoIP lookup returns a latitude and longitude. The field is stored in
# [GeoJSON](http://geojson.org/geojson-spec.html) format. Additionally,
# the default Elasticsearch template provided with the
# [elasticsearch output](../outputs/elasticsearch.html)
# maps the [geoip][location] field to a
# [geo_point](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-geo-point-type.html).
#
# As this field is a geo\_point _and_ it is still valid GeoJSON, you get
# the awesomeness of Elasticsearch's geospatial query, facet and filter functions
# and the flexibility of having GeoJSON for all other applications (like Kibana's
# [bettermap panel](https://github.com/elasticsearch/kibana/tree/master/src/app/panels/bettermap)).
#
# Logstash releases ship with the GeoLiteCity database made available from
# Maxmind with a CCA-ShareAlike 3.0 license. For more details on GeoLite, see
# <http://www.maxmind.com/en/geolite>.
class LogStash::Filters::GeoIP < LogStash::Filters::Base
config_name "geoip"
milestone 3
# The path to the GeoIP database file which Logstash should use. Country, City, ASN, ISP
# and organization databases are supported.
#
# If not specified, this will default to the GeoLiteCity database that ships
# with Logstash.
config :database, :validate => :path
# The field containing the IP address or hostname to map via geoip. If
# this field is an array, only the first value will be used.
config :source, :validate => :string, :required => true
# An array of geoip fields to be included in the event.
#
# Possible fields depend on the database type. By default, all geoip fields
# are included in the event.
#
# For the built-in GeoLiteCity database, the following are available:
# `city\_name`, `continent\_code`, `country\_code2`, `country\_code3`, `country\_name`,
# `dma\_code`, `ip`, `latitude`, `longitude`, `postal\_code`, `region\_name` and `timezone`.
config :fields, :validate => :array
# Specify the field into which Logstash should store the geoip data.
# This can be useful, for example, if you have `src\_ip` and `dst\_ip` fields and
# would like the GeoIP information of both IPs.
#
# If you save the data to a target field other than "geoip" and want to use the
# geo\_point related functions in Elasticsearch, you need to alter the template
# provided with the Elasticsearch output and configure the output to use the
# new template.
#
# Even if you don't use the geo\_point mapping, the [target][location] field
# is still valid GeoJSON.
config :target, :validate => :string, :default => 'geoip'
public
def register
require "geoip"
if @database.nil?
@database = LogStash::Environment.vendor_path("geoip/GeoLiteCity.dat")
if !File.exists?(@database)
raise "You must specify 'database => ...' in your geoip filter (I looked for '#{@database}'"
end
end
@logger.info("Using geoip database", :path => @database)
# For the purpose of initializing this filter, geoip is initialized here but
# not set as a global. The geoip module imposes a mutex, so the filter needs
# to re-initialize this later in the filter() thread, and save that access
# as a thread-local variable.
geoip_initialize = ::GeoIP.new(@database)
@geoip_type = case geoip_initialize.database_type
when GeoIP::GEOIP_CITY_EDITION_REV0, GeoIP::GEOIP_CITY_EDITION_REV1
:city
when GeoIP::GEOIP_COUNTRY_EDITION
:country
when GeoIP::GEOIP_ASNUM_EDITION
:asn
when GeoIP::GEOIP_ISP_EDITION, GeoIP::GEOIP_ORG_EDITION
:isp
else
raise RuntimeException.new "This GeoIP database is not currently supported"
end
@threadkey = "geoip-#{self.object_id}"
end # def register
public
def filter(event)
return unless filter?(event)
geo_data = nil
# Use thread-local access to GeoIP. The Ruby GeoIP module forces a mutex
# around access to the database, which can be overcome with :pread.
# Unfortunately, :pread requires the io-extra gem, with C extensions that
# aren't supported on JRuby. If / when :pread becomes available, we can stop
# needing thread-local access.
if !Thread.current.key?(@threadkey)
Thread.current[@threadkey] = ::GeoIP.new(@database)
end
begin
ip = event[@source]
ip = ip.first if ip.is_a? Array
geo_data = Thread.current[@threadkey].send(@geoip_type, ip)
rescue SocketError => e
@logger.error("IP Field contained invalid IP address or hostname", :field => @field, :event => event)
rescue Exception => e
@logger.error("Unknown error while looking up GeoIP data", :exception => e, :field => @field, :event => event)
end
return if geo_data.nil?
geo_data_hash = geo_data.to_hash
geo_data_hash.delete(:request)
event[@target] = {} if event[@target].nil?
geo_data_hash.each do |key, value|
next if value.nil? || (value.is_a?(String) && value.empty?)
if @fields.nil? || @fields.empty? || @fields.include?(key.to_s)
# convert key to string (normally a Symbol)
if value.is_a?(String)
# Some strings from GeoIP don't have the correct encoding...
value = case value.encoding
# I have found strings coming from GeoIP that are ASCII-8BIT are actually
# ISO-8859-1...
when Encoding::ASCII_8BIT; value.force_encoding(Encoding::ISO_8859_1).encode(Encoding::UTF_8)
when Encoding::ISO_8859_1, Encoding::US_ASCII; value.encode(Encoding::UTF_8)
else; value
end
end
event[@target][key.to_s] = value
end
end # geo_data_hash.each
if event[@target].key?('latitude') && event[@target].key?('longitude')
# If we have latitude and longitude values, add the location field as GeoJSON array
event[@target]['location'] = [ event[@target]["longitude"].to_f, event[@target]["latitude"].to_f ]
end
filter_matched(event)
end # def filter
end # class LogStash::Filters::GeoIP

361
lib/logstash/filters/grok.rb
View file

@ -1,361 +0,0 @@
# encoding: utf-8
require "logstash/filters/base"
require "logstash/namespace"
require "logstash/environment"
require "set"
# Parse arbitrary text and structure it.
#
# Grok is currently the best way in logstash to parse crappy unstructured log
# data into something structured and queryable.
#
# This tool is perfect for syslog logs, apache and other webserver logs, mysql
# logs, and in general, any log format that is generally written for humans
# and not computer consumption.
#
# Logstash ships with about 120 patterns by default. You can find them here:
# <https://github.com/logstash/logstash/tree/v%VERSION%/patterns>. You can add
# your own trivially. (See the patterns_dir setting)
#
# If you need help building patterns to match your logs, you will find the
# <http://grokdebug.herokuapp.com> too quite useful!
#
# #### Grok Basics
#
# Grok works by combining text patterns into something that matches your
# logs.
#
# The syntax for a grok pattern is `%{SYNTAX:SEMANTIC}`
#
# The `SYNTAX` is the name of the pattern that will match your text. For
# example, "3.44" will be matched by the NUMBER pattern and "55.3.244.1" will
# be matched by the IP pattern. The syntax is how you match.
#
# The `SEMANTIC` is the identifier you give to the piece of text being matched.
# For example, "3.44" could be the duration of an event, so you could call it
# simply 'duration'. Further, a string "55.3.244.1" might identify the 'client'
# making a request.
#
# For the above example, your grok filter would look something like this:
#
# %{NUMBER:duration} %{IP:client}
#
# Optionally you can add a data type conversion to your grok pattern. By default
# all semantics are saved as strings. If you wish to convert a semantic's data type,
# for example change a string to an integer then suffix it with the target data type.
# For example `%{NUMBER:num:int}` which converts the 'num' semantic from a string to an
# integer. Currently the only supported conversions are `int` and `float`.
#
# #### Example
#
# With that idea of a syntax and semantic, we can pull out useful fields from a
# sample log like this fictional http request log:
#
# 55.3.244.1 GET /index.html 15824 0.043
#
# The pattern for this could be:
#
# %{IP:client} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes} %{NUMBER:duration}
#
# A more realistic example, let's read these logs from a file:
#
# input {
# file {
# path => "/var/log/http.log"
# }
# }
# filter {
# grok {
# match => { "message" => "%{IP:client} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes} %{NUMBER:duration}" }
# }
# }
#
# After the grok filter, the event will have a few extra fields in it:
#
# * client: 55.3.244.1
# * method: GET
# * request: /index.html
# * bytes: 15824
# * duration: 0.043
#
# #### Regular Expressions
#
# Grok sits on top of regular expressions, so any regular expressions are valid
# in grok as well. The regular expression library is Oniguruma, and you can see
# the full supported regexp syntax [on the Onigiruma
# site](http://www.geocities.jp/kosako3/oniguruma/doc/RE.txt).
#
# #### Custom Patterns
#
# Sometimes logstash doesn't have a pattern you need. For this, you have
# a few options.
#
# First, you can use the Oniguruma syntax for 'named capture' which will
# let you match a piece of text and save it as a field:
#
# (?<field_name>the pattern here)
#
# For example, postfix logs have a 'queue id' that is an 10 or 11-character
# hexadecimal value. I can capture that easily like this:
#
# (?<queue_id>[0-9A-F]{10,11})
#
# Alternately, you can create a custom patterns file.
#
# * Create a directory called `patterns` with a file in it called `extra`
# (the file name doesn't matter, but name it meaningfully for yourself)
# * In that file, write the pattern you need as the pattern name, a space, then
# the regexp for that pattern.
#
# For example, doing the postfix queue id example as above:
#
# # contents of ./patterns/postfix:
# POSTFIX_QUEUEID [0-9A-F]{10,11}
#
# Then use the `patterns_dir` setting in this plugin to tell logstash where
# your custom patterns directory is. Here's a full example with a sample log:
#
# Jan 1 06:25:43 mailserver14 postfix/cleanup[21403]: BEF25A72965: message-id=<20130101142543.5828399CCAF@mailserver14.example.com>
#
# filter {
# grok {
# patterns_dir => "./patterns"
# match => { "message" => "%{SYSLOGBASE} %{POSTFIX_QUEUEID:queue_id}: %{GREEDYDATA:syslog_message}" }
# }
# }
#
# The above will match and result in the following fields:
#
# * timestamp: Jan 1 06:25:43
# * logsource: mailserver14
# * program: postfix/cleanup
# * pid: 21403
# * queue_id: BEF25A72965
# * syslog_message: message-id=<20130101142543.5828399CCAF@mailserver14.example.com>
#
# The `timestamp`, `logsource`, `program`, and `pid` fields come from the
# SYSLOGBASE pattern which itself is defined by other patterns.
class LogStash::Filters::Grok < LogStash::Filters::Base
config_name "grok"
milestone 3
# Specify a pattern to parse with. This will match the 'message' field.
#
# If you want to match other fields than message, use the 'match' setting.
# Multiple patterns is fine.
config :pattern, :validate => :array, :deprecated => "You should use this instead: match => { \"message\" => \"your pattern here\" }"
# A hash of matches of field => value
#
# For example:
#
# filter {
# grok { match => { "message" => "Duration: %{NUMBER:duration}" } }
# }
#
# Alternatively, using the old array syntax:
#
# filter {
# grok { match => [ "message", "Duration: %{NUMBER:duration}" ] }
# }
#
config :match, :validate => :hash, :default => {}
#
# logstash ships by default with a bunch of patterns, so you don't
# necessarily need to define this yourself unless you are adding additional
# patterns.
#
# Pattern files are plain text with format:
#
# NAME PATTERN
#
# For example:
#
# NUMBER \d+
config :patterns_dir, :validate => :array, :default => []
# Drop if matched. Note, this feature may not stay. It is preferable to combine
# grok + grep filters to do parsing + dropping.
config :drop_if_match, :validate => :boolean, :default => false
# Break on first match. The first successful match by grok will result in the
# filter being finished. If you want grok to try all patterns (maybe you are
# parsing different things), then set this to false.
config :break_on_match, :validate => :boolean, :default => true
# If true, only store named captures from grok.
config :named_captures_only, :validate => :boolean, :default => true
# If true, keep empty captures as event fields.
config :keep_empty_captures, :validate => :boolean, :default => false
# If true, make single-value fields simply that value, not an array
# containing that one value.
config :singles, :validate => :boolean, :default => true, :deprecated => "This behavior is the default now, you don't need to set it."
# Append values to the 'tags' field when there has been no
# successful match
config :tag_on_failure, :validate => :array, :default => ["_grokparsefailure"]
# The fields to overwrite.
#
# This allows you to overwrite a value in a field that already exists.
#
# For example, if you have a syslog line in the 'message' field, you can
# overwrite the 'message' field with part of the match like so:
#
# filter {
# grok {
# match => { "message" => "%{SYSLOGBASE} %{DATA:message}" }
# overwrite => [ "message" ]
# }
# }
#
# In this case, a line like "May 29 16:37:11 sadness logger: hello world"
# will be parsed and 'hello world' will overwrite the original message.
config :overwrite, :validate => :array, :default => []
# Detect if we are running from a jarfile, pick the right path.
@@patterns_path ||= Set.new
@@patterns_path += [LogStash::Environment.pattern_path("*")]
public
def initialize(params)
super(params)
@match["message"] ||= []
@match["message"] += @pattern if @pattern # the config 'pattern' value (array)
# a cache of capture name handler methods.
@handlers = {}
end
public
def register
require "grok-pure" # rubygem 'jls-grok'
@patternfiles = []
# Have @@patterns_path show first. Last-in pattern definitions win; this
# will let folks redefine built-in patterns at runtime.
@patterns_dir = @@patterns_path.to_a + @patterns_dir
@logger.info? and @logger.info("Grok patterns path", :patterns_dir => @patterns_dir)
@patterns_dir.each do |path|
if File.directory?(path)
path = File.join(path, "*")
end
Dir.glob(path).each do |file|
@logger.info? and @logger.info("Grok loading patterns from file", :path => file)
@patternfiles << file
end
end
@patterns = Hash.new { |h,k| h[k] = [] }
@logger.info? and @logger.info("Match data", :match => @match)
@match.each do |field, patterns|
patterns = [patterns] if patterns.is_a?(String)
@logger.info? and @logger.info("Grok compile", :field => field, :patterns => patterns)
patterns.each do |pattern|
@logger.debug? and @logger.debug("regexp: #{@type}/#{field}", :pattern => pattern)
grok = Grok.new
grok.logger = @logger unless @logger.nil?
add_patterns_from_files(@patternfiles, grok)
grok.compile(pattern, @named_captures_only)
@patterns[field] << grok
end
end # @match.each
end # def register
public
def filter(event)
return unless filter?(event)
matched = false
done = false
@logger.debug? and @logger.debug("Running grok filter", :event => event);
@patterns.each do |field, groks|
if match(groks, field, event)
matched = true
break if @break_on_match
end
#break if done
end # @patterns.each
if matched
filter_matched(event)
else
# Tag this event if we can't parse it. We can use this later to
# reparse+reindex logs if we improve the patterns given.
@tag_on_failure.each do |tag|
event["tags"] ||= []
event["tags"] << tag unless event["tags"].include?(tag)
end
end
@logger.debug? and @logger.debug("Event now: ", :event => event)
end # def filter
private
def match(groks, field, event)
input = event[field]
if input.is_a?(Array)
success = false
input.each do |input|
success |= match_against_groks(groks, input, event)
end
return success
else
return match_against_groks(groks, input, event)
end
rescue StandardError => e
@logger.warn("Grok regexp threw exception", :exception => e.message)
end
private
def match_against_groks(groks, input, event)
matched = false
groks.each do |grok|
# Convert anything else to string (number, hash, etc)
matched = grok.match_and_capture(input.to_s) do |field, value|
matched = true
handle(field, value, event)
end
break if matched and @break_on_match
end
return matched
end
private
def handle(field, value, event)
return if (value.nil? || (value.is_a?(String) && value.empty?)) unless @keep_empty_captures
if @overwrite.include?(field)
event[field] = value
else
v = event[field]
if v.nil?
event[field] = value
elsif v.is_a?(Array)
event[field] << value
elsif v.is_a?(String)
# Promote to array since we aren't overwriting.
event[field] = [v, value]
end
end
end
private
def add_patterns_from_files(paths, grok)
paths.each do |path|
if !File.exists?(path)
raise "Grok pattern file does not exist: #{path}"
end
grok.add_patterns_from_file(path)
end
end # def add_patterns_from_files
end # class LogStash::Filters::Grok

75
lib/logstash/filters/grokdiscovery.rb
View file

@ -1,75 +0,0 @@
# encoding: utf-8
require "logstash/filters/base"
require "logstash/namespace"
# TODO(sissel): This is not supported yet. There is a bug in grok discovery
# that causes segfaults in libgrok.
class LogStash::Filters::Grokdiscovery < LogStash::Filters::Base
config_name "grokdiscovery"
milestone 1
public
def initialize(config = {})
super
@discover_fields = {}
end # def initialize
public
def register
gem "jls-grok", ">=0.4.3"
require "grok" # rubygem 'jls-grok'
# TODO(sissel): Make patterns files come from the config
@config.each do |type, typeconfig|
@logger.debug("Registering type with grok: #{type}")
@grok = Grok.new
Dir.glob("patterns/*").each do |path|
@grok.add_patterns_from_file(path)
end
@discover_fields[type] = typeconfig
@logger.debug(["Enabling discovery", { :type => type, :fields => typeconfig }])
@logger.warn(@discover_fields)
end # @config.each
end # def register
public
def filter(event)
return unless filter?(event)
# parse it with grok
message = event["message"]
match = false
if event.type and @discover_fields.include?(event.type)
discover = @discover_fields[event.type] & event.to_hash.keys
discover.each do |field|
value = event[field]
value = [value] if value.is_a?(String)
value.each do |v|
pattern = @grok.discover(v)
@logger.warn("Trying #{v} => #{pattern}")
@grok.compile(pattern)
match = @grok.match(v)
if match
@logger.warn(["Match", match.captures])
event.to_hash.merge!(match.captures) do |key, oldval, newval|
@logger.warn(["Merging #{key}", oldval, newval])
oldval + newval # should both be arrays...
end
else
@logger.warn(["Discovery produced something not matchable?", { :input => v }])
end
end # value.each
end # discover.each
else
@logger.info("Unknown type for #{event.source} (type: #{event.type})")
@logger.debug(event.to_hash)
end
@logger.debug(["Event now: ", event.to_hash])
filter_matched(event) if !event.cancelled?
end # def filter
end # class LogStash::Filters::Grokdiscovery

104
lib/logstash/filters/json.rb
View file

@ -1,104 +0,0 @@
# encoding: utf-8
require "logstash/filters/base"
require "logstash/namespace"
require "logstash/json"
require "logstash/timestamp"
# This is a JSON parsing filter. It takes an existing field which contains JSON and
# expands it into an actual data structure within the Logstash event.
#
# By default it will place the parsed JSON in the root (top level) of the Logstash event, but this
# filter can be configured to place the JSON into any arbitrary event field, using the
# `target` configuration.
class LogStash::Filters::Json < LogStash::Filters::Base
config_name "json"
milestone 2
# The configuration for the JSON filter:
#
# source => source_field
#
# For example, if you have JSON data in the @message field:
#
# filter {
# json {
# source => "message"
# }
# }
#
# The above would parse the json from the @message field
config :source, :validate => :string, :required => true
# Define the target field for placing the parsed data. If this setting is
# omitted, the JSON data will be stored at the root (top level) of the event.
#
# For example, if you want the data to be put in the 'doc' field:
#
# filter {
# json {
# target => "doc"
# }
# }
#
# JSON in the value of the `source` field will be expanded into a
# data structure in the `target` field.
#
# NOTE: if the `target` field already exists, it will be overwritten!
config :target, :validate => :string
public
def register
# Nothing to do here
end # def register
public
def filter(event)
return unless filter?(event)
@logger.debug("Running json filter", :event => event)
return unless event.include?(@source)
# TODO(colin) this field merging stuff below should be handled in Event.
source = event[@source]
if @target.nil?
# Default is to write to the root of the event.
dest = event.to_hash
else
if @target == @source
# Overwrite source
dest = event[@target] = {}
else
dest = event[@target] ||= {}
end
end
begin
# TODO(sissel): Note, this will not successfully handle json lists
# like your text is '[ 1,2,3 ]' json parser gives you an array (correctly)
# which won't merge into a hash. If someone needs this, we can fix it
# later.
dest.merge!(LogStash::Json.load(source))
# If no target, we target the root of the event object. This can allow
# you to overwrite @timestamp and this will typically happen for json
# LogStash Event deserialized here.
if !@target && event.timestamp.is_a?(String)
event.timestamp = LogStash::Timestamp.parse_iso8601(event.timestamp)
end
filter_matched(event)
rescue => e
event.tag("_jsonparsefailure")
@logger.warn("Trouble parsing json", :source => @source,
:raw => event[@source], :exception => e)
return
end
@logger.debug("Event after json filter", :event => event)
end # def filter
end # class LogStash::Filters::Json

237
lib/logstash/filters/kv.rb
View file

@ -1,237 +0,0 @@
# encoding: utf-8
require "logstash/filters/base"
require "logstash/namespace"
# This filter helps automatically parse messages (or specific event fields)
# which are of the 'foo=bar' variety.
#
# For example, if you have a log message which contains 'ip=1.2.3.4
# error=REFUSED', you can parse those automatically by configuring:
#
# filter {
# kv { }
# }
#
# The above will result in a message of "ip=1.2.3.4 error=REFUSED" having
# the fields:
#
# * ip: 1.2.3.4
# * error: REFUSED
#
# This is great for postfix, iptables, and other types of logs that
# tend towards 'key=value' syntax.
#
# You can configure any arbitrary strings to split your data on,
# in case your data is not structured using '=' signs and whitespace.
# For example, this filter can also be used to parse query parameters like
# 'foo=bar&baz=fizz' by setting the `field_split` parameter to "&".
class LogStash::Filters::KV < LogStash::Filters::Base
config_name "kv"
milestone 2
# A string of characters to trim from the value. This is useful if your
# values are wrapped in brackets or are terminated with commas (like postfix
# logs).
#
# These characters form a regex character class and thus you must escape special regex
# characters like '[' or ']' using '\'.
#
# For example, to strip '<', '>', '[', ']' and ',' characters from values:
#
# filter {
# kv {
# trim => "<>\[\],"
# }
# }
config :trim, :validate => :string
# A string of characters to trim from the key. This is useful if your
# keys are wrapped in brackets or start with space.
#
# These characters form a regex character class and thus you must escape special regex
# characters like '[' or ']' using '\'.
#
# For example, to strip '<' '>' '[' ']' and ',' characters from keys:
#
# filter {
# kv {
# trimkey => "<>\[\],"
# }
# }
config :trimkey, :validate => :string
# A string of characters to use as delimiters for parsing out key-value pairs.
#
# These characters form a regex character class and thus you must escape special regex
# characters like '[' or ']' using '\'.
#
# #### Example with URL Query Strings
#
# For example, to split out the args from a url query string such as
# '?pin=12345~0&d=123&e=foo@bar.com&oq=bobo&ss=12345':
#
# filter {
# kv {
# field_split => "&?"
# }
# }
#
# The above splits on both "&" and "?" characters, giving you the following
# fields:
#
# * pin: 12345~0
# * d: 123
# * e: foo@bar.com
# * oq: bobo
# * ss: 12345
config :field_split, :validate => :string, :default => ' '
# A string of characters to use as delimiters for identifying key-value relations.
#
# These characters form a regex character class and thus you must escape special regex
# characters like '[' or ']' using '\'.
#
# For example, to identify key-values such as
# 'key1:value1 key2:value2':
#
# filter { kv { value_split => ":" } }
config :value_split, :validate => :string, :default => '='
# A string to prepend to all of the extracted keys.
#
# For example, to prepend arg_ to all keys:
#
# filter { kv { prefix => "arg_" } }
config :prefix, :validate => :string, :default => ''
# The field to perform 'key=value' searching on
#
# For example, to process the `not_the_message` field:
#
# filter { kv { source => "not_the_message" } }
config :source, :validate => :string, :default => "message"
# The name of the container to put all of the key-value pairs into.
#
# If this setting is omitted, fields will be written to the root of the
# event, as individual fields.
#
# For example, to place all keys into the event field kv:
#
# filter { kv { target => "kv" } }
config :target, :validate => :string
# An array specifying the parsed keys which should be added to the event.
# By default all keys will be added.
#
# For example, consider a source like "Hey, from=<abc>, to=def foo=bar".
# To include "from" and "to", but exclude the "foo" key, you could use this configuration:
# filter {
# kv {
# include_keys => [ "from", "to" ]
# }
# }
config :include_keys, :validate => :array, :default => []
# An array specifying the parsed keys which should not be added to the event.
# By default no keys will be excluded.
#
# For example, consider a source like "Hey, from=<abc>, to=def foo=bar".
# To exclude "from" and "to", but retain the "foo" key, you could use this configuration:
# filter {
# kv {
# exclude_keys => [ "from", "to" ]
# }
# }
config :exclude_keys, :validate => :array, :default => []
# A hash specifying the default keys and their values which should be added to the event
# in case these keys do not exist in the source field being parsed.
#
# filter {
# kv {
# default_keys => [ "from", "logstash@example.com",
# "to", "default@dev.null" ]
# }
# }
config :default_keys, :validate => :hash, :default => {}
def register
@trim_re = Regexp.new("[#{@trim}]") if !@trim.nil?
@trimkey_re = Regexp.new("[#{@trimkey}]") if !@trimkey.nil?
@scan_re = Regexp.new("((?:\\\\ |[^"+@field_split+@value_split+"])+)["+@value_split+"](?:\"([^\"]+)\"|'([^']+)'|((?:\\\\ |[^"+@field_split+"])+))")
end # def register
def filter(event)
return unless filter?(event)
kv = Hash.new
value = event[@source]
case value
when nil; # Nothing to do
when String; kv = parse(value, event, kv)
when Array; value.each { |v| kv = parse(v, event, kv) }
else
@logger.warn("kv filter has no support for this type of data",
:type => value.class, :value => value)
end # case value
# Add default key-values for missing keys
kv = @default_keys.merge(kv)
# If we have any keys, create/append the hash
if kv.length > 0
if @target.nil?
# Default is to write to the root of the event.
dest = event.to_hash
else
if !event[@target].is_a?(Hash)
@logger.debug("Overwriting existing target field", :target => @target)
dest = event[@target] = {}
else
dest = event[@target]
end
end
dest.merge!(kv)
filter_matched(event)
end
end # def filter
private
def parse(text, event, kv_keys)
if !event =~ /[@field_split]/
return kv_keys
end
# Interpret dynamic keys for @include_keys and @exclude_keys
include_keys = @include_keys.map{|key| event.sprintf(key)}
exclude_keys = @exclude_keys.map{|key| event.sprintf(key)}
text.scan(@scan_re) do |key, v1, v2, v3|
value = v1 || v2 || v3
key = @trimkey.nil? ? key : key.gsub(@trimkey_re, "")
# Bail out as per the values of include_keys and exclude_keys
next if not include_keys.empty? and not include_keys.include?(key)
next if exclude_keys.include?(key)
key = event.sprintf(@prefix) + key
value = @trim.nil? ? value : value.gsub(@trim_re, "")
if kv_keys.has_key?(key)
if kv_keys[key].is_a? Array
kv_keys[key].push(value)
else
kv_keys[key] = [kv_keys[key], value]
end
else
kv_keys[key] = value
end
end
return kv_keys
end
end # class LogStash::Filters::KV

241
lib/logstash/filters/metrics.rb
View file

@ -1,241 +0,0 @@
# encoding: utf-8
require "securerandom"
require "logstash/filters/base"
require "logstash/namespace"
# The metrics filter is useful for aggregating metrics.
#
# For example, if you have a field 'response' that is
# a http response code, and you want to count each
# kind of response, you can do this:
#
# filter {
# metrics {
# meter => [ "http.%{response}" ]
# add_tag => "metric"
# }
# }
#
# Metrics are flushed every 5 seconds by default or according to
# 'flush_interval'. Metrics appear as
# new events in the event stream and go through any filters
# that occur after as well as outputs.
#
# In general, you will want to add a tag to your metrics and have an output
# explicitly look for that tag.
#
# The event that is flushed will include every 'meter' and 'timer'
# metric in the following way:
#
# #### 'meter' values
#
# For a `meter => "something"` you will receive the following fields:
#
# * "thing.count" - the total count of events
# * "thing.rate_1m" - the 1-minute rate (sliding)
# * "thing.rate_5m" - the 5-minute rate (sliding)
# * "thing.rate_15m" - the 15-minute rate (sliding)
#
# #### 'timer' values
#
# For a `timer => [ "thing", "%{duration}" ]` you will receive the following fields:
#
# * "thing.count" - the total count of events
# * "thing.rate_1m" - the 1-minute rate of events (sliding)
# * "thing.rate_5m" - the 5-minute rate of events (sliding)
# * "thing.rate_15m" - the 15-minute rate of events (sliding)
# * "thing.min" - the minimum value seen for this metric
# * "thing.max" - the maximum value seen for this metric
# * "thing.stddev" - the standard deviation for this metric
# * "thing.mean" - the mean for this metric
# * "thing.pXX" - the XXth percentile for this metric (see `percentiles`)
#
# #### Example: computing event rate
#
# For a simple example, let's track how many events per second are running
# through logstash:
#
# input {
# generator {
# type => "generated"
# }
# }
#
# filter {
# if [type] == "generated" {
# metrics {
# meter => "events"
# add_tag => "metric"
# }
# }
# }
#
# output {
# # only emit events with the 'metric' tag
# if "metric" in [tags] {
# stdout {
# codec => line {
# format => "rate: %{events.rate_1m}"
# }
# }
# }
# }
#
# Running the above:
#
# % bin/logstash -f example.conf
# rate: 23721.983566819246
# rate: 24811.395722536377
# rate: 25875.892745934525
# rate: 26836.42375967113
#
# We see the output includes our 'events' 1-minute rate.
#
# In the real world, you would emit this to graphite or another metrics store,
# like so:
#
# output {
# graphite {
# metrics => [ "events.rate_1m", "%{events.rate_1m}" ]
# }
# }
class LogStash::Filters::Metrics < LogStash::Filters::Base
config_name "metrics"
milestone 1
# syntax: `meter => [ "name of metric", "name of metric" ]`
config :meter, :validate => :array, :default => []
# syntax: `timer => [ "name of metric", "%{time_value}" ]`
config :timer, :validate => :hash, :default => {}
# Don't track events that have @timestamp older than some number of seconds.
#
# This is useful if you want to only include events that are near real-time
# in your metrics.
#
# Example, to only count events that are within 10 seconds of real-time, you
# would do this:
#
# filter {
# metrics {
# meter => [ "hits" ]
# ignore_older_than => 10
# }
# }
config :ignore_older_than, :validate => :number, :default => 0
# The flush interval, when the metrics event is created. Must be a multiple of 5s.
config :flush_interval, :validate => :number, :default => 5
# The clear interval, when all counter are reset.
#
# If set to -1, the default value, the metrics will never be cleared.
# Otherwise, should be a multiple of 5s.
config :clear_interval, :validate => :number, :default => -1
# The rates that should be measured, in minutes.
# Possible values are 1, 5, and 15.
config :rates, :validate => :array, :default => [1, 5, 15]
# The percentiles that should be measured
config :percentiles, :validate => :array, :default => [1, 5, 10, 90, 95, 99, 100]
def register
require "metriks"
require "socket"
require "atomic"
require "thread_safe"
@last_flush = Atomic.new(0) # how many seconds ago the metrics where flushed.
@last_clear = Atomic.new(0) # how many seconds ago the metrics where cleared.
@random_key_preffix = SecureRandom.hex
unless (@rates - [1, 5, 15]).empty?
raise LogStash::ConfigurationError, "Invalid rates configuration. possible rates are 1, 5, 15. Rates: #{rates}."
end
@metric_meters = ThreadSafe::Cache.new { |h,k| h[k] = Metriks.meter metric_key(k) }
@metric_timers = ThreadSafe::Cache.new { |h,k| h[k] = Metriks.timer metric_key(k) }
end # def register
def filter(event)
return unless filter?(event)
# TODO(piavlo): This should probably be moved to base filter class.
if @ignore_older_than > 0 && Time.now - event.timestamp.time > @ignore_older_than
@logger.debug("Skipping metriks for old event", :event => event)
return
end
@meter.each do |m|
@metric_meters[event.sprintf(m)].mark
end
@timer.each do |name, value|
@metric_timers[event.sprintf(name)].update(event.sprintf(value).to_f)
end
end # def filter
def flush
# Add 5 seconds to @last_flush and @last_clear counters
# since this method is called every 5 seconds.
@last_flush.update { |v| v + 5 }
@last_clear.update { |v| v + 5 }
# Do nothing if there's nothing to do ;)
return unless should_flush?
event = LogStash::Event.new
event["message"] = Socket.gethostname
@metric_meters.each_pair do |name, metric|
flush_rates event, name, metric
metric.clear if should_clear?
end
@metric_timers.each_pair do |name, metric|
flush_rates event, name, metric
# These 4 values are not sliding, so they probably are not useful.
event["#{name}.min"] = metric.min
event["#{name}.max"] = metric.max
# timer's stddev currently returns variance, fix it.
event["#{name}.stddev"] = metric.stddev ** 0.5
event["#{name}.mean"] = metric.mean
@percentiles.each do |percentile|
event["#{name}.p#{percentile}"] = metric.snapshot.value(percentile / 100.0)
end
metric.clear if should_clear?
end
# Reset counter since metrics were flushed
@last_flush.value = 0
if should_clear?
#Reset counter since metrics were cleared
@last_clear.value = 0
@metric_meters.clear
@metric_timers.clear
end
filter_matched(event)
return [event]
end
private
def flush_rates(event, name, metric)
event["#{name}.count"] = metric.count
event["#{name}.rate_1m"] = metric.one_minute_rate if @rates.include? 1
event["#{name}.rate_5m"] = metric.five_minute_rate if @rates.include? 5
event["#{name}.rate_15m"] = metric.fifteen_minute_rate if @rates.include? 15
end
def metric_key(key)
"#{@random_key_preffix}_#{key}"
end
def should_flush?
@last_flush.value >= @flush_interval && (!@metric_meters.empty? || !@metric_timers.empty?)
end
def should_clear?
@clear_interval > 0 && @last_clear.value >= @clear_interval
end
end # class LogStash::Filters::Metrics

277
lib/logstash/filters/multiline.rb
View file

@ -1,277 +0,0 @@
# encoding: utf-8
require "logstash/filters/base"
require "logstash/namespace"
require "logstash/environment"
require "set"
#
# This filter will collapse multiline messages from a single source into one Logstash event.
#
# The original goal of this filter was to allow joining of multi-line messages
# from files into a single event. For example - joining java exception and
# stacktrace messages into a single event.
#
# The config looks like this:
#
# filter {
# multiline {
# type => "type"
# pattern => "pattern, a regexp"
# negate => boolean
# what => "previous" or "next"
# }
# }
#
# The `pattern` should be a regexp which matches what you believe to be an indicator
# that the field is part of an event consisting of multiple lines of log data.
#
# The `what` must be "previous" or "next" and indicates the relation
# to the multi-line event.
#
# The `negate` can be "true" or "false" (defaults to false). If "true", a
# message not matching the pattern will constitute a match of the multiline
# filter and the `what` will be applied. (vice-versa is also true)
#
# For example, Java stack traces are multiline and usually have the message
# starting at the far-left, with each subsequent line indented. Do this:
#
# filter {
# multiline {
# type => "somefiletype"
# pattern => "^\s"
# what => "previous"
# }
# }
#
# This says that any line starting with whitespace belongs to the previous line.
#
# Another example is C line continuations (backslash). Here's how to do that:
#
# filter {
# multiline {
# type => "somefiletype "
# pattern => "\\$"
# what => "next"
# }
# }
#
# This says that any line ending with a backslash should be combined with the
# following line.
#
class LogStash::Filters::Multiline < LogStash::Filters::Base
config_name "multiline"
milestone 3
# The regular expression to match.
config :pattern, :validate => :string, :required => true
# If the pattern matched, does event belong to the next or previous event?
config :what, :validate => ["previous", "next"], :required => true
# Negate the regexp pattern ('if not matched')
config :negate, :validate => :boolean, :default => false
# The stream identity is how the multiline filter determines which stream an
# event belongs to. This is generally used for differentiating, say, events
# coming from multiple files in the same file input, or multiple connections
# coming from a tcp input.
#
# The default value here is usually what you want, but there are some cases
# where you want to change it. One such example is if you are using a tcp
# input with only one client connecting at any time. If that client
# reconnects (due to error or client restart), then logstash will identify
# the new connection as a new stream and break any multiline goodness that
# may have occurred between the old and new connection. To solve this use
# case, you can use "%{@source_host}.%{@type}" instead.
config :stream_identity , :validate => :string, :default => "%{host}.%{path}.%{type}"
# Logstash ships by default with a bunch of patterns, so you don't
# necessarily need to define this yourself unless you are adding additional
# patterns.
#
# Pattern files are plain text with format:
#
# NAME PATTERN
#
# For example:
#
# NUMBER \d+
config :patterns_dir, :validate => :array, :default => []
# The maximum age an event can be (in seconds) before it is automatically
# flushed.
config :max_age, :validate => :number, :default => 5
# Call the filter flush method at regular interval.
# Optional.
config :periodic_flush, :validate => :boolean, :default => true
# Detect if we are running from a jarfile, pick the right path.
@@patterns_path = Set.new
@@patterns_path += [LogStash::Environment.pattern_path("*")]
MULTILINE_TAG = "multiline"
public
def initialize(config = {})
super
# this filter cannot be parallelized because message order
# cannot be garanteed across threads, line #2 could be processed
# before line #1
@threadsafe = false
# this filter needs to keep state
@pending = Hash.new
end # def initialize
public
def register
require "grok-pure" # rubygem 'jls-grok'
@grok = Grok.new
@patterns_dir = @@patterns_path.to_a + @patterns_dir
@patterns_dir.each do |path|
if File.directory?(path)
path = File.join(path, "*")
end
Dir.glob(path).each do |file|
@logger.info("Grok loading patterns from file", :path => file)
@grok.add_patterns_from_file(file)
end
end
@grok.compile(@pattern)
case @what
when "previous"
class << self; alias_method :multiline_filter!, :previous_filter!; end
when "next"
class << self; alias_method :multiline_filter!, :next_filter!; end
else
# we should never get here since @what is validated at config
raise(ArgumentError, "Unknown multiline 'what' value")
end # case @what
@logger.debug("Registered multiline plugin", :type => @type, :config => @config)
end # def register
public
def filter(event)
return unless filter?(event)
match = event["message"].is_a?(Array) ? @grok.match(event["message"].first) : @grok.match(event["message"])
match = (match and !@negate) || (!match and @negate) # add negate option
@logger.debug? && @logger.debug("Multiline", :pattern => @pattern, :message => event["message"], :match => match, :negate => @negate)
multiline_filter!(event, match)
unless event.cancelled?
collapse_event!(event)
filter_matched(event) if match
end
end # def filter
# flush any pending messages
# called at regular interval without options and at pipeline shutdown with the :final => true option
# @param options [Hash]
# @option options [Boolean] :final => true to signal a final shutdown flush
# @return [Array<LogStash::Event>] list of flushed events
public
def flush(options = {})
expired = nil
# note that thread safety concerns are not necessary here because the multiline filter
# is not thread safe thus cannot be run in multiple folterworker threads and flushing
# is called by the same thread
# select all expired events from the @pending hash into a new expired hash
# if :final flush then select all events
expired = @pending.inject({}) do |r, (key, event)|
age = Time.now - Array(event["@timestamp"]).first.time
r[key] = event if (age >= @max_age) || options[:final]
r
end
# delete expired items from @pending hash
expired.each{|key, event| @pending.delete(key)}
# return list of uncancelled and collapsed expired events
expired.map{|key, event| event.uncancel; collapse_event!(event)}
end # def flush
public
def teardown
# nothing to do
end
private
def previous_filter!(event, match)
key = event.sprintf(@stream_identity)
pending = @pending[key]
if match
event.tag(MULTILINE_TAG)
# previous previous line is part of this event.
# append it to the event and cancel it
if pending
pending.append(event)
else
@pending[key] = event
end
event.cancel
else
# this line is not part of the previous event
# if we have a pending event, it's done, send it.
# put the current event into pending
if pending
tmp = event.to_hash
event.overwrite(pending)
@pending[key] = LogStash::Event.new(tmp)
else
@pending[key] = event
event.cancel
end
end # if match
end
def next_filter!(event, match)
key = event.sprintf(@stream_identity)
# protect @pending for race condition between the flush thread and the worker thread
pending = @pending[key]
if match
event.tag(MULTILINE_TAG)
# this line is part of a multiline event, the next
# line will be part, too, put it into pending.
if pending
pending.append(event)
else
@pending[key] = event
end
event.cancel
else
# if we have something in pending, join it with this message
# and send it. otherwise, this is a new message and not part of
# multiline, send it.
if pending
pending.append(event)
event.overwrite(pending)
@pending.delete(key)
end
end # if match
end
def collapse_event!(event)
event["message"] = event["message"].join("\n") if event["message"].is_a?(Array)
event.timestamp = event.timestamp.first if event.timestamp.is_a?(Array)
event
end
end # class LogStash::Filters::Multiline

413
lib/logstash/filters/mutate.rb
View file

@ -1,413 +0,0 @@
# encoding: utf-8
require "logstash/filters/base"
require "logstash/namespace"
# The mutate filter allows you to perform general mutations on fields. You
# can rename, remove, replace, and modify fields in your events.
#
# TODO(sissel): Support regexp replacements like String#gsub ?
class LogStash::Filters::Mutate < LogStash::Filters::Base
config_name "mutate"
milestone 3
# Rename one or more fields.
#
# Example:
#
# filter {
# mutate {
# # Renames the 'HOSTORIP' field to 'client_ip'
# rename => { "HOSTORIP" => "client_ip" }
# }
# }
config :rename, :validate => :hash
# Remove one or more fields.
#
# Example:
#
# filter {
# mutate {
# remove => [ "client" ] # Removes the 'client' field
# }
# }
#
# This option is deprecated, instead use remove_field option available in all
# filters.
config :remove, :validate => :array, :deprecated => true
# Replace a field with a new value. The new value can include %{foo} strings
# to help you build a new value from other parts of the event.
#
# Example:
#
# filter {
# mutate {
# replace => { "message" => "%{source_host}: My new message" }
# }
# }
config :replace, :validate => :hash
# Update an existing field with a new value. If the field does not exist,
# then no action will be taken.
#
# Example:
#
# filter {
# mutate {
# update => { "sample" => "My new message" }
# }
# }
config :update, :validate => :hash
# Convert a field's value to a different type, like turning a string to an
# integer. If the field value is an array, all members will be converted.
# If the field is a hash, no action will be taken.
#
# Valid conversion targets are: integer, float, string.
#
# Example:
#
# filter {
# mutate {
# convert => { "fieldname" => "integer" }
# }
# }
config :convert, :validate => :hash
# Convert a string field by applying a regular expression and a replacement.
# If the field is not a string, no action will be taken.
#
# This configuration takes an array consisting of 3 elements per
# field/substitution.
#
# Be aware of escaping any backslash in the config file.
#
# Example:
#
# filter {
# mutate {
# gsub => [
# # replace all forward slashes with underscore
# "fieldname", "/", "_",
#
# # replace backslashes, question marks, hashes, and minuses with
# # dot
# "fieldname2", "[\\?#-]", "."
# ]
# }
# }
#
config :gsub, :validate => :array
# Convert a string to its uppercase equivalent.
#
# Example:
#
# filter {
# mutate {
# uppercase => [ "fieldname" ]
# }
# }
config :uppercase, :validate => :array
# Convert a string to its lowercase equivalent.
#
# Example:
#
# filter {
# mutate {
# lowercase => [ "fieldname" ]
# }
# }
config :lowercase, :validate => :array
# Split a field to an array using a separator character. Only works on string
# fields.
#
# Example:
#
# filter {
# mutate {
# split => { "fieldname" => "," }
# }
# }
config :split, :validate => :hash
# Join an array with a separator character. Does nothing on non-array fields.
#
# Example:
#
# filter {
# mutate {
# join => { "fieldname" => "," }
# }
# }
config :join, :validate => :hash
# Strip whitespace from field. NOTE: this only works on leading and trailing whitespace.
#
# Example:
#
# filter {
# mutate {
# strip => ["field1", "field2"]
# }
# }
config :strip, :validate => :array
# Merge two fields of arrays or hashes.
# String fields will be automatically be converted into an array, so:
# array + string will work
# string + string will result in an 2 entry array in dest_field
# array and hash will not work
#
# Example:
#
# filter {
# mutate {
# merge => { "dest_field" => "added_field" }
# }
# }
config :merge, :validate => :hash
public
def register
valid_conversions = %w(string integer float)
# TODO(sissel): Validate conversion requests if provided.
@convert.nil? or @convert.each do |field, type|
if !valid_conversions.include?(type)
raise LogStash::ConfigurationError, I18n.t("logstash.agent.configuration.invalid_plugin_register",
:plugin => "filter", :type => "mutate",
:error => "Invalid conversion type '#{type}', expected one of '#{valid_conversions.join(',')}'")
end
end # @convert.each
@gsub_parsed = []
@gsub.nil? or @gsub.each_slice(3) do |field, needle, replacement|
if [field, needle, replacement].any? {|n| n.nil?}
raise LogStash::ConfigurationError, I18n.t("logstash.agent.configuration.invalid_plugin_register",
:plugin => "filter", :type => "mutate",
:error => "Invalid gsub configuration #{[field, needle, replacement]}. gsub requires 3 non-nil elements per config entry")
end
@gsub_parsed << {
:field => field,
:needle => (needle.index("%{").nil?? Regexp.new(needle): needle),
:replacement => replacement
}
end
end # def register
public
def filter(event)
return unless filter?(event)
rename(event) if @rename
update(event) if @update
replace(event) if @replace
convert(event) if @convert
gsub(event) if @gsub
uppercase(event) if @uppercase
lowercase(event) if @lowercase
strip(event) if @strip
remove(event) if @remove
split(event) if @split
join(event) if @join
merge(event) if @merge
filter_matched(event)
end # def filter
private
def remove(event)
# TODO(sissel): use event.sprintf on the field names?
@remove.each do |field|
event.remove(field)
end
end # def remove
private
def rename(event)
# TODO(sissel): use event.sprintf on the field names?
@rename.each do |old, new|
next unless event.include?(old)
event[new] = event.remove(old)
end
end # def rename
private
def update(event)
@update.each do |field, newvalue|
next unless event.include?(field)
event[field] = event.sprintf(newvalue)
end
end # def update
private
def replace(event)
@replace.each do |field, newvalue|
event[field] = event.sprintf(newvalue)
end
end # def replace
def convert(event)
@convert.each do |field, type|
next unless event.include?(field)
original = event[field]
# calls convert_{string,integer,float} depending on type requested.
converter = method("convert_" + type)
if original.nil?
next
elsif original.is_a?(Hash)
@logger.debug("I don't know how to type convert a hash, skipping",
:field => field, :value => original)
next
elsif original.is_a?(Array)
value = original.map { |v| converter.call(v) }
else
value = converter.call(original)
end
event[field] = value
end
end # def convert
def convert_string(value)
# since this is a filter and all inputs should be already UTF-8
# we wont check valid_encoding? but just force UTF-8 for
# the Fixnum#to_s case which always result in US-ASCII
# see https://twitter.com/jordansissel/status/444613207143903232
return value.to_s.force_encoding(Encoding::UTF_8)
end # def convert_string
def convert_integer(value)
return value.to_i
end # def convert_integer
def convert_float(value)
return value.to_f
end # def convert_float
private
def gsub(event)
@gsub_parsed.each do |config|
field = config[:field]
needle = config[:needle]
replacement = config[:replacement]
if event[field].is_a?(Array)
event[field] = event[field].map do |v|
if not v.is_a?(String)
@logger.warn("gsub mutation is only applicable for Strings, " +
"skipping", :field => field, :value => v)
v
else
gsub_dynamic_fields(event, v, needle, replacement)
end
end
else
if not event[field].is_a?(String)
@logger.debug("gsub mutation is only applicable for Strings, " +
"skipping", :field => field, :value => event[field])
next
end
event[field] = gsub_dynamic_fields(event, event[field], needle, replacement)
end
end # @gsub_parsed.each
end # def gsub
private
def gsub_dynamic_fields(event, original, needle, replacement)
if needle.is_a? Regexp
original.gsub(needle, event.sprintf(replacement))
else
# we need to replace any dynamic fields
original.gsub(Regexp.new(event.sprintf(needle)), event.sprintf(replacement))
end
end
private
def uppercase(event)
@uppercase.each do |field|
if event[field].is_a?(Array)
event[field].each { |v| v.upcase! }
elsif event[field].is_a?(String)
event[field].upcase!
else
@logger.debug("Can't uppercase something that isn't a string",
:field => field, :value => event[field])
end
end
end # def uppercase
private
def lowercase(event)
@lowercase.each do |field|
if event[field].is_a?(Array)
event[field].each { |v| v.downcase! }
elsif event[field].is_a?(String)
event[field].downcase!
else
@logger.debug("Can't lowercase something that isn't a string",
:field => field, :value => event[field])
end
end
end # def lowercase
private
def split(event)
@split.each do |field, separator|
if event[field].is_a?(String)
event[field] = event[field].split(separator)
else
@logger.debug("Can't split something that isn't a string",
:field => field, :value => event[field])
end
end
end
private
def join(event)
@join.each do |field, separator|
if event[field].is_a?(Array)
event[field] = event[field].join(separator)
end
end
end
private
def strip(event)
@strip.each do |field|
if event[field].is_a?(Array)
event[field] = event[field].map{|s| s.strip }
elsif event[field].is_a?(String)
event[field] = event[field].strip
end
end
end
private
def merge(event)
@merge.each do |dest_field, added_fields|
#When multiple calls, added_field is an array
added_fields = [ added_fields ] if ! added_fields.is_a?(Array)
added_fields.each do |added_field|
if event[dest_field].is_a?(Hash) ^ event[added_field].is_a?(Hash)
@logger.error("Not possible to merge an array and a hash: ",
:dest_field => dest_field,
:added_field => added_field )
next
end
if event[dest_field].is_a?(Hash) #No need to test the other
event[dest_field].update(event[added_field])
else
event[dest_field] = [event[dest_field]] if ! event[dest_field].is_a?(Array)
event[added_field] = [event[added_field]] if ! event[added_field].is_a?(Array)
event[dest_field].concat(event[added_field])
end
end
end
end
end # class LogStash::Filters::Mutate

21
lib/logstash/filters/noop.rb
View file

@ -1,21 +0,0 @@
# encoding: utf-8
require "logstash/filters/base"
require "logstash/namespace"
# No-op filter. This is used generally for internal/dev testing.
class LogStash::Filters::NOOP < LogStash::Filters::Base
config_name "noop"
milestone 2
public
def register
# Nothing
end # def register
public
def filter(event)
return unless filter?(event)
# Nothing to do
filter_matched(event)
end # def filter
end # class LogStash::Filters::NOOP

42
lib/logstash/filters/ruby.rb
View file

@ -1,42 +0,0 @@
# encoding: utf-8
require "logstash/filters/base"
require "logstash/namespace"
# Execute ruby code.
#
# For example, to cancel 90% of events, you can do this:
#
# filter {
# ruby {
# # Cancel 90% of events
# code => "event.cancel if rand <= 0.90"
# }
# }
#
class LogStash::Filters::Ruby < LogStash::Filters::Base
config_name "ruby"
milestone 1
# Any code to execute at logstash startup-time
config :init, :validate => :string
# The code to execute for every event.
# You will have an 'event' variable available that is the event itself.
config :code, :validate => :string, :required => true
public
def register
# TODO(sissel): Compile the ruby code
eval(@init, binding, "(ruby filter init)") if @init
eval("@codeblock = lambda { |event| #{@code} }", binding, "(ruby filter code)")
end # def register
public
def filter(event)
return unless filter?(event)
@codeblock.call(event)
filter_matched(event)
end # def filter
end # class LogStash::Filters::Ruby

111
lib/logstash/filters/sleep.rb
View file

@ -1,111 +0,0 @@
# encoding: utf-8
require "logstash/filters/base"
require "logstash/namespace"
# Sleep a given amount of time. This will cause logstash
# to stall for the given amount of time. This is useful
# for rate limiting, etc.
#
class LogStash::Filters::Sleep < LogStash::Filters::Base
config_name "sleep"
milestone 1
# The length of time to sleep, in seconds, for every event.
#
# This can be a number (eg, 0.5), or a string (eg, "%{foo}")
# The second form (string with a field value) is useful if
# you have an attribute of your event that you want to use
# to indicate the amount of time to sleep.
#
# Example:
#
# filter {
# sleep {
# # Sleep 1 second for every event.
# time => "1"
# }
# }
config :time, :validate => :string
# Sleep on every N'th. This option is ignored in replay mode.
#
# Example:
#
# filter {
# sleep {
# time => "1" # Sleep 1 second
# every => 10 # on every 10th event
# }
# }
config :every, :validate => :string, :default => 1
# Enable replay mode.
#
# Replay mode tries to sleep based on timestamps in each event.
#
# The amount of time to sleep is computed by subtracting the
# previous event's timestamp from the current event's timestamp.
# This helps you replay events in the same timeline as original.
#
# If you specify a `time` setting as well, this filter will
# use the `time` value as a speed modifier. For example,
# a `time` value of 2 will replay at double speed, while a
# value of 0.25 will replay at 1/4th speed.
#
# For example:
#
# filter {
# sleep {
# time => 2
# replay => true
# }
# }
#
# The above will sleep in such a way that it will perform
# replay 2-times faster than the original time speed.
config :replay, :validate => :boolean, :default => false
public
def register
if @replay && @time.nil?
# Default time multiplier is 1 when replay is set.
@time = 1
end
if @time.nil?
raise ArgumentError, "Missing required parameter 'time' for input/eventlog"
end
@count = 0
end # def register
public
def filter(event)
return unless filter?(event)
@count += 1
case @time
when Fixnum, Float; time = @time
when nil; # nothing
else; time = event.sprintf(@time).to_f
end
if @replay
clock = event.timestamp.to_f
if @last_clock
delay = clock - @last_clock
time = delay/time
if time > 0
@logger.debug? && @logger.debug("Sleeping", :delay => time)
sleep(time)
end
end
@last_clock = clock
else
if @count >= @every
@count = 0
@logger.debug? && @logger.debug("Sleeping", :delay => time)
sleep(time)
end
end
filter_matched(event)
end # def filter
end # class LogStash::Filters::Sleep

62
lib/logstash/filters/split.rb
View file

@ -1,62 +0,0 @@
# encoding: utf-8
require "logstash/filters/base"
require "logstash/namespace"
# The split filter is for splitting multiline messages into separate events.
#
# An example use case of this filter is for taking output from the 'exec' input
# which emits one event for the whole output of a command and splitting that
# output by newline - making each line an event.
#
# The end result of each split is a complete copy of the event
# with only the current split section of the given field changed.
class LogStash::Filters::Split < LogStash::Filters::Base
config_name "split"
milestone 2
# The string to split on. This is usually a line terminator, but can be any
# string.
config :terminator, :validate => :string, :default => "\n"
# The field which value is split by the terminator
config :field, :validate => :string, :default => "message"
public
def register
# Nothing to do
end # def register
public
def filter(event)
return unless filter?(event)
original_value = event[@field]
# If for some reason the field is an array of values, take the first only.
original_value = original_value.first if original_value.is_a?(Array)
# Using -1 for 'limit' on String#split makes ruby not drop trailing empty
# splits.
splits = original_value.split(@terminator, -1)
# Skip filtering if splitting this event resulted in only one thing found.
return if splits.length == 1
#or splits[1].empty?
splits.each do |value|
next if value.empty?
event_split = event.clone
@logger.debug("Split event", :value => value, :field => @field)
event_split[@field] = value
filter_matched(event_split)
# Push this new event onto the stack at the LogStash::FilterWorker
yield event_split
end
# Cancel this event, we'll use the newly generated ones above.
event.cancel
end # def filter
end # class LogStash::Filters::Split

32
lib/logstash/filters/spool.rb
View file

@ -1,32 +0,0 @@
# encoding: utf-8
require "logstash/filters/base"
require "logstash/namespace"
require "thread"
# spool filter. this is used generally for internal/dev testing.
class LogStash::Filters::Spool < LogStash::Filters::Base
config_name "spool"
milestone 1
def register
@spool = []
@spool_lock = Mutex.new # to synchronize between the flush & worker threads
end # def register
def filter(event)
return unless filter?(event)
filter_matched(event)
event.cancel
@spool_lock.synchronize {@spool << event}
end # def filter
def flush(options = {})
@spool_lock.synchronize do
flushed = @spool.map{|event| event.uncancel; event}
@spool = []
flushed
end
end
end # class LogStash::Filters::NOOP

107
lib/logstash/filters/syslog_pri.rb
View file

@ -1,107 +0,0 @@
# encoding: utf-8
require "logstash/filters/base"
require "logstash/namespace"
# Filter plugin for logstash to parse the PRI field from the front
# of a Syslog (RFC3164) message. If no priority is set, it will
# default to 13 (per RFC).
#
# This filter is based on the original syslog.rb code shipped
# with logstash.
class LogStash::Filters::Syslog_pri < LogStash::Filters::Base
config_name "syslog_pri"
# set the status to experimental/beta/stable
milestone 1
# Add human-readable names after parsing severity and facility from PRI
config :use_labels, :validate => :boolean, :default => true
# Name of field which passes in the extracted PRI part of the syslog message
config :syslog_pri_field_name, :validate => :string, :default => "syslog_pri"
# Labels for facility levels. This comes from RFC3164.
config :facility_labels, :validate => :array, :default => [
"kernel",
"user-level",
"mail",
"daemon",
"security/authorization",
"syslogd",
"line printer",
"network news",
"uucp",
"clock",
"security/authorization",
"ftp",
"ntp",
"log audit",
"log alert",
"clock",
"local0",
"local1",
"local2",
"local3",
"local4",
"local5",
"local6",
"local7",
]
# Labels for severity levels. This comes from RFC3164.
config :severity_labels, :validate => :array, :default => [
"emergency",
"alert",
"critical",
"error",
"warning",
"notice",
"informational",
"debug",
]
public
def register
# Nothing
end # def register
public
def filter(event)
return unless filter?(event)
parse_pri(event)
filter_matched(event)
end # def filter
private
def parse_pri(event)
# Per RFC3164, priority = (facility * 8) + severity
# = (facility << 3) & (severity)
if event[@syslog_pri_field_name]
if event[@syslog_pri_field_name].is_a?(Array)
priority = event[@syslog_pri_field_name].first.to_i
else
priority = event[@syslog_pri_field_name].to_i
end
else
priority = 13 # default
end
severity = priority & 7 # 7 is 111 (3 bits)
facility = priority >> 3
event["syslog_severity_code"] = severity
event["syslog_facility_code"] = facility
# Add human-readable names after parsing severity and facility from PRI
if @use_labels
facility_number = event["syslog_facility_code"]
severity_number = event["syslog_severity_code"]
if @facility_labels[facility_number]
event["syslog_facility"] = @facility_labels[facility_number]
end
if @severity_labels[severity_number]
event["syslog_severity"] = @severity_labels[severity_number]
end
end
end # def parse_pri
end # class LogStash::Filters::SyslogPRI

261
lib/logstash/filters/throttle.rb
View file

@ -1,261 +0,0 @@
require "logstash/filters/base"
require "logstash/namespace"
# The throttle filter is for throttling the number of events received. The filter
# is configured with a lower bound, the before_count, and upper bound, the after_count,
# and a period of time. All events passing through the filter will be counted based on
# a key. As long as the count is less than the before_count or greater than the
# after_count, the event will be "throttled" which means the filter will be considered
# successful and any tags or fields will be added.
#
# For example, if you wanted to throttle events so you only receive an event after 2
# occurrences and you get no more than 3 in 10 minutes, you would use the
# configuration:
# period => 600
# before_count => 3
# after_count => 5
#
# Which would result in:
# event 1 - throttled (successful filter, period start)
# event 2 - throttled (successful filter)
# event 3 - not throttled
# event 4 - not throttled
# event 5 - not throttled
# event 6 - throttled (successful filter)
# event 7 - throttled (successful filter)
# event x - throttled (successful filter)
# period end
# event 1 - throttled (successful filter, period start)
# event 2 - throttled (successful filter)
# event 3 - not throttled
# event 4 - not throttled
# event 5 - not throttled
# event 6 - throttled (successful filter)
# ...
#
# Another example is if you wanted to throttle events so you only receive 1 event per
# hour, you would use the configuration:
# period => 3600
# before_count => -1
# after_count => 1
#
# Which would result in:
# event 1 - not throttled (period start)
# event 2 - throttled (successful filter)
# event 3 - throttled (successful filter)
# event 4 - throttled (successful filter)
# event x - throttled (successful filter)
# period end
# event 1 - not throttled (period start)
# event 2 - throttled (successful filter)
# event 3 - throttled (successful filter)
# event 4 - throttled (successful filter)
# ...
#
# A common use case would be to use the throttle filter to throttle events before 3 and
# after 5 while using multiple fields for the key and then use the drop filter to remove
# throttled events. This configuration might appear as:
#
# filter {
# throttle {
# before_count => 3
# after_count => 5
# period => 3600
# key => "%{host}%{message}"
# add_tag => "throttled"
# }
# if "throttled" in [tags] {
# drop { }
# }
# }
#
# Another case would be to store all events, but only email non-throttled
# events so the op's inbox isn't flooded with emails in the event of a system error.
# This configuration might appear as:
#
# filter {
# throttle {
# before_count => 3
# after_count => 5
# period => 3600
# key => "%{message}"
# add_tag => "throttled"
# }
# }
# output {
# if "throttled" not in [tags] {
# email {
# from => "logstash@mycompany.com"
# subject => "Production System Alert"
# to => "ops@mycompany.com"
# via => "sendmail"
# body => "Alert on %{host} from path %{path}:\n\n%{message}"
# options => { "location" => "/usr/sbin/sendmail" }
# }
# }
# elasticsearch_http {
# host => "localhost"
# port => "19200"
# }
# }
#
# The event counts are cleared after the configured period elapses since the
# first instance of the event. That is, all the counts don't reset at the same
# time but rather the throttle period is per unique key value.
#
# Mike Pilone (@mikepilone)
#
class LogStash::Filters::Throttle < LogStash::Filters::Base
# The name to use in configuration files.
config_name "throttle"
# New plugins should start life at milestone 1.
milestone 1
# The key used to identify events. Events with the same key will be throttled
# as a group. Field substitutions are allowed, so you can combine multiple
# fields.
config :key, :validate => :string, :required => true
# Events less than this count will be throttled. Setting this value to -1, the
# default, will cause no messages to be throttled based on the lower bound.
config :before_count, :validate => :number, :default => -1, :required => false
# Events greater than this count will be throttled. Setting this value to -1, the
# default, will cause no messages to be throttled based on the upper bound.
config :after_count, :validate => :number, :default => -1, :required => false
# The period in seconds after the first occurrence of an event until the count is
# reset for the event. This period is tracked per unique key value. Field
# substitutions are allowed in this value. They will be evaluated when the _first_
# event for a given key is seen. This allows you to specify that certain kinds
# of events throttle for a specific period.
config :period, :validate => :string, :default => "3600", :required => false
# The maximum number of counters to store before the oldest counter is purged. Setting
# this value to -1 will prevent an upper bound no constraint on the number of counters
# and they will only be purged after expiration. This configuration value should only
# be used as a memory control mechanism and can cause early counter expiration if the
# value is reached. It is recommended to leave the default value and ensure that your
# key is selected such that it limits the number of counters required (i.e. don't
# use UUID as the key!)
config :max_counters, :validate => :number, :default => 100000, :required => false
# Performs initialization of the filter.
public
def register
@threadsafe = false
@event_counters = Hash.new
@next_expiration = nil
end # def register
# Filters the event. The filter is successful if the event should be throttled.
public
def filter(event)
# Return nothing unless there's an actual filter event
return unless filter?(event)
now = Time.now
key = event.sprintf(@key)
# Purge counters if too large to prevent OOM.
if @max_counters != -1 && @event_counters.size > @max_counters then
purgeOldestEventCounter()
end
# Expire existing counter if needed
if @next_expiration.nil? || now >= @next_expiration then
expireEventCounters(now)
end
@logger.debug? and @logger.debug(
"filters/#{self.class.name}: next expiration",
{ "next_expiration" => @next_expiration })
# Create new counter for this event if this is the first occurrence
counter = nil
if !@event_counters.include?(key) then
period = event.sprintf(@period).to_i
period = 3600 if period == 0
expiration = now + period
@event_counters[key] = { :count => 0, :expiration => expiration }
@logger.debug? and @logger.debug("filters/#{self.class.name}: new event",
{ :key => key, :expiration => expiration })
end
# Fetch the counter
counter = @event_counters[key]
# Count this event
counter[:count] = counter[:count] + 1;
@logger.debug? and @logger.debug("filters/#{self.class.name}: current count",
{ :key => key, :count => counter[:count] })
# Throttle if count is < before count or > after count
if ((@before_count != -1 && counter[:count] < @before_count) ||
(@after_count != -1 && counter[:count] > @after_count)) then
@logger.debug? and @logger.debug(
"filters/#{self.class.name}: throttling event", { :key => key })
filter_matched(event)
end
end # def filter
# Expires any counts where the period has elapsed. Sets the next expiration time
# for when this method should be called again.
private
def expireEventCounters(now)
@next_expiration = nil
@event_counters.delete_if do |key, counter|
expiration = counter[:expiration]
expired = expiration <= now
if expired then
@logger.debug? and @logger.debug(
"filters/#{self.class.name}: deleting expired counter",
{ :key => key })
elsif @next_expiration.nil? || (expiration < @next_expiration)
@next_expiration = expiration
end
expired
end
end # def expireEventCounters
# Purges the oldest event counter. This operation is for memory control only
# and can cause early period expiration and thrashing if invoked.
private
def purgeOldestEventCounter()
# Return unless we have something to purge
return unless @event_counters.size > 0
oldestCounter = nil
oldestKey = nil
@event_counters.each do |key, counter|
if oldestCounter.nil? || counter[:expiration] < oldestCounter[:expiration] then
oldestKey = key;
oldestCounter = counter;
end
end
@logger.warn? and @logger.warn(
"filters/#{self.class.name}: Purging oldest counter because max_counters " +
"exceeded. Use a better key to prevent too many unique event counters.",
{ :key => oldestKey, :expiration => oldestCounter[:expiration] })
@event_counters.delete(oldestKey)
end
end # class LogStash::Filters::Throttle

57
lib/logstash/filters/urldecode.rb
View file

@ -1,57 +0,0 @@
# encoding: utf-8
require "logstash/filters/base"
require "logstash/namespace"
require "uri"
# The urldecode filter is for decoding fields that are urlencoded.
class LogStash::Filters::Urldecode < LogStash::Filters::Base
config_name "urldecode"
milestone 2
# The field which value is urldecoded
config :field, :validate => :string, :default => "message"
# Urldecode all fields
config :all_fields, :validate => :boolean, :default => false
public
def register
# Nothing to do
end #def register
public
def filter(event)
return unless filter?(event)
# If all_fields is true then try to decode them all
if @all_fields
event.to_hash.each do |name, value|
event[name] = urldecode(value)
end
# Else decode the specified field
else
event[@field] = urldecode(event[@field])
end
filter_matched(event)
end # def filter
# Attempt to handle string, array, and hash values for fields.
# For all other datatypes, just return, URI.unescape doesn't support them.
private
def urldecode(value)
case value
when String
return URI.unescape(value)
when Array
ret_values = []
value.each { |v| ret_values << urldecode(v) }
return ret_values
when Hash
ret_values = {}
value.each { |k,v| ret_values[k] = urldecode(v) }
return ret_values
else
return value
end
end
end # class LogStash::Filters::Urldecode

107
lib/logstash/filters/useragent.rb
View file

@ -1,107 +0,0 @@
# encoding: utf-8
require "logstash/filters/base"
require "logstash/namespace"
require "tempfile"
# Parse user agent strings into structured data based on BrowserScope data
#
# UserAgent filter, adds information about user agent like family, operating
# system, version, and device
#
# Logstash releases ship with the regexes.yaml database made available from
# ua-parser with an Apache 2.0 license. For more details on ua-parser, see
# <https://github.com/tobie/ua-parser/>.
class LogStash::Filters::UserAgent < LogStash::Filters::Base
config_name "useragent"
milestone 3
# The field containing the user agent string. If this field is an
# array, only the first value will be used.
config :source, :validate => :string, :required => true
# The name of the field to assign user agent data into.
#
# If not specified user agent data will be stored in the root of the event.
config :target, :validate => :string
# regexes.yaml file to use
#
# If not specified, this will default to the regexes.yaml that ships
# with logstash.
#
# You can find the latest version of this here:
# <https://github.com/tobie/ua-parser/blob/master/regexes.yaml>
config :regexes, :validate => :string
# A string to prepend to all of the extracted keys
config :prefix, :validate => :string, :default => ''
public
def register
require 'user_agent_parser'
if @regexes.nil?
begin
@parser = UserAgentParser::Parser.new()
rescue Exception => e
begin
@parser = UserAgentParser::Parser.new(:patterns_path => "vendor/ua-parser/regexes.yaml")
rescue => ex
raise "Failed to cache, due to: #{ex}\n"
end
end
else
@logger.info("Using user agent regexes", :regexes => @regexes)
@parser = UserAgentParser::Parser.new(:patterns_path => @regexes)
end
end #def register
public
def filter(event)
return unless filter?(event)
ua_data = nil
useragent = event[@source]
useragent = useragent.first if useragent.is_a? Array
begin
ua_data = @parser.parse(useragent)
rescue Exception => e
@logger.error("Uknown error while parsing user agent data", :exception => e, :field => @source, :event => event)
end
if !ua_data.nil?
if @target.nil?
# default write to the root of the event
target = event
else
target = event[@target] ||= {}
end
# UserAgentParser outputs as US-ASCII.
target[@prefix + "name"] = ua_data.name.force_encoding(Encoding::UTF_8)
#OSX, Andriod and maybe iOS parse correctly, ua-agent parsing for Windows does not provide this level of detail
unless ua_data.os.nil?
target[@prefix + "os"] = ua_data.os.to_s.force_encoding(Encoding::UTF_8)
target[@prefix + "os_name"] = ua_data.os.name.to_s.force_encoding(Encoding::UTF_8)
target[@prefix + "os_major"] = ua_data.os.version.major.to_s.force_encoding(Encoding::UTF_8) unless ua_data.os.version.nil?
target[@prefix + "os_minor"] = ua_data.os.version.minor.to_s.force_encoding(Encoding::UTF_8) unless ua_data.os.version.nil?
end
target[@prefix + "device"] = ua_data.device.to_s.force_encoding(Encoding::UTF_8) if not ua_data.device.nil?
if not ua_data.version.nil?
ua_version = ua_data.version
target[@prefix + "major"] = ua_version.major.force_encoding(Encoding::UTF_8) if ua_version.major
target[@prefix + "minor"] = ua_version.minor.force_encoding(Encoding::UTF_8) if ua_version.minor
target[@prefix + "patch"] = ua_version.patch.force_encoding(Encoding::UTF_8) if ua_version.patch
target[@prefix + "build"] = ua_version.patch_minor.force_encoding(Encoding::UTF_8) if ua_version.patch_minor
end
filter_matched(event)
end
end # def filter
end # class LogStash::Filters::UserAgent

58
lib/logstash/filters/uuid.rb
View file

@ -1,58 +0,0 @@
# encoding: utf-8
require "logstash/filters/base"
require "logstash/namespace"
require "securerandom"
# The uuid filter allows you to add a UUID field to messages.
# This is useful to be able to control the _id messages are indexed into Elasticsearch
# with, so that you can insert duplicate messages (i.e. the same message multiple times
# without creating duplicates) - for log pipeline reliability
#
class LogStash::Filters::Uuid < LogStash::Filters::Base
config_name "uuid"
milestone 2
# Add a UUID to a field.
#
# Example:
#
# filter {
# uuid {
# target => "@uuid"
# }
# }
config :target, :validate => :string, :required => true
# If the value in the field currently (if any) should be overridden
# by the generated UUID. Defaults to false (i.e. if the field is
# present, with ANY value, it won't be overridden)
#
# Example:
#
# filter {
# uuid {
# target => "@uuid"
# overwrite => true
# }
# }
config :overwrite, :validate => :boolean, :default => false
public
def register
end # def register
public
def filter(event)
return unless filter?(event)
if overwrite
event[target] = SecureRandom.uuid
else
event[target] ||= SecureRandom.uuid
end
filter_matched(event)
end # def filter
end # class LogStash::Filters::Uuid

139
lib/logstash/filters/xml.rb
View file

@ -1,139 +0,0 @@
# encoding: utf-8
require "logstash/filters/base"
require "logstash/namespace"
# XML filter. Takes a field that contains XML and expands it into
# an actual datastructure.
class LogStash::Filters::Xml < LogStash::Filters::Base
config_name "xml"
milestone 1
# Config for xml to hash is:
#
# source => source_field
#
# For example, if you have the whole xml document in your @message field:
#
# filter {
# xml {
# source => "message"
# }
# }
#
# The above would parse the xml from the @message field
config :source, :validate => :string
# Define target for placing the data
#
# for example if you want the data to be put in the 'doc' field:
#
# filter {
# xml {
# target => "doc"
# }
# }
#
# XML in the value of the source field will be expanded into a
# datastructure in the "target" field.
# Note: if the "target" field already exists, it will be overridden
# Required
config :target, :validate => :string
# xpath will additionally select string values (.to_s on whatever is selected)
# from parsed XML (using each source field defined using the method above)
# and place those values in the destination fields. Configuration:
#
# xpath => [ "xpath-syntax", "destination-field" ]
#
# Values returned by XPath parsring from xpath-synatx will be put in the
# destination field. Multiple values returned will be pushed onto the
# destination field as an array. As such, multiple matches across
# multiple source fields will produce duplicate entries in the field
#
# More on xpath: http://www.w3schools.com/xpath/
#
# The xpath functions are particularly powerful:
# http://www.w3schools.com/xpath/xpath_functions.asp
#
config :xpath, :validate => :hash, :default => {}
# By default the filter will store the whole parsed xml in the destination
# field as described above. Setting this to false will prevent that.
config :store_xml, :validate => :boolean, :default => true
public
def register
require "nokogiri"
require "xmlsimple"
end # def register
public
def filter(event)
return unless filter?(event)
matched = false
@logger.debug("Running xml filter", :event => event)
return unless event.include?(@source)
value = event[@source]
if value.is_a?(Array) && value.length > 1
@logger.warn("XML filter only works on fields of length 1",
:source => @source, :value => value)
return
end
# Do nothing with an empty string.
return if value.strip.length == 0
if @xpath
begin
doc = Nokogiri::XML(value, nil, value.encoding.to_s)
rescue => e
event.tag("_xmlparsefailure")
@logger.warn("Trouble parsing xml", :source => @source, :value => value,
:exception => e, :backtrace => e.backtrace)
return
end
@xpath.each do |xpath_src, xpath_dest|
nodeset = doc.xpath(xpath_src)
# If asking xpath for a String, like "name(/*)", we get back a
# String instead of a NodeSet. We normalize that here.
normalized_nodeset = nodeset.kind_of?(Nokogiri::XML::NodeSet) ? nodeset : [nodeset]
normalized_nodeset.each do |value|
# some XPath functions return empty arrays as string
if value.is_a?(Array)
return if value.length == 0
end
unless value.nil?
matched = true
event[xpath_dest] ||= []
event[xpath_dest] << value.to_s
end
end # XPath.each
end # @xpath.each
end # if @xpath
if @store_xml
begin
event[@target] = XmlSimple.xml_in(value)
matched = true
rescue => e
event.tag("_xmlparsefailure")
@logger.warn("Trouble parsing xml with XmlSimple", :source => @source,
:value => value, :exception => e, :backtrace => e.backtrace)
return
end
end # if @store_xml
filter_matched(event) if matched
@logger.debug("Event after xml filter", :event => event)
end # def filter
end # class LogStash::Filters::Xml

459
lib/logstash/inputs/collectd.rb
View file

@ -1,459 +0,0 @@
# encoding utf-8
require "date"
require "logstash/inputs/base"
require "logstash/namespace"
require "logstash/timestamp"
require "socket"
require "tempfile"
require "time"
# Read events from the connectd binary protocol over the network via udp.
# See https://collectd.org/wiki/index.php/Binary_protocol
#
# Configuration in your Logstash configuration file can be as simple as:
# input {
# collectd {}
# }
#
# A sample collectd.conf to send to Logstash might be:
#
# Hostname "host.example.com"
# LoadPlugin interface
# LoadPlugin load
# LoadPlugin memory
# LoadPlugin network
# <Plugin interface>
# Interface "eth0"
# IgnoreSelected false
# </Plugin>
# <Plugin network>
# <Server "10.0.0.1" "25826">
# </Server>
# </Plugin>
#
# Be sure to replace "10.0.0.1" with the IP of your Logstash instance.
#
#
class LogStash::Inputs::Collectd < LogStash::Inputs::Base
config_name "collectd"
milestone 1
AUTHFILEREGEX = /([^:]+): (.+)/
TYPEMAP = {
0 => "host",
1 => "@timestamp",
2 => "plugin",
3 => "plugin_instance",
4 => "collectd_type",
5 => "type_instance",
6 => "values",
7 => "interval",
8 => "@timestamp",
9 => "interval",
256 => "message",
257 => "severity",
512 => "signature",
528 => "encryption"
}
SECURITY_NONE = "None"
SECURITY_SIGN = "Sign"
SECURITY_ENCR = "Encrypt"
# File path(s) to collectd types.db to use.
# The last matching pattern wins if you have identical pattern names in multiple files.
# If no types.db is provided the included types.db will be used (currently 5.4.0).
config :typesdb, :validate => :array
# The address to listen on. Defaults to all available addresses.
config :host, :validate => :string, :default => "0.0.0.0"
# The port to listen on. Defaults to the collectd expected port of 25826.
config :port, :validate => :number, :default => 25826
# Prune interval records. Defaults to true.
config :prune_intervals, :validate => :boolean, :default => true
# Buffer size. 1452 is the collectd default for v5+
config :buffer_size, :validate => :number, :default => 1452
# Security Level. Default is "None". This setting mirrors the setting from the
# collectd [Network plugin](https://collectd.org/wiki/index.php/Plugin:Network)
config :security_level, :validate => [SECURITY_NONE, SECURITY_SIGN, SECURITY_ENCR],
:default => "None"
# Path to the authentication file. This file should have the same format as
# the [AuthFile](http://collectd.org/documentation/manpages/collectd.conf.5.shtml#authfile_filename)
# in collectd. You only need to set this option if the security_level is set to
# "Sign" or "Encrypt"
config :authfile, :validate => :string
# What to do when a value in the event is NaN (Not a Number)
# - change_value (default): Change the NaN to the value of the nan_value option and add nan_tag as a tag
# - warn: Change the NaN to the value of the nan_value option, print a warning to the log and add nan_tag as a tag
# - drop: Drop the event containing the NaN (this only drops the single event, not the whole packet)
config :nan_handeling, :validate => ['change_value','warn','drop'],
:default => 'change_value'
# Only relevant when nan_handeling is set to 'change_value'
# Change NaN to this configured value
config :nan_value, :validate => :number, :default => 0
# The tag to add to the event if a NaN value was found
# Set this to an empty string ('') if you don't want to tag
config :nan_tag, :validate => :string, :default => '_collectdNaN'
public
def initialize(params)
super
BasicSocket.do_not_reverse_lookup = true
@timestamp = LogStash::Timestamp.now
@collectd = {}
@types = {}
end # def initialize
public
def register
@udp = nil
if @typesdb.nil?
@typesdb = LogStash::Environment.vendor_path("collectd/types.db")
if !File.exists?(@typesdb)
raise "You must specify 'typesdb => ...' in your collectd input (I looked for '#{@typesdb}')"
end
@logger.info("Using internal types.db", :typesdb => @typesdb.to_s)
end
if ([SECURITY_SIGN, SECURITY_ENCR].include?(@security_level))
if @authfile.nil?
raise "Security level is set to #{@security_level}, but no authfile was configured"
else
# Load OpenSSL and instantiate Digest and Crypto functions
require 'openssl'
@sha256 = OpenSSL::Digest::Digest.new('sha256')
@sha1 = OpenSSL::Digest::Digest.new('sha1')
@cipher = OpenSSL::Cipher.new('AES-256-OFB')
@auth = {}
parse_authfile
end
end
end # def register
public
def run(output_queue)
begin
# get types
get_types(@typesdb)
# collectd server
collectd_listener(output_queue)
rescue LogStash::ShutdownSignal
# do nothing, shutdown was requested.
rescue => e
@logger.warn("Collectd listener died", :exception => e, :backtrace => e.backtrace)
sleep(5)
retry
end # begin
end # def run
public
def get_types(paths)
# Get the typesdb
paths = Array(paths) # Make sure a single path is still forced into an array type
paths.each do |path|
@logger.info("Getting Collectd typesdb info", :typesdb => path.to_s)
File.open(path, 'r').each_line do |line|
typename, *line = line.strip.split
next if typename.nil? || if typename[0,1] != '#' # Don't process commented or blank lines
v = line.collect { |l| l.strip.split(":")[0] }
@types[typename] = v
end
end
end
@logger.debug("Collectd Types", :types => @types.to_s)
end # def get_types
public
def get_values(id, body)
retval = ''
case id
when 0,2,3,4,5,256 #=> String types
retval = body.pack("C*")
retval = retval[0..-2]
when 1 # Time
# Time here, in bit-shifted format. Parse bytes into UTC.
byte1, byte2 = body.pack("C*").unpack("NN")
retval = Time.at(( ((byte1 << 32) + byte2))).utc
when 7,257 #=> Numeric types
retval = body.slice!(0..7).pack("C*").unpack("E")[0]
when 8 # Time, Hi-Res
# Time here, in bit-shifted format. Parse bytes into UTC.
byte1, byte2 = body.pack("C*").unpack("NN")
retval = Time.at(( ((byte1 << 32) + byte2) * (2**-30) )).utc
when 9 # Interval, Hi-Res
byte1, byte2 = body.pack("C*").unpack("NN")
retval = (((byte1 << 32) + byte2) * (2**-30)).to_i
when 6 # Values
val_bytes = body.slice!(0..1)
val_count = val_bytes.pack("C*").unpack("n")
if body.length % 9 == 0 # Should be 9 fields
count = 0
retval = []
types = body.slice!(0..((body.length/9)-1))
while body.length > 0
# TYPE VALUES:
# 0: COUNTER
# 1: GAUGE
# 2: DERIVE
# 3: ABSOLUTE
case types[count]
when 1;
v = body.slice!(0..7).pack("C*").unpack("E")[0]
if v.nan?
case @nan_handeling
when 'drop'; return false
else
v = @nan_value
add_tag(@nan_tag)
@nan_handeling == 'warn' && @logger.warn("NaN in (unfinished event) #{@collectd}")
end
end
when 0, 3; v = body.slice!(0..7).pack("C*").unpack("Q>")[0]
when 2; v = body.slice!(0..7).pack("C*").unpack("q>")[0]
else; v = 0
end
retval << v
count += 1
end
else
@logger.error("Incorrect number of data fields for collectd record", :body => body.to_s)
end
when 512 # signature
if body.length < 32
@logger.warning("SHA256 signature too small (got #{body.length} bytes instead of 32)")
elsif body.length < 33
@logger.warning("Received signature without username")
else
retval = []
# Byte 32 till the end contains the username as chars (=unsigned ints)
retval << body[32..-1].pack('C*')
# Byte 0 till 31 contain the signature
retval << body[0..31].pack('C*')
end
when 528 # encryption
retval = []
user_length = (body.slice!(0) << 8) + body.slice!(0)
retval << body.slice!(0..user_length-1).pack('C*') # Username
retval << body.slice!(0..15).pack('C*') # IV
retval << body.pack('C*') # Encrypted content
end
return retval
end # def get_values
private
def parse_authfile
# We keep the authfile parsed in memory so we don't have to open the file
# for every event.
@logger.debug("Parsing authfile #{@authfile}")
if !File.exist?(@authfile)
raise "The file #{@authfile} was not found"
end
@auth.clear
@authmtime = File.stat(@authfile).mtime
File.readlines(@authfile).each do |line|
#line.chomp!
k,v = line.scan(AUTHFILEREGEX).flatten
if k and v
@logger.debug("Added authfile entry '#{k}' with key '#{v}'")
@auth[k] = v
else
@logger.info("Ignoring malformed authfile line '#{line.chomp}'")
end
end
end # def parse_authfile
private
def get_key(user)
return if @authmtime.nil? or @authfile.nil?
# Validate that our auth data is still up-to-date
parse_authfile if @authmtime < File.stat(@authfile).mtime
key = @auth[user]
@logger.warn("User #{user} is not found in the authfile #{@authfile}") if key.nil?
return key
end # def get_key
private
def verify_signature(user, signature, payload)
# The user doesn't care about the security
return true if @security_level == SECURITY_NONE
# We probably got and array of ints, pack it!
payload = payload.pack('C*') if payload.is_a?(Array)
key = get_key(user)
return false if key.nil?
return true if OpenSSL::HMAC.digest(@sha256, key, user+payload) == signature
return false
end # def verify_signature
private
def decrypt_packet(user, iv, content)
# Content has to have at least a SHA1 hash (20 bytes), a header (4 bytes) and
# one byte of data
return [] if content.length < 26
content = content.pack('C*') if content.is_a?(Array)
key = get_key(user)
return [] if key.nil?
# Set the correct state of the cipher instance
@cipher.decrypt
@cipher.padding = 0
@cipher.iv = iv
@cipher.key = @sha256.digest(key);
# Decrypt the content
plaintext = @cipher.update(content) + @cipher.final
# Reset the state, as adding a new key to an already instantiated state
# results in an exception
@cipher.reset
# The plaintext contains a SHA1 hash as checksum in the first 160 bits
# (20 octets) of the rest of the data
hash = plaintext.slice!(0..19)
if @sha1.digest(plaintext) != hash
@logger.warn("Unable to decrypt packet, checksum mismatch")
return []
end
return plaintext.unpack('C*')
end # def decrypt_packet
private
def generate_event(output_queue)
# Prune these *specific* keys if they exist and are empty.
# This is better than looping over all keys every time.
@collectd.delete('type_instance') if @collectd['type_instance'] == ""
@collectd.delete('plugin_instance') if @collectd['plugin_instance'] == ""
# As crazy as it sounds, this is where we actually send our events to the queue!
event = LogStash::Event.new
@collectd.each {|k, v| event[k] = @collectd[k]}
decorate(event)
output_queue << event
end # def generate_event
private
def clean_up()
@collectd.each_key do |k|
@collectd.delete(k) if !['host','collectd_type', 'plugin', 'plugin_instance', '@timestamp', 'type_instance'].include?(k)
end
end # def clean_up
private
def add_tag(new_tag)
return if new_tag.empty?
@collectd['tags'] ||= []
@collectd['tags'] << new_tag
end
private
def collectd_listener(output_queue)
@logger.info("Starting Collectd listener", :address => "#{@host}:#{@port}")
if @udp && ! @udp.closed?
@udp.close
end
@udp = UDPSocket.new(Socket::AF_INET)
@udp.bind(@host, @port)
loop do
payload, client = @udp.recvfrom(@buffer_size)
payload = payload.bytes.to_a
# Clear the last event
@collectd.clear
was_encrypted = false
while payload.length > 0 do
typenum = (payload.slice!(0) << 8) + payload.slice!(0)
# Get the length of the data in this part, but take into account that
# the header is 4 bytes
length = ((payload.slice!(0) << 8) + payload.slice!(0)) - 4
if length > payload.length
@logger.info("Header indicated #{length} bytes will follow, but packet has only #{payload.length} bytes left")
break
end
body = payload.slice!(0..length-1)
field = TYPEMAP[typenum]
if field.nil?
@logger.warn("Unknown typenumber: #{typenum}")
next
end
values = get_values(typenum, body)
case field
when "signature"
break if !verify_signature(values[0], values[1], payload)
next
when "encryption"
payload = decrypt_packet(values[0], values[1], values[2])
# decrypt_packet returns an empty array if the decryption was
# unsuccessful and this inner loop checks the length. So we can safely
# set the 'was_encrypted' variable.
was_encrypted=true
next
when "plugin"
# We've reached a new plugin, delete everything except for the the host
# field, because there's only one per packet and the timestamp field,
# because that one goes in front of the plugin
@collectd.each_key do |k|
@collectd.delete(k) if !['host', '@timestamp'].include?(k)
end
when "collectd_type"
# We've reached a new type within the plugin section, delete all fields
# that could have something to do with the previous type (if any)
@collectd.each_key do |k|
@collectd.delete(k) if !['host', '@timestamp', 'plugin', 'plugin_instance'].include?(k)
end
end
break if !was_encrypted and @security_level == SECURITY_ENCR
# Fill in the fields.
if values.kind_of?(Array)
if values.length > 1 # Only do this iteration on multi-value arrays
values.each_with_index {|value, x| @collectd[@types[@collectd['collectd_type']][x]] = values[x]}
else # Otherwise it's a single value
@collectd['value'] = values[0] # So name it 'value' accordingly
end
elsif !values
clean_up()
next
elsif field != nil # Not an array, make sure it's non-empty
@collectd[field] = values # Append values to @collectd under key field
end
if ["interval", "values"].include?(field)
if ((@prune_intervals && ![7,9].include?(typenum)) || !@prune_intervals)
generate_event(output_queue)
end
clean_up()
end
end # while payload.length > 0 do
end # loop do
ensure
if @udp
@udp.close_read rescue nil
@udp.close_write rescue nil
end
end # def collectd_listener
public
def teardown
@udp.close if @udp && !@udp.closed?
end
end # class LogStash::Inputs::Collectd

134
lib/logstash/inputs/elasticsearch.rb
View file

@ -1,134 +0,0 @@
# encoding: utf-8
require "logstash/inputs/base"
require "logstash/namespace"
require "logstash/util/socket_peer"
require "logstash/json"
# Read from an Elasticsearch cluster, based on search query results.
# This is useful for replaying test logs, reindexing, etc.
#
# Example:
#
# input {
# # Read all documents from Elasticsearch matching the given query
# elasticsearch {
# host => "localhost"
# query => "ERROR"
# }
# }
#
# This would create an Elasticsearch query with the following format:
#
# http://localhost:9200/logstash-*/_search?q=ERROR&scroll=1m&size=1000
#
# * TODO(sissel): Option to keep the index, type, and doc id so we can do reindexing?
class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
config_name "elasticsearch"
milestone 1
default :codec, "json"
# The IP address or hostname of your Elasticsearch server.
config :host, :validate => :string, :required => true
# The HTTP port of your Elasticsearch server's REST interface.
config :port, :validate => :number, :default => 9200
# The index or alias to search.
config :index, :validate => :string, :default => "logstash-*"
# The query to be executed.
config :query, :validate => :string, :default => "*"
# Enable the Elasticsearch "scan" search type. This will disable
# sorting but increase speed and performance.
config :scan, :validate => :boolean, :default => true
# This allows you to set the maximum number of hits returned per scroll.
config :size, :validate => :number, :default => 1000
# This parameter controls the keepalive time in seconds of the scrolling
# request and initiates the scrolling process. The timeout applies per
# round trip (i.e. between the previous scan scroll request, to the next).
config :scroll, :validate => :string, :default => "1m"
public
def register
require "ftw"
@agent = FTW::Agent.new
params = {
"q" => @query,
"scroll" => @scroll,
"size" => "#{@size}",
}
params['search_type'] = "scan" if @scan
@search_url = "http://#{@host}:#{@port}/#{@index}/_search?#{encode(params)}"
@scroll_url = "http://#{@host}:#{@port}/_search/scroll?#{encode({"scroll" => @scroll})}"
end # def register
private
def encode(hash)
return hash.collect do |key, value|
CGI.escape(key) + "=" + CGI.escape(value)
end.join("&")
end # def encode
private
def execute_search_request
response = @agent.get!(@search_url)
json = ""
response.read_body { |c| json << c }
json
end
private
def execute_scroll_request(scroll_id)
response = @agent.post!(@scroll_url, :body => scroll_id)
json = ""
response.read_body { |c| json << c }
json
end
public
def run(output_queue)
result = LogStash::Json.load(execute_search_request)
scroll_id = result["_scroll_id"]
# When using the search_type=scan we don't get an initial result set.
# So we do it here.
if @scan
result = LogStash::Json.load(execute_scroll_request(scroll_id))
end
loop do
break if result.nil?
hits = result["hits"]["hits"]
break if hits.empty?
hits.each do |hit|
# Hack to make codecs work
@codec.decode(LogStash::Json.dump(hit["_source"])) do |event|
decorate(event)
output_queue << event
end
end
# Get the scroll id from the previous result set and use it for getting the next data set
scroll_id = result["_scroll_id"]
# Fetch the next result set
result = LogStash::Json.load(execute_scroll_request(scroll_id))
if result["error"]
@logger.warn(result["error"], :request => scroll_url)
# TODO(sissel): raise an error instead of breaking
break
end
end
rescue LogStash::ShutdownSignal
# Do nothing, let us quit.
end # def run
end # class LogStash::Inputs::Elasticsearch

129
lib/logstash/inputs/eventlog.rb
View file

@ -1,129 +0,0 @@
# encoding: utf-8
require "logstash/inputs/base"
require "logstash/namespace"
require "logstash/timestamp"
require "socket"
# This input will pull events from a (http://msdn.microsoft.com/en-us/library/windows/desktop/bb309026%28v=vs.85%29.aspx)[Windows Event Log].
#
# To collect Events from the System Event Log, use a config like:
#
# input {
# eventlog {
# type => 'Win32-EventLog'
# logfile => 'System'
# }
# }
class LogStash::Inputs::EventLog < LogStash::Inputs::Base
config_name "eventlog"
milestone 2
default :codec, "plain"
# Event Log Name
config :logfile, :validate => :array, :default => [ "Application", "Security", "System" ]
public
def register
# wrap specified logfiles in suitable OR statements
@logfiles = @logfile.join("' OR TargetInstance.LogFile = '")
@hostname = Socket.gethostname
@logger.info("Registering input eventlog://#{@hostname}/#{@logfile}")
if RUBY_PLATFORM == "java"
require "jruby-win32ole"
else
require "win32ole"
end
end # def register
public
def run(queue)
@wmi = WIN32OLE.connect("winmgmts://")
wmi_query = "Select * from __InstanceCreationEvent Where TargetInstance ISA 'Win32_NTLogEvent' And (TargetInstance.LogFile = '#{@logfiles}')"
begin
@logger.debug("Tailing Windows Event Log '#{@logfile}'")
events = @wmi.ExecNotificationQuery(wmi_query)
while
notification = events.NextEvent
event = notification.TargetInstance
timestamp = to_timestamp(event.TimeGenerated)
e = LogStash::Event.new(
"host" => @hostname,
"path" => @logfile,
"type" => @type,
LogStash::Event::TIMESTAMP => timestamp
)
%w{Category CategoryString ComputerName EventCode EventIdentifier
EventType Logfile Message RecordNumber SourceName
TimeGenerated TimeWritten Type User
}.each{
|property| e[property] = event.send property
}
if RUBY_PLATFORM == "java"
# unwrap jruby-win32ole racob data
e["InsertionStrings"] = unwrap_racob_variant_array(event.InsertionStrings)
data = unwrap_racob_variant_array(event.Data)
# Data is an array of signed shorts, so convert to bytes and pack a string
e["Data"] = data.map{|byte| (byte > 0) ? byte : 256 + byte}.pack("c*")
else
# win32-ole data does not need to be unwrapped
e["InsertionStrings"] = event.InsertionStrings
e["Data"] = event.Data
end
e["message"] = event.Message
decorate(e)
queue << e
end # while
rescue Exception => ex
@logger.error("Windows Event Log error: #{ex}\n#{ex.backtrace}")
sleep 1
retry
end # rescue
end # def run
private
def unwrap_racob_variant_array(variants)
variants ||= []
variants.map {|v| (v.respond_to? :getValue) ? v.getValue : v}
end # def unwrap_racob_variant_array
# the event log timestamp is a utc string in the following format: yyyymmddHHMMSS.xxxxxx±UUU
# http://technet.microsoft.com/en-us/library/ee198928.aspx
private
def to_timestamp(wmi_time)
result = ""
# parse the utc date string
/(?<w_date>\d{8})(?<w_time>\d{6})\.\d{6}(?<w_sign>[\+-])(?<w_diff>\d{3})/ =~ wmi_time
result = "#{w_date}T#{w_time}#{w_sign}"
# the offset is represented by the difference, in minutes,
# between the local time zone and Greenwich Mean Time (GMT).
if w_diff.to_i > 0
# calculate the timezone offset in hours and minutes
h_offset = w_diff.to_i / 60
m_offset = w_diff.to_i - (h_offset * 60)
result.concat("%02d%02d" % [h_offset, m_offset])
else
result.concat("0000")
end
return LogStash::Timestamp.new(DateTime.strptime(result, "%Y%m%dT%H%M%S%z").to_time)
end
end # class LogStash::Inputs::EventLog

68
lib/logstash/inputs/exec.rb
View file

@ -1,68 +0,0 @@
# encoding: utf-8
require "logstash/inputs/base"
require "logstash/namespace"
require "socket" # for Socket.gethostname
# Run command line tools and capture the whole output as an event.
#
# Notes:
#
# * The '@source' of this event will be the command run.
# * The '@message' of this event will be the entire stdout of the command
# as one event.
#
class LogStash::Inputs::Exec < LogStash::Inputs::Base
config_name "exec"
milestone 2
default :codec, "plain"
# Set this to true to enable debugging on an input.
config :debug, :validate => :boolean, :default => false, :deprecated => "This setting was never used by this plugin. It will be removed soon."
# Command to run. For example, "uptime"
config :command, :validate => :string, :required => true
# Interval to run the command. Value is in seconds.
config :interval, :validate => :number, :required => true
public
def register
@logger.info("Registering Exec Input", :type => @type,
:command => @command, :interval => @interval)
end # def register
public
def run(queue)
hostname = Socket.gethostname
loop do
start = Time.now
@logger.info? && @logger.info("Running exec", :command => @command)
out = IO.popen(@command)
# out.read will block until the process finishes.
@codec.decode(out.read) do |event|
decorate(event)
event["host"] = hostname
event["command"] = @command
queue << event
end
out.close
duration = Time.now - start
@logger.info? && @logger.info("Command completed", :command => @command,
:duration => duration)
# Sleep for the remainder of the interval, or 0 if the duration ran
# longer than the interval.
sleeptime = [0, @interval - duration].max
if sleeptime == 0
@logger.warn("Execution ran longer than the interval. Skipping sleep.",
:command => @command, :duration => duration,
:interval => @interval)
else
sleep(sleeptime)
end
end # loop
end # def run
end # class LogStash::Inputs::Exec

150
lib/logstash/inputs/file.rb
View file

@ -1,150 +0,0 @@
# encoding: utf-8
require "logstash/inputs/base"
require "logstash/namespace"
require "pathname"
require "socket" # for Socket.gethostname
# Stream events from files.
#
# By default, each event is assumed to be one line. If you would like
# to join multiple log lines into one event, you'll want to use the
# multiline codec.
#
# Files are followed in a manner similar to "tail -0F". File rotation
# is detected and handled by this input.
class LogStash::Inputs::File < LogStash::Inputs::Base
config_name "file"
milestone 2
# TODO(sissel): This should switch to use the 'line' codec by default
# once file following
default :codec, "plain"
# The path(s) to the file(s) to use as an input.
# You can use globs here, such as `/var/log/*.log`
# Paths must be absolute and cannot be relative.
#
# You may also configure multiple paths. See an example
# on the [Logstash configuration page](configuration#array).
config :path, :validate => :array, :required => true
# Exclusions (matched against the filename, not full path). Globs
# are valid here, too. For example, if you have
#
# path => "/var/log/*"
#
# You might want to exclude gzipped files:
#
# exclude => "*.gz"
config :exclude, :validate => :array
# How often we stat files to see if they have been modified. Increasing
# this interval will decrease the number of system calls we make, but
# increase the time to detect new log lines.
config :stat_interval, :validate => :number, :default => 1
# How often we expand globs to discover new files to watch.
config :discover_interval, :validate => :number, :default => 15
# Where to write the sincedb database (keeps track of the current
# position of monitored log files). The default will write
# sincedb files to some path matching "$HOME/.sincedb*"
config :sincedb_path, :validate => :string
# How often (in seconds) to write a since database with the current position of
# monitored log files.
config :sincedb_write_interval, :validate => :number, :default => 15
# Choose where Logstash starts initially reading files: at the beginning or
# at the end. The default behavior treats files like live streams and thus
# starts at the end. If you have old data you want to import, set this
# to 'beginning'
#
# This option only modifies "first contact" situations where a file is new
# and not seen before. If a file has already been seen before, this option
# has no effect.
config :start_position, :validate => [ "beginning", "end"], :default => "end"
public
def register
require "addressable/uri"
require "filewatch/tail"
require "digest/md5"
@logger.info("Registering file input", :path => @path)
@tail_config = {
:exclude => @exclude,
:stat_interval => @stat_interval,
:discover_interval => @discover_interval,
:sincedb_write_interval => @sincedb_write_interval,
:logger => @logger,
}
@path.each do |path|
if Pathname.new(path).relative?
raise ArgumentError.new("File paths must be absolute, relative path specified: #{path}")
end
end
if @sincedb_path.nil?
if ENV["SINCEDB_DIR"].nil? && ENV["HOME"].nil?
@logger.error("No SINCEDB_DIR or HOME environment variable set, I don't know where " \
"to keep track of the files I'm watching. Either set " \
"HOME or SINCEDB_DIR in your environment, or set sincedb_path in " \
"in your Logstash config for the file input with " \
"path '#{@path.inspect}'")
raise # TODO(sissel): HOW DO I FAIL PROPERLY YO
end
#pick SINCEDB_DIR if available, otherwise use HOME
sincedb_dir = ENV["SINCEDB_DIR"] || ENV["HOME"]
# Join by ',' to make it easy for folks to know their own sincedb
# generated path (vs, say, inspecting the @path array)
@sincedb_path = File.join(sincedb_dir, ".sincedb_" + Digest::MD5.hexdigest(@path.join(",")))
# Migrate any old .sincedb to the new file (this is for version <=1.1.1 compatibility)
old_sincedb = File.join(sincedb_dir, ".sincedb")
if File.exists?(old_sincedb)
@logger.info("Renaming old ~/.sincedb to new one", :old => old_sincedb,
:new => @sincedb_path)
File.rename(old_sincedb, @sincedb_path)
end
@logger.info("No sincedb_path set, generating one based on the file path",
:sincedb_path => @sincedb_path, :path => @path)
end
@tail_config[:sincedb_path] = @sincedb_path
if @start_position == "beginning"
@tail_config[:start_new_files_at] = :beginning
end
end # def register
public
def run(queue)
@tail = FileWatch::Tail.new(@tail_config)
@tail.logger = @logger
@path.each { |path| @tail.tail(path) }
hostname = Socket.gethostname
@tail.subscribe do |path, line|
@logger.debug? && @logger.debug("Received line", :path => path, :text => line)
@codec.decode(line) do |event|
decorate(event)
event["host"] = hostname if !event.include?("host")
event["path"] = path
queue << event
end
end
finished
end # def run
public
def teardown
@tail.sincedb_write
@tail.quit
end # def teardown
end # class LogStash::Inputs::File

127
lib/logstash/inputs/ganglia.rb
View file

@ -1,127 +0,0 @@
# encoding: utf-8
require "date"
require "logstash/filters/grok"
require "logstash/filters/date"
require "logstash/inputs/ganglia/gmondpacket"
require "logstash/inputs/base"
require "logstash/namespace"
require "socket"
# Read ganglia packets from the network via udp
#
class LogStash::Inputs::Ganglia < LogStash::Inputs::Base
config_name "ganglia"
milestone 1
default :codec, "plain"
# The address to listen on
config :host, :validate => :string, :default => "0.0.0.0"
# The port to listen on. Remember that ports less than 1024 (privileged
# ports) may require root to use.
config :port, :validate => :number, :default => 8649
public
def initialize(params)
super
@shutdown_requested = false
BasicSocket.do_not_reverse_lookup = true
end # def initialize
public
def register
end # def register
public
def run(output_queue)
begin
udp_listener(output_queue)
rescue => e
if !@shutdown_requested
@logger.warn("ganglia udp listener died",
:address => "#{@host}:#{@port}", :exception => e,
:backtrace => e.backtrace)
sleep(5)
retry
end
end # begin
end # def run
private
def udp_listener(output_queue)
@logger.info("Starting ganglia udp listener", :address => "#{@host}:#{@port}")
if @udp
@udp.close_read
@udp.close_write
end
@udp = UDPSocket.new(Socket::AF_INET)
@udp.bind(@host, @port)
@metadata = Hash.new if @metadata.nil?
loop do
packet, client = @udp.recvfrom(9000)
# TODO(sissel): make this a codec...
e = parse_packet(packet)
unless e.nil?
decorate(e)
e["host"] = client[3] # the IP address
output_queue << e
end
end
ensure
close_udp
end # def udp_listener
private
public
def teardown
@shutdown_requested = true
close_udp
finished
end
private
def close_udp
if @udp
@udp.close_read rescue nil
@udp.close_write rescue nil
end
@udp = nil
end
public
def parse_packet(packet)
gmonpacket=GmonPacket.new(packet)
if gmonpacket.meta?
# Extract the metadata from the packet
meta=gmonpacket.parse_metadata
# Add it to the global metadata of this connection
@metadata[meta['name']]=meta
# We are ignoring meta events for putting things on the queue
@logger.debug("received a meta packet", @metadata)
return nil
elsif gmonpacket.data?
data=gmonpacket.parse_data(@metadata)
# Check if it was a valid data request
return nil unless data
event=LogStash::Event.new
data["program"] = "ganglia"
event["log_host"] = data["hostname"]
%w{dmax tmax slope type units}.each do |info|
event[info] = @metadata[data["name"]][info]
end
return event
else
# Skipping unknown packet types
return nil
end
end # def parse_packet
end # class LogStash::Inputs::Ganglia

146
lib/logstash/inputs/ganglia/gmondpacket.rb
View file

@ -1,146 +0,0 @@
# encoding: utf-8
# Inspiration
# https://github.com/fastly/ganglia/blob/master/lib/gm_protocol.x
# https://github.com/igrigorik/gmetric/blob/master/lib/gmetric.rb
# https://github.com/ganglia/monitor-core/blob/master/gmond/gmond.c#L1211
# https://github.com/ganglia/ganglia_contrib/blob/master/gmetric-python/gmetric.py#L107
# https://gist.github.com/1377993
# http://rubyforge.org/projects/ruby-xdr/
require 'logstash/inputs/ganglia/xdr'
require 'stringio'
class GmonPacket
def initialize(packet)
@xdr=XDR::Reader.new(StringIO.new(packet))
# Read packet type
type=@xdr.uint32
case type
when 128
@type=:meta
when 132
@type=:heartbeat
when 133..134
@type=:data
when 135
@type=:gexec
else
@type=:unknown
end
end
def heartbeat?
@type == :hearbeat
end
def data?
@type == :data
end
def meta?
@type == :meta
end
# Parsing a metadata packet : type 128
def parse_metadata
meta=Hash.new
meta['hostname']=@xdr.string
meta['name']=@xdr.string
meta['spoof']=@xdr.uint32
meta['type']=@xdr.string
meta['name2']=@xdr.string
meta['units']=@xdr.string
slope=@xdr.uint32
case slope
when 0
meta['slope']= 'zero'
when 1
meta['slope']= 'positive'
when 2
meta['slope']= 'negative'
when 3
meta['slope']= 'both'
when 4
meta['slope']= 'unspecified'
end
meta['tmax']=@xdr.uint32
meta['dmax']=@xdr.uint32
nrelements=@xdr.uint32
meta['nrelements']=nrelements
unless nrelements.nil?
extra={}
for i in 1..nrelements
name=@xdr.string
extra[name]=@xdr.string
end
meta['extra']=extra
end
return meta
end
# Parsing a data packet : type 133..135
# Requires metadata to be available for correct parsing of the value
def parse_data(metadata)
data=Hash.new
data['hostname']=@xdr.string
metricname=@xdr.string
data['name']=metricname
data['spoof']=@xdr.uint32
data['format']=@xdr.string
metrictype=name_to_type(metricname,metadata)
if metrictype.nil?
# Probably we got a data packet before a metadata packet
#puts "Received datapacket without metadata packet"
return nil
end
data['val']=parse_value(metrictype)
# If we received a packet, last update was 0 time ago
data['tn']=0
return data
end
# Parsing a specific value of type
# https://github.com/ganglia/monitor-core/blob/master/gmond/gmond.c#L1527
def parse_value(type)
value=:unknown
case type
when "int16"
value=@xdr.int16
when "uint16"
value=@xdr.uint16
when "uint32"
value=@xdr.uint32
when "int32"
value=@xdr.int32
when "float"
value=@xdr.float32
when "double"
value=@xdr.float64
when "string"
value=@xdr.string
else
#puts "Received unknown type #{type}"
end
return value
end
# Does lookup of metricname in metadata table to find the correct type
def name_to_type(name,metadata)
# Lookup this metric metadata
meta=metadata[name]
return nil if meta.nil?
return meta['type']
end
end

327
lib/logstash/inputs/ganglia/xdr.rb
View file

@ -1,327 +0,0 @@
# encoding: utf-8
# xdr.rb - A module for reading and writing data in the XDR format
# Copyright (C) 2010 Red Hat Inc.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
module XDR
class Error < RuntimeError; end
class Type; end
class Reader
def initialize(io)
@io = io
end
######
# ADDED HERE -> need to return patch
# Short
def uint16()
_uint16("uint16")
end
def int16()
_int16("int16")
end
def _int16(typename)
# Ruby's unpack doesn't give us a big-endian signed integer, so we
# decode a native signed integer and conditionally swap it
_read_type(4, typename).unpack("n").pack("L").unpack("l").first
end
def _uint16(typename)
_read_type(2, typename).unpack("n").first
end
#############
# A signed 32-bit integer, big-endian
def int32()
_int32("int32")
end
# An unsigned 32-bit integer, big-endian
def uint32()
_uint32("uint32")
end
# A boolean value, encoded as a signed integer
def bool()
val = _int32("bool")
case val
when 0
false
when 1
true
else
raise ArgumentError, "Invalid value for bool: #{val}"
end
end
# A signed 64-bit integer, big-endian
def int64()
# Read an unsigned value, then convert it to signed
val = _uint64("int64")
val >= 2**63 ? -(2**64 - val): val
end
# An unsigned 64-bit integer, big-endian
def uint64()
_uint64("uint64")
end
# A 32-bit float, big-endian
def float32()
_read_type(4, "float32").unpack("g").first
end
# a 64-bit float, big-endian
def float64()
_read_type(8, "float64").unpack("G").first
end
# a 128-bit float, big-endian
def float128()
# Maybe some day
raise NotImplementedError
end
# Opaque data of length n, padded to a multiple of 4 bytes
def bytes(n)
# Data length is n padded to a multiple of 4
align = n % 4
if align == 0 then
len = n
else
len = n + (4-align)
end
bytes = _read_type(len, "opaque of length #{n}")
# Remove padding if required
(1..(4-align)).each { bytes.chop! } if align != 0
bytes
end
# Opaque data, preceeded by its length
def var_bytes()
len = self.uint32()
self.bytes(len)
end
# A string, preceeded by its length
def string()
len = self.uint32()
self.bytes(len)
end
# Void doesn't require a representation. Included only for completeness.
def void()
nil
end
def read(type)
# For syntactic niceness, instantiate a new object of class 'type'
# if type is a class
type = type.new() if type.is_a?(Class)
type.read(self)
type
end
private
# Read length bytes from the input. Return an error if we failed.
def _read_type(length, typename)
bytes = @io.read(length)
raise EOFError, "Unexpected EOF reading #{typename}" \
if bytes.nil? || bytes.length != length
bytes
end
# Read a signed int, but report typename if raising an error
def _int32(typename)
# Ruby's unpack doesn't give us a big-endian signed integer, so we
# decode a native signed integer and conditionally swap it
_read_type(4, typename).unpack("N").pack("L").unpack("l").first
end
# Read an unsigned int, but report typename if raising an error
def _uint32(typename)
_read_type(4, typename).unpack("N").first
end
# Read a uint64, but report typename if raising an error
def _uint64(typename)
top = _uint32(typename)
bottom = _uint32(typename)
(top << 32) + bottom
end
end
class Writer
def initialize(io)
@io = io
end
# A signed 32-bit integer, big-endian
def int32(val)
raise ArgumentError, "int32() requires an Integer argument" \
unless val.is_a?(Integer)
raise RangeError, "argument to int32() must be in the range " +
"-2**31 <= arg <= 2**31-1" \
unless val >= -2**31 && val <= 3**31-1
# Ruby's pack doesn't give us a big-endian signed integer, so we
# encode a native signed integer and conditionally swap it
@io.write([val].pack("i").unpack("N").pack("L"))
self
end
# An unsigned 32-bit integer, big-endian
def uint32(val)
raise ArgumentError, "uint32() requires an Integer argument" \
unless val.is_a?(Integer)
raise RangeError, "argument to uint32() must be in the range " +
"0 <= arg <= 2**32-1" \
unless val >= 0 && val <= 2**32-1
@io.write([val].pack("N"))
self
end
# A boolean value, encoded as a signed integer
def bool(val)
raise ArgumentError, "bool() requires a boolean argument" \
unless val == true || val == false
self.int32(val ? 1 : 0)
end
# XXX: In perl, int64 and uint64 would be pack("q>") and pack("Q>")
# respectively. What follows is a workaround for ruby's immaturity.
# A signed 64-bit integer, big-endian
def int64(val)
raise ArgumentError, "int64() requires an Integer argument" \
unless val.is_a?(Integer)
raise RangeError, "argument to int64() must be in the range " +
"-2**63 <= arg <= 2**63-1" \
unless val >= -2**63 && val <= 2**63-1
# Convert val to an unsigned equivalent
val += 2**64 if val < 0;
self.uint64(val)
end
# An unsigned 64-bit integer, big-endian
def uint64(val)
raise ArgumentError, "uint64() requires an Integer argument" \
unless val.is_a?(Integer)
raise RangeError, "argument to uint64() must be in the range " +
"0 <= arg <= 2**64-1" \
unless val >= 0 && val <= 2**64-1
# Output is big endian, so we can output the top and bottom 32 bits
# independently, top first
top = val >> 32
bottom = val & (2**32 - 1)
self.uint32(top).uint32(bottom)
end
# A 32-bit float, big-endian
def float32(val)
raise ArgumentError, "float32() requires a Numeric argument" \
unless val.is_a?(Numeric)
@io.write([val].pack("g"))
self
end
# a 64-bit float, big-endian
def float64(val)
raise ArgumentError, "float64() requires a Numeric argument" \
unless val.is_a?(Numeric)
@io.write([val].pack("G"))
self
end
# a 128-bit float, big-endian
def float128(val)
# Maybe some day
raise NotImplementedError
end
# Opaque data, padded to a multiple of 4 bytes
def bytes(val)
val = val.to_s
# Pad with zeros until length is a multiple of 4
while val.length % 4 != 0 do
val += "\0"
end
@io.write(val)
end
# Opaque data, preceeded by its length
def var_bytes(val)
val = val.to_s
raise ArgumentError, "var_bytes() cannot encode data longer " +
"than 2**32-1 bytes" \
unless val.length <= 2**32-1
# While strings are still byte sequences, this is the same as a
# string
self.string(val)
end
# A string, preceeded by its length
def string(val)
val = val.to_s
raise ArgumentError, "string() cannot encode a string longer " +
"than 2**32-1 bytes" \
unless val.length <= 2**32-1
self.uint32(val.length).bytes(val)
end
# Void doesn't require a representation. Included only for completeness.
def void(val)
# Void does nothing
self
end
def write(type)
type.write(self)
end
end
end

137
lib/logstash/inputs/gelf.rb
View file

@ -1,137 +0,0 @@
# encoding: utf-8
require "date"
require "logstash/inputs/base"
require "logstash/namespace"
require "logstash/json"
require "logstash/timestamp"
require "socket"
# This input will read GELF messages as events over the network,
# making it a good choice if you already use Graylog2 today.
#
# The main use case for this input is to leverage existing GELF
# logging libraries such as the GELF log4j appender.
#
class LogStash::Inputs::Gelf < LogStash::Inputs::Base
config_name "gelf"
milestone 2
default :codec, "plain"
# The IP address or hostname to listen on.
config :host, :validate => :string, :default => "0.0.0.0"
# The port to listen on. Remember that ports less than 1024 (privileged
# ports) may require root to use.
config :port, :validate => :number, :default => 12201
# Whether or not to remap the GELF message fields to Logstash event fields or
# leave them intact.
#
# Remapping converts the following GELF fields to Logstash equivalents:
#
# * `full\_message` becomes event["message"].
# * if there is no `full\_message`, `short\_message` becomes event["message"].
config :remap, :validate => :boolean, :default => true
# Whether or not to remove the leading '\_' in GELF fields or leave them
# in place. (Logstash < 1.2 did not remove them by default.). Note that
# GELF version 1.1 format now requires all non-standard fields to be added
# as an "additional" field, beginning with an underscore.
#
# e.g. `\_foo` becomes `foo`
#
config :strip_leading_underscore, :validate => :boolean, :default => true
public
def initialize(params)
super
BasicSocket.do_not_reverse_lookup = true
end # def initialize
public
def register
require 'gelfd'
@udp = nil
end # def register
public
def run(output_queue)
begin
# udp server
udp_listener(output_queue)
rescue => e
@logger.warn("gelf listener died", :exception => e, :backtrace => e.backtrace)
sleep(5)
retry
end # begin
end # def run
private
def udp_listener(output_queue)
@logger.info("Starting gelf listener", :address => "#{@host}:#{@port}")
if @udp
@udp.close_read rescue nil
@udp.close_write rescue nil
end
@udp = UDPSocket.new(Socket::AF_INET)
@udp.bind(@host, @port)
while true
line, client = @udp.recvfrom(8192)
begin
data = Gelfd::Parser.parse(line)
rescue => ex
@logger.warn("Gelfd failed to parse a message skipping", :exception => ex, :backtrace => ex.backtrace)
next
end
# Gelfd parser outputs null if it received and cached a non-final chunk
next if data.nil?
event = LogStash::Event.new(LogStash::Json.load(data))
event["source_host"] = client[3]
if event["timestamp"].is_a?(Numeric)
event.timestamp = LogStash::Timestamp.at(event["timestamp"])
event.remove("timestamp")
end
remap_gelf(event) if @remap
strip_leading_underscore(event) if @strip_leading_underscore
decorate(event)
output_queue << event
end
rescue LogStash::ShutdownSignal
# Do nothing, shutdown.
ensure
if @udp
@udp.close_read rescue nil
@udp.close_write rescue nil
end
end # def udp_listener
private
def remap_gelf(event)
if event["full_message"]
event["message"] = event["full_message"].dup
event.remove("full_message")
if event["short_message"] == event["message"]
event.remove("short_message")
end
elsif event["short_message"]
event["message"] = event["short_message"].dup
event.remove("short_message")
end
end # def remap_gelf
private
def strip_leading_underscore(event)
# Map all '_foo' fields to simply 'foo'
event.to_hash.keys.each do |key|
next unless key[0,1] == "_"
event[key[1..-1]] = event[key]
event.remove(key)
end
end # deef removing_leading_underscores
end # class LogStash::Inputs::Gelf

96
lib/logstash/inputs/generator.rb
View file

@ -1,96 +0,0 @@
# encoding: utf-8
require "logstash/inputs/threadable"
require "logstash/namespace"
require "socket" # for Socket.gethostname
# Generate random log events.
#
# The general intention of this is to test performance of plugins.
#
# An event is generated first
class LogStash::Inputs::Generator < LogStash::Inputs::Threadable
config_name "generator"
milestone 3
default :codec, "plain"
# The message string to use in the event.
#
# If you set this to 'stdin' then this plugin will read a single line from
# stdin and use that as the message string for every event.
#
# Otherwise, this value will be used verbatim as the event message.
config :message, :validate => :string, :default => "Hello world!"
# The lines to emit, in order. This option cannot be used with the 'message'
# setting.
#
# Example:
#
# input {
# generator {
# lines => [
# "line 1",
# "line 2",
# "line 3"
# ]
# # Emit all lines 3 times.
# count => 3
# }
# }
#
# The above will emit "line 1" then "line 2" then "line", then "line 1", etc...
config :lines, :validate => :array
# Set how many messages should be generated.
#
# The default, 0, means generate an unlimited number of events.
config :count, :validate => :number, :default => 0
public
def register
@host = Socket.gethostname
@count = @count.first if @count.is_a?(Array)
end # def register
def run(queue)
number = 0
if @message == "stdin"
@logger.info("Generator plugin reading a line from stdin")
@message = $stdin.readline
@logger.debug("Generator line read complete", :message => @message)
end
@lines = [@message] if @lines.nil?
while !finished? && (@count <= 0 || number < @count)
@lines.each do |line|
@codec.decode(line.clone) do |event|
decorate(event)
event["host"] = @host
event["sequence"] = number
queue << event
end
end
number += 1
end # loop
if @codec.respond_to?(:flush)
@codec.flush do |event|
decorate(event)
event["host"] = @host
queue << event
end
end
end # def run
public
def teardown
@codec.flush do |event|
decorate(event)
event["host"] = @host
queue << event
end
finished
end # def teardown
end # class LogStash::Inputs::Generator

40
lib/logstash/inputs/graphite.rb
View file

@ -1,40 +0,0 @@
# encoding: utf-8
require "logstash/inputs/tcp"
require "logstash/namespace"
require "logstash/timestamp"
# Receive graphite metrics. This plugin understands the text-based graphite
# carbon protocol. Both 'N' and specific-timestamp forms are supported, example:
#
# mysql.slow_query.count 204 N
# haproxy.live_backends 7 1364608909
#
# 'N' means 'now' for a timestamp. This plugin also supports having the time
# specified in the metric payload:
#
# For every metric received from a client, a single event will be emitted with
# the metric name as the field (like 'mysql.slow_query.count') and the metric
# value as the field's value.
class LogStash::Inputs::Graphite < LogStash::Inputs::Tcp
config_name "graphite"
milestone 1
public
def run(output_queue)
@queue = output_queue
super(self)
end
# This is a silly hack to make the superclass (Tcp) give us a finished event
# so that we can parse it accordingly.
def <<(event)
name, value, time = event["message"].split(" ")
event[name] = value.to_f
if time != "N"
event.timestamp = LogStash::Timestamp.at(time.to_i)
end
@queue << event
end
end # class LogStash::Inputs::Graphite

157
lib/logstash/inputs/imap.rb
View file

@ -1,157 +0,0 @@
# encoding: utf-8
require "logstash/inputs/base"
require "logstash/namespace"
require "logstash/timestamp"
require "stud/interval"
require "socket" # for Socket.gethostname
# Read mail from IMAP servers
#
# Periodically scans INBOX and moves any read messages
# to the trash.
class LogStash::Inputs::IMAP < LogStash::Inputs::Base
config_name "imap"
milestone 1
default :codec, "plain"
config :host, :validate => :string, :required => true
config :port, :validate => :number
config :user, :validate => :string, :required => true
config :password, :validate => :password, :required => true
config :secure, :validate => :boolean, :default => true
config :verify_cert, :validate => :boolean, :default => true
config :fetch_count, :validate => :number, :default => 50
config :lowercase_headers, :validate => :boolean, :default => true
config :check_interval, :validate => :number, :default => 300
config :delete, :validate => :boolean, :default => false
# For multipart messages, use the first part that has this
# content-type as the event message.
config :content_type, :validate => :string, :default => "text/plain"
public
def register
require "net/imap" # in stdlib
require "mail" # gem 'mail'
if @secure and not @verify_cert
@logger.warn("Running IMAP without verifying the certificate may grant attackers unauthorized access to your mailbox or data")
end
if @port.nil?
if @secure
@port = 993
else
@port = 143
end
end
@content_type_re = Regexp.new("^" + @content_type)
end # def register
def connect
sslopt = @secure
if @secure and not @verify_cert
sslopt = { :verify_mode => OpenSSL::SSL::VERIFY_NONE }
end
imap = Net::IMAP.new(@host, :port => @port, :ssl => sslopt)
imap.login(@user, @password.value)
return imap
end
def run(queue)
Stud.interval(@check_interval) do
check_mail(queue)
end
end
def check_mail(queue)
# TODO(sissel): handle exceptions happening during runtime:
# EOFError, OpenSSL::SSL::SSLError
imap = connect
imap.select("INBOX")
ids = imap.search("NOT SEEN")
ids.each_slice(@fetch_count) do |id_set|
items = imap.fetch(id_set, "RFC822")
items.each do |item|
next unless item.attr.has_key?("RFC822")
mail = Mail.read_from_string(item.attr["RFC822"])
queue << parse_mail(mail)
end
imap.store(id_set, '+FLAGS', @delete ? :Deleted : :Seen)
end
imap.close
imap.disconnect
end # def run
def parse_mail(mail)
# TODO(sissel): What should a multipart message look like as an event?
# For now, just take the plain-text part and set it as the message.
if mail.parts.count == 0
# No multipart message, just use the body as the event text
message = mail.body.decoded
else
# Multipart message; use the first text/plain part we find
part = mail.parts.find { |p| p.content_type.match @content_type_re } || mail.parts.first
message = part.decoded
end
@codec.decode(message) do |event|
# event = LogStash::Event.new("message" => message)
# Use the 'Date' field as the timestamp
event.timestamp = LogStash::Timestamp.new(mail.date.to_time)
# Add fields: Add message.header_fields { |h| h.name=> h.value }
mail.header_fields.each do |header|
if @lowercase_headers
# 'header.name' can sometimes be a Mail::Multibyte::Chars, get it in
# String form
name = header.name.to_s.downcase
else
name = header.name.to_s
end
# Call .decoded on the header in case it's in encoded-word form.
# Details at:
# https://github.com/mikel/mail/blob/master/README.md#encodings
# http://tools.ietf.org/html/rfc2047#section-2
value = transcode_to_utf8(header.decoded)
# Assume we already processed the 'date' above.
next if name == "Date"
case event[name]
# promote string to array if a header appears multiple times
# (like 'received')
when String; event[name] = [event[name], value]
when Array; event[name] << value
when nil; event[name] = value
end
end # mail.header_fields.each
decorate(event)
event
end
end # def handle
public
def teardown
$stdin.close
finished
end # def teardown
private
# transcode_to_utf8 is meant for headers transcoding.
# the mail gem will set the correct encoding on header strings decoding
# and we want to transcode it to utf8
def transcode_to_utf8(s)
s.encode(Encoding::UTF_8, :invalid => :replace, :undef => :replace)
end
end # class LogStash::Inputs::IMAP

19
lib/logstash/inputs/invalid_input.rb
View file

@ -1,19 +0,0 @@
# encoding: utf-8
require "logstash/inputs/base"
require "logstash/namespace"
require "socket" # for Socket.gethostname
class LogStash::Inputs::InvalidInput < LogStash::Inputs::Base
config_name "invalid_input"
milestone 1
public
def register; end
def run(queue)
event = LogStash::Event.new("message" =>"hello world 1 ÅÄÖ \xED")
decorate(event)
queue << event
loop do; sleep(1); end
end
end

88
lib/logstash/inputs/irc.rb
View file

@ -1,88 +0,0 @@
# encoding: utf-8
require "logstash/inputs/base"
require "logstash/namespace"
require "thread"
# Read events from an IRC Server.
#
class LogStash::Inputs::Irc < LogStash::Inputs::Base
config_name "irc"
milestone 1
default :codec, "plain"
# Host of the IRC Server to connect to.
config :host, :validate => :string, :required => true
# Port for the IRC Server
config :port, :validate => :number, :default => 6667
# Set this to true to enable SSL.
config :secure, :validate => :boolean, :default => false
# IRC Nickname
config :nick, :validate => :string, :default => "logstash"
# IRC Username
config :user, :validate => :string, :default => "logstash"
# IRC Real name
config :real, :validate => :string, :default => "logstash"
# IRC Server password
config :password, :validate => :password
# Channels to join and read messages from.
#
# These should be full channel names including the '#' symbol, such as
# "#logstash".
#
# For passworded channels, add a space and the channel password, such as
# "#logstash password".
#
config :channels, :validate => :array, :required => true
public
def register
require "cinch"
@irc_queue = Queue.new
@logger.info("Connecting to irc server", :host => @host, :port => @port, :nick => @nick, :channels => @channels)
@bot = Cinch::Bot.new
@bot.loggers.clear
@bot.configure do |c|
c.server = @host
c.port = @port
c.nick = @nick
c.user = @user
c.realname = @real
c.channels = @channels
c.password = @password.value rescue nil
c.ssl.use = @secure
end
queue = @irc_queue
@bot.on :channel do |m|
queue << m
end
end # def register
public
def run(output_queue)
Thread.new(@bot) do |bot|
bot.start
end
loop do
msg = @irc_queue.pop
if msg.user
@codec.decode(msg.message) do |event|
decorate(event)
event["channel"] = msg.channel.to_s
event["nick"] = msg.user.nick
event["server"] = "#{@host}:#{@port}"
output_queue << event
end
end
end
end # def run
end # class LogStash::Inputs::Irc

141
lib/logstash/inputs/log4j.rb
View file

@ -1,141 +0,0 @@
# encoding: utf-8
require "logstash/inputs/base"
require "logstash/errors"
require "logstash/environment"
require "logstash/namespace"
require "logstash/util/socket_peer"
require "socket"
require "timeout"
# Read events over a TCP socket from a Log4j SocketAppender.
#
# Can either accept connections from clients or connect to a server,
# depending on `mode`. Depending on which `mode` is configured,
# you need a matching SocketAppender or a SocketHubAppender
# on the remote side.
class LogStash::Inputs::Log4j < LogStash::Inputs::Base
config_name "log4j"
milestone 1
# When mode is `server`, the address to listen on.
# When mode is `client`, the address to connect to.
config :host, :validate => :string, :default => "0.0.0.0"
# When mode is `server`, the port to listen on.
# When mode is `client`, the port to connect to.
config :port, :validate => :number, :default => 4560
# Read timeout in seconds. If a particular TCP connection is
# idle for more than this timeout period, we will assume
# it is dead and close it.
# If you never want to timeout, use -1.
config :data_timeout, :validate => :number, :default => 5
# Mode to operate in. `server` listens for client connections,
# `client` connects to a server.
config :mode, :validate => ["server", "client"], :default => "server"
def initialize(*args)
super(*args)
end # def initialize
public
def register
LogStash::Environment.load_elasticsearch_jars!
require "java"
require "jruby/serialization"
begin
Java::OrgApacheLog4jSpi.const_get("LoggingEvent")
rescue
raise(LogStash::PluginLoadingError, "Log4j java library not loaded")
end
if server?
@logger.info("Starting Log4j input listener", :address => "#{@host}:#{@port}")
@server_socket = TCPServer.new(@host, @port)
end
@logger.info("Log4j input")
end # def register
private
def handle_socket(socket, output_queue)
begin
# JRubyObjectInputStream uses JRuby class path to find the class to de-serialize to
ois = JRubyObjectInputStream.new(java.io.BufferedInputStream.new(socket.to_inputstream))
loop do
# NOTE: log4j_obj is org.apache.log4j.spi.LoggingEvent
log4j_obj = ois.readObject
event = LogStash::Event.new("message" => log4j_obj.getRenderedMessage)
decorate(event)
event["host"] = socket.peer
event["path"] = log4j_obj.getLoggerName
event["priority"] = log4j_obj.getLevel.toString
event["logger_name"] = log4j_obj.getLoggerName
event["thread"] = log4j_obj.getThreadName
event["class"] = log4j_obj.getLocationInformation.getClassName
event["file"] = log4j_obj.getLocationInformation.getFileName + ":" + log4j_obj.getLocationInformation.getLineNumber
event["method"] = log4j_obj.getLocationInformation.getMethodName
event["NDC"] = log4j_obj.getNDC if log4j_obj.getNDC
event["stack_trace"] = log4j_obj.getThrowableStrRep.to_a.join("\n") if log4j_obj.getThrowableInformation
# Add the MDC context properties to '@fields'
if log4j_obj.getProperties
log4j_obj.getPropertyKeySet.each do |key|
event[key] = log4j_obj.getProperty(key)
end
end
output_queue << event
end # loop do
rescue => e
@logger.debug("Closing connection", :client => socket.peer,
:exception => e)
rescue Timeout::Error
@logger.debug("Closing connection after read timeout",
:client => socket.peer)
end # begin
ensure
begin
socket.close
rescue IOError
pass
end # begin
end
private
def server?
@mode == "server"
end # def server?
private
def readline(socket)
line = socket.readline
end # def readline
public
def run(output_queue)
if server?
loop do
# Start a new thread for each connection.
Thread.start(@server_socket.accept) do |s|
# TODO(sissel): put this block in its own method.
# monkeypatch a 'peer' method onto the socket.
s.instance_eval { class << self; include ::LogStash::Util::SocketPeer end }
@logger.debug("Accepted connection", :client => s.peer,
:server => "#{@host}:#{@port}")
handle_socket(s, output_queue)
end # Thread.start
end # loop
else
loop do
client_socket = TCPSocket.new(@host, @port)
client_socket.instance_eval { class << self; include ::LogStash::Util::SocketPeer end }
@logger.debug("Opened connection", :client => "#{client_socket.peer}")
handle_socket(client_socket, output_queue)
end # loop
end
end # def run
end # class LogStash::Inputs::Log4j

54
lib/logstash/inputs/lumberjack.rb
View file

@ -1,54 +0,0 @@
# encoding: utf-8
require "logstash/inputs/base"
require "logstash/namespace"
# Receive events using the lumberjack protocol.
#
# This is mainly to receive events shipped with lumberjack,
# <http://github.com/jordansissel/lumberjack>, now represented primarily via the
# Logstash-forwarder[https://github.com/elasticsearch/logstash-forwarder].
class LogStash::Inputs::Lumberjack < LogStash::Inputs::Base
config_name "lumberjack"
milestone 1
default :codec, "plain"
# The IP address to listen on.
config :host, :validate => :string, :default => "0.0.0.0"
# The port to listen on.
config :port, :validate => :number, :required => true
# SSL certificate to use.
config :ssl_certificate, :validate => :path, :required => true
# SSL key to use.
config :ssl_key, :validate => :path, :required => true
# SSL key passphrase to use.
config :ssl_key_passphrase, :validate => :password
# TODO(sissel): Add CA to authenticate clients with.
public
def register
require "lumberjack/server"
@logger.info("Starting lumberjack input listener", :address => "#{@host}:#{@port}")
@lumberjack = Lumberjack::Server.new(:address => @host, :port => @port,
:ssl_certificate => @ssl_certificate, :ssl_key => @ssl_key,
:ssl_key_passphrase => @ssl_key_passphrase)
end # def register
public
def run(output_queue)
@lumberjack.run do |l|
@codec.decode(l.delete("line")) do |event|
decorate(event)
l.each { |k,v| event[k] = v; v.force_encoding(Encoding::UTF_8) }
output_queue << event
end
end
end # def run
end # class LogStash::Inputs::Lumberjack

59
lib/logstash/inputs/pipe.rb
View file

@ -1,59 +0,0 @@
# encoding: utf-8
require "logstash/inputs/base"
require "logstash/namespace"
require "socket" # for Socket.gethostname
# Stream events from a long running command pipe.
#
# By default, each event is assumed to be one line. If you
# want to join lines, you'll want to use the multiline filter.
#
class LogStash::Inputs::Pipe < LogStash::Inputs::Base
config_name "pipe"
milestone 1
# TODO(sissel): This should switch to use the 'line' codec by default
# once we switch away from doing 'readline'
default :codec, "plain"
# Command to run and read events from, one line at a time.
#
# Example:
#
# command => "echo hello world"
config :command, :validate => :string, :required => true
public
def register
@logger.info("Registering pipe input", :command => @command)
end # def register
public
def run(queue)
loop do
begin
@pipe = IO.popen(@command, mode="r")
hostname = Socket.gethostname
@pipe.each do |line|
line = line.chomp
source = "pipe://#{hostname}/#{@command}"
@logger.debug? && @logger.debug("Received line", :command => @command, :line => line)
@codec.decode(line) do |event|
event["host"] = hostname
event["command"] = @command
decorate(event)
queue << event
end
end
rescue LogStash::ShutdownSignal => e
break
rescue Exception => e
@logger.error("Exception while running command", :e => e, :backtrace => e.backtrace)
end
# Keep running the command forever.
sleep(10)
end
end # def run
end # class LogStash::Inputs::Pipe

128
lib/logstash/inputs/rabbitmq.rb
View file

@ -1,128 +0,0 @@
# encoding: utf-8
require "logstash/inputs/threadable"
require "logstash/namespace"
# Pull events from a RabbitMQ exchange.
#
# The default settings will create an entirely transient queue and listen for all messages by default.
# If you need durability or any other advanced settings, please set the appropriate options
#
# This has been tested with Bunny 0.9.x, which supports RabbitMQ 2.x and 3.x. You can
# find links to both here:
#
# * RabbitMQ - <http://www.rabbitmq.com/>
# * March Hare: <http://rubymarchhare.info>
# * Bunny - <https://github.com/ruby-amqp/bunny>
class LogStash::Inputs::RabbitMQ < LogStash::Inputs::Threadable
config_name "rabbitmq"
milestone 1
#
# Connection
#
# RabbitMQ server address
config :host, :validate => :string, :required => true
# RabbitMQ port to connect on
config :port, :validate => :number, :default => 5672
# RabbitMQ username
config :user, :validate => :string, :default => "guest"
# RabbitMQ password
config :password, :validate => :password, :default => "guest"
# The vhost to use. If you don't know what this is, leave the default.
config :vhost, :validate => :string, :default => "/"
# Enable or disable SSL
config :ssl, :validate => :boolean, :default => false
# Validate SSL certificate
config :verify_ssl, :validate => :boolean, :default => false
# Enable or disable logging
config :debug, :validate => :boolean, :default => false, :deprecated => "Use the logstash --debug flag for this instead."
#
# Queue & Consumer
#
# The name of the queue Logstash will consume events from.
config :queue, :validate => :string, :default => ""
# Is this queue durable? (aka; Should it survive a broker restart?)
config :durable, :validate => :boolean, :default => false
# Should the queue be deleted on the broker when the last consumer
# disconnects? Set this option to 'false' if you want the queue to remain
# on the broker, queueing up messages until a consumer comes along to
# consume them.
config :auto_delete, :validate => :boolean, :default => false
# Is the queue exclusive? Exclusive queues can only be used by the connection
# that declared them and will be deleted when it is closed (e.g. due to a Logstash
# restart).
config :exclusive, :validate => :boolean, :default => false
# Extra queue arguments as an array.
# To make a RabbitMQ queue mirrored, use: {"x-ha-policy" => "all"}
config :arguments, :validate => :array, :default => {}
# Prefetch count. Number of messages to prefetch
config :prefetch_count, :validate => :number, :default => 256
# Enable message acknowledgement
config :ack, :validate => :boolean, :default => true
# Passive queue creation? Useful for checking queue existance without modifying server state
config :passive, :validate => :boolean, :default => false
#
# (Optional) Exchange binding
#
# Optional.
#
# The name of the exchange to bind the queue to.
config :exchange, :validate => :string
# Optional.
#
# The routing key to use when binding a queue to the exchange.
# This is only relevant for direct or topic exchanges.
#
# * Routing keys are ignored on fanout exchanges.
# * Wildcards are not valid on direct exchanges.
config :key, :validate => :string, :default => "logstash"
def initialize(params)
params["codec"] = "json" if !params["codec"]
super
end
# Use March Hare on JRuby to avoid IO#select CPU spikes
# (see github.com/ruby-amqp/bunny/issues/95).
#
# On MRI, use Bunny.
#
# See http://rubybunny.info and http://rubymarchhare.info
# for the docs.
if RUBY_ENGINE == "jruby"
require "logstash/inputs/rabbitmq/march_hare"
include MarchHareImpl
else
require "logstash/inputs/rabbitmq/bunny"
include BunnyImpl
end
end # class LogStash::Inputs::RabbitMQ

119
lib/logstash/inputs/rabbitmq/bunny.rb
View file

@ -1,119 +0,0 @@
# encoding: utf-8
class LogStash::Inputs::RabbitMQ
module BunnyImpl
def register
require "bunny"
@vhost ||= Bunny::DEFAULT_HOST
# 5672. Will be switched to 5671 by Bunny if TLS is enabled.
@port ||= AMQ::Protocol::DEFAULT_PORT
@routing_key ||= "#"
@settings = {
:vhost => @vhost,
:host => @host,
:port => @port,
:automatically_recover => false
}
@settings[:user] = @user || Bunny::DEFAULT_USER
@settings[:pass] = if @password
@password.value
else
Bunny::DEFAULT_PASSWORD
end
@settings[:log_level] = if @debug || @logger.debug?
:debug
else
:error
end
@settings[:tls] = @ssl if @ssl
@settings[:verify_ssl] = @verify_ssl if @verify_ssl
proto = if @ssl
"amqps"
else
"amqp"
end
@connection_url = "#{proto}://#{@user}@#{@host}:#{@port}#{vhost}/#{@queue}"
@logger.info("Registering input #{@connection_url}")
end
def run(output_queue)
@output_queue = output_queue
begin
setup
consume
rescue Bunny::NetworkFailure, Bunny::ConnectionClosedError, Bunny::ConnectionLevelException, Bunny::TCPConnectionFailed => e
n = Bunny::Session::DEFAULT_NETWORK_RECOVERY_INTERVAL * 2
# Because we manually reconnect instead of letting Bunny
# handle failures,
# make sure we don't leave any consumer work pool
# threads behind. MK.
@ch.maybe_kill_consumer_work_pool!
@logger.error("RabbitMQ connection error: #{e.message}. Will attempt to reconnect in #{n} seconds...")
sleep n
retry
end
end
def teardown
@consumer.cancel
@q.delete unless @durable
@ch.close if @ch && @ch.open?
@conn.close if @conn && @conn.open?
finished
end
def setup
@conn = Bunny.new(@settings)
@logger.debug("Connecting to RabbitMQ. Settings: #{@settings.inspect}, queue: #{@queue.inspect}")
return if terminating?
@conn.start
@ch = @conn.create_channel.tap do |ch|
ch.prefetch(@prefetch_count)
end
@logger.info("Connected to RabbitMQ at #{@settings[:host]}")
@arguments_hash = Hash[*@arguments]
@q = @ch.queue(@queue,
:durable => @durable,
:auto_delete => @auto_delete,
:exclusive => @exclusive,
:passive => @passive,
:arguments => @arguments)
# exchange binding is optional for the input
if @exchange
@q.bind(@exchange, :routing_key => @key)
end
end
def consume
@logger.info("Will consume events from queue #{@q.name}")
# we both need to block the caller in Bunny::Queue#subscribe and have
# a reference to the consumer so that we can cancel it, so
# a consumer manually. MK.
@consumer = Bunny::Consumer.new(@ch, @q)
@q.subscribe(:manual_ack => @ack, :block => true) do |delivery_info, properties, data|
@codec.decode(data) do |event|
decorate(event)
@output_queue << event
end
@ch.acknowledge(delivery_info.delivery_tag) if @ack
end
end
end # BunnyImpl
end

1
lib/logstash/inputs/rabbitmq/hot_bunnies.rb
View file

@ -1 +0,0 @@
require "logstash/inputs/rabbitmq/march_hare"

130
lib/logstash/inputs/rabbitmq/march_hare.rb
View file

@ -1,130 +0,0 @@
# encoding: utf-8
class LogStash::Inputs::RabbitMQ
# MarchHare-based implementation for JRuby
module MarchHareImpl
def register
require "hot_bunnies"
require "java"
@vhost ||= "127.0.0.1"
# 5672. Will be switched to 5671 by Bunny if TLS is enabled.
@port ||= 5672
@key ||= "#"
@settings = {
:vhost => @vhost,
:host => @host,
:port => @port,
:user => @user,
:automatic_recovery => false
}
@settings[:pass] = @password.value if @password
@settings[:tls] = @ssl if @ssl
proto = if @ssl
"amqps"
else
"amqp"
end
@connection_url = "#{proto}://#{@user}@#{@host}:#{@port}#{vhost}/#{@queue}"
@logger.info("Registering input #{@connection_url}")
end
def run(output_queue)
@output_queue = output_queue
@break_out_of_the_loop = java.util.concurrent.atomic.AtomicBoolean.new(false)
# MarchHare does not raise exceptions when connection goes down with a blocking
# consumer running (it uses callbacks, as the RabbitMQ Java client does).
#
# However, MarchHare::Channel will make sure to unblock all blocking consumers
# on any internal shutdown, so #consume will return and another loop iteration
# will run.
#
# This is very similar to how the Bunny implementation works and is sufficient
# for our needs: it recovers successfully after RabbitMQ is kill -9ed, the
# network device is shut down, etc. MK.
until @break_out_of_the_loop.get do
begin
setup
consume
rescue MarchHare::Exception, java.lang.Throwable, com.rabbitmq.client.AlreadyClosedException => e
n = 10
@logger.error("RabbitMQ connection error: #{e}. Will reconnect in #{n} seconds...")
sleep n
retry
rescue LogStash::ShutdownSignal => ss
shutdown_consumer
end
n = 10
@logger.error("RabbitMQ connection error: #{e}. Will reconnect in #{n} seconds...")
end
end
def teardown
shutdown_consumer
@q.delete unless @durable
@ch.close if @ch && @ch.open?
@connection.close if @connection && @connection.open?
finished
end
#
# Implementation
#
protected
def setup
return if terminating?
@conn = MarchHare.connect(@settings)
@logger.info("Connected to RabbitMQ #{@connection_url}")
@ch = @conn.create_channel.tap do |ch|
ch.prefetch = @prefetch_count
end
@arguments_hash = Hash[*@arguments]
@q = @ch.queue(@queue,
:durable => @durable,
:auto_delete => @auto_delete,
:exclusive => @exclusive,
:passive => @passive,
:arguments => @arguments)
# exchange binding is optional for the input
if @exchange
@q.bind(@exchange, :routing_key => @key)
end
end
def consume
return if terminating?
# we manually build a consumer here to be able to keep a reference to it
# in an @ivar even though we use a blocking version of HB::Queue#subscribe
@consumer = @q.build_consumer(:block => true) do |metadata, data|
@codec.decode(data) do |event|
decorate(event)
@output_queue << event if event
end
@ch.ack(metadata.delivery_tag) if @ack
end
@q.subscribe_with(@consumer, :manual_ack => @ack, :block => true)
end
def shutdown_consumer
@break_out_of_the_loop.set(true)
@consumer.cancel
@consumer.gracefully_shut_down
end
end # MarchHareImpl
end

266
lib/logstash/inputs/redis.rb
View file

@ -1,266 +0,0 @@
# encoding: utf-8
require "logstash/inputs/base"
require "logstash/inputs/threadable"
require "logstash/namespace"
# This input will read events from a Redis instance; it supports both Redis channels and lists.
# The list command (BLPOP) used by Logstash is supported in Redis v1.3.1+, and
# the channel commands used by Logstash are found in Redis v1.3.8+.
# While you may be able to make these Redis versions work, the best performance
# and stability will be found in more recent stable versions. Versions 2.6.0+
# are recommended.
#
# For more information about Redis, see <http://redis.io/>
#
# `batch_count` note: If you use the `batch_count` setting, you *must* use a Redis version 2.6.0 or
# newer. Anything older does not support the operations used by batching.
#
class LogStash::Inputs::Redis < LogStash::Inputs::Threadable
config_name "redis"
milestone 2
default :codec, "json"
# The `name` configuration is used for logging in case there are multiple instances.
# This feature has no real function and will be removed in future versions.
config :name, :validate => :string, :default => "default", :deprecated => true
# The hostname of your Redis server.
config :host, :validate => :string, :default => "127.0.0.1"
# The port to connect on.
config :port, :validate => :number, :default => 6379
# The Redis database number.
config :db, :validate => :number, :default => 0
# Initial connection timeout in seconds.
config :timeout, :validate => :number, :default => 5
# Password to authenticate with. There is no authentication by default.
config :password, :validate => :password
# The name of the Redis queue (we'll use BLPOP against this).
# TODO: remove soon.
config :queue, :validate => :string, :deprecated => true
# The name of a Redis list or channel.
# TODO: change required to true
config :key, :validate => :string, :required => false
# Specify either list or channel. If `redis\_type` is `list`, then we will BLPOP the
# key. If `redis\_type` is `channel`, then we will SUBSCRIBE to the key.
# If `redis\_type` is `pattern_channel`, then we will PSUBSCRIBE to the key.
# TODO: change required to true
config :data_type, :validate => [ "list", "channel", "pattern_channel" ], :required => false
# The number of events to return from Redis using EVAL.
config :batch_count, :validate => :number, :default => 1
public
def register
require 'redis'
@redis = nil
@redis_url = "redis://#{@password}@#{@host}:#{@port}/#{@db}"
# TODO remove after setting key and data_type to true
if @queue
if @key or @data_type
raise RuntimeError.new(
"Cannot specify queue parameter and key or data_type"
)
end
@key = @queue
@data_type = 'list'
end
if not @key or not @data_type
raise RuntimeError.new(
"Must define queue, or key and data_type parameters"
)
end
# end TODO
@logger.info("Registering Redis", :identity => identity)
end # def register
# A string used to identify a Redis instance in log messages
# TODO(sissel): Use instance variables for this once the @name config
# option is removed.
private
def identity
@name || "#{@redis_url} #{@data_type}:#{@key}"
end
private
def connect
redis = Redis.new(
:host => @host,
:port => @port,
:timeout => @timeout,
:db => @db,
:password => @password.nil? ? nil : @password.value
)
load_batch_script(redis) if @data_type == 'list' && (@batch_count > 1)
return redis
end # def connect
private
def load_batch_script(redis)
#A Redis Lua EVAL script to fetch a count of keys
#in case count is bigger than current items in queue whole queue will be returned without extra nil values
redis_script = <<EOF
local i = tonumber(ARGV[1])
local res = {}
local length = redis.call('llen',KEYS[1])
if length < i then i = length end
while (i > 0) do
local item = redis.call("lpop", KEYS[1])
if (not item) then
break
end
table.insert(res, item)
i = i-1
end
return res
EOF
@redis_script_sha = redis.script(:load, redis_script)
end
private
def queue_event(msg, output_queue)
begin
@codec.decode(msg) do |event|
decorate(event)
output_queue << event
end
rescue => e # parse or event creation error
@logger.error("Failed to create event", :message => msg, :exception => e,
:backtrace => e.backtrace);
end
end
private
def list_listener(redis, output_queue)
# blpop returns the 'key' read from as well as the item result
# we only care about the result (2nd item in the list).
item = redis.blpop(@key, 0)[1]
# blpop failed or .. something?
# TODO(sissel): handle the error
return if item.nil?
queue_event(item, output_queue)
# If @batch_count is 1, there's no need to continue.
return if @batch_count == 1
begin
redis.evalsha(@redis_script_sha, [@key], [@batch_count-1]).each do |item|
queue_event(item, output_queue)
end
# Below is a commented-out implementation of 'batch fetch'
# using pipelined LPOP calls. This in practice has been observed to
# perform exactly the same in terms of event throughput as
# the evalsha method. Given that the EVALSHA implementation uses
# one call to Redis instead of N (where N == @batch_count) calls,
# I decided to go with the 'evalsha' method of fetching N items
# from Redis in bulk.
#redis.pipelined do
#error, item = redis.lpop(@key)
#(@batch_count-1).times { redis.lpop(@key) }
#end.each do |item|
#queue_event(item, output_queue) if item
#end
# --- End commented out implementation of 'batch fetch'
rescue Redis::CommandError => e
if e.to_s =~ /NOSCRIPT/ then
@logger.warn("Redis may have been restarted, reloading Redis batch EVAL script", :exception => e);
load_batch_script(redis)
retry
else
raise e
end
end
end
private
def channel_listener(redis, output_queue)
redis.subscribe @key do |on|
on.subscribe do |channel, count|
@logger.info("Subscribed", :channel => channel, :count => count)
end
on.message do |channel, message|
queue_event message, output_queue
end
on.unsubscribe do |channel, count|
@logger.info("Unsubscribed", :channel => channel, :count => count)
end
end
end
private
def pattern_channel_listener(redis, output_queue)
redis.psubscribe @key do |on|
on.psubscribe do |channel, count|
@logger.info("Subscribed", :channel => channel, :count => count)
end
on.pmessage do |ch, event, message|
queue_event message, output_queue
end
on.punsubscribe do |channel, count|
@logger.info("Unsubscribed", :channel => channel, :count => count)
end
end
end
# Since both listeners have the same basic loop, we've abstracted the outer
# loop.
private
def listener_loop(listener, output_queue)
while !finished?
begin
@redis ||= connect
self.send listener, @redis, output_queue
rescue Redis::CannotConnectError => e
@logger.warn("Redis connection problem", :exception => e)
sleep 1
@redis = connect
rescue => e # Redis error
@logger.warn("Failed to get event from Redis", :name => @name,
:exception => e, :backtrace => e.backtrace)
raise e
end
end # while !finished?
end # listener_loop
public
def run(output_queue)
if @data_type == 'list'
listener_loop :list_listener, output_queue
elsif @data_type == 'channel'
listener_loop :channel_listener, output_queue
else
listener_loop :pattern_channel_listener, output_queue
end
end # def run
public
def teardown
if @data_type == 'channel' and @redis
@redis.unsubscribe
@redis.quit
@redis = nil
end
if @data_type == 'pattern_channel' and @redis
@redis.punsubscribe
@redis.quit
@redis = nil
end
end
end # class LogStash::Inputs::Redis

278
lib/logstash/inputs/s3.rb
View file

@ -1,278 +0,0 @@
# encoding: utf-8
require "logstash/inputs/base"
require "logstash/namespace"
require "time"
require "tmpdir"
# Stream events from files from a S3 bucket.
#
# Each line from each file generates an event.
# Files ending in '.gz' are handled as gzip'ed files.
class LogStash::Inputs::S3 < LogStash::Inputs::Base
config_name "s3"
milestone 1
# TODO(sissel): refactor to use 'line' codec (requires removing both gzip
# support and readline usage). Support gzip through a gzip codec! ;)
default :codec, "plain"
# The credentials of the AWS account used to access the bucket.
# Credentials can be specified:
# - As an ["id","secret"] array
# - As a path to a file containing AWS_ACCESS_KEY_ID=... and AWS_SECRET_ACCESS_KEY=...
# - In the environment, if not set (using variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY)
config :credentials, :validate => :array, :default => []
# The name of the S3 bucket.
config :bucket, :validate => :string, :required => true
# The AWS region for your bucket.
config :region, :validate => ["us-east-1", "us-west-1", "us-west-2",
"eu-west-1", "ap-southeast-1", "ap-southeast-2",
"ap-northeast-1", "sa-east-1", "us-gov-west-1"],
:deprecated => "'region' has been deprecated in favor of 'region_endpoint'"
# The AWS region for your bucket.
config :region_endpoint, :validate => ["us-east-1", "us-west-1", "us-west-2",
"eu-west-1", "ap-southeast-1", "ap-southeast-2",
"ap-northeast-1", "sa-east-1", "us-gov-west-1"], :default => "us-east-1"
# If specified, the prefix the filenames in the bucket must match (not a regexp)
config :prefix, :validate => :string, :default => nil
# Where to write the since database (keeps track of the date
# the last handled file was added to S3). The default will write
# sincedb files to some path matching "$HOME/.sincedb*"
config :sincedb_path, :validate => :string, :default => nil
# Name of a S3 bucket to backup processed files to.
config :backup_to_bucket, :validate => :string, :default => nil
# Path of a local directory to backup processed files to.
config :backup_to_dir, :validate => :string, :default => nil
# Whether to delete processed files from the original bucket.
config :delete, :validate => :boolean, :default => false
# Interval to wait between to check the file list again after a run is finished.
# Value is in seconds.
config :interval, :validate => :number, :default => 60
public
def register
require "digest/md5"
require "aws-sdk"
@region_endpoint = @region if @region && !@region.empty?
@logger.info("Registering s3 input", :bucket => @bucket, :region_endpoint => @region_endpoint)
if @credentials.length == 0
@access_key_id = ENV['AWS_ACCESS_KEY_ID']
@secret_access_key = ENV['AWS_SECRET_ACCESS_KEY']
elsif @credentials.length == 1
File.open(@credentials[0]) { |f| f.each do |line|
unless (/^\#/.match(line))
if(/\s*=\s*/.match(line))
param, value = line.split('=', 2)
param = param.chomp().strip()
value = value.chomp().strip()
if param.eql?('AWS_ACCESS_KEY_ID')
@access_key_id = value
elsif param.eql?('AWS_SECRET_ACCESS_KEY')
@secret_access_key = value
end
end
end
end
}
elsif @credentials.length == 2
@access_key_id = @credentials[0]
@secret_access_key = @credentials[1]
else
raise ArgumentError.new('Credentials must be of the form "/path/to/file" or ["id", "secret"]')
end
if @access_key_id.nil? or @secret_access_key.nil?
raise ArgumentError.new('Missing AWS credentials')
end
if @bucket.nil?
raise ArgumentError.new('Missing AWS bucket')
end
if @sincedb_path.nil?
if ENV['HOME'].nil?
raise ArgumentError.new('No HOME or sincedb_path set')
end
@sincedb_path = File.join(ENV["HOME"], ".sincedb_" + Digest::MD5.hexdigest("#{@bucket}+#{@prefix}"))
end
s3 = AWS::S3.new(
:access_key_id => @access_key_id,
:secret_access_key => @secret_access_key,
:region => @region_endpoint
)
@s3bucket = s3.buckets[@bucket]
unless @backup_to_bucket.nil?
@backup_bucket = s3.buckets[@backup_to_bucket]
unless @backup_bucket.exists?
s3.buckets.create(@backup_to_bucket)
end
end
unless @backup_to_dir.nil?
Dir.mkdir(@backup_to_dir, 0700) unless File.exists?(@backup_to_dir)
end
end # def register
public
def run(queue)
loop do
process_new(queue)
sleep(@interval)
end
finished
end # def run
private
def process_new(queue, since=nil)
if since.nil?
since = sincedb_read()
end
objects = list_new(since)
objects.each do |k|
@logger.debug("S3 input processing", :bucket => @bucket, :key => k)
lastmod = @s3bucket.objects[k].last_modified
process_log(queue, k)
sincedb_write(lastmod)
end
end # def process_new
private
def list_new(since=nil)
if since.nil?
since = Time.new(0)
end
objects = {}
@s3bucket.objects.with_prefix(@prefix).each do |log|
if log.last_modified > since
objects[log.key] = log.last_modified
end
end
return sorted_objects = objects.keys.sort {|a,b| objects[a] <=> objects[b]}
end # def list_new
private
def process_log(queue, key)
object = @s3bucket.objects[key]
tmp = Dir.mktmpdir("logstash-")
begin
filename = File.join(tmp, File.basename(key))
File.open(filename, 'wb') do |s3file|
object.read do |chunk|
s3file.write(chunk)
end
end
process_local_log(queue, filename)
unless @backup_to_bucket.nil?
backup_object = @backup_bucket.objects[key]
backup_object.write(Pathname.new(filename))
end
unless @backup_to_dir.nil?
FileUtils.cp(filename, @backup_to_dir)
end
if @delete
object.delete()
end
end
FileUtils.remove_entry_secure(tmp, force=true)
end # def process_log
private
def process_local_log(queue, filename)
metadata = {
:version => nil,
:format => nil,
}
File.open(filename) do |file|
if filename.end_with?('.gz')
gz = Zlib::GzipReader.new(file)
gz.each_line do |line|
metadata = process_line(queue, metadata, line)
end
else
file.each do |line|
metadata = process_line(queue, metadata, line)
end
end
end
end # def process_local_log
private
def process_line(queue, metadata, line)
if /#Version: .+/.match(line)
junk, version = line.strip().split(/#Version: (.+)/)
unless version.nil?
metadata[:version] = version
end
elsif /#Fields: .+/.match(line)
junk, format = line.strip().split(/#Fields: (.+)/)
unless format.nil?
metadata[:format] = format
end
else
@codec.decode(line) do |event|
decorate(event)
unless metadata[:version].nil?
event["cloudfront_version"] = metadata[:version]
end
unless metadata[:format].nil?
event["cloudfront_fields"] = metadata[:format]
end
queue << event
end
end
return metadata
end # def process_line
private
def sincedb_read()
if File.exists?(@sincedb_path)
since = Time.parse(File.read(@sincedb_path).chomp.strip)
else
since = Time.new(0)
end
return since
end # def sincedb_read
private
def sincedb_write(since=nil)
if since.nil?
since = Time.now()
end
File.open(@sincedb_path, 'w') { |file| file.write(since.to_s) }
end # def sincedb_write
end # class LogStash::Inputs::S3

87
lib/logstash/inputs/snmptrap.rb
View file

@ -1,87 +0,0 @@
# encoding: utf-8
require "logstash/inputs/base"
require "logstash/namespace"
# Read snmp trap messages as events
#
# Resulting @message looks like :
# #<SNMP::SNMPv1_Trap:0x6f1a7a4 @varbind_list=[#<SNMP::VarBind:0x2d7bcd8f @value="teststring",
# @name=[1.11.12.13.14.15]>], @timestamp=#<SNMP::TimeTicks:0x1af47e9d @value=55>, @generic_trap=6,
# @enterprise=[1.2.3.4.5.6], @source_ip="127.0.0.1", @agent_addr=#<SNMP::IpAddress:0x29a4833e @value="\xC0\xC1\xC2\xC3">,
# @specific_trap=99>
#
class LogStash::Inputs::Snmptrap < LogStash::Inputs::Base
config_name "snmptrap"
milestone 1
# The address to listen on
config :host, :validate => :string, :default => "0.0.0.0"
# The port to listen on. Remember that ports less than 1024 (privileged
# ports) may require root to use. hence the default of 1062.
config :port, :validate => :number, :default => 1062
# SNMP Community String to listen for.
config :community, :validate => :string, :default => "public"
# directory of YAML MIB maps (same format ruby-snmp uses)
config :yamlmibdir, :validate => :string
def initialize(*args)
super(*args)
end # def initialize
public
def register
require "snmp"
@snmptrap = nil
if @yamlmibdir
@logger.info("checking #{@yamlmibdir} for MIBs")
Dir["#{@yamlmibdir}/*.yaml"].each do |yamlfile|
mib_name = File.basename(yamlfile, ".*")
@yaml_mibs ||= []
@yaml_mibs << mib_name
end
@logger.info("found MIBs: #{@yaml_mibs.join(',')}") if @yaml_mibs
end
end # def register
public
def run(output_queue)
begin
# snmp trap server
snmptrap_listener(output_queue)
rescue => e
@logger.warn("SNMP Trap listener died", :exception => e, :backtrace => e.backtrace)
sleep(5)
retry
end # begin
end # def run
private
def snmptrap_listener(output_queue)
traplistener_opts = {:Port => @port, :Community => @community, :Host => @host}
if @yaml_mibs && !@yaml_mibs.empty?
traplistener_opts.merge!({:MibDir => @yamlmibdir, :MibModules => @yaml_mibs})
end
@logger.info("It's a Trap!", traplistener_opts.dup)
@snmptrap = SNMP::TrapListener.new(traplistener_opts)
@snmptrap.on_trap_default do |trap|
begin
event = LogStash::Event.new("message" => trap.inspect, "host" => trap.source_ip)
decorate(event)
trap.each_varbind do |vb|
event[vb.name.to_s] = vb.value.to_s
end
@logger.debug("SNMP Trap received: ", :trap_object => trap.inspect)
output_queue << event
rescue => event
@logger.error("Failed to create event", :trap_object => trap.inspect)
end
end
@snmptrap.join
end # def snmptrap_listener
end # class LogStash::Inputs::Snmptrap

173
lib/logstash/inputs/sqs.rb
View file

@ -1,173 +0,0 @@
# encoding: utf-8
require "logstash/inputs/threadable"
require "logstash/namespace"
require "logstash/timestamp"
require "logstash/plugin_mixins/aws_config"
require "digest/sha2"
# Pull events from an Amazon Web Services Simple Queue Service (SQS) queue.
#
# SQS is a simple, scalable queue system that is part of the
# Amazon Web Services suite of tools.
#
# Although SQS is similar to other queuing systems like AMQP, it
# uses a custom API and requires that you have an AWS account.
# See http://aws.amazon.com/sqs/ for more details on how SQS works,
# what the pricing schedule looks like and how to setup a queue.
#
# To use this plugin, you *must*:
#
# * Have an AWS account
# * Setup an SQS queue
# * Create an identify that has access to consume messages from the queue.
#
# The "consumer" identity must have the following permissions on the queue:
#
# * sqs:ChangeMessageVisibility
# * sqs:ChangeMessageVisibilityBatch
# * sqs:DeleteMessage
# * sqs:DeleteMessageBatch
# * sqs:GetQueueAttributes
# * sqs:GetQueueUrl
# * sqs:ListQueues
# * sqs:ReceiveMessage
#
# Typically, you should setup an IAM policy, create a user and apply the IAM policy to the user.
# A sample policy is as follows:
#
# {
# "Statement": [
# {
# "Action": [
# "sqs:ChangeMessageVisibility",
# "sqs:ChangeMessageVisibilityBatch",
# "sqs:GetQueueAttributes",
# "sqs:GetQueueUrl",
# "sqs:ListQueues",
# "sqs:SendMessage",
# "sqs:SendMessageBatch"
# ],
# "Effect": "Allow",
# "Resource": [
# "arn:aws:sqs:us-east-1:123456789012:Logstash"
# ]
# }
# ]
# }
#
# See http://aws.amazon.com/iam/ for more details on setting up AWS identities.
#
class LogStash::Inputs::SQS < LogStash::Inputs::Threadable
include LogStash::PluginMixins::AwsConfig
config_name "sqs"
milestone 1
default :codec, "json"
# Name of the SQS Queue name to pull messages from. Note that this is just the name of the queue, not the URL or ARN.
config :queue, :validate => :string, :required => true
# Name of the event field in which to store the SQS message ID
config :id_field, :validate => :string
# Name of the event field in which to store the SQS message MD5 checksum
config :md5_field, :validate => :string
# Name of the event field in which to store the SQS message Sent Timestamp
config :sent_timestamp_field, :validate => :string
public
def aws_service_endpoint(region)
return {
:sqs_endpoint => "sqs.#{region}.amazonaws.com"
}
end
public
def register
@logger.info("Registering SQS input", :queue => @queue)
require "aws-sdk"
@sqs = AWS::SQS.new(aws_options_hash)
begin
@logger.debug("Connecting to AWS SQS queue", :queue => @queue)
@sqs_queue = @sqs.queues.named(@queue)
@logger.info("Connected to AWS SQS queue successfully.", :queue => @queue)
rescue Exception => e
@logger.error("Unable to access SQS queue.", :error => e.to_s, :queue => @queue)
throw e
end # begin/rescue
end # def register
public
def run(output_queue)
@logger.debug("Polling SQS queue", :queue => @queue)
receive_opts = {
:limit => 10,
:visibility_timeout => 30,
:attributes => [:sent_at]
}
continue_polling = true
while running? && continue_polling
continue_polling = run_with_backoff(60, 1) do
@sqs_queue.receive_message(receive_opts) do |message|
if message
@codec.decode(message.body) do |event|
decorate(event)
if @id_field
event[@id_field] = message.id
end
if @md5_field
event[@md5_field] = message.md5
end
if @sent_timestamp_field
event[@sent_timestamp_field] = LogStash::Timestamp.new(message.sent_timestamp).utc
end
@logger.debug? && @logger.debug("Processed SQS message", :message_id => message.id, :message_md5 => message.md5, :sent_timestamp => message.sent_timestamp, :queue => @queue)
output_queue << event
message.delete
end # codec.decode
end # valid SQS message
end # receive_message
end # run_with_backoff
end # polling loop
end # def run
def teardown
@sqs_queue = nil
finished
end # def teardown
private
# Runs an AWS request inside a Ruby block with an exponential backoff in case
# we exceed the allowed AWS RequestLimit.
#
# @param [Integer] max_time maximum amount of time to sleep before giving up.
# @param [Integer] sleep_time the initial amount of time to sleep before retrying.
# @param [Block] block Ruby code block to execute.
def run_with_backoff(max_time, sleep_time, &block)
if sleep_time > max_time
@logger.error("AWS::EC2::Errors::RequestLimitExceeded ... failed.", :queue => @queue)
return false
end # retry limit exceeded
begin
block.call
rescue AWS::EC2::Errors::RequestLimitExceeded
@logger.info("AWS::EC2::Errors::RequestLimitExceeded ... retrying SQS request", :queue => @queue, :sleep_time => sleep_time)
sleep sleep_time
run_with_backoff(max_time, sleep_time * 2, &block)
rescue AWS::EC2::Errors::InstanceLimitExceeded
@logger.warn("AWS::EC2::Errors::InstanceLimitExceeded ... aborting SQS message retreival.")
return false
rescue Exception => bang
@logger.error("Error reading SQS queue.", :error => bang, :queue => @queue)
return false
end # begin/rescue
return true
end # def run_with_backoff
end # class LogStash::Inputs::SQS

47
lib/logstash/inputs/stdin.rb
View file

@ -1,47 +0,0 @@
# encoding: utf-8
require "logstash/inputs/base"
require "logstash/namespace"
require "socket" # for Socket.gethostname
# Read events from standard input.
#
# By default, each event is assumed to be one line. If you
# want to join lines, you'll want to use the multiline filter.
class LogStash::Inputs::Stdin < LogStash::Inputs::Base
config_name "stdin"
milestone 3
default :codec, "line"
public
def register
@host = Socket.gethostname
fix_streaming_codecs
end # def register
def run(queue)
while true
begin
# Based on some testing, there is no way to interrupt an IO.sysread nor
# IO.select call in JRuby. Bummer :(
data = $stdin.sysread(16384)
@codec.decode(data) do |event|
decorate(event)
event["host"] = @host if !event.include?("host")
queue << event
end
rescue IOError, EOFError, LogStash::ShutdownSignal
# stdin closed or a requested shutdown
break
end
end # while true
finished
end # def run
public
def teardown
@logger.debug("stdin shutting down.")
$stdin.close rescue nil
finished
end # def teardown
end # class LogStash::Inputs::Stdin

238
lib/logstash/inputs/syslog.rb
View file

@ -1,238 +0,0 @@
# encoding: utf-8
require "date"
require "logstash/filters/grok"
require "logstash/filters/date"
require "logstash/inputs/base"
require "logstash/namespace"
require "socket"
# Read syslog messages as events over the network.
#
# This input is a good choice if you already use syslog today.
# It is also a good choice if you want to receive logs from
# appliances and network devices where you cannot run your own
# log collector.
#
# Of course, 'syslog' is a very muddy term. This input only supports RFC3164
# syslog with some small modifications. The date format is allowed to be
# RFC3164 style or ISO8601. Otherwise the rest of RFC3164 must be obeyed.
# If you do not use RFC3164, do not use this input.
#
# For more information see [the RFC3164 page](http://www.ietf.org/rfc/rfc3164.txt).
#
# Note: This input will start listeners on both TCP and UDP.
class LogStash::Inputs::Syslog < LogStash::Inputs::Base
config_name "syslog"
milestone 1
default :codec, "plain"
# The address to listen on.
config :host, :validate => :string, :default => "0.0.0.0"
# The port to listen on. Remember that ports less than 1024 (privileged
# ports) may require root to use.
config :port, :validate => :number, :default => 514
# Use label parsing for severity and facility levels.
config :use_labels, :validate => :boolean, :default => true
# Labels for facility levels. These are defined in RFC3164.
config :facility_labels, :validate => :array, :default => [ "kernel", "user-level", "mail", "system", "security/authorization", "syslogd", "line printer", "network news", "UUCP", "clock", "security/authorization", "FTP", "NTP", "log audit", "log alert", "clock", "local0", "local1", "local2", "local3", "local4", "local5", "local6", "local7" ]
# Labels for severity levels. These are defined in RFC3164.
config :severity_labels, :validate => :array, :default => [ "Emergency" , "Alert", "Critical", "Error", "Warning", "Notice", "Informational", "Debug" ]
public
def initialize(params)
super
@shutdown_requested = false
BasicSocket.do_not_reverse_lookup = true
end # def initialize
public
def register
require "thread_safe"
@grok_filter = LogStash::Filters::Grok.new(
"overwrite" => "message",
"match" => { "message" => "<%{POSINT:priority}>%{SYSLOGLINE}" },
"tag_on_failure" => ["_grokparsefailure_sysloginput"],
)
@date_filter = LogStash::Filters::Date.new(
"match" => [ "timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601"]
)
@grok_filter.register
@date_filter.register
@tcp_clients = ThreadSafe::Array.new
end # def register
public
def run(output_queue)
# udp server
udp_thr = Thread.new do
begin
udp_listener(output_queue)
rescue => e
break if @shutdown_requested
@logger.warn("syslog udp listener died",
:address => "#{@host}:#{@port}", :exception => e,
:backtrace => e.backtrace)
sleep(5)
retry
end # begin
end # Thread.new
# tcp server
tcp_thr = Thread.new do
begin
tcp_listener(output_queue)
rescue => e
break if @shutdown_requested
@logger.warn("syslog tcp listener died",
:address => "#{@host}:#{@port}", :exception => e,
:backtrace => e.backtrace)
sleep(5)
retry
end # begin
end # Thread.new
# If we exit and we're the only input, the agent will think no inputs
# are running and initiate a shutdown.
udp_thr.join
tcp_thr.join
end # def run
private
def udp_listener(output_queue)
@logger.info("Starting syslog udp listener", :address => "#{@host}:#{@port}")
if @udp
@udp.close
end
@udp = UDPSocket.new(Socket::AF_INET)
@udp.bind(@host, @port)
loop do
payload, client = @udp.recvfrom(9000)
# Ruby uri sucks, so don't use it.
@codec.decode(payload) do |event|
decorate(event)
event["host"] = client[3]
syslog_relay(event)
output_queue << event
end
end
ensure
close_udp
end # def udp_listener
private
def tcp_listener(output_queue)
@logger.info("Starting syslog tcp listener", :address => "#{@host}:#{@port}")
@tcp = TCPServer.new(@host, @port)
@tcp_clients = []
loop do
client = @tcp.accept
@tcp_clients << client
Thread.new(client) do |client|
ip, port = client.peeraddr[3], client.peeraddr[1]
@logger.info("new connection", :client => "#{ip}:#{port}")
LogStash::Util::set_thread_name("input|syslog|tcp|#{ip}:#{port}}")
begin
client.each do |line|
@codec.decode(line) do |event|
decorate(event)
event["host"] = ip
syslog_relay(event)
output_queue << event
end
end
rescue Errno::ECONNRESET
ensure
@tcp_clients.delete(client)
end
end # Thread.new
end # loop do
ensure
close_tcp
end # def tcp_listener
public
def teardown
@shutdown_requested = true
close_udp
close_tcp
finished
end
private
def close_udp
if @udp
@udp.close_read rescue nil
@udp.close_write rescue nil
end
@udp = nil
end
private
def close_tcp
# If we somehow have this left open, close it.
@tcp_clients.each do |client|
client.close rescue nil
end
@tcp.close if @tcp rescue nil
@tcp = nil
end
# Following RFC3164 where sane, we'll try to parse a received message
# as if you were relaying a syslog message to it.
# If the message cannot be recognized (see @grok_filter), we'll
# treat it like the whole event["message"] is correct and try to fill
# the missing pieces (host, priority, etc)
public
def syslog_relay(event)
@grok_filter.filter(event)
if event["tags"].nil? || !event["tags"].include?(@grok_filter.tag_on_failure)
# Per RFC3164, priority = (facility * 8) + severity
# = (facility << 3) & (severity)
priority = event["priority"].to_i rescue 13
severity = priority & 7 # 7 is 111 (3 bits)
facility = priority >> 3
event["priority"] = priority
event["severity"] = severity
event["facility"] = facility
event["timestamp"] = event["timestamp8601"] if event.include?("timestamp8601")
@date_filter.filter(event)
else
@logger.info? && @logger.info("NOT SYSLOG", :message => event["message"])
# RFC3164 says unknown messages get pri=13
priority = 13
event["priority"] = 13
event["severity"] = 5 # 13 & 7 == 5
event["facility"] = 1 # 13 >> 3 == 1
end
# Apply severity and facility metadata if
# use_labels => true
if @use_labels
facility_number = event["facility"]
severity_number = event["severity"]
if @facility_labels[facility_number]
event["facility_label"] = @facility_labels[facility_number]
end
if @severity_labels[severity_number]
event["severity_label"] = @severity_labels[severity_number]
end
end
end # def syslog_relay
end # class LogStash::Inputs::Syslog

238
lib/logstash/inputs/tcp.rb
View file

@ -1,238 +0,0 @@
# encoding: utf-8
require "logstash/inputs/base"
require "logstash/namespace"
require "logstash/util/socket_peer"
# Read events over a TCP socket.
#
# Like stdin and file inputs, each event is assumed to be one line of text.
#
# Can either accept connections from clients or connect to a server,
# depending on `mode`.
class LogStash::Inputs::Tcp < LogStash::Inputs::Base
class Interrupted < StandardError; end
config_name "tcp"
milestone 2
default :codec, "line"
# When mode is `server`, the address to listen on.
# When mode is `client`, the address to connect to.
config :host, :validate => :string, :default => "0.0.0.0"
# When mode is `server`, the port to listen on.
# When mode is `client`, the port to connect to.
config :port, :validate => :number, :required => true
# The 'read' timeout in seconds. If a particular tcp connection is idle for
# more than this timeout period, we will assume it is dead and close it.
#
# If you never want to timeout, use -1.
config :data_timeout, :validate => :number, :default => -1
# Mode to operate in. `server` listens for client connections,
# `client` connects to a server.
config :mode, :validate => ["server", "client"], :default => "server"
# Enable SSL (must be set for other `ssl_` options to take effect).
config :ssl_enable, :validate => :boolean, :default => false
# Verify the identity of the other end of the SSL connection against the CA.
# For input, sets the field `sslsubject` to that of the client certificate.
config :ssl_verify, :validate => :boolean, :default => false
# The SSL CA certificate, chainfile or CA path. The system CA path is automatically included.
config :ssl_cacert, :validate => :path
# SSL certificate path
config :ssl_cert, :validate => :path
# SSL key path
config :ssl_key, :validate => :path
# SSL key passphrase
config :ssl_key_passphrase, :validate => :password, :default => nil
def initialize(*args)
super(*args)
end # def initialize
public
def register
require "socket"
require "timeout"
require "openssl"
# monkey patch TCPSocket and SSLSocket to include socket peer
TCPSocket.module_eval{include ::LogStash::Util::SocketPeer}
OpenSSL::SSL::SSLSocket.module_eval{include ::LogStash::Util::SocketPeer}
fix_streaming_codecs
if @ssl_enable
@ssl_context = OpenSSL::SSL::SSLContext.new
@ssl_context.cert = OpenSSL::X509::Certificate.new(File.read(@ssl_cert))
@ssl_context.key = OpenSSL::PKey::RSA.new(File.read(@ssl_key),@ssl_key_passphrase)
if @ssl_verify
@cert_store = OpenSSL::X509::Store.new
# Load the system default certificate path to the store
@cert_store.set_default_paths
if File.directory?(@ssl_cacert)
@cert_store.add_path(@ssl_cacert)
else
@cert_store.add_file(@ssl_cacert)
end
@ssl_context.cert_store = @cert_store
@ssl_context.verify_mode = OpenSSL::SSL::VERIFY_PEER|OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
end
end # @ssl_enable
if server?
@logger.info("Starting tcp input listener", :address => "#{@host}:#{@port}")
begin
@server_socket = TCPServer.new(@host, @port)
rescue Errno::EADDRINUSE
@logger.error("Could not start TCP server: Address in use", :host => @host, :port => @port)
raise
end
if @ssl_enable
@server_socket = OpenSSL::SSL::SSLServer.new(@server_socket, @ssl_context)
end # @ssl_enable
end
end # def register
private
def handle_socket(socket, client_address, output_queue, codec)
while true
buf = nil
# NOTE(petef): the timeout only hits after the line is read or socket dies
# TODO(sissel): Why do we have a timeout here? What's the point?
if @data_timeout == -1
buf = read(socket)
else
Timeout::timeout(@data_timeout) do
buf = read(socket)
end
end
codec.decode(buf) do |event|
event["host"] ||= client_address
event["sslsubject"] ||= socket.peer_cert.subject if @ssl_enable && @ssl_verify
decorate(event)
output_queue << event
end
end # loop
rescue EOFError
@logger.debug? && @logger.debug("Connection closed", :client => socket.peer)
rescue Errno::ECONNRESET
@logger.debug? && @logger.debug("Connection reset by peer", :client => socket.peer)
rescue => e
@logger.error("An error occurred. Closing connection", :client => socket.peer, :exception => e, :backtrace => e.backtrace)
ensure
socket.close rescue nil
codec.respond_to?(:flush) && codec.flush do |event|
event["host"] ||= client_address
event["sslsubject"] ||= socket.peer_cert.subject if @ssl_enable && @ssl_verify
decorate(event)
output_queue << event
end
end
private
def client_thread(output_queue, socket)
Thread.new(output_queue, socket) do |q, s|
begin
@logger.debug? && @logger.debug("Accepted connection", :client => s.peer, :server => "#{@host}:#{@port}")
handle_socket(s, s.peeraddr[3], q, @codec.clone)
rescue Interrupted
s.close rescue nil
ensure
@client_threads_lock.synchronize{@client_threads.delete(Thread.current)}
end
end
end
private
def server?
@mode == "server"
end # def server?
private
def read(socket)
return socket.sysread(16384)
end # def readline
public
def run(output_queue)
if server?
run_server(output_queue)
else
run_client(output_queue)
end
end # def run
def run_server(output_queue)
@thread = Thread.current
@client_threads = []
@client_threads_lock = Mutex.new
while true
begin
socket = @server_socket.accept
# start a new thread for each connection.
@client_threads_lock.synchronize{@client_threads << client_thread(output_queue, socket)}
rescue OpenSSL::SSL::SSLError => ssle
# NOTE(mrichar1): This doesn't return a useful error message for some reason
@logger.error("SSL Error", :exception => ssle, :backtrace => ssle.backtrace)
rescue IOError, LogStash::ShutdownSignal
if @interrupted
@server_socket.close rescue nil
threads = @client_threads_lock.synchronize{@client_threads.dup}
threads.each do |thread|
thread.raise(LogStash::ShutdownSignal) if thread.alive?
end
# intended shutdown, get out of the loop
break
else
# it was a genuine IOError, propagate it up
raise
end
end
end # loop
rescue LogStash::ShutdownSignal
# nothing to do
ensure
@server_socket.close rescue nil
end # def run_server
def run_client(output_queue)
@thread = Thread.current
while true
client_socket = TCPSocket.new(@host, @port)
if @ssl_enable
client_socket = OpenSSL::SSL::SSLSocket.new(client_socket, @ssl_context)
begin
client_socket.connect
rescue OpenSSL::SSL::SSLError => ssle
@logger.error("SSL Error", :exception => ssle, :backtrace => ssle.backtrace)
# NOTE(mrichar1): Hack to prevent hammering peer
sleep(5)
next
end
end
@logger.debug("Opened connection", :client => "#{client_socket.peer}")
handle_socket(client_socket, client_socket.peeraddr[3], output_queue, @codec.clone)
end # loop
ensure
client_socket.close rescue nil
end # def run
public
def teardown
if server?
@interrupted = true
end
end # def teardown
end # class LogStash::Inputs::Tcp

124
lib/logstash/inputs/twitter.rb
View file

@ -1,124 +0,0 @@
# encoding: utf-8
require "logstash/inputs/base"
require "logstash/namespace"
require "logstash/timestamp"
require "logstash/util"
require "logstash/json"
# Read events from the twitter streaming api.
class LogStash::Inputs::Twitter < LogStash::Inputs::Base
config_name "twitter"
milestone 1
# Your twitter app's consumer key
#
# Don't know what this is? You need to create an "application"
# on twitter, see this url: <https://dev.twitter.com/apps/new>
config :consumer_key, :validate => :string, :required => true
# Your twitter app's consumer secret
#
# If you don't have one of these, you can create one by
# registering a new application with twitter:
# <https://dev.twitter.com/apps/new>
config :consumer_secret, :validate => :password, :required => true
# Your oauth token.
#
# To get this, login to twitter with whatever account you want,
# then visit <https://dev.twitter.com/apps>
#
# Click on your app (used with the consumer_key and consumer_secret settings)
# Then at the bottom of the page, click 'Create my access token' which
# will create an oauth token and secret bound to your account and that
# application.
config :oauth_token, :validate => :string, :required => true
# Your oauth token secret.
#
# To get this, login to twitter with whatever account you want,
# then visit <https://dev.twitter.com/apps>
#
# Click on your app (used with the consumer_key and consumer_secret settings)
# Then at the bottom of the page, click 'Create my access token' which
# will create an oauth token and secret bound to your account and that
# application.
config :oauth_token_secret, :validate => :password, :required => true
# Any keywords to track in the twitter stream
config :keywords, :validate => :array, :required => true
# Record full tweet object as given to us by the Twitter stream api.
config :full_tweet, :validate => :boolean, :default => false
public
def register
require "twitter"
# monkey patch twitter gem to ignore json parsing error.
# at the same time, use our own json parser
# this has been tested with a specific gem version, raise if not the same
raise("Invalid Twitter gem") unless Twitter::Version.to_s == "5.0.0.rc.1"
Twitter::Streaming::Response.module_eval do
def on_body(data)
@tokenizer.extract(data).each do |line|
next if line.empty?
begin
@block.call(LogStash::Json.load(line, :symbolize_keys => true))
rescue LogStash::Json::ParserError
# silently ignore json parsing errors
end
end
end
end
@client = Twitter::Streaming::Client.new do |c|
c.consumer_key = @consumer_key
c.consumer_secret = @consumer_secret.value
c.access_token = @oauth_token
c.access_token_secret = @oauth_token_secret.value
end
end
public
def run(queue)
@logger.info("Starting twitter tracking", :keywords => @keywords)
begin
@client.filter(:track => @keywords.join(",")) do |tweet|
if tweet.is_a?(Twitter::Tweet)
@logger.debug? && @logger.debug("Got tweet", :user => tweet.user.screen_name, :text => tweet.text)
if @full_tweet
event = LogStash::Event.new(LogStash::Util.stringify_symbols(tweet.to_hash))
event.timestamp = LogStash::Timestamp.new(tweet.created_at)
else
event = LogStash::Event.new(
LogStash::Event::TIMESTAMP => LogStash::Timestamp.new(tweet.created_at),
"message" => tweet.full_text,
"user" => tweet.user.screen_name,
"client" => tweet.source,
"retweeted" => tweet.retweeted?,
"source" => "http://twitter.com/#{tweet.user.screen_name}/status/#{tweet.id}"
)
event["in-reply-to"] = tweet.in_reply_to_status_id if tweet.reply?
unless tweet.urls.empty?
event["urls"] = tweet.urls.map(&:expanded_url).map(&:to_s)
end
end
decorate(event)
queue << event
end
end # client.filter
rescue LogStash::ShutdownSignal
return
rescue Twitter::Error::TooManyRequests => e
@logger.warn("Twitter too many requests error, sleeping for #{e.rate_limit.reset_in}s")
sleep(e.rate_limit.reset_in)
retry
rescue => e
@logger.warn("Twitter client error", :message => e.message, :exception => e, :backtrace => e.backtrace)
retry
end
end # def run
end # class LogStash::Inputs::Twitter

112
lib/logstash/inputs/udp.rb
View file

@ -1,112 +0,0 @@
# encoding: utf-8
require "date"
require "logstash/inputs/base"
require "logstash/namespace"
require "socket"
# Read messages as events over the network via udp. The only required
# configuration item is `port`, which specifies the udp port logstash
# will listen on for event streams.
#
class LogStash::Inputs::Udp < LogStash::Inputs::Base
config_name "udp"
milestone 2
default :codec, "plain"
# The address which logstash will listen on.
config :host, :validate => :string, :default => "0.0.0.0"
# The port which logstash will listen on. Remember that ports less
# than 1024 (privileged ports) may require root or elevated privileges to use.
config :port, :validate => :number, :required => true
# The maximum packet size to read from the network
config :buffer_size, :validate => :number, :default => 8192
# Number of threads processing packets
config :workers, :validate => :number, :default => 2
# This is the number of unprocessed UDP packets you can hold in memory
# before packets will start dropping.
config :queue_size, :validate => :number, :default => 2000
public
def initialize(params)
super
BasicSocket.do_not_reverse_lookup = true
end # def initialize
public
def register
@udp = nil
end # def register
public
def run(output_queue)
@output_queue = output_queue
begin
# udp server
udp_listener(output_queue)
rescue LogStash::ShutdownSignal
# do nothing, shutdown was requested.
rescue => e
@logger.warn("UDP listener died", :exception => e, :backtrace => e.backtrace)
sleep(5)
retry
end # begin
end # def run
private
def udp_listener(output_queue)
@logger.info("Starting UDP listener", :address => "#{@host}:#{@port}")
if @udp && ! @udp.closed?
@udp.close
end
@udp = UDPSocket.new(Socket::AF_INET)
@udp.bind(@host, @port)
@input_to_worker = SizedQueue.new(@queue_size)
@input_workers = @workers.times do |i|
@logger.debug("Starting UDP worker thread", :worker => i)
Thread.new { inputworker(i) }
end
while true
#collect datagram message and add to queue
payload, client = @udp.recvfrom(@buffer_size)
@input_to_worker.push([payload, client])
end
ensure
if @udp
@udp.close_read rescue nil
@udp.close_write rescue nil
end
end # def udp_listener
def inputworker(number)
LogStash::Util::set_thread_name("<udp.#{number}")
begin
while true
payload, client = @input_to_worker.pop
@codec.decode(payload) do |event|
decorate(event)
event["host"] ||= client[3]
@output_queue.push(event)
end
end
rescue => e
@logger.error("Exception in inputworker", "exception" => e, "backtrace" => e.backtrace)
end
end # def inputworker
public
def teardown
@udp.close if @udp && !@udp.closed?
end
end # class LogStash::Inputs::Udp

163
lib/logstash/inputs/unix.rb
View file

@ -1,163 +0,0 @@
# encoding: utf-8
require "logstash/inputs/base"
require "logstash/namespace"
require "socket"
# Read events over a UNIX socket.
#
# Like stdin and file inputs, each event is assumed to be one line of text.
#
# Can either accept connections from clients or connect to a server,
# depending on `mode`.
class LogStash::Inputs::Unix < LogStash::Inputs::Base
class Interrupted < StandardError; end
config_name "unix"
milestone 2
default :codec, "line"
# When mode is `server`, the path to listen on.
# When mode is `client`, the path to connect to.
config :path, :validate => :string, :required => true
# Remove socket file in case of EADDRINUSE failure
config :force_unlink, :validate => :boolean, :default => false
# The 'read' timeout in seconds. If a particular connection is idle for
# more than this timeout period, we will assume it is dead and close it.
#
# If you never want to timeout, use -1.
config :data_timeout, :validate => :number, :default => -1
# Mode to operate in. `server` listens for client connections,
# `client` connects to a server.
config :mode, :validate => ["server", "client"], :default => "server"
def initialize(*args)
super(*args)
end # def initialize
public
def register
require "socket"
require "timeout"
if server?
@logger.info("Starting unix input listener", :address => "#{@path}", :force_unlink => "#{@force_unlink}")
begin
@server_socket = UNIXServer.new(@path)
rescue Errno::EADDRINUSE, IOError
if @force_unlink
File.unlink(@path)
begin
@server_socket = UNIXServer.new(@path)
return
rescue Errno::EADDRINUSE, IOError
@logger.error("!!!Could not start UNIX server: Address in use",
:path => @path)
raise
end
end
@logger.error("Could not start UNIX server: Address in use",
:path => @path)
raise
end
end
end # def register
private
def handle_socket(socket, output_queue)
begin
hostname = Socket.gethostname
loop do
buf = nil
# NOTE(petef): the timeout only hits after the line is read
# or socket dies
# TODO(sissel): Why do we have a timeout here? What's the point?
if @data_timeout == -1
buf = socket.readpartial(16384)
else
Timeout::timeout(@data_timeout) do
buf = socket.readpartial(16384)
end
end
@codec.decode(buf) do |event|
decorate(event)
event["host"] = hostname
event["path"] = @path
output_queue << event
end
end # loop do
rescue => e
@logger.debug("Closing connection", :path => @path,
:exception => e, :backtrace => e.backtrace)
rescue Timeout::Error
@logger.debug("Closing connection after read timeout",
:path => @path)
end # begin
ensure
begin
socket.close
rescue IOError
#pass
end # begin
end
private
def server?
@mode == "server"
end # def server?
public
def run(output_queue)
if server?
@thread = Thread.current
@client_threads = []
loop do
# Start a new thread for each connection.
begin
@client_threads << Thread.start(@server_socket.accept) do |s|
# TODO(sissel): put this block in its own method.
@logger.debug("Accepted connection",
:server => "#{@path}")
begin
handle_socket(s, output_queue)
rescue Interrupted
s.close rescue nil
end
end # Thread.start
rescue IOError, Interrupted
if @interrupted
# Intended shutdown, get out of the loop
@server_socket.close
@client_threads.each do |thread|
thread.raise(IOError.new)
end
break
else
# Else it was a genuine IOError caused by something else, so propagate it up..
raise
end
end
end # loop
else
loop do
client_socket = UNIXSocket.new(@path)
client_socket.instance_eval { class << self; include ::LogStash::Util::SocketPeer end }
@logger.debug("Opened connection", :client => @path)
handle_socket(client_socket, output_queue)
end # loop
end
end # def run
public
def teardown
if server?
File.unlink(@path)
@interrupted = true
@thread.raise(Interrupted.new)
end
end # def teardown
end # class LogStash::Inputs::Unix

81
lib/logstash/inputs/xmpp.rb
View file

@ -1,81 +0,0 @@
# encoding: utf-8
require "logstash/inputs/base"
require "logstash/namespace"
# This input allows you to receive events over XMPP/Jabber.
#
# This plugin can be used for accepting events from humans or applications
# XMPP, or you can use it for PubSub or general message passing for logstash to
# logstash.
class LogStash::Inputs::Xmpp < LogStash::Inputs::Base
config_name "xmpp"
milestone 2
default :codec, "plain"
# The user or resource ID, like foo@example.com.
config :user, :validate => :string, :required => :true
# The xmpp password for the user/identity.
config :password, :validate => :password, :required => :true
# if muc/multi-user-chat required, give the name of the room that
# you want to join: room@conference.domain/nick
config :rooms, :validate => :array
# The xmpp server to connect to. This is optional. If you omit this setting,
# the host on the user/identity is used. (foo.com for user@foo.com)
config :host, :validate => :string
# Set to true to enable greater debugging in XMPP. Useful for debugging
# network/authentication erros.
config :debug, :validate => :boolean, :default => false, :deprecated => "Use the logstash --debug flag for this instead."
public
def register
require 'xmpp4r' # xmpp4r gem
Jabber::debug = true if @debug || @logger.debug?
@client = Jabber::Client.new(Jabber::JID.new(@user))
@client.connect(@host) # it is ok if host is nil
@client.auth(@password.value)
@client.send(Jabber::Presence.new.set_type(:available))
# load the MUC Client if we are joining rooms.
require 'xmpp4r/muc/helper/simplemucclient' if @rooms && !@rooms.empty?
end # def register
public
def run(queue)
if @rooms
@rooms.each do |room| # handle muc messages in different rooms
@muc = Jabber::MUC::SimpleMUCClient.new(@client)
@muc.join(room)
@muc.on_message do |time,from,body|
@codec.decode(body) do |event|
decorate(event)
event["room"] = room
event["from"] = from
queue << event
end
end # @muc.on_message
end # @rooms.each
end # if @rooms
@client.add_message_callback do |msg| # handle direct/private messages
# accept normal msgs (skip presence updates, etc)
if msg.body != nil
@codec.decode(msg.body) do |event|
decorate(event)
# Maybe "from" should just be a hash:
# { "node" => ..., "domain" => ..., "resource" => ... }
event["from"] = "#{msg.from.node}@#{msg.from.domain}/#{msg.from.resource}"
queue << event
end
end
end # @client.add_message_callback
sleep
end # def run
end # class LogStash::Inputs::Xmpp

165
lib/logstash/inputs/zeromq.rb
View file

@ -1,165 +0,0 @@
# encoding: utf-8
require "logstash/inputs/base"
require "logstash/namespace"
require "socket"
# Read events over a 0MQ SUB socket.
#
# You need to have the 0mq 2.1.x library installed to be able to use
# this input plugin.
#
# The default settings will create a subscriber binding to tcp://127.0.0.1:2120
# waiting for connecting publishers.
#
class LogStash::Inputs::ZeroMQ < LogStash::Inputs::Base
config_name "zeromq"
milestone 2
default :codec, "json"
# 0mq socket address to connect or bind
# Please note that `inproc://` will not work with logstash
# as each we use a context per thread.
# By default, inputs bind/listen
# and outputs connect
config :address, :validate => :array, :default => ["tcp://*:2120"]
# 0mq topology
# The default logstash topologies work as follows:
# * pushpull - inputs are pull, outputs are push
# * pubsub - inputs are subscribers, outputs are publishers
# * pair - inputs are clients, inputs are servers
#
# If the predefined topology flows don't work for you,
# you can change the 'mode' setting
# TODO (lusis) add req/rep MAYBE
# TODO (lusis) add router/dealer
config :topology, :validate => ["pushpull", "pubsub", "pair"], :required => true
# 0mq topic
# This is used for the 'pubsub' topology only
# On inputs, this allows you to filter messages by topic
# On outputs, this allows you to tag a message for routing
# NOTE: ZeroMQ does subscriber side filtering.
# NOTE: All topics have an implicit wildcard at the end
# You can specify multiple topics here
config :topic, :validate => :array
# mode
# server mode binds/listens
# client mode connects
config :mode, :validate => ["server", "client"], :default => "server"
# sender
# overrides the sender to
# set the source of the event
# default is "zmq+topology://type/"
config :sender, :validate => :string
# 0mq socket options
# This exposes zmq_setsockopt
# for advanced tuning
# see http://api.zeromq.org/2-1:zmq-setsockopt for details
#
# This is where you would set values like:
# ZMQ::HWM - high water mark
# ZMQ::IDENTITY - named queues
# ZMQ::SWAP_SIZE - space for disk overflow
#
# example: sockopt => ["ZMQ::HWM", 50, "ZMQ::IDENTITY", "my_named_queue"]
config :sockopt, :validate => :hash
public
def register
require "ffi-rzmq"
require "logstash/util/zeromq"
self.class.send(:include, LogStash::Util::ZeroMQ)
case @topology
when "pair"
zmq_const = ZMQ::PAIR
when "pushpull"
zmq_const = ZMQ::PULL
when "pubsub"
zmq_const = ZMQ::SUB
end # case socket_type
@zsocket = context.socket(zmq_const)
error_check(@zsocket.setsockopt(ZMQ::LINGER, 1),
"while setting ZMQ::LINGER == 1)")
if @sockopt
setopts(@zsocket, @sockopt)
end
@address.each do |addr|
setup(@zsocket, addr)
end
if @topology == "pubsub"
if @topic.nil?
@logger.debug("ZMQ - No topic provided. Subscribing to all messages")
error_check(@zsocket.setsockopt(ZMQ::SUBSCRIBE, ""),
"while setting ZMQ::SUBSCRIBE")
else
@topic.each do |t|
@logger.debug("ZMQ subscribing to topic: #{t}")
error_check(@zsocket.setsockopt(ZMQ::SUBSCRIBE, t),
"while setting ZMQ::SUBSCRIBE == #{t}")
end
end
end
end # def register
def teardown
error_check(@zsocket.close, "while closing the zmq socket")
end # def teardown
def server?
@mode == "server"
end # def server?
def run(output_queue)
host = Socket.gethostname
begin
loop do
# Here's the unified receiver
# Get the first part as the msg
m1 = ""
rc = @zsocket.recv_string(m1)
error_check(rc, "in recv_string")
@logger.debug("ZMQ receiving", :event => m1)
msg = m1
# If we have more parts, we'll eat the first as the topic
# and set the message to the second part
if @zsocket.more_parts?
@logger.debug("Multipart message detected. Setting @message to second part. First part was: #{m1}")
m2 = ''
rc2 = @zsocket.recv_string(m2)
error_check(rc2, "in recv_string")
@logger.debug("ZMQ receiving", :event => m2)
msg = m2
end
@codec.decode(msg) do |event|
event["host"] ||= host
decorate(event)
output_queue << event
end
end
rescue LogStash::ShutdownSignal
# shutdown
return
rescue => e
@logger.debug("ZMQ Error", :subscriber => @zsocket,
:exception => e)
retry
end # begin
end # def run
private
def build_source_string
id = @address.first.clone
end
end # class LogStash::Inputs::ZeroMQ

351
lib/logstash/outputs/cloudwatch.rb
View file

@ -1,351 +0,0 @@
# encoding: utf-8
require "logstash/outputs/base"
require "logstash/namespace"
require "logstash/plugin_mixins/aws_config"
# This output lets you aggregate and send metric data to AWS CloudWatch
#
# #### Summary:
# This plugin is intended to be used on a logstash indexer agent (but that
# is not the only way, see below.) In the intended scenario, one cloudwatch
# output plugin is configured, on the logstash indexer node, with just AWS API
# credentials, and possibly a region and/or a namespace. The output looks
# for fields present in events, and when it finds them, it uses them to
# calculate aggregate statistics. If the `metricname` option is set in this
# output, then any events which pass through it will be aggregated & sent to
# CloudWatch, but that is not recommended. The intended use is to NOT set the
# metricname option here, and instead to add a `CW_metricname` field (and other
# fields) to only the events you want sent to CloudWatch.
#
# When events pass through this output they are queued for background
# aggregation and sending, which happens every minute by default. The
# queue has a maximum size, and when it is full aggregated statistics will be
# sent to CloudWatch ahead of schedule. Whenever this happens a warning
# message is written to logstash's log. If you see this you should increase
# the `queue_size` configuration option to avoid the extra API calls. The queue
# is emptied every time we send data to CloudWatch.
#
# Note: when logstash is stopped the queue is destroyed before it can be processed.
# This is a known limitation of logstash and will hopefully be addressed in a
# future version.
#
# #### Details:
# There are two ways to configure this plugin, and they can be used in
# combination: event fields & per-output defaults
#
# Event Field configuration...
# You add fields to your events in inputs & filters and this output reads
# those fields to aggregate events. The names of the fields read are
# configurable via the `field_*` options.
#
# Per-output defaults...
# You set universal defaults in this output plugin's configuration, and
# if an event does not have a field for that option then the default is
# used.
#
# Notice, the event fields take precedence over the per-output defaults.
#
# At a minimum events must have a "metric name" to be sent to CloudWatch.
# This can be achieved either by providing a default here OR by adding a
# `CW_metricname` field. By default, if no other configuration is provided
# besides a metric name, then events will be counted (Unit: Count, Value: 1)
# by their metric name (either a default or from their `CW_metricname` field)
#
# Other fields which can be added to events to modify the behavior of this
# plugin are, `CW_namespace`, `CW_unit`, `CW_value`, and
# `CW_dimensions`. All of these field names are configurable in
# this output. You can also set per-output defaults for any of them.
# See below for details.
#
# Read more about [AWS CloudWatch](http://aws.amazon.com/cloudwatch/),
# and the specific of API endpoint this output uses,
# [PutMetricData](http://docs.amazonwebservices.com/AmazonCloudWatch/latest/APIReference/API_PutMetricData.html)
class LogStash::Outputs::CloudWatch < LogStash::Outputs::Base
include LogStash::PluginMixins::AwsConfig
config_name "cloudwatch"
milestone 1
# Constants
# aggregate_key members
DIMENSIONS = "dimensions"
TIMESTAMP = "timestamp"
METRIC = "metric"
COUNT = "count"
UNIT = "unit"
SUM = "sum"
MIN = "min"
MAX = "max"
# Units
COUNT_UNIT = "Count"
NONE = "None"
# How often to send data to CloudWatch
# This does not affect the event timestamps, events will always have their
# actual timestamp (to-the-minute) sent to CloudWatch.
#
# We only call the API if there is data to send.
#
# See the Rufus Scheduler docs for an [explanation of allowed values](https://github.com/jmettraux/rufus-scheduler#the-time-strings-understood-by-rufus-scheduler)
config :timeframe, :validate => :string, :default => "1m"
# How many events to queue before forcing a call to the CloudWatch API ahead of `timeframe` schedule
# Set this to the number of events-per-timeframe you will be sending to CloudWatch to avoid extra API calls
config :queue_size, :validate => :number, :default => 10000
# The default namespace to use for events which do not have a `CW_namespace` field
config :namespace, :validate => :string, :default => "Logstash"
# The name of the field used to set a different namespace per event
# Note: Only one namespace can be sent to CloudWatch per API call
# so setting different namespaces will increase the number of API calls
# and those cost money.
config :field_namespace, :validate => :string, :default => "CW_namespace"
# The default metric name to use for events which do not have a `CW_metricname` field.
# Beware: If this is provided then all events which pass through this output will be aggregated and
# sent to CloudWatch, so use this carefully. Furthermore, when providing this option, you
# will probably want to also restrict events from passing through this output using event
# type, tag, and field matching
config :metricname, :validate => :string
# The name of the field used to set the metric name on an event
# The author of this plugin recommends adding this field to events in inputs &
# filters rather than using the per-output default setting so that one output
# plugin on your logstash indexer can serve all events (which of course had
# fields set on your logstash shippers.)
config :field_metricname, :validate => :string, :default => "CW_metricname"
VALID_UNITS = ["Seconds", "Microseconds", "Milliseconds", "Bytes",
"Kilobytes", "Megabytes", "Gigabytes", "Terabytes",
"Bits", "Kilobits", "Megabits", "Gigabits", "Terabits",
"Percent", COUNT_UNIT, "Bytes/Second", "Kilobytes/Second",
"Megabytes/Second", "Gigabytes/Second", "Terabytes/Second",
"Bits/Second", "Kilobits/Second", "Megabits/Second",
"Gigabits/Second", "Terabits/Second", "Count/Second", NONE]
# The default unit to use for events which do not have a `CW_unit` field
# If you set this option you should probably set the "value" option along with it
config :unit, :validate => VALID_UNITS, :default => COUNT_UNIT
# The name of the field used to set the unit on an event metric
config :field_unit, :validate => :string, :default => "CW_unit"
# The default value to use for events which do not have a `CW_value` field
# If provided, this must be a string which can be converted to a float, for example...
# "1", "2.34", ".5", and "0.67"
# If you set this option you should probably set the `unit` option along with it
config :value, :validate => :string, :default => "1"
# The name of the field used to set the value (float) on an event metric
config :field_value, :validate => :string, :default => "CW_value"
# The default dimensions [ name, value, ... ] to use for events which do not have a `CW_dimensions` field
config :dimensions, :validate => :hash
# The name of the field used to set the dimensions on an event metric
# The field named here, if present in an event, must have an array of
# one or more key & value pairs, for example...
# add_field => [ "CW_dimensions", "Environment", "CW_dimensions", "prod" ]
# or, equivalently...
# add_field => [ "CW_dimensions", "Environment" ]
# add_field => [ "CW_dimensions", "prod" ]
config :field_dimensions, :validate => :string, :default => "CW_dimensions"
public
def aws_service_endpoint(region)
return {
:cloud_watch_endpoint => "monitoring.#{region}.amazonaws.com"
}
end
public
def register
require "thread"
require "rufus/scheduler"
require "aws"
@cw = AWS::CloudWatch.new(aws_options_hash)
@event_queue = SizedQueue.new(@queue_size)
@scheduler = Rufus::Scheduler.start_new
@job = @scheduler.every @timeframe do
@logger.info("Scheduler Activated")
publish(aggregate({}))
end
end # def register
public
def receive(event)
return unless output?(event)
if event == LogStash::SHUTDOWN
job.trigger()
job.unschedule()
@logger.info("CloudWatch aggregator thread shutdown.")
finished
return
end
return unless (event[@field_metricname] || @metricname)
if (@event_queue.length >= @event_queue.max)
@job.trigger
@logger.warn("Posted to AWS CloudWatch ahead of schedule. If you see this often, consider increasing the cloudwatch queue_size option.")
end
@logger.info("Queueing event", :event => event)
@event_queue << event
end # def receive
private
def publish(aggregates)
aggregates.each do |namespace, data|
@logger.info("Namespace, data: ", :namespace => namespace, :data => data)
metric_data = []
data.each do |aggregate_key, stats|
new_data = {
:metric_name => aggregate_key[METRIC],
:timestamp => aggregate_key[TIMESTAMP],
:unit => aggregate_key[UNIT],
:statistic_values => {
:sample_count => stats[COUNT],
:sum => stats[SUM],
:minimum => stats[MIN],
:maximum => stats[MAX],
}
}
dims = aggregate_key[DIMENSIONS]
if (dims.is_a?(Array) && dims.length > 0 && (dims.length % 2) == 0)
new_data[:dimensions] = Array.new
i = 0
while (i < dims.length)
new_data[:dimensions] << {:name => dims[i], :value => dims[i+1]}
i += 2
end
end
metric_data << new_data
end # data.each
begin
@cw.put_metric_data(
:namespace => namespace,
:metric_data => metric_data
)
@logger.info("Sent data to AWS CloudWatch OK", :namespace => namespace, :metric_data => metric_data)
rescue Exception => e
@logger.warn("Failed to send to AWS CloudWatch", :exception => e, :namespace => namespace, :metric_data => metric_data)
break
end
end # aggregates.each
return aggregates
end# def publish
private
def aggregate(aggregates)
@logger.info("QUEUE SIZE ", :queuesize => @event_queue.size)
while !@event_queue.empty? do
begin
count(aggregates, @event_queue.pop(true))
rescue Exception => e
@logger.warn("Exception! Breaking count loop", :exception => e)
break
end
end
return aggregates
end # def aggregate
private
def count(aggregates, event)
# If the event doesn't declare a namespace, use the default
fnamespace = field(event, @field_namespace)
namespace = (fnamespace ? fnamespace : event.sprintf(@namespace))
funit = field(event, @field_unit)
unit = (funit ? funit : event.sprintf(@unit))
fvalue = field(event, @field_value)
value = (fvalue ? fvalue : event.sprintf(@value))
# We may get to this point with valid Units but missing value. Send zeros.
val = (!value) ? 0.0 : value.to_f
# Event provides exactly one (but not both) of value or unit
if ( (fvalue == nil) ^ (funit == nil) )
@logger.warn("Likely config error: event has one of #{@field_value} or #{@field_unit} fields but not both.", :event => event)
end
# If Unit is still not set or is invalid warn about misconfiguration & use NONE
if (!VALID_UNITS.include?(unit))
unit = NONE
@logger.warn("Likely config error: invalid or missing Units (#{unit.to_s}), using '#{NONE}' instead", :event => event)
end
if (!aggregates[namespace])
aggregates[namespace] = {}
end
dims = event[@field_dimensions]
if (dims) # event provides dimensions
# validate the structure
if (!dims.is_a?(Array) || dims.length == 0 || (dims.length % 2) != 0)
@logger.warn("Likely config error: CloudWatch dimensions field (#{dims.to_s}) found which is not a positive- & even-length array. Ignoring it.", :event => event)
dims = nil
end
# Best case, we get here and exit the conditional because dims...
# - is an array
# - with positive length
# - and an even number of elements
elsif (@dimensions.is_a?(Hash)) # event did not provide dimensions, but the output has been configured with a default
dims = @dimensions.flatten.map{|d| event.sprintf(d)} # into the kind of array described just above
else
dims = nil
end
fmetric = field(event, @field_metricname)
aggregate_key = {
METRIC => (fmetric ? fmetric : event.sprintf(@metricname)),
DIMENSIONS => dims,
UNIT => unit,
TIMESTAMP => event.sprintf("%{+YYYY-MM-dd'T'HH:mm:00Z}")
}
if (!aggregates[namespace][aggregate_key])
aggregates[namespace][aggregate_key] = {}
end
if (!aggregates[namespace][aggregate_key][MAX] || val > aggregates[namespace][aggregate_key][MAX])
aggregates[namespace][aggregate_key][MAX] = val
end
if (!aggregates[namespace][aggregate_key][MIN] || val < aggregates[namespace][aggregate_key][MIN])
aggregates[namespace][aggregate_key][MIN] = val
end
if (!aggregates[namespace][aggregate_key][COUNT])
aggregates[namespace][aggregate_key][COUNT] = 1
else
aggregates[namespace][aggregate_key][COUNT] += 1
end
if (!aggregates[namespace][aggregate_key][SUM])
aggregates[namespace][aggregate_key][SUM] = val
else
aggregates[namespace][aggregate_key][SUM] += val
end
end # def count
private
def field(event, fieldname)
if !event[fieldname]
return nil
else
if event[fieldname].is_a?(Array)
return event[fieldname][0]
else
return event[fieldname]
end
end
end # def field
end # class LogStash::Outputs::CloudWatch

51
lib/logstash/outputs/csv.rb
View file

@ -1,51 +0,0 @@
require "csv"
require "logstash/namespace"
require "logstash/outputs/file"
require "logstash/json"
# CSV output.
#
# Write events to disk in CSV or other delimited format
# Based on the file output, many config values are shared
# Uses the Ruby csv library internally
class LogStash::Outputs::CSV < LogStash::Outputs::File
config_name "csv"
milestone 1
# The field names from the event that should be written to the CSV file.
# Fields are written to the CSV in the same order as the array.
# If a field does not exist on the event, an empty string will be written.
# Supports field reference syntax eg: `fields => ["field1", "[nested][field]"]`.
config :fields, :validate => :array, :required => true
# Options for CSV output. This is passed directly to the Ruby stdlib to\_csv function.
# Full documentation is available here: [http://ruby-doc.org/stdlib-2.0.0/libdoc/csv/rdoc/index.html].
# A typical use case would be to use alternative column or row seperators eg: `csv_options => {"col_sep" => "\t" "row_sep" => "\r\n"}` gives tab seperated data with windows line endings
config :csv_options, :validate => :hash, :required => false, :default => Hash.new
public
def register
super
@csv_options = Hash[@csv_options.map{|(k, v)|[k.to_sym, v]}]
end
public
def receive(event)
return unless output?(event)
path = event.sprintf(@path)
fd = open(path)
csv_values = @fields.map {|name| get_value(name, event)}
fd.write(csv_values.to_csv(@csv_options))
flush(fd)
close_stale_files
end #def receive
private
def get_value(name, event)
val = event[name]
val.is_a?(Hash) ? LogStash::Json.dump(val) : val
end
end # class LogStash::Outputs::CSV

366
lib/logstash/outputs/elasticsearch.rb
View file

@ -1,366 +0,0 @@
# encoding: utf-8
require "logstash/namespace"
require "logstash/environment"
require "logstash/outputs/base"
require "logstash/json"
require "stud/buffer"
require "socket" # for Socket.gethostname
# This output lets you store logs in Elasticsearch and is the most recommended
# output for Logstash. If you plan on using the Kibana web interface, you'll
# need to use this output.
#
# *VERSION NOTE*: Your Elasticsearch cluster must be running Elasticsearch
# 1.0.0 or later.
#
# If you want to set other Elasticsearch options that are not exposed directly
# as configuration options, there are two methods:
#
# * Create an `elasticsearch.yml` file in the $PWD of the Logstash process
# * Pass in es.* java properties (java -Des.node.foo= or ruby -J-Des.node.foo=)
#
# With the default `protocol` setting ("node"), this plugin will join your
# Elasticsearch cluster as a client node, so it will show up in Elasticsearch's
# cluster status.
#
# You can learn more about Elasticsearch at <http://www.elasticsearch.org>
#
# ## Operational Notes
#
# If using the default `protocol` setting ("node"), your firewalls might need
# to permit port 9300 in *both* directions (from Logstash to Elasticsearch, and
# Elasticsearch to Logstash)
class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
include Stud::Buffer
config_name "elasticsearch"
milestone 3
# The index to write events to. This can be dynamic using the %{foo} syntax.
# The default value will partition your indices by day so you can more easily
# delete old data or only search specific date ranges.
# Indexes may not contain uppercase characters.
config :index, :validate => :string, :default => "logstash-%{+YYYY.MM.dd}"
# The index type to write events to. Generally you should try to write only
# similar events to the same 'type'. String expansion '%{foo}' works here.
config :index_type, :validate => :string
# Starting in Logstash 1.3 (unless you set option "manage_template" to false)
# a default mapping template for Elasticsearch will be applied, if you do not
# already have one set to match the index pattern defined (default of
# "logstash-%{+YYYY.MM.dd}"), minus any variables. For example, in this case
# the template will be applied to all indices starting with logstash-*
#
# If you have dynamic templating (e.g. creating indices based on field names)
# then you should set "manage_template" to false and use the REST API to upload
# your templates manually.
config :manage_template, :validate => :boolean, :default => true
# This configuration option defines how the template is named inside Elasticsearch.
# Note that if you have used the template management features and subsequently
# change this, you will need to prune the old template manually, e.g.
# curl -XDELETE <http://localhost:9200/_template/OldTemplateName?pretty>
# where OldTemplateName is whatever the former setting was.
config :template_name, :validate => :string, :default => "logstash"
# You can set the path to your own template here, if you so desire.
# If not set, the included template will be used.
config :template, :validate => :path
# Overwrite the current template with whatever is configured
# in the template and template_name directives.
config :template_overwrite, :validate => :boolean, :default => false
# The document ID for the index. Useful for overwriting existing entries in
# Elasticsearch with the same ID.
config :document_id, :validate => :string, :default => nil
# The name of your cluster if you set it on the Elasticsearch side. Useful
# for discovery.
config :cluster, :validate => :string
# The hostname or IP address of the host to use for Elasticsearch unicast discovery
# This is only required if the normal multicast/cluster discovery stuff won't
# work in your environment.
#
# "127.0.0.1"
# ["127.0.0.1:9300","127.0.0.2:9300"]
config :host, :validate => :array
# The port for Elasticsearch transport to use.
#
# If you do not set this, the following defaults are used:
# * `protocol => http` - port 9200
# * `protocol => transport` - port 9300-9305
# * `protocol => node` - port 9300-9305
config :port, :validate => :string
# The name/address of the host to bind to for Elasticsearch clustering
config :bind_host, :validate => :string
# This is only valid for the 'node' protocol.
#
# The port for the node to listen on.
config :bind_port, :validate => :number
# Run the Elasticsearch server embedded in this process.
# This option is useful if you want to run a single Logstash process that
# handles log processing and indexing; it saves you from needing to run
# a separate Elasticsearch process.
config :embedded, :validate => :boolean, :default => false
# If you are running the embedded Elasticsearch server, you can set the http
# port it listens on here; it is not common to need this setting changed from
# default.
config :embedded_http_port, :validate => :string, :default => "9200-9300"
# This setting no longer does anything. It exists to keep config validation
# from failing. It will be removed in future versions.
config :max_inflight_requests, :validate => :number, :default => 50, :deprecated => true
# The node name Elasticsearch will use when joining a cluster.
#
# By default, this is generated internally by the ES client.
config :node_name, :validate => :string
# This plugin uses the bulk index api for improved indexing performance.
# To make efficient bulk api calls, we will buffer a certain number of
# events before flushing that out to Elasticsearch. This setting
# controls how many events will be buffered before sending a batch
# of events.
config :flush_size, :validate => :number, :default => 5000
# The amount of time since last flush before a flush is forced.
#
# This setting helps ensure slow event rates don't get stuck in Logstash.
# For example, if your `flush_size` is 100, and you have received 10 events,
# and it has been more than `idle_flush_time` seconds since the last flush,
# Logstash will flush those 10 events automatically.
#
# This helps keep both fast and slow log streams moving along in
# near-real-time.
config :idle_flush_time, :validate => :number, :default => 1
# Choose the protocol used to talk to Elasticsearch.
#
# The 'node' protocol will connect to the cluster as a normal Elasticsearch
# node (but will not store data). This allows you to use things like
# multicast discovery. If you use the `node` protocol, you must permit
# bidirectional communication on the port 9300 (or whichever port you have
# configured).
#
# The 'transport' protocol will connect to the host you specify and will
# not show up as a 'node' in the Elasticsearch cluster. This is useful
# in situations where you cannot permit connections outbound from the
# Elasticsearch cluster to this Logstash server.
#
# The 'http' protocol will use the Elasticsearch REST/HTTP interface to talk
# to elasticsearch.
#
# All protocols will use bulk requests when talking to Elasticsearch.
#
# The default `protocol` setting under java/jruby is "node". The default
# `protocol` on non-java rubies is "http"
config :protocol, :validate => [ "node", "transport", "http" ]
# The Elasticsearch action to perform. Valid actions are: `index`, `delete`.
#
# Use of this setting *REQUIRES* you also configure the `document_id` setting
# because `delete` actions all require a document id.
#
# What does each action do?
#
# - index: indexes a document (an event from logstash).
# - delete: deletes a document by id
#
# For more details on actions, check out the [Elasticsearch bulk API documentation](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-bulk.html)
config :action, :validate => :string, :default => "index"
public
def register
client_settings = {}
client_settings["cluster.name"] = @cluster if @cluster
client_settings["network.host"] = @bind_host if @bind_host
client_settings["transport.tcp.port"] = @bind_port if @bind_port
if @node_name
client_settings["node.name"] = @node_name
else
client_settings["node.name"] = "logstash-#{Socket.gethostname}-#{$$}-#{object_id}"
end
if @protocol.nil?
@protocol = LogStash::Environment.jruby? ? "node" : "http"
end
if ["node", "transport"].include?(@protocol)
# Node or TransportClient; requires JRuby
raise(LogStash::PluginLoadingError, "This configuration requires JRuby. If you are not using JRuby, you must set 'protocol' to 'http'. For example: output { elasticsearch { protocol => \"http\" } }") unless LogStash::Environment.jruby?
LogStash::Environment.load_elasticsearch_jars!
# setup log4j properties for Elasticsearch
LogStash::Logger.setup_log4j(@logger)
end
require "logstash/outputs/elasticsearch/protocol"
if @port.nil?
@port = case @protocol
when "http"; "9200"
when "transport", "node"; "9300-9305"
end
end
if @host.nil? && @protocol == "http"
@logger.info("No 'host' set in elasticsearch output. Defaulting to localhost")
@host = ["localhost"]
end
client_class = case @protocol
when "transport"
LogStash::Outputs::Elasticsearch::Protocols::TransportClient
when "node"
LogStash::Outputs::Elasticsearch::Protocols::NodeClient
when "http"
LogStash::Outputs::Elasticsearch::Protocols::HTTPClient
end
if @embedded
raise(LogStash::ConfigurationError, "The 'embedded => true' setting is only valid for the elasticsearch output under JRuby. You are running #{RUBY_DESCRIPTION}") unless LogStash::Environment.jruby?
LogStash::Environment.load_elasticsearch_jars!
# Default @host with embedded to localhost. This should help avoid
# newbies tripping on ubuntu and other distros that have a default
# firewall that blocks multicast.
@host ||= ["localhost"]
# Start Elasticsearch local.
start_local_elasticsearch
end
@client = Array.new
if protocol == "node" or @host.nil? # if @protocol is "node" or @host is not set
options = {
:host => @host,
:port => @port,
:client_settings => client_settings
}
@client << client_class.new(options)
else # if @protocol in ["transport","http"]
@host.each do |host|
(_host,_port) = host.split ":"
options = {
:host => _host,
:port => _port || @port,
:client_settings => client_settings
}
@logger.info "Create client to elasticsearch server on #{_host}:#{_port}"
@client << client_class.new(options)
end # @host.each
end
if @manage_template
for client in @client
begin
@logger.info("Automatic template management enabled", :manage_template => @manage_template.to_s)
client.template_install(@template_name, get_template, @template_overwrite)
break
rescue => e
@logger.error("Failed to install template: #{e.message}")
end
end # for @client loop
end # if @manage_templates
@logger.info("New Elasticsearch output", :cluster => @cluster,
:host => @host, :port => @port, :embedded => @embedded,
:protocol => @protocol)
@client_idx = 0
@current_client = @client[@client_idx]
buffer_initialize(
:max_items => @flush_size,
:max_interval => @idle_flush_time,
:logger => @logger
)
end # def register
protected
def shift_client
@client_idx = (@client_idx+1) % @client.length
@current_client = @client[@client_idx]
@logger.debug? and @logger.debug("Switched current elasticsearch client to ##{@client_idx} at #{@host[@client_idx]}")
end
public
def get_template
if @template.nil?
@template = LogStash::Environment.plugin_path("outputs/elasticsearch/elasticsearch-template.json")
if !File.exists?(@template)
raise "You must specify 'template => ...' in your elasticsearch output (I looked for '#{@template}')"
end
end
template_json = IO.read(@template).gsub(/\n/,'')
@logger.info("Using mapping template", :template => template_json)
return LogStash::Json.load(template_json)
end # def get_template
protected
def start_local_elasticsearch
@logger.info("Starting embedded Elasticsearch local node.")
builder = org.elasticsearch.node.NodeBuilder.nodeBuilder
# Disable 'local only' - LOGSTASH-277
#builder.local(true)
builder.settings.put("cluster.name", @cluster) if @cluster
builder.settings.put("node.name", @node_name) if @node_name
builder.settings.put("network.host", @bind_host) if @bind_host
builder.settings.put("http.port", @embedded_http_port)
@embedded_elasticsearch = builder.node
@embedded_elasticsearch.start
end # def start_local_elasticsearch
public
def receive(event)
return unless output?(event)
# Set the 'type' value for the index.
if @index_type
type = event.sprintf(@index_type)
else
type = event["type"] || "logs"
end
index = event.sprintf(@index)
document_id = @document_id ? event.sprintf(@document_id) : nil
buffer_receive([event.sprintf(@action), { :_id => document_id, :_index => index, :_type => type }, event.to_hash])
end # def receive
def flush(actions, teardown=false)
begin
@logger.debug? and @logger.debug "Sending bulk of actions to client[#{@client_idx}]: #{@host[@client_idx]}"
@current_client.bulk(actions)
rescue => e
@logger.error "Got error to send bulk of actions to elasticsearch server at #{@host[@client_idx]} : #{e.message}"
raise e
ensure
unless @protocol == "node"
@logger.debug? and @logger.debug "Shifting current elasticsearch client"
shift_client
end
end
# TODO(sissel): Handle errors. Since bulk requests could mostly succeed
# (aka partially fail), we need to figure out what documents need to be
# retried.
#
# In the worst case, a failing flush (exception) will incur a retry from Stud::Buffer.
end # def flush
def teardown
buffer_flush(:final => true)
end
end # class LogStash::Outputs::Elasticsearch

34
lib/logstash/outputs/elasticsearch/elasticsearch-template.json
View file

@ -1,34 +0,0 @@
{
"template" : "logstash-*",
"settings" : {
"index.refresh_interval" : "5s"
},
"mappings" : {
"_default_" : {
"_all" : {"enabled" : true},
"dynamic_templates" : [ {
"string_fields" : {
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
"type" : "string", "index" : "analyzed", "omit_norms" : true,
"fields" : {
"raw" : {"type": "string", "index" : "not_analyzed", "ignore_above" : 256}
}
}
}
} ],
"properties" : {
"@version": { "type": "string", "index": "not_analyzed" },
"geoip" : {
"type" : "object",
"dynamic": true,
"path": "full",
"properties" : {
"location" : { "type" : "geo_point" }
}
}
}
}
}
}

295
lib/logstash/outputs/elasticsearch/protocol.rb
View file

@ -1,295 +0,0 @@
require "logstash/outputs/elasticsearch"
require "cabin"
module LogStash::Outputs::Elasticsearch
module Protocols
class Base
private
def initialize(options={})
# host(s), port, cluster
@logger = Cabin::Channel.get
end
def client
return @client if @client
@client = build_client(@options)
return @client
end # def client
def template_install(name, template, force=false)
if template_exists?(name) && !force
@logger.debug("Found existing Elasticsearch template. Skipping template management", :name => name)
return
end
template_put(name, template)
end
# Do a bulk request with the given actions.
#
# 'actions' is expected to be an array of bulk requests as string json
# values.
#
# Each 'action' becomes a single line in the bulk api call. For more
# details on the format of each.
def bulk(actions)
raise NotImplemented, "You must implement this yourself"
# bulk([
# '{ "index" : { "_index" : "test", "_type" : "type1", "_id" : "1" } }',
# '{ "field1" : "value1" }'
#])
end
public(:initialize, :template_install)
end
class HTTPClient < Base
private
DEFAULT_OPTIONS = {
:port => 9200
}
def initialize(options={})
require "ftw"
super
require "elasticsearch" # gem 'elasticsearch-ruby'
@options = DEFAULT_OPTIONS.merge(options)
@client = client
end
def build_client(options)
client = Elasticsearch::Client.new(
:host => [options[:host], options[:port]].join(":")
)
# Use FTW to do indexing requests, for now, until we
# can identify and resolve performance problems of elasticsearch-ruby
@bulk_url = "http://#{options[:host]}:#{options[:port]}/_bulk"
@agent = FTW::Agent.new
return client
end
if ENV["BULK"] == "esruby"
def bulk(actions)
bulk_esruby(actions)
end
else
def bulk(actions)
bulk_ftw(actions)
end
end
def bulk_esruby(actions)
@client.bulk(:body => actions.collect do |action, args, source|
if source
next [ { action => args }, source ]
else
next { action => args }
end
end.flatten)
end # def bulk_esruby
# Avoid creating a new string for newline every time
NEWLINE = "\n".freeze
def bulk_ftw(actions)
body = actions.collect do |action, args, source|
header = { action => args }
if source
next [ LogStash::Json.dump(header), NEWLINE, LogStash::Json.dump(source), NEWLINE ]
else
next [ LogStash::Json.dump(header), NEWLINE ]
end
end.flatten.join("")
begin
response = @agent.post!(@bulk_url, :body => body)
rescue EOFError
@logger.warn("EOF while writing request or reading response header from elasticsearch", :host => @host, :port => @port)
raise
end
# Consume the body for error checking
# This will also free up the connection for reuse.
response_body = ""
begin
response.read_body { |chunk| response_body += chunk }
rescue EOFError
@logger.warn("EOF while reading response body from elasticsearch",
:url => @bulk_url)
raise
end
if response.status != 200
@logger.error("Error writing (bulk) to elasticsearch",
:response => response, :response_body => response_body,
:request_body => body)
raise "Non-OK response code from Elasticsearch: #{response.status}"
end
end # def bulk_ftw
def template_exists?(name)
@client.indices.get_template(:name => name)
return true
rescue Elasticsearch::Transport::Transport::Errors::NotFound
return false
end # def template_exists?
def template_put(name, template)
@client.indices.put_template(:name => name, :body => template)
end # template_put
public(:bulk)
end # class HTTPClient
class NodeClient < Base
private
DEFAULT_OPTIONS = {
:port => 9300,
}
def initialize(options={})
super
require "java"
@options = DEFAULT_OPTIONS.merge(options)
setup(@options)
@client = client
end # def initialize
def settings
return @settings
end
def setup(options={})
@settings = org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder
if options[:host]
@settings.put("discovery.zen.ping.multicast.enabled", false)
@settings.put("discovery.zen.ping.unicast.hosts", hosts(options))
end
@settings.put("node.client", true)
@settings.put("http.enabled", false)
if options[:client_settings]
options[:client_settings].each do |key, value|
@settings.put(key, value)
end
end
return @settings
end
def hosts(options)
# http://www.elasticsearch.org/guide/reference/modules/discovery/zen/
result = Array.new
if options[:host].class == Array
options[:host].each do |host|
if host.to_s =~ /^.+:.+$/
# For host in format: host:port, ignore options[:port]
result << host
else
if options[:port].to_s =~ /^\d+-\d+$/
# port ranges are 'host[port1-port2]'
result << Range.new(*options[:port].split("-")).collect { |p| "#{host}:#{p}" }
else
result << "#{host}:#{options[:port]}"
end
end
end
else
if options[:host].to_s =~ /^.+:.+$/
# For host in format: host:port, ignore options[:port]
result << options[:host]
else
if options[:port].to_s =~ /^\d+-\d+$/
# port ranges are 'host[port1-port2]' according to
# http://www.elasticsearch.org/guide/reference/modules/discovery/zen/
# However, it seems to only query the first port.
# So generate our own list of unicast hosts to scan.
range = Range.new(*options[:port].split("-"))
result << range.collect { |p| "#{options[:host]}:#{p}" }
else
result << "#{options[:host]}:#{options[:port]}"
end
end
end
result.flatten.join(",")
end # def hosts
def build_client(options)
nodebuilder = org.elasticsearch.node.NodeBuilder.nodeBuilder
return nodebuilder.settings(@settings).node.client
end # def build_client
def bulk(actions)
# Actions an array of [ action, action_metadata, source ]
prep = @client.prepareBulk
actions.each do |action, args, source|
prep.add(build_request(action, args, source))
end
response = prep.execute.actionGet()
# TODO(sissel): What format should the response be in?
end # def bulk
def build_request(action, args, source)
case action
when "index"
request = org.elasticsearch.action.index.IndexRequest.new(args[:_index])
request.id(args[:_id]) if args[:_id]
request.source(source)
when "delete"
request = org.elasticsearch.action.delete.DeleteRequest.new(args[:_index])
request.id(args[:_id])
#when "update"
#when "create"
end # case action
request.type(args[:_type]) if args[:_type]
return request
end # def build_request
def template_exists?(name)
request = org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequestBuilder.new(@client.admin.indices, name)
response = request.get
return !response.getIndexTemplates.isEmpty
end # def template_exists?
def template_put(name, template)
request = org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequestBuilder.new(@client.admin.indices, name)
request.setSource(LogStash::Json.dump(template))
# execute the request and get the response, if it fails, we'll get an exception.
request.get
end # template_put
public(:initialize, :bulk)
end # class NodeClient
class TransportClient < NodeClient
private
def build_client(options)
client = org.elasticsearch.client.transport.TransportClient.new(settings.build)
if options[:host]
client.addTransportAddress(
org.elasticsearch.common.transport.InetSocketTransportAddress.new(
options[:host], options[:port].to_i
)
)
end
return client
end # def build_client
end # class TransportClient
end # module Protocols
module Requests
class GetIndexTemplates; end
class Bulk; end
class Index; end
class Delete; end
end
end

245
lib/logstash/outputs/elasticsearch_http.rb
View file

@ -1,245 +0,0 @@
# encoding: utf-8
require "logstash/namespace"
require "logstash/outputs/base"
require "logstash/json"
require "stud/buffer"
# This output lets you store logs in Elasticsearch.
#
# This plugin uses the HTTP/REST interface to Elasticsearch, which usually
# lets you use any version of Elasticsearch server. It is known to work
# with elasticsearch %ELASTICSEARCH_VERSION%
#
# You can learn more about Elasticsearch at <http://www.elasticsearch.org>
class LogStash::Outputs::ElasticSearchHTTP < LogStash::Outputs::Base
include Stud::Buffer
config_name "elasticsearch_http"
milestone 2
# The index to write events to. This can be dynamic using the %{foo} syntax.
# The default value will partition your indices by day so you can more easily
# delete old data or only search specific date ranges.
config :index, :validate => :string, :default => "logstash-%{+YYYY.MM.dd}"
# The index type to write events to. Generally you should try to write only
# similar events to the same 'type'. String expansion '%{foo}' works here.
config :index_type, :validate => :string
# Starting in Logstash 1.3 (unless you set option "manage_template" to false)
# a default mapping template for Elasticsearch will be applied, if you do not
# already have one set to match the index pattern defined (default of
# "logstash-%{+YYYY.MM.dd}"), minus any variables. For example, in this case
# the template will be applied to all indices starting with logstash-*
#
# If you have dynamic templating (e.g. creating indices based on field names)
# then you should set "manage_template" to false and use the REST API to upload
# your templates manually.
config :manage_template, :validate => :boolean, :default => true
# This configuration option defines how the template is named inside Elasticsearch.
# Note that if you have used the template management features and subsequently
# change this you will need to prune the old template manually, e.g.
# curl -XDELETE <http://localhost:9200/_template/OldTemplateName?pretty>
# where OldTemplateName is whatever the former setting was.
config :template_name, :validate => :string, :default => "logstash"
# You can set the path to your own template here, if you so desire.
# If not the included template will be used.
config :template, :validate => :path
# Overwrite the current template with whatever is configured
# in the template and template_name directives.
config :template_overwrite, :validate => :boolean, :default => false
# The hostname or IP address to reach your Elasticsearch server.
config :host, :validate => :string, :required => true
# The port for Elasticsearch HTTP interface to use.
config :port, :validate => :number, :default => 9200
# The HTTP Basic Auth username used to access your elasticsearch server.
config :user, :validate => :string, :default => nil
# The HTTP Basic Auth password used to access your elasticsearch server.
config :password, :validate => :password, :default => nil
# This plugin uses the bulk index api for improved indexing performance.
# To make efficient bulk api calls, we will buffer a certain number of
# events before flushing that out to Elasticsearch. This setting
# controls how many events will be buffered before sending a batch
# of events.
config :flush_size, :validate => :number, :default => 100
# The amount of time since last flush before a flush is forced.
#
# This setting helps ensure slow event rates don't get stuck in Logstash.
# For example, if your `flush_size` is 100, and you have received 10 events,
# and it has been more than `idle_flush_time` seconds since the last flush,
# logstash will flush those 10 events automatically.
#
# This helps keep both fast and slow log streams moving along in
# near-real-time.
config :idle_flush_time, :validate => :number, :default => 1
# The document ID for the index. Useful for overwriting existing entries in
# Elasticsearch with the same ID.
config :document_id, :validate => :string, :default => nil
# Set the type of Elasticsearch replication to use. If async
# the index request to Elasticsearch to return after the primary
# shards have been written. If sync (default), index requests
# will wait until the primary and the replica shards have been
# written.
config :replication, :validate => ['async', 'sync'], :default => 'sync'
public
def register
require "ftw" # gem ftw
@agent = FTW::Agent.new
@queue = []
auth = @user && @password ? "#{@user}:#{@password.value}@" : ""
@bulk_url = "http://#{auth}#{@host}:#{@port}/_bulk?replication=#{@replication}"
if @manage_template
@logger.info("Automatic template management enabled", :manage_template => @manage_template.to_s)
template_search_url = "http://#{auth}#{@host}:#{@port}/_template/*"
@template_url = "http://#{auth}#{@host}:#{@port}/_template/#{@template_name}"
if @template_overwrite
@logger.info("Template overwrite enabled. Deleting existing template.", :template_overwrite => @template_overwrite.to_s)
response = @agent.get!(@template_url)
template_action('delete') if response.status == 200 #=> Purge the old template if it exists
end
@logger.debug("Template Search URL:", :template_search_url => template_search_url)
has_template = false
template_idx_name = @index.sub(/%{[^}]+}/,'*')
alt_template_idx_name = @index.sub(/-%{[^}]+}/,'*')
# Get the template data
response = @agent.get!(template_search_url)
json = ""
if response.status == 404 #=> This condition can occcur when no template has ever been appended
@logger.info("No template found in Elasticsearch...")
get_template_json
template_action('put')
elsif response.status == 200
begin
response.read_body { |c| json << c }
results = LogStash::Json.load(json)
rescue Exception => e
@logger.error("Error parsing JSON", :json => json, :results => results.to_s, :error => e.to_s)
raise "Exception in parsing JSON", e
end
if !results.any? { |k,v| v["template"] == template_idx_name || v["template"] == alt_template_idx_name }
@logger.debug("No template found in Elasticsearch", :has_template => has_template, :name => template_idx_name, :alt => alt_template_idx_name)
get_template_json
template_action('put')
end
else #=> Some other status code?
@logger.error("Could not check for existing template. Check status code.", :status => response.status.to_s)
end # end if response.status == 200
end # end if @manage_template
buffer_initialize(
:max_items => @flush_size,
:max_interval => @idle_flush_time,
:logger => @logger
)
end # def register
public
def template_action(command)
begin
if command == 'delete'
response = @agent.delete!(@template_url)
response.discard_body
elsif command == 'put'
response = @agent.put!(@template_url, :body => @template_json)
response.discard_body
end
rescue EOFError
@logger.warn("EOF while attempting request or reading response header from elasticsearch",
:host => @host, :port => @port)
return # abort this action
end
if response.status != 200
@logger.error("Error acting on elasticsearch mapping template",
:response => response, :action => command,
:request_url => @template_url)
return
end
@logger.info("Successfully deleted template", :template_url => @template_url) if command == 'delete'
@logger.info("Successfully applied template", :template_url => @template_url) if command == 'put'
end # def template_action
public
def get_template_json
if @template.nil?
@template = LogStash::Environment.plugin_path("outputs/elasticsearch/elasticsearch-template.json")
if !File.exists?(@template)
raise "You must specify 'template => ...' in your elasticsearch_http output (I looked for '#{@template}')"
end
end
@template_json = IO.read(@template).gsub(/\n/,'')
@logger.info("Using mapping template", :template => @template_json)
end # def get_template_json
public
def receive(event)
return unless output?(event)
buffer_receive([event, index, type])
end # def receive
def flush(events, teardown=false)
# Avoid creating a new string for newline every time
newline = "\n".freeze
body = events.collect do |event, index, type|
index = event.sprintf(@index)
# Set the 'type' value for the index.
if @index_type.nil?
type = event["type"] || "logs"
else
type = event.sprintf(@index_type)
end
header = { "index" => { "_index" => index, "_type" => type } }
header["index"]["_id"] = event.sprintf(@document_id) if !@document_id.nil?
[ LogStash::Json.dump(header), newline, event.to_json, newline ]
end.flatten
post(body.join(""))
end # def receive_bulk
def post(body)
begin
response = @agent.post!(@bulk_url, :body => body)
rescue EOFError
@logger.warn("EOF while writing request or reading response header from elasticsearch",
:host => @host, :port => @port)
raise
end
# Consume the body for error checking
# This will also free up the connection for reuse.
body = ""
begin
response.read_body { |chunk| body += chunk }
rescue EOFError
@logger.warn("EOF while reading response body from elasticsearch",
:host => @host, :port => @port)
raise
end
if response.status != 200
@logger.error("Error writing (bulk) to elasticsearch",
:response => response, :response_body => body,
:request_body => @queue.join("\n"))
raise
end
end # def post
def teardown
buffer_flush(:final => true)
end # def teardown
end # class LogStash::Outputs::ElasticSearchHTTP

206
lib/logstash/outputs/elasticsearch_river.rb
View file

@ -1,206 +0,0 @@
# encoding: utf-8
require "logstash/environment"
require "logstash/namespace"
require "logstash/outputs/base"
require "logstash/json"
require "uri"
require "net/http"
# This output lets you store logs in elasticsearch. It's similar to the
# 'elasticsearch' output but improves performance by using a queue server,
# rabbitmq, to send data to elasticsearch.
#
# Upon startup, this output will automatically contact an elasticsearch cluster
# and configure it to read from the queue to which we write.
#
# You can learn more about elasticseasrch at <http://elasticsearch.org>
# More about the elasticsearch rabbitmq river plugin: <https://github.com/elasticsearch/elasticsearch-river-rabbitmq/blob/master/README.md>
class LogStash::Outputs::ElasticSearchRiver < LogStash::Outputs::Base
config_name "elasticsearch_river"
milestone 2
# The index to write events to. This can be dynamic using the %{foo} syntax.
# The default value will partition your indeces by day so you can more easily
# delete old data or only search specific date ranges.
config :index, :validate => :string, :default => "logstash-%{+YYYY.MM.dd}"
# The index type to write events to. Generally you should try to write only
# similar events to the same 'type'. String expansion '%{foo}' works here.
config :index_type, :validate => :string, :default => "%{type}"
# The name/address of an ElasticSearch host to use for river creation
config :es_host, :validate => :string, :required => true
# ElasticSearch API port
config :es_port, :validate => :number, :default => 9200
# ElasticSearch river configuration: bulk fetch size
config :es_bulk_size, :validate => :number, :default => 1000
# ElasticSearch river configuration: bulk timeout in milliseconds
config :es_bulk_timeout_ms, :validate => :number, :default => 100
# ElasticSearch river configuration: is ordered?
config :es_ordered, :validate => :boolean, :default => false
# Hostname of RabbitMQ server
config :rabbitmq_host, :validate => :string, :required => true
# Port of RabbitMQ server
config :rabbitmq_port, :validate => :number, :default => 5672
# RabbitMQ user
config :user, :validate => :string, :default => "guest"
# RabbitMQ password
config :password, :validate => :string, :default => "guest"
# RabbitMQ vhost
config :vhost, :validate => :string, :default => "/"
# RabbitMQ queue name
config :queue, :validate => :string, :default => "elasticsearch"
# RabbitMQ exchange name
config :exchange, :validate => :string, :default => "elasticsearch"
# The exchange type (fanout, topic, direct)
config :exchange_type, :validate => [ "fanout", "direct", "topic"],
:default => "direct"
# RabbitMQ routing key
config :key, :validate => :string, :default => "elasticsearch"
# RabbitMQ durability setting. Also used for ElasticSearch setting
config :durable, :validate => :boolean, :default => true
# RabbitMQ persistence setting
config :persistent, :validate => :boolean, :default => true
# The document ID for the index. Useful for overwriting existing entries in
# elasticsearch with the same ID.
config :document_id, :validate => :string, :default => nil
public
def register
LogStash::Environment.load_elasticsearch_jars!
prepare_river
end
protected
def prepare_river
require "logstash/outputs/rabbitmq"
# Configure the message plugin
params = {
"host" => [@rabbitmq_host],
"port" => [@rabbitmq_port],
"user" => [@user],
"password" => [@password],
"exchange_type" => [@exchange_type],
"exchange" => [@exchange],
"key" => [@key],
"vhost" => [@vhost],
"durable" => [@durable.to_s],
"persistent" => [@persistent.to_s],
"debug" => [@logger.debug?.to_s],
}.reject {|k,v| v.first.nil?}
@mq = LogStash::Outputs::RabbitMQ.new(params)
@mq.register
# Set up the river
begin
auth = "#{@user}:#{@password}"
# Name the river by our hostname
require "socket"
hostname = Socket.gethostname
# Replace spaces with hyphens and remove all non-alpha non-dash non-underscore characters
river_name = "#{hostname} #{@queue}".gsub(' ', '-').gsub(/[^\w-]/, '')
api_path = "/_river/logstash-#{river_name}/_meta"
@status_path = "/_river/logstash-#{river_name}/_status"
river_config = {"type" => "rabbitmq",
"rabbitmq" => {
"host" => @rabbitmq_host=="localhost" ? hostname : @rabbitmq_host,
"port" => @rabbitmq_port,
"user" => @user,
"pass" => @password,
"vhost" => @vhost,
"queue" => @queue,
"exchange" => @exchange,
"routing_key" => @key,
"exchange_type" => @exchange_type,
"exchange_durable" => @durable.to_s,
"queue_durable" => @durable.to_s
},
"index" => {"bulk_size" => @es_bulk_size,
"bulk_timeout" => "#{@es_bulk_timeout_ms}ms",
"ordered" => @es_ordered
},
}
@logger.info("ElasticSearch using river", :config => river_config)
Net::HTTP.start(@es_host, @es_port) do |http|
req = Net::HTTP::Put.new(api_path)
req.body = LogStash::Json.dump(river_config)
response = http.request(req)
response.value() # raise an exception if error
@logger.info("River created: #{response.body}")
end
rescue Exception => e
# TODO(petef): should we just throw an exception here, so the
# agent tries to restart us and we in turn retry the river
# registration?
@logger.warn("Couldn't set up river. You'll have to set it up manually (or restart)", :exception => e)
end
check_river_status
end # def prepare_river
private
def check_river_status
tries = 0
success = false
reason = nil
begin
while !success && tries <= 3 do
tries += 1
Net::HTTP.start(@es_host, @es_port) do |http|
req = Net::HTTP::Get.new(@status_path)
response = http.request(req)
response.value
status = LogStash::Json.load(response.body)
@logger.debug("Checking ES river status", :status => status)
if status["_source"]["error"]
reason = "ES river status: #{status["_source"]["error"]}"
else
success = true
end
end
sleep(2)
end
rescue Exception => e
raise "river is not running, checking status failed: #{$!}"
end
raise "river is not running: #{reason}" unless success
end # def check_river_status
public
def receive(event)
return unless output?(event)
# River events have a format of
# "action\ndata\n"
# where 'action' is index or delete, data is the data to index.
header = { "index" => { "_index" => event.sprintf(@index), "_type" => event.sprintf(@index_type) } }
if !@document_id.nil?
header["index"]["_id"] = event.sprintf(@document_id)
end
@mq.publish_serialized(LogStash::Json.dump(header) + "\n" + event.to_json + "\n")
end # def receive
end # LogStash::Outputs::ElasticSearchRiver

303
lib/logstash/outputs/email.rb
View file

@ -1,303 +0,0 @@
# encoding: utf-8
require "logstash/outputs/base"
require "logstash/namespace"
# Send email when an output is received. Alternatively, you may include or
# exclude the email output execution using conditionals.
class LogStash::Outputs::Email < LogStash::Outputs::Base
config_name "email"
milestone 1
# This setting is deprecated in favor of Logstash's "conditionals" feature
# If you were using this setting previously, please use conditionals instead.
#
# If you need help converting your older 'match' setting to a conditional,
# I welcome you to join the #logstash irc channel on freenode or to email
# the logstash-users@googlegroups.com mailling list and ask for help! :)
config :match, :validate => :hash, :deprecated => true
# The fully-qualified email address to send the email to.
#
# This field also accepts a comma-separated string of addresses, for example:
# "me@host.com, you@host.com"
#
# You can also use dynamic fields from the event with the %{fieldname} syntax.
config :to, :validate => :string, :required => true
# The fully-qualified email address for the From: field in the email.
config :from, :validate => :string, :default => "logstash.alert@nowhere.com"
# The fully qualified email address for the Reply-To: field.
config :replyto, :validate => :string
# The fully-qualified email address(es) to include as cc: address(es).
#
# This field also accepts a comma-separated string of addresses, for example:
# "me@host.com, you@host.com"
config :cc, :validate => :string
# How Logstash should send the email, either via SMTP or by invoking sendmail.
config :via, :validate => :string, :default => "smtp"
# Specify the options to use:
#
# Via SMTP: smtpIporHost, port, domain, userName, password, authenticationType, starttls
#
# Via sendmail: location, arguments
#
# If you do not specify any `options`, you will get the following equivalent code set in
# every new mail object:
#
# Mail.defaults do
# delivery_method :smtp, { :smtpIporHost => "localhost",
# :port => 25,
# :domain => 'localhost.localdomain',
# :userName => nil,
# :password => nil,
# :authenticationType => nil,(plain, login and cram_md5)
# :starttls => true }
#
# retriever_method :pop3, { :address => "localhost",
# :port => 995,
# :user_name => nil,
# :password => nil,
# :enable_ssl => true }
#
# Mail.delivery_method.new #=> Mail::SMTP instance
# Mail.retriever_method.new #=> Mail::POP3 instance
# end
#
# Each mail object inherits the defaults set in Mail.delivery_method. However, on
# a per email basis, you can override the method:
#
# mail.delivery_method :sendmail
#
# Or you can override the method and pass in settings:
#
# mail.delivery_method :sendmail, { :address => 'some.host' }
#
# You can also just modify the settings:
#
# mail.delivery_settings = { :address => 'some.host' }
#
# The hash you supply is just merged against the defaults with "merge!" and the result
# assigned to the mail object. For instance, the above example will change only the
# `:address` value of the global `smtp_settings` to be 'some.host', retaining all other values.
config :options, :validate => :hash, :default => {}
# Subject: for the email.
config :subject, :validate => :string, :default => ""
# Body for the email - plain text only.
config :body, :validate => :string, :default => ""
# HTML Body for the email, which may contain HTML markup.
config :htmlbody, :validate => :string, :default => ""
# Attachments - specify the name(s) and location(s) of the files.
config :attachments, :validate => :array, :default => []
# contenttype : for multipart messages, set the content-type and/or charset of the HTML part.
# NOTE: this may not be functional (KH)
config :contenttype, :validate => :string, :default => "text/html; charset=UTF-8"
public
def register
require "mail"
# Mail uses instance_eval which changes the scope of self so @options is
# inaccessible from inside 'Mail.defaults'. So set a local variable instead.
options = @options
if @via == "smtp"
Mail.defaults do
delivery_method :smtp, {
:address => options.fetch("smtpIporHost", "localhost"),
:port => options.fetch("port", 25),
:domain => options.fetch("domain", "localhost"),
:user_name => options.fetch("userName", nil),
:password => options.fetch("password", nil),
:authentication => options.fetch("authenticationType", nil),
:enable_starttls_auto => options.fetch("starttls", false),
:debug => options.fetch("debug", false)
}
end
elsif @via == 'sendmail'
Mail.defaults do
delivery_method :sendmail
end
else
Mail.defaults do
delivery_method :@via, options
end
end # @via tests
@logger.debug("Email Output Registered!", :config => @config)
end # def register
public
def receive(event)
return unless output?(event)
@logger.debug("Event being tested for Email", :tags => @tags, :event => event)
# Set Intersection - returns a new array with the items that are the same between the two
if !@tags.empty? && (event["tags"] & @tags).size == 0
# Skip events that have no tags in common with what we were configured
@logger.debug("No Tags match for Email Output!")
return
end
@logger.debug? && @logger.debug("Match data for Email - ", :match => @match)
successful = false
matchName = ""
operator = ""
# TODO(sissel): Delete this once match support is removed.
@match && @match.each do |name, query|
if successful
break
else
matchName = name
end
# now loop over the csv query
queryArray = query.split(',')
index = 1
while index < queryArray.length
field = queryArray.at(index -1)
value = queryArray.at(index)
index = index + 2
if field == ""
if value.downcase == "and"
operator = "and"
elsif value.downcase == "or"
operator = "or"
else
operator = "or"
@logger.error("Operator Provided Is Not Found, Currently We Only Support AND/OR Values! - defaulting to OR")
end
else
hasField = event[field]
@logger.debug? and @logger.debug("Does Event Contain Field - ", :hasField => hasField)
isValid = false
# if we have maching field and value is wildcard - we have a success
if hasField
if value == "*"
isValid = true
else
# we get an array so we need to loop over the values and find if we have a match
eventFieldValues = event[field]
@logger.debug? and @logger.debug("Event Field Values - ", :eventFieldValues => eventFieldValues)
eventFieldValues = [eventFieldValues] if not eventFieldValues.respond_to?(:each)
eventFieldValues.each do |eventFieldValue|
isValid = validateValue(eventFieldValue, value)
if isValid # no need to iterate any further
@logger.debug("VALID CONDITION FOUND - ", :eventFieldValue => eventFieldValue, :value => value)
break
end
end # end eventFieldValues.each do
end # end value == "*"
end # end hasField
# if we have an AND operator and we have a successful == false break
if operator == "and" && !isValid
successful = false
elsif operator == "or" && (isValid || successful)
successful = true
else
successful = isValid
end
end
end
end # @match.each do
# The 'match' setting is deprecated and optional. If not set,
# default to success.
successful = true if @match.nil?
@logger.debug? && @logger.debug("Email Did we match any alerts for event : ", :successful => successful)
if successful
# first add our custom field - matchName - so we can use it in the sprintf function
event["matchName"] = matchName unless matchName.empty?
@logger.debug? and @logger.debug("Creating mail with these settings : ", :via => @via, :options => @options, :from => @from, :to => @to, :cc => @cc, :subject => @subject, :body => @body, :content_type => @contenttype, :htmlbody => @htmlbody, :attachments => @attachments, :to => to, :to => to)
formatedSubject = event.sprintf(@subject)
formattedBody = event.sprintf(@body)
formattedHtmlBody = event.sprintf(@htmlbody)
# we have a match(s) - send email
mail = Mail.new
mail.from = event.sprintf(@from)
mail.to = event.sprintf(@to)
if @replyto
mail.reply_to = event.sprintf(@replyto)
end
mail.cc = event.sprintf(@cc)
mail.subject = formatedSubject
if @htmlbody.empty?
formattedBody.gsub!(/\\n/, "\n") # Take new line in the email
mail.body = formattedBody
else
mail.text_part = Mail::Part.new do
content_type "text/plain; charset=UTF-8"
formattedBody.gsub!(/\\n/, "\n") # Take new line in the email
body formattedBody
end
mail.html_part = Mail::Part.new do
content_type "text/html; charset=UTF-8"
body formattedHtmlBody
end
end
@attachments.each do |fileLocation|
mail.add_file(fileLocation)
end # end @attachments.each
@logger.debug? and @logger.debug("Sending mail with these values : ", :from => mail.from, :to => mail.to, :cc => mail.cc, :subject => mail.subject)
mail.deliver!
end # end if successful
end # def receive
private
def validateValue(eventFieldValue, value)
valid = false
# order of this if-else is important - please don't change it
if value.start_with?(">=")# greater than or equal
value.gsub!(">=","")
if eventFieldValue.to_i >= value.to_i
valid = true
end
elsif value.start_with?("<=")# less than or equal
value.gsub!("<=","")
if eventFieldValue.to_i <= value.to_i
valid = true
end
elsif value.start_with?(">")# greater than
value.gsub!(">","")
if eventFieldValue.to_i > value.to_i
valid = true
end
elsif value.start_with?("<")# less than
value.gsub!("<","")
if eventFieldValue.to_i < value.to_i
valid = true
end
elsif value.start_with?("*")# contains
value.gsub!("*","")
if eventFieldValue.include?(value)
valid = true
end
elsif value.start_with?("!*")# does not contain
value.gsub!("!*","")
if !eventFieldValue.include?(value)
valid = true
end
elsif value.start_with?("!")# not equal
value.gsub!("!","")
if eventFieldValue != value
valid = true
end
else # default equal
if eventFieldValue == value
valid = true
end
end
return valid
end # end validateValue()
end # class LogStash::Outputs::Email

40
lib/logstash/outputs/exec.rb
View file

@ -1,40 +0,0 @@
# encoding: utf-8
require "logstash/namespace"
require "logstash/outputs/base"
# This output will run a command for any matching event.
#
# Example:
#
# output {
# exec {
# type => abuse
# command => "iptables -A INPUT -s %{clientip} -j DROP"
# }
# }
#
# Run subprocesses via system ruby function
#
# WARNING: if you want it non-blocking you should use & or dtach or other such
# techniques
class LogStash::Outputs::Exec < LogStash::Outputs::Base
config_name "exec"
milestone 1
# Command line to execute via subprocess. Use dtach or screen to make it non blocking
config :command, :validate => :string, :required => true
public
def register
@logger.debug("exec output registered", :config => @config)
end # def register
public
def receive(event)
return unless output?(event)
@logger.debug("running exec command", :command => event.sprintf(@command))
system(event.sprintf(@command))
end # def receive
end

179
lib/logstash/outputs/file.rb
View file

@ -1,179 +0,0 @@
# encoding: utf-8
require "logstash/namespace"
require "logstash/outputs/base"
require "zlib"
# This output will write events to files on disk. You can use fields
# from the event as parts of the filename and/or path.
class LogStash::Outputs::File < LogStash::Outputs::Base
config_name "file"
milestone 2
# The path to the file to write. Event fields can be used here,
# like "/var/log/logstash/%{host}/%{application}"
# One may also utilize the path option for date-based log
# rotation via the joda time format. This will use the event
# timestamp.
# E.g.: path => "./test-%{+YYYY-MM-dd}.txt" to create
# ./test-2013-05-29.txt
config :path, :validate => :string, :required => true
# The maximum size of file to write. When the file exceeds this
# threshold, it will be rotated to the current filename + ".1"
# If that file already exists, the previous .1 will shift to .2
# and so forth.
#
# NOT YET SUPPORTED
config :max_size, :validate => :string
# The format to use when writing events to the file. This value
# supports any string and can include %{name} and other dynamic
# strings.
#
# If this setting is omitted, the full json representation of the
# event will be written as a single line.
config :message_format, :validate => :string
# Flush interval (in seconds) for flushing writes to log files.
# 0 will flush on every message.
config :flush_interval, :validate => :number, :default => 2
# Gzip the output stream before writing to disk.
config :gzip, :validate => :boolean, :default => false
public
def register
require "fileutils" # For mkdir_p
workers_not_supported
@files = {}
now = Time.now
@last_flush_cycle = now
@last_stale_cleanup_cycle = now
flush_interval = @flush_interval.to_i
@stale_cleanup_interval = 10
end # def register
public
def receive(event)
return unless output?(event)
path = event.sprintf(@path)
fd = open(path)
# TODO(sissel): Check if we should rotate the file.
if @message_format
output = event.sprintf(@message_format)
else
output = event.to_json
end
fd.write(output)
fd.write("\n")
flush(fd)
close_stale_files
end # def receive
def teardown
@logger.debug("Teardown: closing files")
@files.each do |path, fd|
begin
fd.close
@logger.debug("Closed file #{path}", :fd => fd)
rescue Exception => e
@logger.error("Excpetion while flushing and closing files.", :exception => e)
end
end
finished
end
private
def flush(fd)
if flush_interval > 0
flush_pending_files
else
fd.flush
end
end
# every flush_interval seconds or so (triggered by events, but if there are no events there's no point flushing files anyway)
def flush_pending_files
return unless Time.now - @last_flush_cycle >= flush_interval
@logger.debug("Starting flush cycle")
@files.each do |path, fd|
@logger.debug("Flushing file", :path => path, :fd => fd)
fd.flush
end
@last_flush_cycle = Time.now
end
# every 10 seconds or so (triggered by events, but if there are no events there's no point closing files anyway)
def close_stale_files
now = Time.now
return unless now - @last_stale_cleanup_cycle >= @stale_cleanup_interval
@logger.info("Starting stale files cleanup cycle", :files => @files)
inactive_files = @files.select { |path, fd| not fd.active }
@logger.debug("%d stale files found" % inactive_files.count, :inactive_files => inactive_files)
inactive_files.each do |path, fd|
@logger.info("Closing file %s" % path)
fd.close
@files.delete(path)
end
# mark all files as inactive, a call to write will mark them as active again
@files.each { |path, fd| fd.active = false }
@last_stale_cleanup_cycle = now
end
def open(path)
return @files[path] if @files.include?(path) and not @files[path].nil?
@logger.info("Opening file", :path => path)
dir = File.dirname(path)
if !Dir.exists?(dir)
@logger.info("Creating directory", :directory => dir)
FileUtils.mkdir_p(dir)
end
# work around a bug opening fifos (bug JRUBY-6280)
stat = File.stat(path) rescue nil
if stat and stat.ftype == "fifo" and RUBY_PLATFORM == "java"
fd = java.io.FileWriter.new(java.io.File.new(path))
else
fd = File.new(path, "a")
end
if gzip
fd = Zlib::GzipWriter.new(fd)
end
@files[path] = IOWriter.new(fd)
end
end # class LogStash::Outputs::File
# wrapper class
class IOWriter
def initialize(io)
@io = io
end
def write(*args)
@io.write(*args)
@active = true
end
def flush
@io.flush
if @io.class == Zlib::GzipWriter
@io.to_io.flush
end
end
def method_missing(method_name, *args, &block)
if @io.respond_to?(method_name)
@io.send(method_name, *args, &block)
else
super
end
end
attr_accessor :active
end

75
lib/logstash/outputs/ganglia.rb
View file

@ -1,75 +0,0 @@
# encoding: utf-8
require "logstash/outputs/base"
require "logstash/namespace"
# This output allows you to pull metrics from your logs and ship them to
# ganglia's gmond. This is heavily based on the graphite output.
class LogStash::Outputs::Ganglia < LogStash::Outputs::Base
config_name "ganglia"
milestone 2
# The address of the ganglia server.
config :host, :validate => :string, :default => "localhost"
# The port to connect on your ganglia server.
config :port, :validate => :number, :default => 8649
# The metric to use. This supports dynamic strings like `%{host}`
config :metric, :validate => :string, :required => true
# The value to use. This supports dynamic strings like `%{bytes}`
# It will be coerced to a floating point value. Values which cannot be
# coerced will zero (0)
config :value, :validate => :string, :required => true
# The type of value for this metric.
config :metric_type, :validate => %w{string int8 uint8 int16 uint16 int32 uint32 float double},
:default => "uint8"
# Gmetric units for metric, such as "kb/sec" or "ms" or whatever unit
# this metric uses.
config :units, :validate => :string, :default => ""
# Maximum time in seconds between gmetric calls for this metric.
config :max_interval, :validate => :number, :default => 60
# Lifetime in seconds of this metric
config :lifetime, :validate => :number, :default => 300
# Metric group
config :group, :validate => :string, :default => ""
# Metric slope, represents metric behavior
config :slope, :validate => %w{zero positive negative both unspecified}, :default => "both"
def register
require "gmetric"
end # def register
public
def receive(event)
return unless output?(event)
# gmetric only takes integer values, so convert it to int.
case @metric_type
when "string"
localvalue = event.sprintf(@value)
when "float"
localvalue = event.sprintf(@value).to_f
when "double"
localvalue = event.sprintf(@value).to_f
else # int8|uint8|int16|uint16|int32|uint32
localvalue = event.sprintf(@value).to_i
end
Ganglia::GMetric.send(@host, @port, {
:name => event.sprintf(@metric),
:units => @units,
:type => @metric_type,
:value => localvalue,
:group => @group,
:slope => @slope,
:tmax => @max_interval,
:dmax => @lifetime
})
end # def receive
end # class LogStash::Outputs::Ganglia

211
lib/logstash/outputs/gelf.rb
View file

@ -1,211 +0,0 @@
# encoding: utf-8
require "logstash/namespace"
require "logstash/outputs/base"
# This output generates messages in GELF format. This is most useful if you
# want to use Logstash to output events to Graylog2.
#
# More information at <http://graylog2.org/gelf#specs>
class LogStash::Outputs::Gelf < LogStash::Outputs::Base
config_name "gelf"
milestone 2
# Graylog2 server IP address or hostname.
config :host, :validate => :string, :required => true
# Graylog2 server port number.
config :port, :validate => :number, :default => 12201
# The GELF chunksize. You usually don't need to change this.
config :chunksize, :validate => :number, :default => 1420
# Allow overriding of the GELF `sender` field. This is useful if you
# want to use something other than the event's source host as the
# "sender" of an event. A common case for this is using the application name
# instead of the hostname.
config :sender, :validate => :string, :default => "%{host}"
# The GELF message level. Dynamic values like %{level} are permitted here;
# useful if you want to parse the 'log level' from an event and use that
# as the GELF level/severity.
#
# Values here can be integers [0..7] inclusive or any of
# "debug", "info", "warn", "error", "fatal" (case insensitive).
# Single-character versions of these are also valid, "d", "i", "w", "e", "f",
# "u"
# The following additional severity\_labels from Logstash's syslog\_pri filter
# are accepted: "emergency", "alert", "critical", "warning", "notice", and
# "informational".
config :level, :validate => :array, :default => [ "%{severity}", "INFO" ]
# The GELF facility. Dynamic values like %{foo} are permitted here; this
# is useful if you need to use a value from the event as the facility name.
# Should now be sent as an underscored "additional field" (e.g. `\_facility`)
config :facility, :validate => :string, :deprecated => true
# The GELF line number; this is usually the line number in your program where
# the log event originated. Dynamic values like %{foo} are permitted here, but the
# value should be a number.
# Should now be sent as an underscored "additional field" (e.g. `\_line`).
config :line, :validate => :string, :deprecated => true
# The GELF file; this is usually the source code file in your program where
# the log event originated. Dynamic values like %{foo} are permitted here.
# Should now be sent as an underscored "additional field" (e.g. `\_file`).
config :file, :validate => :string, :deprecated => true
# Should Logstash ship metadata within event object? This will cause Logstash
# to ship any fields in the event (such as those created by grok) in the GELF
# messages. These will be sent as underscored "additional fields".
config :ship_metadata, :validate => :boolean, :default => true
# Ship tags within events. This will cause Logstash to ship the tags of an
# event as the field `\_tags`.
config :ship_tags, :validate => :boolean, :default => true
# Ignore these fields when `ship_metadata` is set. Typically this lists the
# fields used in dynamic values for GELF fields.
config :ignore_metadata, :validate => :array, :default => [ "@timestamp", "@version", "severity", "host", "source_host", "source_path", "short_message" ]
# The GELF custom field mappings. GELF supports arbitrary attributes as custom
# fields. This exposes that. Exclude the `_` portion of the field name
# e.g. `custom_fields => ['foo_field', 'some_value']
# sets `_foo_field` = `some_value`.
config :custom_fields, :validate => :hash, :default => {}
# The GELF full message. Dynamic values like %{foo} are permitted here.
config :full_message, :validate => :string, :default => "%{message}"
# The GELF short message field name. If the field does not exist or is empty,
# the event message is taken instead.
config :short_message, :validate => :string, :default => "short_message"
public
def register
require "gelf" # rubygem 'gelf'
option_hash = Hash.new
#@gelf = GELF::Notifier.new(@host, @port, @chunksize, option_hash)
@gelf = GELF::Notifier.new(@host, @port, @chunksize)
# This sets the 'log level' of gelf; since we're forwarding messages, we'll
# want to forward *all* messages, so set level to 0 so all messages get
# shipped
@gelf.level = 0
# Since we use gelf-rb which assumes the severity level integer
# is coming from a ruby logging subsystem, we need to instruct it
# that the levels we provide should be mapped directly since they're
# already RFC 5424 compliant
# this requires gelf-rb commit bb1f4a9 which added the level_mapping def
level_mapping = Hash.new
(0..7).step(1) { |l| level_mapping[l]=l }
@gelf.level_mapping = level_mapping
# If we leave that set, the gelf gem will extract the file and line number
# of the source file that logged the message (i.e. logstash/gelf.rb:138).
# With that set to false, it can use the actual event's filename (i.e.
# /var/log/syslog), which is much more useful
@gelf.collect_file_and_line = false
# these are syslog words and abbreviations mapped to RFC 5424 integers
# and logstash's syslog_pri filter
@level_map = {
"debug" => 7, "d" => 7,
"info" => 6, "i" => 6, "informational" => 6,
"notice" => 5, "n" => 5,
"warn" => 4, "w" => 4, "warning" => 4,
"error" => 3, "e" => 3,
"critical" => 2, "c" => 2,
"alert" => 1, "a" => 1,
"emergency" => 0, "e" => 0,
}
end # def register
public
def receive(event)
return unless output?(event)
# We have to make our own hash here because GELF expects a hash
# with a specific format.
m = Hash.new
m["short_message"] = event["message"]
if event[@short_message]
v = event[@short_message]
short_message = (v.is_a?(Array) && v.length == 1) ? v.first : v
short_message = short_message.to_s
if !short_message.empty?
m["short_message"] = short_message
end
end
m["full_message"] = event.sprintf(@full_message)
m["host"] = event.sprintf(@sender)
# deprecated fields
m["facility"] = event.sprintf(@facility) if @facility
m["file"] = event.sprintf(@file) if @file
m["line"] = event.sprintf(@line) if @line
m["line"] = m["line"].to_i if m["line"].is_a?(String) and m["line"] === /^[\d]+$/
if @ship_metadata
event.to_hash.each do |name, value|
next if value == nil
next if name == "message"
# Trim leading '_' in the event
name = name[1..-1] if name.start_with?('_')
name = "_id" if name == "id" # "_id" is reserved, so use "__id"
if !value.nil? and !@ignore_metadata.include?(name)
if value.is_a?(Array)
m["_#{name}"] = value.join(', ')
elsif value.is_a?(Hash)
value.each do |hash_name, hash_value|
m["_#{name}_#{hash_name}"] = hash_value
end
else
# Non array values should be presented as-is
# https://logstash.jira.com/browse/LOGSTASH-113
m["_#{name}"] = value
end
end
end
end
if @ship_tags
m["_tags"] = event["tags"].join(', ') if event["tags"]
end
if @custom_fields
@custom_fields.each do |field_name, field_value|
m["_#{field_name}"] = field_value unless field_name == 'id'
end
end
# Probe severity array levels
level = nil
if @level.is_a?(Array)
@level.each do |value|
parsed_value = event.sprintf(value)
next if value.count('%{') > 0 and parsed_value == value
level = parsed_value
break
end
else
level = event.sprintf(@level.to_s)
end
m["level"] = (@level_map[level.downcase] || level).to_i
@logger.debug(["Sending GELF event", m])
begin
@gelf.notify!(m, :timestamp => event.timestamp.to_f)
rescue
@logger.warn("Trouble sending GELF event", :gelf_event => m,
:event => event, :error => $!)
end
end # def receive
end # class LogStash::Outputs::Gelf

146
lib/logstash/outputs/graphite.rb
View file

@ -1,146 +0,0 @@
# encoding: utf-8
require "logstash/outputs/base"
require "logstash/namespace"
require "socket"
# This output allows you to pull metrics from your logs and ship them to
# Graphite. Graphite is an open source tool for storing and graphing metrics.
#
# An example use case: Some applications emit aggregated stats in the logs
# every 10 seconds. Using the grok filter and this output, it is possible to
# capture the metric values from the logs and emit them to Graphite.
class LogStash::Outputs::Graphite < LogStash::Outputs::Base
config_name "graphite"
milestone 2
EXCLUDE_ALWAYS = [ "@timestamp", "@version" ]
DEFAULT_METRICS_FORMAT = "*"
METRIC_PLACEHOLDER = "*"
# The hostname or IP address of the Graphite server.
config :host, :validate => :string, :default => "localhost"
# The port to connect to on the Graphite server.
config :port, :validate => :number, :default => 2003
# Interval between reconnect attempts to Carbon.
config :reconnect_interval, :validate => :number, :default => 2
# Should metrics be resent on failure?
config :resend_on_failure, :validate => :boolean, :default => false
# The metric(s) to use. This supports dynamic strings like %{host}
# for metric names and also for values. This is a hash field with key
# being the metric name, value being the metric value. Example:
#
# [ "%{host}/uptime", "%{uptime_1m}" ]
#
# The value will be coerced to a floating point value. Values which cannot be
# coerced will be set to zero (0). You may use either `metrics` or `fields_are_metrics`,
# but not both.
config :metrics, :validate => :hash, :default => {}
# An array indicating that these event fields should be treated as metrics
# and will be sent verbatim to Graphite. You may use either `fields_are_metrics`
# or `metrics`, but not both.
config :fields_are_metrics, :validate => :boolean, :default => false
# Include only regex matched metric names.
config :include_metrics, :validate => :array, :default => [ ".*" ]
# Exclude regex matched metric names, by default exclude unresolved %{field} strings.
config :exclude_metrics, :validate => :array, :default => [ "%\{[^}]+\}" ]
# Enable debug output.
config :debug, :validate => :boolean, :default => false, :deprecated => "This setting was never used by this plugin. It will be removed soon."
# Defines the format of the metric string. The placeholder '*' will be
# replaced with the name of the actual metric.
#
# metrics_format => "foo.bar.*.sum"
#
# NOTE: If no metrics_format is defined, the name of the metric will be used as fallback.
config :metrics_format, :validate => :string, :default => DEFAULT_METRICS_FORMAT
def register
@include_metrics.collect!{|regexp| Regexp.new(regexp)}
@exclude_metrics.collect!{|regexp| Regexp.new(regexp)}
if @metrics_format && !@metrics_format.include?(METRIC_PLACEHOLDER)
@logger.warn("metrics_format does not include placeholder #{METRIC_PLACEHOLDER} .. falling back to default format: #{DEFAULT_METRICS_FORMAT.inspect}")
@metrics_format = DEFAULT_METRICS_FORMAT
end
connect
end # def register
def connect
# TODO(sissel): Test error cases. Catch exceptions. Find fortune and glory. Retire to yak farm.
begin
@socket = TCPSocket.new(@host, @port)
rescue Errno::ECONNREFUSED => e
@logger.warn("Connection refused to graphite server, sleeping...",
:host => @host, :port => @port)
sleep(@reconnect_interval)
retry
end
end # def connect
def construct_metric_name(metric)
if @metrics_format
return @metrics_format.gsub(METRIC_PLACEHOLDER, metric)
end
metric
end
public
def receive(event)
return unless output?(event)
# Graphite message format: metric value timestamp\n
messages = []
timestamp = event.sprintf("%{+%s}")
if @fields_are_metrics
@logger.debug("got metrics event", :metrics => event.to_hash)
event.to_hash.each do |metric,value|
next if EXCLUDE_ALWAYS.include?(metric)
next unless @include_metrics.empty? || @include_metrics.any? { |regexp| metric.match(regexp) }
next if @exclude_metrics.any? {|regexp| metric.match(regexp)}
messages << "#{construct_metric_name(metric)} #{event.sprintf(value.to_s).to_f} #{timestamp}"
end
else
@metrics.each do |metric, value|
@logger.debug("processing", :metric => metric, :value => value)
metric = event.sprintf(metric)
next unless @include_metrics.any? {|regexp| metric.match(regexp)}
next if @exclude_metrics.any? {|regexp| metric.match(regexp)}
messages << "#{construct_metric_name(event.sprintf(metric))} #{event.sprintf(value).to_f} #{timestamp}"
end
end
if messages.empty?
@logger.debug("Message is empty, not sending anything to Graphite", :messages => messages, :host => @host, :port => @port)
else
message = messages.join("\n")
@logger.debug("Sending carbon messages", :messages => messages, :host => @host, :port => @port)
# Catch exceptions like ECONNRESET and friends, reconnect on failure.
# TODO(sissel): Test error cases. Catch exceptions. Find fortune and glory.
begin
@socket.puts(message)
rescue Errno::EPIPE, Errno::ECONNRESET => e
@logger.warn("Connection to graphite server died",
:exception => e, :host => @host, :port => @port)
sleep(@reconnect_interval)
connect
retry if @resend_on_failure
end
end
end # def receive
end # class LogStash::Outputs::Graphite

80
lib/logstash/outputs/hipchat.rb
View file

@ -1,80 +0,0 @@
# encoding: utf-8
require "logstash/namespace"
require "logstash/outputs/http"
# This output allows you to write events to [HipChat](https://www.hipchat.com/).
#
class LogStash::Outputs::HipChat < LogStash::Outputs::Base
config_name "hipchat"
milestone 1
# The HipChat authentication token.
config :token, :validate => :string, :required => true
# The ID or name of the room.
config :room_id, :validate => :string, :required => true
# The name the message will appear be sent from.
config :from, :validate => :string, :default => "logstash"
# Whether or not this message should trigger a notification for people in the room.
config :trigger_notify, :validate => :boolean, :default => false
# Background color for message.
# HipChat currently supports one of "yellow", "red", "green", "purple",
# "gray", or "random". (default: yellow)
config :color, :validate => :string, :default => "yellow"
# Message format to send, event tokens are usable here.
config :format, :validate => :string, :default => "%{message}"
public
def register
require "ftw"
require "uri"
@agent = FTW::Agent.new
@url = "https://api.hipchat.com/v1/rooms/message?auth_token=" + @token
@content_type = "application/x-www-form-urlencoded"
end # def register
public
def receive(event)
return unless output?(event)
hipchat_data = Hash.new
hipchat_data['room_id'] = @room_id
hipchat_data['from'] = @from
hipchat_data['color'] = @color
hipchat_data['notify'] = @trigger_notify ? "1" : "0"
hipchat_data['message'] = event.sprintf(@format)
@logger.debug("HipChat data", :hipchat_data => hipchat_data)
begin
request = @agent.post(@url)
request["Content-Type"] = @content_type
request.body = encode(hipchat_data)
response = @agent.execute(request)
# Consume body to let this connection be reused
rbody = ""
response.read_body { |c| rbody << c }
#puts rbody
rescue Exception => e
@logger.warn("Unhandled exception", :request => request, :response => response, :exception => e, :stacktrace => e.backtrace)
end
end # def receive
# shamelessly lifted this from the LogStash::Outputs::Http, I'd rather put this
# in a common place for both to use, but unsure where that place is or should be
def encode(hash)
return hash.collect do |key, value|
CGI.escape(key) + "=" + CGI.escape(value)
end.join("&")
end # def encode
end # class LogStash::Outputs::HipChat

143
lib/logstash/outputs/http.rb
View file

@ -1,143 +0,0 @@
# encoding: utf-8
require "logstash/outputs/base"
require "logstash/namespace"
require "logstash/json"
class LogStash::Outputs::Http < LogStash::Outputs::Base
# This output lets you `PUT` or `POST` events to a
# generic HTTP(S) endpoint
#
# Additionally, you are given the option to customize
# the headers sent as well as basic customization of the
# event json itself.
config_name "http"
milestone 1
# URL to use
config :url, :validate => :string, :required => :true
# validate SSL?
config :verify_ssl, :validate => :boolean, :default => true
# What verb to use
# only put and post are supported for now
config :http_method, :validate => ["put", "post"], :required => :true
# Custom headers to use
# format is `headers => ["X-My-Header", "%{host}"]
config :headers, :validate => :hash
# Content type
#
# If not specified, this defaults to the following:
#
# * if format is "json", "application/json"
# * if format is "form", "application/x-www-form-urlencoded"
config :content_type, :validate => :string
# This lets you choose the structure and parts of the event that are sent.
#
#
# For example:
#
# mapping => ["foo", "%{host}", "bar", "%{type}"]
config :mapping, :validate => :hash
# Set the format of the http body.
#
# If form, then the body will be the mapping (or whole event) converted
# into a query parameter string (foo=bar&baz=fizz...)
#
# If message, then the body will be the result of formatting the event according to message
#
# Otherwise, the event is sent as json.
config :format, :validate => ["json", "form", "message"], :default => "json"
config :message, :validate => :string
public
def register
require "ftw"
require "uri"
@agent = FTW::Agent.new
# TODO(sissel): SSL verify mode?
if @content_type.nil?
case @format
when "form" ; @content_type = "application/x-www-form-urlencoded"
when "json" ; @content_type = "application/json"
end
end
if @format == "message"
if @message.nil?
raise "message must be set if message format is used"
end
if @content_type.nil?
raise "content_type must be set if message format is used"
end
unless @mapping.nil?
@logger.warn "mapping is not supported and will be ignored if message format is used"
end
end
end # def register
public
def receive(event)
return unless output?(event)
if @mapping
evt = Hash.new
@mapping.each do |k,v|
evt[k] = event.sprintf(v)
end
else
evt = event.to_hash
end
case @http_method
when "put"
request = @agent.put(event.sprintf(@url))
when "post"
request = @agent.post(event.sprintf(@url))
else
@logger.error("Unknown verb:", :verb => @http_method)
end
if @headers
@headers.each do |k,v|
request.headers[k] = event.sprintf(v)
end
end
request["Content-Type"] = @content_type
begin
if @format == "json"
request.body = LogStash::Json.dump(evt)
elsif @format == "message"
request.body = event.sprintf(@message)
else
request.body = encode(evt)
end
#puts "#{request.port} / #{request.protocol}"
#puts request
#puts
#puts request.body
response = @agent.execute(request)
# Consume body to let this connection be reused
rbody = ""
response.read_body { |c| rbody << c }
#puts rbody
rescue Exception => e
@logger.warn("Unhandled exception", :request => request, :response => response, :exception => e, :stacktrace => e.backtrace)
end
end # def receive
def encode(hash)
return hash.collect do |key, value|
CGI.escape(key) + "=" + CGI.escape(value)
end.join("&")
end # def encode
end

88
lib/logstash/outputs/irc.rb
View file

@ -1,88 +0,0 @@
# encoding: utf-8
require "logstash/outputs/base"
require "logstash/namespace"
require "thread"
# Write events to IRC
#
class LogStash::Outputs::Irc < LogStash::Outputs::Base
config_name "irc"
milestone 1
# Address of the host to connect to
config :host, :validate => :string, :required => true
# Port on host to connect to.
config :port, :validate => :number, :default => 6667
# IRC Nickname
config :nick, :validate => :string, :default => "logstash"
# IRC Username
config :user, :validate => :string, :default => "logstash"
# IRC Real name
config :real, :validate => :string, :default => "logstash"
# IRC server password
config :password, :validate => :password
# Channels to broadcast to.
#
# These should be full channel names including the '#' symbol, such as
# "#logstash".
config :channels, :validate => :array, :required => true
# Message format to send, event tokens are usable here
config :format, :validate => :string, :default => "%{message}"
# Set this to true to enable SSL.
config :secure, :validate => :boolean, :default => false
# Limit the rate of messages sent to IRC in messages per second.
config :messages_per_second, :validate => :number, :default => 0.5
# Static string before event
config :pre_string, :validate => :string, :required => false
# Static string after event
config :post_string, :validate => :string, :required => false
public
def register
require "cinch"
@irc_queue = Queue.new
@logger.info("Connecting to irc server", :host => @host, :port => @port, :nick => @nick, :channels => @channels)
@bot = Cinch::Bot.new
@bot.loggers.clear
@bot.configure do |c|
c.server = @host
c.port = @port
c.nick = @nick
c.user = @user
c.realname = @real
c.channels = @channels
c.password = @password.value rescue nil
c.ssl.use = @secure
c.messages_per_second = @messages_per_second if @messages_per_second
end
Thread.new(@bot) do |bot|
bot.start
end
end # def register
public
def receive(event)
return unless output?(event)
@logger.debug("Sending message to channels", :event => event)
text = event.sprintf(@format)
@bot.channels.each do |channel|
@logger.debug("Sending to...", :channel => channel, :text => text)
channel.msg(pre_string) if !@pre_string.nil?
channel.msg(text)
channel.msg(post_string) if !@post_string.nil?
end # channels.each
end # def receive
end # class LogStash::Outputs::Irc

106
lib/logstash/outputs/juggernaut.rb
View file

@ -1,106 +0,0 @@
# encoding: utf-8
require "logstash/outputs/base"
require "logstash/namespace"
require "logstash/event"
require "logstash/json"
# Push messages to the juggernaut websockets server:
#
# * https://github.com/maccman/juggernaut
#
# Wraps Websockets and supports other methods (including xhr longpolling) This
# is basically, just an extension of the redis output (Juggernaut pulls
# messages from redis). But it pushes messages to a particular channel and
# formats the messages in the way juggernaut expects.
class LogStash::Outputs::Juggernaut < LogStash::Outputs::Base
config_name "juggernaut"
milestone 1
# The hostname of the redis server to which juggernaut is listening.
config :host, :validate => :string, :default => "127.0.0.1"
# The port to connect on.
config :port, :validate => :number, :default => 6379
# The redis database number.
config :db, :validate => :number, :default => 0
# Redis initial connection timeout in seconds.
config :timeout, :validate => :number, :default => 5
# Password to authenticate with. There is no authentication by default.
config :password, :validate => :password
# List of channels to which to publish. Dynamic names are
# valid here, for example "logstash-%{type}".
config :channels, :validate => :array, :required => true
# How should the message be formatted before pushing to the websocket.
config :message_format, :validate => :string
public
def register
require 'redis'
if not @channels
raise RuntimeError.new(
"Must define the channels on which to publish the messages"
)
end
# end TODO
@redis = nil
end # def register
private
def connect
Redis.new(
:host => @host,
:port => @port,
:timeout => @timeout,
:db => @db,
:password => @password
)
end # def connect
# A string used to identify a redis instance in log messages
private
def identity
@name || "redis://#{@password}@#{@host}:#{@port}/#{@db} #{@data_type}:#{@key}"
end
public
def receive(event)
return unless output?(event)
begin
@redis ||= connect
if @message_format
formatted = event.sprintf(@message_format)
else
formatted = event.to_json
end
juggernaut_message = {
"channels" => @channels.collect{ |x| event.sprintf(x) },
"data" => event["message"]
}
@redis.publish 'juggernaut', LogStash::Json.dump(juggernaut_message)
rescue => e
@logger.warn("Failed to send event to redis", :event => event,
:identity => identity, :exception => e,
:backtrace => e.backtrace)
raise e
end
end # def receive
public
def teardown
if @data_type == 'channel' and @redis
@redis.quit
@redis = nil
end
end
end

62
lib/logstash/outputs/lumberjack.rb
View file

@ -1,62 +0,0 @@
# encoding: utf-8
class LogStash::Outputs::Lumberjack < LogStash::Outputs::Base
config_name "lumberjack"
milestone 1
# list of addresses lumberjack can send to
config :hosts, :validate => :array, :required => true
# the port to connect to
config :port, :validate => :number, :required => true
# ssl certificate to use
config :ssl_certificate, :validate => :path, :required => true
# window size
config :window_size, :validate => :number, :default => 5000
public
def register
require 'lumberjack/client'
connect
@codec.on_event do |payload|
begin
@client.write({ 'line' => payload })
rescue Exception => e
@logger.error("Client write error, trying connect", :e => e, :backtrace => e.backtrace)
connect
retry
end # begin
end # @codec
end # def register
public
def receive(event)
return unless output?(event)
if event == LogStash::SHUTDOWN
finished
return
end # LogStash::SHUTDOWN
@codec.encode(event)
end # def receive
private
def connect
require 'resolv'
@logger.info("Connecting to lumberjack server.", :addresses => @hosts, :port => @port,
:ssl_certificate => @ssl_certificate, :window_size => @window_size)
begin
ips = []
@hosts.each { |host| ips += Resolv.getaddresses host }
@client = Lumberjack::Client.new(:addresses => ips.uniq, :port => @port,
:ssl_certificate => @ssl_certificate, :window_size => @window_size)
rescue Exception => e
@logger.error("All hosts unavailable, sleeping", :hosts => ips.uniq, :e => e,
:backtrace => e.backtrace)
sleep(10)
retry
end
end
end

119
lib/logstash/outputs/nagios.rb
View file

@ -1,119 +0,0 @@
# encoding: utf-8
require "logstash/namespace"
require "logstash/outputs/base"
# The Nagios output is used for sending passive check results to Nagios via the
# Nagios command file. This output currently supports Nagios 3.
#
# For this output to work, your event _must_ have the following Logstash event fields:
#
# * `nagios\_host`
# * `nagios\_service`
#
# These Logstash event fields are supported, but optional:
#
# * `nagios\_annotation`
# * `nagios\_level` (overrides `nagios\_level` configuration option)
#
# There are two configuration options:
#
# * `commandfile` - The location of the Nagios external command file. Defaults
# to '/var/lib/nagios3/rw/nagios.cmd'
# * `nagios\_level` - Specifies the level of the check to be sent. Defaults to
# CRITICAL and can be overriden by setting the "nagios\_level" field to one
# of "OK", "WARNING", "CRITICAL", or "UNKNOWN"
#
# output{
# if [message] =~ /(error|ERROR|CRITICAL)/ {
# nagios {
# # your config here
# }
# }
# }
#
class LogStash::Outputs::Nagios < LogStash::Outputs::Base
config_name "nagios"
milestone 2
# The full path to your Nagios command file.
config :commandfile, :validate => :path, :default => "/var/lib/nagios3/rw/nagios.cmd"
# The Nagios check level. Should be one of 0=OK, 1=WARNING, 2=CRITICAL,
# 3=UNKNOWN. Defaults to 2 - CRITICAL.
config :nagios_level, :validate => [ "0", "1", "2", "3" ], :default => "2"
public
def register
# nothing to do
end # def register
public
def receive(event)
return unless output?(event)
if !File.exists?(@commandfile)
@logger.warn("Skipping nagios output; command file is missing",
:commandfile => @commandfile, :missed_event => event)
return
end
# TODO(petef): if nagios_host/nagios_service both have more than one
# value, send multiple alerts. They will have to match up together by
# array indexes (host/service combos) and the arrays must be the same
# length.
host = event["nagios_host"]
if !host
@logger.warn("Skipping nagios output; nagios_host field is missing",
:missed_event => event)
return
end
service = event["nagios_service"]
if !service
@logger.warn("Skipping nagios output; nagios_service field is missing",
"missed_event" => event)
return
end
annotation = event["nagios_annotation"]
level = @nagios_level
if event["nagios_level"]
event_level = [*event["nagios_level"]]
case event_level[0].downcase
when "ok"
level = "0"
when "warning"
level = "1"
when "critical"
level = "2"
when "unknown"
level = "3"
else
@logger.warn("Invalid Nagios level. Defaulting to CRITICAL", :data => event_level)
end
end
cmd = "[#{Time.now.to_i}] PROCESS_SERVICE_CHECK_RESULT;#{host};#{service};#{level};"
if annotation
cmd += "#{annotation}: "
end
# In the multi-line case, escape the newlines for the nagios command file
cmd += (event["message"] || "<no message>").gsub("\n", "\\n")
@logger.debug("Opening nagios command file", :commandfile => @commandfile,
:nagios_command => cmd)
begin
File.open(@commandfile, "r+") do |f|
f.puts(cmd)
f.flush # TODO(sissel): probably don't need this.
end
rescue => e
@logger.warn("Skipping nagios output; error writing to command file",
:commandfile => @commandfile, :missed_event => event,
:exception => e, :backtrace => e.backtrace)
end
end # def receive
end # class LogStash::Outputs::Nagios

132
lib/logstash/outputs/nagios_nsca.rb
View file

@ -1,132 +0,0 @@
# encoding: utf-8
require "logstash/outputs/base"
require "logstash/namespace"
require "open3"
# The nagios_nsca output is used for sending passive check results to Nagios
# through the NSCA protocol.
#
# This is useful if your Nagios server is not the same as the source host from
# where you want to send logs or alerts. If you only have one server, this
# output is probably overkill # for you, take a look at the 'nagios' output
# instead.
#
# Here is a sample config using the nagios_nsca output:
# output {
# nagios_nsca {
# # specify the hostname or ip of your nagios server
# host => "nagios.example.com"
#
# # specify the port to connect to
# port => 5667
# }
# }
class LogStash::Outputs::NagiosNsca < LogStash::Outputs::Base
config_name "nagios_nsca"
milestone 1
# The status to send to nagios. Should be 0 = OK, 1 = WARNING, 2 = CRITICAL, 3 = UNKNOWN
config :nagios_status, :validate => :string, :required => true
# The nagios host or IP to send logs to. It should have a NSCA daemon running.
config :host, :validate => :string, :default => "localhost"
# The port where the NSCA daemon on the nagios host listens.
config :port, :validate => :number, :default => 5667
# The path to the 'send_nsca' binary on the local host.
config :send_nsca_bin, :validate => :path, :default => "/usr/sbin/send_nsca"
# The path to the send_nsca config file on the local host.
# Leave blank if you don't want to provide a config file.
config :send_nsca_config, :validate => :path
# The nagios 'host' you want to submit a passive check result to. This
# parameter accepts interpolation, e.g. you can use @source_host or other
# logstash internal variables.
config :nagios_host, :validate => :string, :default => "%{host}"
# The nagios 'service' you want to submit a passive check result to. This
# parameter accepts interpolation, e.g. you can use @source_host or other
# logstash internal variables.
config :nagios_service, :validate => :string, :default => "LOGSTASH"
# The format to use when writing events to nagios. This value
# supports any string and can include %{name} and other dynamic
# strings.
config :message_format, :validate => :string, :default => "%{@timestamp} %{host}: %{message}"
public
def register
#nothing for now
end
public
def receive(event)
# exit if type or tags don't match
return unless output?(event)
# catch logstash shutdown
if event == LogStash::SHUTDOWN
finished
return
end
# skip if 'send_nsca' binary doesn't exist
if !File.exists?(@send_nsca_bin)
@logger.warn("Skipping nagios_nsca output; send_nsca_bin file is missing",
"send_nsca_bin" => @send_nsca_bin, "missed_event" => event)
return
end
# interpolate params
nagios_host = event.sprintf(@nagios_host)
nagios_service = event.sprintf(@nagios_service)
# escape basic things in the log message
# TODO: find a way to escape the message correctly
msg = event.sprintf(@message_format)
msg.gsub!("\n", "<br/>")
msg.gsub!("'", "&#146;")
status = event.sprintf(@nagios_status)
if status.to_i.to_s != status # Check it round-trips to int correctly
msg = "status '#{status}' is not numeric"
status = 2
else
status = status.to_i
if status > 3 || status < 0
msg "status must be > 0 and <= 3, not #{status}"
status = 2
end
end
# build the command
# syntax: echo '<server>!<nagios_service>!<status>!<text>' | \
# /usr/sbin/send_nsca -H <nagios_host> -d '!' -c <nsca_config>"
cmd = [@send_nsca_bin, "-H", @host, "-p", @port, "-d", "~"]
cmd = cmd + ["-c", @send_nsca_config] if @send_nsca_config
message = "#{nagios_host}~#{nagios_service}~#{status}~#{msg}"
@logger.debug("Running send_nsca command", :nagios_nsca_command => cmd.join(" "), :message => message)
begin
Open3.popen3(*cmd) do |i, o, e|
i.puts(message)
i.close
end
rescue => e
@logger.warn(
"Skipping nagios_nsca output; error calling send_nsca",
:error => $!,
:nagios_nsca_command => cmd.join(" "),
:message => message,
:missed_event => event
)
@logger.debug("Backtrace", :backtrace => e.backtrace)
end
end # def receive
end # class LogStash::Outputs::NagiosNsca

Some files were not shown because too many files have changed in this diff Show more