mirror of
https://github.com/elastic/logstash.git
synced 2025-04-24 06:37:19 -04:00
Merge branch 'master' of github.com:logstash/logstash
This commit is contained in:
Jordan Sissel
commit
976e9952dd
5 changed files with 134 additions and 335 deletions
1
Gemfile
1
Gemfile
|
|
@ -24,6 +24,7 @@ gem "gelf" # outputs/gelf, # License: MIT-style
|
|||
gem "statsd-ruby", "~> 0.3.0" # outputs/statsd, # License: As-Is
|
||||
gem "gmetric", "~> 0.1.3" # outputs/ganglia, # License: MIT
|
||||
gem "xmpp4r", "~> 0.5" # outputs/xmpp, # License: As-Is
|
||||
gem "gelfd", "~> 0.1.0" #inputs/gelf, # License: Apache 2.0
|
||||
|
||||
# For testing/dev
|
||||
group :development do
|
||||
|
|
|
|||
37
Makefile
37
Makefile
|
|
@ -1,23 +1,31 @@
|
|||
VERSION=$(shell ruby -r./VERSION -e 'puts LOGSTASH_VERSION')
|
||||
|
||||
JRUBY_VERSION=1.6.4
|
||||
JRUBY_CMD=build/jruby/jruby-$(JRUBY_VERSION)/bin/jruby
|
||||
WITH_JRUBY=$(JRUBY_CMD) -S
|
||||
VERSION=$(shell ruby -r./VERSION -e 'puts LOGSTASH_VERSION')
|
||||
JRUBY_URL=http://repository.codehaus.org/org/jruby/jruby-complete/$(JRUBY_VERSION)
|
||||
JRUBY=vendor/jar/jruby-complete-$(JRUBY_VERSION).jar
|
||||
JRUBYC=java -Djruby.compat.version=RUBY1_9 -jar $(PWD)/$(JRUBY) -S jrubyc
|
||||
ELASTICSEARCH_VERSION=0.17.6
|
||||
ELASTICSEARCH_URL=http://github.com/downloads/elasticsearch/elasticsearch
|
||||
ELASTICSEARCH=vendor/jar/elasticsearch-$(ELASTICSEARCH_VERSION)
|
||||
|
||||
PLUGIN_FILES=$(shell git ls-files | egrep '^lib/logstash/(inputs|outputs|filters)/' | egrep -v '/base.rb$$')
|
||||
|
||||
# OS-specific options
|
||||
UNAME := $(shell uname)
|
||||
ifeq ($(UNAME), Darwin)
|
||||
TAR_OPTS=
|
||||
else
|
||||
TAR_OPTS=--wildcards
|
||||
endif
|
||||
|
||||
default: jar
|
||||
|
||||
.PHONY: pre-flight-check
|
||||
pre-flight-check: check-ruby-is-jruby
|
||||
|
||||
.PHONY: check-ruby-is-jruby
|
||||
check-ruby-is-jruby:
|
||||
ruby -e 'if RUBY_ENGINE != "jruby"; puts "JRuby is required to build."; exit 1; else; puts "JRuby OK"; end'
|
||||
check-ruby-is-jruby:
|
||||
$(JRUBY_CMD) -e 'if RUBY_ENGINE != "jruby"; puts "JRuby is required to build."; exit 1; else; puts "JRuby OK"; end'
|
||||
|
||||
debug:
|
||||
echo $(JRUBY)
|
||||
|
|
@ -57,10 +65,9 @@ vendor:
|
|||
vendor/jar: | vendor pre-flight-check
|
||||
mkdir $@
|
||||
|
||||
.PHONY: build-jruby
|
||||
build-jruby: $(JRUBY)
|
||||
|
||||
$(JRUBY): build/jruby/jruby-1.6.4/lib/jruby-complete.jar | vendor/jar
|
||||
$(JRUBY): build/jruby/jruby-$(JRUBY_VERSION)/lib/jruby-complete.jar | vendor/jar
|
||||
cp $< $@
|
||||
|
||||
build/jruby: build
|
||||
|
|
@ -72,7 +79,7 @@ build/jruby/jruby-1.6.4/lib/jruby-complete.jar: build/jruby/jruby-$(JRUBY_VERSIO
|
|||
(cd $<; ant jar-jruby-complete)
|
||||
|
||||
build/jruby/jruby-$(JRUBY_VERSION): build/jruby/jruby-src-$(JRUBY_VERSION).tar.gz
|
||||
tar -C build/jruby/ -zxf $<
|
||||
tar -C build/jruby/ $(TAR_OPTS) -zxf $<
|
||||
# Build jruby from source targeted at 1.9
|
||||
|
||||
build/jruby/jruby-src-$(JRUBY_VERSION).tar.gz: | build/jruby
|
||||
|
|
@ -88,7 +95,7 @@ vendor/jar/elasticsearch-$(ELASTICSEARCH_VERSION).tar.gz: | vendor/jar
|
|||
vendor-elasticsearch: $(ELASTICSEARCH)
|
||||
$(ELASTICSEARCH): $(ELASTICSEARCH).tar.gz | vendor/jar
|
||||
@echo "Pulling the jars out of $<"
|
||||
tar -C $(shell dirname $@) -xf $< --exclude '*sigar*' --wildcards \
|
||||
tar -C $(shell dirname $@) -xf $< $(TAR_OPTS) --exclude '*sigar*' \
|
||||
'elasticsearch-$(ELASTICSEARCH_VERSION)/lib/*.jar'
|
||||
|
||||
# Always run vendor/bundle
|
||||
|
|
@ -102,7 +109,7 @@ vendor-gems: | vendor/bundle
|
|||
.PHONY: vendor/bundle
|
||||
vendor/bundle: | fix-bundler
|
||||
@echo "=> Installing gems to vendor/bundle/..."
|
||||
bundle install --path vendor/bundle
|
||||
$(WITH_JRUBY) bundle install --deployment
|
||||
|
||||
gem: logstash-$(VERSION).gem
|
||||
|
||||
|
|
@ -136,12 +143,13 @@ build/monolith: compile copy-ruby-files
|
|||
|
||||
# Learned how to do pack gems up into the jar mostly from here:
|
||||
# http://blog.nicksieger.com/articles/2009/01/10/jruby-1-1-6-gems-in-a-jar
|
||||
VENDOR_DIR := `ls -d vendor/bundle/*ruby/*`
|
||||
jar: build/logstash-$(VERSION)-monolithic.jar
|
||||
build/logstash-$(VERSION)-monolithic.jar: | build/monolith
|
||||
build/logstash-$(VERSION)-monolithic.jar: JAR_ARGS=-C build/ruby .
|
||||
build/logstash-$(VERSION)-monolithic.jar: JAR_ARGS+=-C build/monolith .
|
||||
build/logstash-$(VERSION)-monolithic.jar: JAR_ARGS+=-C vendor/bundle/jruby/1.8 gems
|
||||
build/logstash-$(VERSION)-monolithic.jar: JAR_ARGS+=-C vendor/bundle/jruby/1.8 specifications
|
||||
build/logstash-$(VERSION)-monolithic.jar: JAR_ARGS+=-C $(VENDOR_DIR) gems
|
||||
build/logstash-$(VERSION)-monolithic.jar: JAR_ARGS+=-C $(VENDOR_DIR) specifications
|
||||
build/logstash-$(VERSION)-monolithic.jar: JAR_ARGS+=-C lib logstash/web/public
|
||||
build/logstash-$(VERSION)-monolithic.jar: JAR_ARGS+=-C lib logstash/web/views
|
||||
build/logstash-$(VERSION)-monolithic.jar: JAR_ARGS+=patterns
|
||||
|
|
@ -151,7 +159,7 @@ build/logstash-$(VERSION)-monolithic.jar:
|
|||
|
||||
.PHONY: test
|
||||
test:
|
||||
ruby bin/logstash test
|
||||
$(JRUBY_CMD) bin/logstash test
|
||||
|
||||
.PHONY: docs
|
||||
docs: docgen doccopy docindex
|
||||
|
|
@ -167,6 +175,7 @@ build/docs: build
|
|||
build/docs/inputs build/docs/filters build/docs/outputs: | build/docs
|
||||
-mkdir $@
|
||||
|
||||
# bluecloth gem doesn't work on jruby. Use ruby.
|
||||
build/docs/inputs/%.html: lib/logstash/inputs/%.rb | build/docs/inputs
|
||||
ruby docs/docgen.rb -o build/docs $<
|
||||
build/docs/filters/%.html: lib/logstash/filters/%.rb | build/docs/filters
|
||||
|
|
@ -180,7 +189,7 @@ build/docs/%: docs/%
|
|||
|
||||
build/docs/index.html: $(addprefix build/docs/,$(subst lib/logstash/,,$(subst .rb,.html,$(PLUGIN_FILES))))
|
||||
build/docs/index.html: docs/generate_index.rb
|
||||
ruby $< build/docs > $@
|
||||
$(JRUBY_CMD) $< build/docs > $@
|
||||
|
||||
publish: | gem
|
||||
gem push logstash-$(VERSION).gem
|
||||
|
|
|
|||
320
Rakefile
320
Rakefile
|
|
@ -1,320 +0,0 @@
|
|||
require "tempfile"
|
||||
require "fileutils"
|
||||
|
||||
require File.join(File.dirname(__FILE__), "VERSION") # For LOGSTASH_VERSION
|
||||
|
||||
# Compile config grammar (ragel -> ruby)
|
||||
file "lib/logstash/config/grammar.rb" => ["lib/logstash/config/grammar.rl"] do
|
||||
sh "make -C lib/logstash/config grammar.rb"
|
||||
end
|
||||
|
||||
# Taken from 'jrubyc'
|
||||
# Currently this code is commented out because jruby emits this:
|
||||
# Failure during compilation of file logstash/web/helpers/require_param.rb:
|
||||
# java.lang.RuntimeException: java.io.FileNotFoundException: File path
|
||||
# /home/jls/projects/logstash/logstash/web/helpers/require_param.rb
|
||||
# does not start with parent path /home/jls/projects/logstash/lib
|
||||
#
|
||||
# org/jruby/util/JavaNameMangler.java:105:in `mangleFilenameForClasspath'
|
||||
# org/jruby/util/JavaNameMangler.java:32:in `mangleFilenameForClasspath'
|
||||
#require 'jruby/jrubyc'
|
||||
##args = [ "-p", "net.logstash" ]
|
||||
#args = ["-d", "build"]
|
||||
#args += Dir.glob("**/*.rb")
|
||||
#status = JRuby::Compiler::compile_argv(args)
|
||||
#if (status != 0)
|
||||
#puts "Compilation FAILED: #{status} error(s) encountered"
|
||||
#exit status
|
||||
#end
|
||||
|
||||
task :clean do
|
||||
sh "rm -rf .bundle || true"
|
||||
#sh "rm -rf build-jar-thin"
|
||||
#sh "rm -rf build-jar"
|
||||
sh "rm -rf build"
|
||||
sh "rm -rf vendor"
|
||||
end
|
||||
|
||||
task :compile => "lib/logstash/config/grammar.rb" do |t|
|
||||
target = "build/ruby"
|
||||
mkdir_p target if !File.directory?(target)
|
||||
#sh "rm -rf lib/net"
|
||||
Dir.chdir("lib") do
|
||||
rel_target = File.join("..", target)
|
||||
sh "jrubyc", "-t", rel_target, "logstash/runner.rb"
|
||||
files = Dir.glob("**/*.rb")
|
||||
files.each do |file|
|
||||
d = File.join(rel_target, File.dirname(file))
|
||||
mkdir_p d if !File.directory?(d)
|
||||
cp file, File.join(d, File.basename(file))
|
||||
end
|
||||
end
|
||||
|
||||
Dir.chdir("test") do
|
||||
rel_target = File.join("..", target)
|
||||
files = Dir.glob("**/*.rb")
|
||||
files.each do |file|
|
||||
d = File.join(rel_target, File.dirname(file))
|
||||
mkdir_p d if !File.directory?(d)
|
||||
cp file, File.join(d, File.basename(file))
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
task :jar => [ "package:monolith:jar" ] do |t|
|
||||
# Nothing
|
||||
end
|
||||
|
||||
VERSIONS = {
|
||||
:jruby => "1.6.3", # Any of CPL1.0/GPL2.0/LGPL2.1 ? Confusing, but OK.
|
||||
:elasticsearch => "0.17.6", # Apache 2.0 license
|
||||
|
||||
# We don't need to include joda ourselves, JRuby ships with it :)
|
||||
#:joda => "1.6.2", # Apache 2.0 license
|
||||
}
|
||||
|
||||
namespace :vendor do
|
||||
file "vendor/jar" do |t|
|
||||
mkdir_p t.name if !File.directory?(t.name)
|
||||
end
|
||||
|
||||
# Download jruby.jar
|
||||
file "vendor/jar/jruby-complete-#{VERSIONS[:jruby]}.jar" => "vendor/jar" do |t|
|
||||
baseurl = "http://repository.codehaus.org/org/jruby/jruby-complete"
|
||||
if !File.exists?(t.name)
|
||||
sh "wget -O #{t.name} #{baseurl}/#{VERSIONS[:jruby]}/#{File.basename(t.name)}"
|
||||
end
|
||||
end # jruby
|
||||
|
||||
task :jruby => "vendor/jar/jruby-complete-#{VERSIONS[:jruby]}.jar" do
|
||||
# nothing to do, the dep does it.
|
||||
end
|
||||
|
||||
# Download elasticsearch + deps (for outputs/elasticsearch)
|
||||
task :elasticsearch => "vendor/jar" do
|
||||
version = VERSIONS[:elasticsearch]
|
||||
tarball = "elasticsearch-#{version}.tar.gz"
|
||||
url = "http://github.com/downloads/elasticsearch/elasticsearch/#{tarball}"
|
||||
if !File.exists?(tarball)
|
||||
# --no-check-certificate is for github and wget not supporting wildcard
|
||||
# certs sanely.
|
||||
sh "wget -O #{tarball} --no-check-certificate #{url}"
|
||||
end
|
||||
|
||||
sh "tar -zxf #{tarball} -C vendor/jar/ elasticsearch-#{version}/lib"
|
||||
end # elasticsearch
|
||||
|
||||
task :joda => "vendor/jar" do
|
||||
version = VERSIONS[:joda]
|
||||
baseurl = "http://sourceforge.net/projects/joda-time/files/joda-time"
|
||||
tarball = "joda-time-#{version}-bin.tar.gz"
|
||||
url = "#{baseurl}/#{version}/#{tarball}/download"
|
||||
|
||||
if !File.exists?(tarball)
|
||||
sh "wget -O #{tarball} #{url}"
|
||||
end
|
||||
|
||||
sh "tar -zxf #{tarball} -C vendor/jar/ joda-time-#{version}/joda-time-#{version}.jar"
|
||||
end # joda
|
||||
|
||||
task :gems => "vendor/jar" do
|
||||
puts "=> Installing gems to vendor/bundle/..."
|
||||
sh "bundle install --path #{File.join("vendor", "bundle")}"
|
||||
end
|
||||
end # vendor namespace
|
||||
|
||||
namespace :package do
|
||||
task :gem do
|
||||
sh "gem build logstash.gemspec"
|
||||
end
|
||||
|
||||
monolith_deps = [ "vendor:jruby", "vendor:gems", "vendor:elasticsearch", "compile" ]
|
||||
|
||||
namespace :monolith do
|
||||
|
||||
task :jar => monolith_deps do
|
||||
builddir = "build/monolith-jar"
|
||||
mkdir_p builddir if !File.directory?(builddir)
|
||||
|
||||
# Unpack all the 3rdparty jars and any jars in gems
|
||||
Dir.glob("vendor/{bundle,jar}/**/*.jar").each do |jar|
|
||||
if jar =~ /sigar.*\.jar$/
|
||||
puts "=> Skipping #{jar} (sigar not needed)"
|
||||
next
|
||||
end
|
||||
|
||||
puts "=> Unpacking #{jar} into #{builddir}/"
|
||||
relative_path = File.join(builddir.split(File::SEPARATOR).collect { |a| ".." })
|
||||
Dir.chdir(builddir) do
|
||||
sh "jar xf #{relative_path}/#{jar}"
|
||||
end
|
||||
end
|
||||
|
||||
# We compile stuff to build/...
|
||||
# TODO(sissel): Could probably just use 'jar uf' for this?
|
||||
Dir.glob("build/ruby/**/*.class").each do |file|
|
||||
target = File.join(builddir, file.gsub("build/ruby/", ""))
|
||||
mkdir_p File.dirname(target)
|
||||
puts "=> Copying #{file} => #{target}"
|
||||
FileUtils.copy(file, target)
|
||||
end
|
||||
|
||||
# Purge any extra files we don't need in META-INF (like manifests and
|
||||
# jar signatures)
|
||||
["INDEX.LIST", "MANIFEST.MF", "ECLIPSEF.RSA", "ECLIPSEF.SF"].each do |file|
|
||||
File.delete(File.join(builddir, "META-INF", file)) rescue nil
|
||||
end
|
||||
|
||||
output = "logstash-#{LOGSTASH_VERSION}-monolithic.jar"
|
||||
sh "jar cfe #{output} logstash.runner -C #{builddir} ."
|
||||
|
||||
jar_update_args = []
|
||||
|
||||
# Learned how to do this mostly from here:
|
||||
# http://blog.nicksieger.com/articles/2009/01/10/jruby-1-1-6-gems-in-a-jar
|
||||
#
|
||||
# Add bundled gems to the jar
|
||||
# Skip the 'cache' dir which is just the original .gem files
|
||||
gem_dirs = %w{bin doc gems specifications}
|
||||
gem_root = File.join(%w{vendor bundle jruby 1.8})
|
||||
# for each dir, build args: -C vendor/bundle/jruby/1.8 bin, etc
|
||||
gem_jar_args = gem_dirs.collect { |d| ["-C", gem_root, d ] }.flatten
|
||||
jar_update_args += gem_jar_args
|
||||
|
||||
# Add compiled our compiled ruby code
|
||||
jar_update_args += %w{ -C build/ruby . }
|
||||
|
||||
# Add web stuff
|
||||
jar_update_args += %w{ -C lib logstash/web/public }
|
||||
jar_update_args += %w{ -C lib logstash/web/views }
|
||||
|
||||
# Add test code
|
||||
#jar_update_args += %w{ -C test logstsah }
|
||||
|
||||
# Add grok patterns
|
||||
jar_update_args << "patterns"
|
||||
|
||||
# Update with other files and also build an index.
|
||||
sh "jar uf #{output} #{jar_update_args.join(" ")}"
|
||||
sh "jar i #{output}"
|
||||
end # task package:monolith:jar
|
||||
end # namespace monolith
|
||||
|
||||
task :jar => [ "vendor:jruby", "vendor:gems", "compile" ] do
|
||||
builddir = "build/thin-jar"
|
||||
mkdir_p builddir if !File.directory?(builddir)
|
||||
|
||||
# Unpack jruby
|
||||
relative_path = File.join(builddir.split(File::SEPARATOR).collect { |a| ".." })
|
||||
Dir.glob("vendor/jar/jruby-complete-1.6.3.jar").each do |jar|
|
||||
puts "=> Unpacking #{jar} into #{builddir}/"
|
||||
Dir.chdir(builddir) do
|
||||
sh "jar xf #{relative_path}/#{jar}"
|
||||
end
|
||||
end
|
||||
|
||||
["INDEX.LIST", "MANIFEST.MF", "ECLIPSEF.RSA", "ECLIPSEF.SF"].each do |file|
|
||||
File.delete(File.join(builddir, "META-INF", file)) rescue nil
|
||||
end
|
||||
|
||||
output = "logstash-#{LOGSTASH_VERSION}.jar"
|
||||
sh "jar cfe #{output} logstash.runner -C #{builddir} ."
|
||||
|
||||
jar_update_args = []
|
||||
|
||||
# Learned how to do this mostly from here:
|
||||
# http://blog.nicksieger.com/articles/2009/01/10/jruby-1-1-6-gems-in-a-jar
|
||||
#
|
||||
# Add bundled gems to the jar
|
||||
# Skip the 'cache' dir which is just the original .gem files
|
||||
gem_dirs = %w{bin doc gems specifications}
|
||||
gem_root = File.join(%w{vendor bundle jruby 1.8})
|
||||
# for each dir, build args: -C vendor/bundle/jruby/1.8 bin, etc
|
||||
gem_jar_args = gem_dirs.collect { |dir| ["-C", gem_root, dir ] }.flatten
|
||||
jar_update_args += gem_jar_args
|
||||
|
||||
# Add compiled our compiled ruby code
|
||||
jar_update_args += %w{ -C build . }
|
||||
|
||||
# Add web stuff
|
||||
jar_update_args += %w{ -C lib logstash/web/public }
|
||||
jar_update_args += %w{ -C lib logstash/web/views }
|
||||
|
||||
# Add test code
|
||||
#jar_update_args += %w{ -C test logstsah }
|
||||
|
||||
# Add grok patterns
|
||||
jar_update_args << "patterns"
|
||||
|
||||
# Update with other files and also build an index.
|
||||
sh "jar uf #{output} #{jar_update_args.join(" ")}"
|
||||
sh "jar i #{output}"
|
||||
end # task package:jar
|
||||
end # namespace package
|
||||
|
||||
task :test do
|
||||
sh "cd test; ruby logstash_test_runner.rb"
|
||||
end
|
||||
|
||||
task :docs => [:docgen, :doccopy, :docindex ] do
|
||||
end
|
||||
|
||||
task :require_output_env do
|
||||
if ENV["output"].nil?
|
||||
raise "No output variable set. Run like: 'rake docs output=path/to/output'"
|
||||
end
|
||||
end
|
||||
|
||||
task :doccopy => [:require_output_env] do
|
||||
if ENV["output"].nil?
|
||||
raise "No output variable set. Run like: 'rake docs output=path/to/output'"
|
||||
end
|
||||
output = ENV["output"].gsub("VERSION", LOGSTASH_VERSION)
|
||||
|
||||
Dir.glob("docs/**/*").each do |doc|
|
||||
dir = File.join(output, File.dirname(doc).gsub(/docs\/?/, ""))
|
||||
mkdir_p dir if !File.directory?(dir)
|
||||
if File.directory?(doc)
|
||||
mkdir_p doc
|
||||
else
|
||||
puts "Copy #{doc} => #{dir}"
|
||||
system("sed -re 's/%VERSION%/#{LOGSTASH_VERSION}/' #{doc} > #{dir}/#{File.basename(doc)}")
|
||||
#cp(doc, dir)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
task :docindex => [:require_output_env] do
|
||||
output = ENV["output"].gsub("VERSION", LOGSTASH_VERSION)
|
||||
sh "ruby docs/generate_index.rb #{output} > #{output}/index.html"
|
||||
end
|
||||
|
||||
task :docgen => [:require_output_env] do
|
||||
if ENV["output"].nil?
|
||||
raise "No output variable set. Run like: 'rake docgen output=path/to/output'"
|
||||
end
|
||||
output = ENV["output"].gsub("VERSION", LOGSTASH_VERSION)
|
||||
|
||||
sh "find lib/logstash/inputs lib/logstash/filters lib/logstash/outputs -type f -not -name 'base.rb' -a -name '*.rb'| xargs ruby docs/docgen.rb -o #{output}"
|
||||
end
|
||||
|
||||
task :publish do
|
||||
latest_gem = %x{ls -t logstash-[0-9]*.gem}.split("\n").first
|
||||
sh "gem push #{latest_gem}"
|
||||
end
|
||||
|
||||
task :release do
|
||||
docs_dir = File.join(File.dirname(__FILE__), "..", "logstash.github.com",
|
||||
"docs", LOGSTASH_VERSION)
|
||||
ENV["output"] = docs_dir
|
||||
sh "sed -i -Re 's/1.0.[0-9]/#{LOGSTASH_VERSION}/'"
|
||||
sh "git tag v#{LOGSTASH_VERSION}"
|
||||
#Rake::Task["docs"].invoke
|
||||
Rake::Task["package:gem"].invoke
|
||||
Rake::Task["package:monolith:jar"].invoke
|
||||
|
||||
puts "Packaging complete."
|
||||
|
||||
puts "Run the following under ruby 1.8.7 (require bluecloth)"
|
||||
puts "> rake docs output=#{docs_dir}"
|
||||
end
|
||||
|
|
@ -34,5 +34,6 @@ name, like --grok-foo.
|
|||
<dt> --port PORT </dt>
|
||||
<dd> Port on which to start webserver. Default is 9292. </dd>
|
||||
<dt> --backend URL </dt>
|
||||
<dd> The backend URL to use. Default is elasticsearch://localhost:9200/ </dd>
|
||||
<dd>The backend URL to use. Default is elasticsearch:/// (assumes multicast discovery).
|
||||
You can specify elasticsearch://[host][:port]/[clustername]</dd>
|
||||
</dl>
|
||||
|
|
|
|||
108
lib/logstash/inputs/gelf.rb
Normal file
108
lib/logstash/inputs/gelf.rb
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
require "date"
|
||||
require "logstash/inputs/base"
|
||||
require "logstash/namespace"
|
||||
require "logstash/time" # should really use the filters/date.rb bits
|
||||
require "socket"
|
||||
|
||||
# Read gelf messages as events over the network.
|
||||
#
|
||||
# This input is a good choice if you already use graylog2 today.
|
||||
#
|
||||
# The main reasoning for this input is to leverage existing GELF
|
||||
# logging libraries such as the gelf log4j appender
|
||||
#
|
||||
class LogStash::Inputs::Gelf < LogStash::Inputs::Base
|
||||
config_name "gelf"
|
||||
|
||||
# The address to listen on
|
||||
config :host, :validate => :string, :default => "0.0.0.0"
|
||||
|
||||
# The port to listen on. Remember that ports less than 1024 (privileged
|
||||
# ports) may require root to use.
|
||||
config :port, :validate => :number, :default => 12201
|
||||
|
||||
# Whether or not to remap the gelf message fields
|
||||
# to logstash event fields or leave them
|
||||
# intact.
|
||||
#
|
||||
# Default is true
|
||||
#
|
||||
# Remapping converts the following:
|
||||
# full_message => event.message
|
||||
# timestamp => event.timestamp
|
||||
# host + file => event.source
|
||||
config :remap, :validate => :boolean, :default => true
|
||||
|
||||
public
|
||||
def initialize(params)
|
||||
super
|
||||
BasicSocket.do_not_reverse_lookup = true
|
||||
|
||||
# nothing else makes sense here
|
||||
# gelf messages ARE json
|
||||
@format = ["json"]
|
||||
end # def initialize
|
||||
|
||||
public
|
||||
def register
|
||||
require 'gelfd'
|
||||
end # def register
|
||||
|
||||
public
|
||||
def run(output_queue)
|
||||
# udp server
|
||||
Thread.new do
|
||||
LogStash::Util::set_thread_name("input|gelf")
|
||||
begin
|
||||
udp_listener(output_queue)
|
||||
rescue => e
|
||||
@logger.warn("gelf listener died: #{$!}")
|
||||
@logger.debug(["Backtrace", e.backtrace])
|
||||
sleep(5)
|
||||
retry
|
||||
end # begin
|
||||
end # Thread.new
|
||||
end # def run
|
||||
|
||||
private
|
||||
def udp_listener(output_queue)
|
||||
@logger.info("Starting gelf listener on #{@host}:#{@port}")
|
||||
|
||||
if @udp
|
||||
@udp.close_read
|
||||
@udp.close_write
|
||||
end
|
||||
|
||||
@udp = UDPSocket.new(Socket::AF_INET)
|
||||
@udp.bind(@host, @port)
|
||||
|
||||
loop do
|
||||
line, client = @udp.recvfrom(8192)
|
||||
# Ruby uri sucks, so don't use it.
|
||||
source = "gelf://#{client[3]}/"
|
||||
data = Gelfd::Parser.parse(line)
|
||||
# The nil guard is needed
|
||||
# to deal with chunked messages.
|
||||
# Gelfd::Parser.parse will only return the message
|
||||
# when all chunks are completed
|
||||
e = to_event(data, source) unless data.nil?
|
||||
if e
|
||||
remap_gelf(e) if @remap
|
||||
output_queue << e
|
||||
end
|
||||
end
|
||||
ensure
|
||||
if @udp
|
||||
@udp.close_read rescue nil
|
||||
@udp.close_write rescue nil
|
||||
end
|
||||
end # def udp_listener
|
||||
|
||||
private
|
||||
def remap_gelf(event)
|
||||
event.message = event.fields["full_message"]
|
||||
event.timestamp = LogStash::Time.to_iso8601(
|
||||
DateTime.strptime(event.fields["timestamp"].to_s, "%s.%L" ))
|
||||
event.source = "gelf://#{event.fields["host"]}#{event.fields["file"]}"
|
||||
end # def remap_gelf
|
||||
end # class LogStash::Inputs::Gelf
|
||||
Loading…
Add table
Add a link
Reference in a new issue