Merge pull request #1325 from jordansissel/purge-JRUBY-6970

Purge jar-specific handling that is not needed anymore.
This commit is contained in:
Jordan Sissel 2014-05-02 11:21:29 -07:00
commit b1e1df4cc5
16 changed files with 28 additions and 387 deletions

View file

@ -1,22 +0,0 @@
# encoding: utf-8
# TODO(sissel): require "openssl" takes *ages* from the logstash jar
# TODO(sissel): monkeypatch Kernel.require to apply this monkeypatch only after
# a 'require "openssl" has occurred.
class OpenSSL::SSL::SSLContext
alias_method :ca_path_JRUBY_6970=, :ca_path=
alias_method :ca_file_JRUBY_6970=, :ca_file=
def ca_file=(arg)
if arg =~ /^jar:file:\//
return ca_file_JRUBY_6970=(arg.gsub(/^jar:/, ""))
end
return ca_file_JRUBY_6970=(arg)
end
def ca_path=(arg)
if arg =~ /^jar:file:\//
return ca_path_JRUBY_6970=(arg.gsub(/^jar:/, ""))
end
return ca_path_JRUBY_6970=(arg)
end
end

View file

@ -1,102 +0,0 @@
# encoding: utf-8
# Monkeypatch for JRUBY-6970
module Kernel
alias_method :require_JRUBY_6970_hack, :require
def require(path)
old_load_path = nil
if path =~ /^jar:file:.+!.+/
path = path.gsub(/^jar:/, "")
puts "JRUBY-6970: require(#{path})" if ENV["REQUIRE_DEBUG"] == "1"
end
# Work around slow openssl load times in flatjar. (LOGSTASH-1223)
# I don't know why this works, I don't care either. This problem only
# exists in the 'jar' builds of logstash which are going to be going away
# soon in favor of the much-better tarball/zip releases!
if __FILE__ =~ /^(?:jar:)?file:.+!.+/ && path == "openssl"
# Loading shared/jruby-openssl first seems to make openssl load faster
# I have no idea. Computers.
require_JRUBY_6970_hack "shared/jruby-openssl"
return require_JRUBY_6970_hack "openssl"
end
# JRUBY-7065
path = File.expand_path(path) if path.include?("/../")
rc = require_JRUBY_6970_hack(path)
# Only monkeypatch openssl after it's been loaded.
if path == "openssl"
require "logstash/JRUBY-6970-openssl"
end
return rc
end
end
# Work around for a bug in File.expand_path that doesn't account for resources
# in jar paths.
#
# Should solve this error:
# Exception in thread "LogStash::Runner" org.jruby.exceptions.RaiseException:
# (Errno::ENOENT) file:/home/jls/projects/logstash/build/data/unicode.data
class File
class << self
alias_method :expand_path_JRUBY_6970, :expand_path
def expand_path(path, dir=nil)
#p :expand_path => [path, dir]
if path =~ /(jar:)?file:\/.*\.jar!/
#p :expand_path_path => [path, dir]
jar, resource = path.split("!", 2)
#p :expand_path => [jar, resource]
if resource.nil? || resource == ""
# Nothing after the "!", nothing special to handle.
return expand_path_JRUBY_6970(path, dir)
else
resource = expand_path_JRUBY_6970(resource, dir)
return fix_jar_path(jar, resource)
end
elsif dir =~ /(jar:)?file:\/.*\.jar!/
jar, dir = dir.split("!", 2)
if dir.empty?
# sometimes the original dir is just 'file:/foo.jar!'
return File.join("#{jar}!", path)
end
dir = expand_path_JRUBY_6970(path, dir)
return fix_jar_path(jar, dir)
else
return expand_path_JRUBY_6970(path, dir)
end
end
end
protected
def self.fix_jar_path(jar, resource)
# TODO(sissel): use LogStash::Util::UNAME
if RbConfig::CONFIG["host_os"] == "mswin32"
# 'expand_path' on "/" will return "C:/" on windows.
# So like.. we don't want that because technically this
# is the root of the jar, not of a disk.
#puts :fix_jar_path => ["#{jar}!#{resource.gsub(/^[A-Za-z]:/, "")}"]
return "#{jar}!#{resource.gsub(/^[A-Za-z]:/, "")}"
else
return "#{jar}!#{resource}"
end
end
end
class Dir
class << self
alias_method :glob_JRUBY_6970_hack, :glob
def glob(path, flags=nil)
if path =~ /^jar:file:/
# Strip leading 'jar:' (LOGSTASH-1316)
return glob_JRUBY_6970_hack(path.gsub(/^jar:/, ""))
else
return glob_JRUBY_6970_hack(path)
end
end
end
end

View file

@ -119,22 +119,12 @@ class LogStash::Codecs::Multiline < LogStash::Codecs::Base
require "grok-pure" # rubygem 'jls-grok'
# Detect if we are running from a jarfile, pick the right path.
patterns_path = []
if __FILE__ =~ /file:\/.*\.jar!.*/
patterns_path += ["#{File.dirname(__FILE__)}/../../patterns/*"]
else
patterns_path += ["#{File.dirname(__FILE__)}/../../../patterns/*"]
end
patterns_path += ["#{File.dirname(__FILE__)}/../../../patterns/*"]
@grok = Grok.new
@patterns_dir = patterns_path.to_a + @patterns_dir
@patterns_dir.each do |path|
# Can't read relative paths from jars, try to normalize away '../'
while path =~ /file:\/.*\.jar!.*\/\.\.\//
# replace /foo/bar/../baz => /foo/baz
path = path.gsub(/[^\/]+\/\.\.\//, "")
end
if File.directory?(path)
path = File.join(path, "*")
end

View file

@ -27,5 +27,9 @@ module LogStash
def jruby?
RUBY_PLATFORM == "java"
end
def vendor_path(path)
return ::File.join(LOGSTASH_HOME, "vendor", relative_path)
end
end
end

View file

@ -64,25 +64,9 @@ class LogStash::Filters::GeoIP < LogStash::Filters::Base
def register
require "geoip"
if @database.nil?
if __FILE__ =~ /^(jar:)?file:\/.+!.+/
begin
# Running from a jar, assume GeoLiteCity.dat is at the root.
jar_path = [__FILE__.split("!").first, "/GeoLiteCity.dat"].join("!")
tmp_file = Tempfile.new('logstash-geoip')
tmp_file.write(File.read(jar_path))
tmp_file.close # this file is reaped when ruby exits
@database = tmp_file.path
rescue => ex
raise "Failed to cache, due to: #{ex}\n#{ex.backtrace}"
end
else
if File.exists?("GeoLiteCity.dat")
@database = "GeoLiteCity.dat"
elsif File.exists?("vendor/geoip/GeoLiteCity.dat")
@database = "vendor/geoip/GeoLiteCity.dat"
else
raise "You must specify 'database => ...' in your geoip filter"
end
@database = LogStash::Environment.vendor_path("geoip/GeoLiteCity.dat")
if !File.exists?(@database)
raise "You must specify 'database => ...' in your geoip filter (I looked for '#{@database}'"
end
end
@logger.info("Using geoip database", :path => @database)

View file

@ -216,11 +216,7 @@ class LogStash::Filters::Grok < LogStash::Filters::Base
# Detect if we are running from a jarfile, pick the right path.
@@patterns_path ||= Set.new
if __FILE__ =~ /file:\/.*\.jar!.*/
@@patterns_path += ["#{File.dirname(__FILE__)}/../../patterns/*"]
else
@@patterns_path += ["#{File.dirname(__FILE__)}/../../../patterns/*"]
end
@@patterns_path += ["#{File.dirname(__FILE__)}/../../../patterns/*"]
public
def initialize(params)
@ -242,13 +238,6 @@ class LogStash::Filters::Grok < LogStash::Filters::Base
@patterns_dir = @@patterns_path.to_a + @patterns_dir
@logger.info? and @logger.info("Grok patterns path", :patterns_dir => @patterns_dir)
@patterns_dir.each do |path|
# Can't read relative paths from jars, try to normalize away '../'
while path =~ /file:\/.*\.jar!.*\/\.\.\//
# replace /foo/bar/../baz => /foo/baz
path = path.gsub(/[^\/]+\/\.\.\//, "")
@logger.debug? and @logger.debug("In-jar path to read", :path => path)
end
if File.directory?(path)
path = File.join(path, "*")
end
@ -407,21 +396,6 @@ class LogStash::Filters::Grok < LogStash::Filters::Base
private
def add_patterns_from_file(path, pile)
# Check if the file path is a jar, if so, we'll have to read it ourselves
# since libgrok won't know what to do with it.
if path =~ /file:\/.*\.jar!.*/
File.new(path).each do |line|
next if line =~ /^(?:\s*#|\s*$)/
# In some cases I have seen 'file.each' yield lines with newlines at
# the end. I don't know if this is a bug or intentional, but we need
# to chomp it.
name, pattern = line.chomp.split(/\s+/, 2)
@logger.debug? and @logger.debug("Adding pattern from file", :name => name,
:pattern => pattern, :path => path)
pile.add_pattern(name, pattern)
end
else
pile.add_patterns_from_file(path)
end
pile.add_patterns_from_file(path)
end # def add_patterns_from_file
end # class LogStash::Filters::Grok

View file

@ -102,11 +102,7 @@ class LogStash::Filters::Multiline < LogStash::Filters::Base
# Detect if we are running from a jarfile, pick the right path.
@@patterns_path = Set.new
if __FILE__ =~ /file:\/.*\.jar!.*/
@@patterns_path += ["#{File.dirname(__FILE__)}/../../patterns/*"]
else
@@patterns_path += ["#{File.dirname(__FILE__)}/../../../patterns/*"]
end
@@patterns_path += ["#{File.dirname(__FILE__)}/../../../patterns/*"]
public
def initialize(config = {})
@ -127,12 +123,6 @@ class LogStash::Filters::Multiline < LogStash::Filters::Base
@patterns_dir = @@patterns_path.to_a + @patterns_dir
@patterns_dir.each do |path|
# Can't read relative paths from jars, try to normalize away '../'
while path =~ /file:\/.*\.jar!.*\/\.\.\//
# replace /foo/bar/../baz => /foo/baz
path = path.gsub(/[^\/]+\/\.\.\//, "")
end
if File.directory?(path)
path = File.join(path, "*")
end

View file

@ -44,14 +44,7 @@ class LogStash::Filters::UserAgent < LogStash::Filters::Base
@parser = UserAgentParser::Parser.new()
rescue Exception => e
begin
if __FILE__ =~ /file:\/.*\.jar!/
# Running from a flatjar which has a different layout
regexes_file = [__FILE__.split("!").first, "/vendor/ua-parser/regexes.yaml"].join("!")
@parser = UserAgentParser::Parser.new(:patterns_path => regexes_file)
else
# assume operating from the git checkout
@parser = UserAgentParser::Parser.new(:patterns_path => "vendor/ua-parser/regexes.yaml")
end
@parser = UserAgentParser::Parser.new(:patterns_path => "vendor/ua-parser/regexes.yaml")
rescue => ex
raise "Failed to cache, due to: #{ex}\n"
end

View file

@ -116,25 +116,12 @@ class LogStash::Inputs::Collectd < LogStash::Inputs::Base
def register
@udp = nil
if @typesdb.nil?
if __FILE__ =~ /^file:\/.+!.+/
begin
# Running from a jar, assume types.db is at the root.
jar_path = [__FILE__.split("!").first, "/types.db"].join("!")
@typesdb = [jar_path]
rescue => ex
raise "Failed to cache, due to: #{ex}\n#{ex.backtrace}"
end
else
if File.exists?("types.db")
@typesdb = ["types.db"]
elsif File.exists?("vendor/collectd/types.db")
@typesdb = ["vendor/collectd/types.db"]
else
raise "You must specify 'typesdb => ...' in your collectd input"
end
@typesdb = LogStash::Environment.vendor_path("collectd/types.db")
if !File.exists?(@typesdb)
raise "You must specify 'typesdb => ...' in your collectd input (I looked for '#{@typesdb}')"
end
@logger.info("Using internal types.db", :typesdb => @typesdb.to_s)
end
@logger.info("Using internal types.db", :typesdb => @typesdb.to_s)
if ([SECURITY_SIGN, SECURITY_ENCR].include?(@security_level))
if @authfile.nil?

View file

@ -33,7 +33,6 @@ class LogStash::Inputs::EventLog < LogStash::Inputs::Base
@logger.info("Registering input eventlog://#{@hostname}/#{@logfile}")
if RUBY_PLATFORM == "java"
require "logstash/inputs/eventlog/racob_fix"
require "jruby-win32ole"
else
require "win32ole"

View file

@ -1,44 +0,0 @@
# encoding: utf-8
# The jruby-win32ole gem uses 'java.lang.System.set_property' to
# tell java(?) where to find the racob dll.
#
# However, it fails when it tries to load the racob dll from the jar
# (UnsatisfiedLinkError).
#
# So easy fix, right? Monkeypatch the set_property to do two things:
# - extract the racob dll somewhere
# - set the property to the extracted path
#
require "fileutils"
require "tmpdir"
class << java.lang.System
alias_method :set_property_seriously, :set_property
def set_property(key, value)
if key == "racob.dll.path" && value =~ /file:.*\.jar!\//
# Path is set in a jar, we'll need to extract it to a
# temporary location, then load it.
filename = File.basename(value)
extracted_path = File.join(Dir.tmpdir, filename)
# FileUtils.cp (and copy, and copy_file) are broken
# when copying this file. I have not debugged it other
# than to see it doesn't work. racob_x86.dll is 105kb,
# but FileUtils.cp only copies 4kb of it.
# open both files in 'binary' mode (sets encoding to BINARY aka
# ASCII-8BIT).
input = File.new(value, "rb")
output = File.new(extracted_path, "wb")
while chunk = input.read(16384)
output.write(chunk)
end
input.close
output.close
return set_property_seriously(key, extracted_path)
else
return set_property_seriously(key, value)
end
end
end

View file

@ -1,2 +0,0 @@
# encoding: utf-8
require "logstash/JRUBY-6970" if RUBY_PLATFORM == "java"

View file

@ -1,66 +0,0 @@
# encoding: utf-8
if RUBY_PLATFORM == "java"
class << File
# mpp == monkey patch for performance
alias_method :mpp_file?, :file?
alias_method :mpp_exist?, :exist?
alias_method :mpp_exists?, :exists?
JAR_RE = /^(?:jar:)?file:(\/.*\.jar)!(\/.*$)/
def file?(path)
#return mpp_file?(path)
# If path is in a jar (file://blah/foo.jar!/some/path)
# - create a cache for this jar of all files
# - return cached results only
if RUBY_PLATFORM == "java"
m = JAR_RE.match(path)
return mpp_file?(path) if !m # not a jar file
c = __zipcache(m[1], m[2]) # m[1] == the jar path
# ZipEntry has only 'isDirectory()' so I assume any
# non-directories are files.
rc = (!c.nil? && !c.isDirectory)
#p path => rc
return rc
end
return mpp_file?(path)
end
def exist?(path)
#return mpp_exist?(path)
# If path is in a jar (file://blah/foo.jar!/some/path)
# - create a cache for this jar of all files
# - return cached results only
if RUBY_PLATFORM == "java"
m = JAR_RE.match(path)
return mpp_exists?(path) if !m # not a jar file
c = __zipcache(m[1], m[2]) # m[1] == the jar path
return !c.nil?
end
return mpp_exists?(path)
end
def exists?(path)
return exist?(path)
end
def __zipcache(jarpath, path)
@jarcache ||= Hash.new { |h,k| h[k] = {} }
if @jarcache[jarpath].empty?
#puts "Caching file entries for #{jarpath}"
s = Time.now
zip = java.util.zip.ZipFile.new(jarpath)
zip.entries.each do |entry|
#puts "Caching file entries for #{jarpath}: /#{entry.name}"
# Prefix entry name with "/" because that's what the jar path looks
# like in jruby: file://some.jar!/some/path
@jarcache[jarpath]["/" + entry.name] = entry
end
end
entry = @jarcache[jarpath][path]
#puts "Serving cached file info #{path}: #{entry}"
return entry
end
end
end

View file

@ -269,24 +269,14 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
public
def get_template
if @template.nil?
if __FILE__ =~ /^(jar:)?file:\/.+!.+/
begin
# Running from a jar, assume types.db is at the root.
jar_path = [__FILE__.split("!").first, "/elasticsearch-template.json"].join("!")
@template = jar_path
rescue => ex
raise "Failed to cache, due to: #{ex}\n#{ex.backtrace}"
end
if File.exists?("elasticsearch-template.json")
@template = "elasticsearch-template.json"
else
if File.exists?("elasticsearch-template.json")
@template = "elasticsearch-template.json"
path = File.join(File.dirname(__FILE__), "elasticsearch/elasticsearch-template.json")
if File.exists?(path)
@template = path
else
path = File.join(File.dirname(__FILE__), "elasticsearch/elasticsearch-template.json")
if File.exists?(path)
@template = path
else
raise "You must specify 'template => ...' in your elasticsearch_http output"
end
raise "You must specify 'template => ...' in your elasticsearch_http output"
end
end
end

View file

@ -173,22 +173,12 @@ class LogStash::Outputs::ElasticSearchHTTP < LogStash::Outputs::Base
public
def get_template_json
if @template.nil?
if __FILE__ =~ /^(jar:)?file:\/.+!.+/
begin
# Running from a jar, assume types.db is at the root.
jar_path = [__FILE__.split("!").first, "/elasticsearch-template.json"].join("!")
@template = jar_path
rescue => ex
raise "Failed to cache, due to: #{ex}\n#{ex.backtrace}"
end
if File.exists?("elasticsearch-template.json")
@template = "elasticsearch-template.json"
elsif File.exists?("lib/logstash/outputs/elasticsearch/elasticsearch-template.json")
@template = "lib/logstash/outputs/elasticsearch/elasticsearch-template.json"
else
if File.exists?("elasticsearch-template.json")
@template = "elasticsearch-template.json"
elsif File.exists?("lib/logstash/outputs/elasticsearch/elasticsearch-template.json")
@template = "lib/logstash/outputs/elasticsearch/elasticsearch-template.json"
else
raise "You must specify 'template => ...' in your elasticsearch_http output"
end
raise "You must specify 'template => ...' in your elasticsearch_http output"
end
end
@template_json = IO.read(@template).gsub(/\n/,'')

View file

@ -40,10 +40,6 @@ if ENV["PROFILE_BAD_LOG_CALLS"] || $DEBUGLIST.include?("log")
end
end # PROFILE_BAD_LOG_CALLS
if __FILE__ =~ /^(jar:)?file:\//
require "logstash/monkeypatches-for-performance"
end
require "logstash/monkeypatches-for-bugs"
require "logstash/monkeypatches-for-debugging"
require "logstash/namespace"
require "logstash/program"
@ -55,27 +51,7 @@ I18n.load_path << File.expand_path(
class LogStash::RSpecsRunner
def initialize(args)
@args = args.collect do |arg|
# if the arg ends in .rb or has a "/" in it, assume it's a path.
if arg =~ /\.rb$/ || arg =~ /\//
# check if it's a file, if not, try inside the jar if we are in it.
if !File.exists?(arg) && __FILE__ =~ /file:.*\.jar!\//
# Try inside the jar.
jar_root = __FILE__.gsub(/!.*/,"!")
newpath = File.join(jar_root, arg)
# Strip leading 'jar:' path (JRUBY_6970)
newpath.gsub!(/^jar:/, "")
if File.exists?(newpath)
# Add the 'spec' dir to the load path so specs can run
specpath = File.join(jar_root, "spec")
$LOAD_PATH << specpath unless $LOAD_PATH.include?(specpath)
next newpath
end
end
end
next arg
end # args.collect
@args = args
end
def run