github-mirrors/logstash
1
0
Fork 0
mirror of https://github.com/elastic/logstash.git synced 2025-04-24 14:47:19 -04:00
Code Issues Projects Releases Packages Wiki Activity

Allow the installation of a plugin from a .gem, refactor the plugin manager classes.

Browse source 
do not change the gemfile or the .lock

Fixes #2946
This commit is contained in:
Pier-Hugues Pellerin 2015-03-27 16:07:16 -04:00 • committed by Jordan Sissel
parent 53b3f7579d
commit 7e44f4ea16
13 changed files with 310 additions and 151 deletions
Download patch file Download diff file Expand all files Collapse all files

35
lib/logstash/bundler.rb
View file

@ -23,6 +23,28 @@ end
module LogStash
module Bundler
# Take a gem package and extract it to a specific target
# @param [String] Gem file, this must be a path
# @param [String, String] Return a Gem::Package and the installed path
def self.unpack(file, path)
require "rubygems/package"
require "securerandom"
# We are creating a random directory per extract,
# if we dont do this bundler will not trigger download of the dependencies.
# Use case is:
# - User build his own gem with a fix
# - User doesnt increment the version
# - User install the same version but different code or dependencies multiple times..
basename = ::File.basename(file, '.gem')
unique = SecureRandom.hex(4)
target_path = ::File.expand_path(::File.join(path, unique, basename))
package = ::Gem::Package.new(file)
package.extract_files(target_path)
return [package, target_path]
end
# capture any $stdout from the passed block. also trap any exception in that block, in which case the trapped exception will be returned
# @param [Proc] the code block to execute
@ -53,6 +75,9 @@ module LogStash
ENV["GEM_PATH"] = LogStash::Environment.logstash_gem_home
# force Rubygems sources to our Gemfile sources
::Gem.sources = options[:rubygems_source] if options[:rubygems_source]
::Bundler.settings[:path] = LogStash::Environment::BUNDLE_DIR
::Bundler.settings[:gemfile] = LogStash::Environment::GEMFILE_PATH
::Bundler.settings[:without] = options[:without].join(":")
@ -60,7 +85,7 @@ module LogStash
try = 0
# capture_stdout also traps any raised exception and pass them back as the function return [output, exception]
capture_stdout do
output, exception = capture_stdout do
loop do
begin
::Bundler.reset!
@ -81,11 +106,15 @@ module LogStash
try += 1
$stderr.puts("Error #{e.class}, retrying #{try}/#{options[:max_tries]}")
$stderr.puts(e.message) if ENV["DEBUG"]
$stderr.puts(e.message)
sleep(0.5)
end
end
end
raise exception if exception
return output
end
# build Bundler::CLI.start arguments array from the given options hash
@ -107,4 +136,4 @@ module LogStash
arguments.flatten
end
end
end
end

1
lib/logstash/environment.rb
View file

@ -50,6 +50,7 @@ module LogStash
GEMFILE_PATH = ::File.join(LOGSTASH_HOME, "Gemfile")
BUNDLE_CONFIG_PATH = ::File.join(LOGSTASH_HOME, ".bundle", "config")
BOOTSTRAP_GEM_PATH = ::File.join(LOGSTASH_HOME, 'build', 'bootstrap')
LOCAL_GEM_PATH = ::File.join(LOGSTASH_HOME, 'vendor', 'locally_installed_gem')
LOGSTASH_ENV = (ENV["LS_ENV"] || 'production').to_s.freeze

20
lib/logstash/gemfile.rb
View file

@ -1,3 +1,4 @@
require "logstash/util"
module LogStash
class GemfileError < StandardError; end
@ -17,6 +18,7 @@ module LogStash
def load
@gemset ||= DSL.parse(@io.read)
backup
self
end
@ -51,6 +53,23 @@ module LogStash
def remove(name)
@gemset.remove_gem(name)
end
def backup
@orignal_backup = @gemset.copy
end
def restore
@gemset = @orignal_backup
end
def restore!
restore
save
end
def locally_installed_gems
@gemset.gems.select { |gem| gem.options.include?(:path) }
end
end
class Gemset
@ -101,7 +120,6 @@ module LogStash
def copy
Marshal.load(Marshal.dump(self))
end
private
def sources_to_s

0
lib/logstash/pluginmanager/maven_tools_patch.rb → lib/logstash/patches/maven_tools_patch.rb
View file

37
lib/logstash/pluginmanager/base.rb Normal file
View file

@ -0,0 +1,37 @@
class LogStash::PluginManager::Base < Clamp::Command
def gemfile
@gemfile ||= LogStash::Gemfile.new(File.new(LogStash::Environment::GEMFILE_PATH, 'r+')).load
end
# If set in debug mode we will raise an exception and display the stacktrace
def report_exception(readable_message, exception)
if ENV["DEBUG"]
raise exception
else
signal_error("#{readable_message}, message: #{exception.message}")
end
end
def display_bundler_output(output)
if ENV['DEBUG'] && output
# Display what bundler did in the last run
$stderr.puts("Bundler output")
$stderr.puts(output)
end
end
# Each plugin install for a gemfile create a path with a unique id.
# we must clear what is not currently used in the
def remove_unused_locally_installed_gems!
used_path = gemfile.locally_installed_gems.collect { |gem| gem.options[:path] }
Dir.glob(File.join(LogStash::Environment::LOCAL_GEM_PATH, '*')) do |path|
FileUtils.rm_rf(relative_path(path)) if used_path.none? { |p| p.start_with?(relative_path(path)) }
end
end
def relative_path(path)
Pathname.new(path).relative_path_from(Pathname.new(LogStash::Environment::LOGSTASH_HOME)).to_s
end
end

177
lib/logstash/pluginmanager/install.rb
View file

@ -1,15 +1,16 @@
require 'clamp'
require 'logstash/namespace'
require 'logstash/environment'
require 'logstash/pluginmanager/util'
require 'jar-dependencies'
require 'jar_install_post_install_hook'
require 'file-dependencies/gem'
require "clamp"
require "logstash/namespace"
require "logstash/environment"
require "logstash/pluginmanager/util"
require "logstash/pluginmanager/base"
require "jar-dependencies"
require "jar_install_post_install_hook"
require "file-dependencies/gem"
require "logstash/gemfile"
require "logstash/bundler"
require "fileutils"
class LogStash::PluginManager::Install < Clamp::Command
class LogStash::PluginManager::Install < LogStash::PluginManager::Base
parameter "[PLUGIN] ...", "plugin name(s) or file"
option "--version", "VERSION", "version of the plugin to install"
option "--[no-]verify", :flag, "verify plugin validity before installation", :default => true
@ -18,95 +19,119 @@ class LogStash::PluginManager::Install < Clamp::Command
# the install logic below support installing multiple plugins with each a version specification
# but the argument parsing does not support it for now so currently if specifying --version only
# one plugin name can be also specified.
#
# TODO: find right syntax to allow specifying list of plugins with optional version specification for each
def execute
if development?
raise(LogStash::PluginManager::Error, "Cannot specify plugin(s) with --development, it will add the development dependencies of the currently installed plugins") unless plugin_list.empty?
validate_cli_options!
if local_gems?
gems = extract_local_gems_plugins
elsif development?
gems = plugins_development_gems
else
raise(LogStash::PluginManager::Error, "No plugin specified") if plugin_list.empty? && verify?
# temporary until we fullfil TODO ^^
raise(LogStash::PluginManager::Error, "Only 1 plugin name can be specified with --version") if version && plugin_list.size > 1
gems = plugins_gems
verify!(gems)
end
raise(LogStash::PluginManager::Error, "File #{LogStash::Environment::GEMFILE_PATH} does not exist or is not writable, aborting") unless File.writable?(LogStash::Environment::GEMFILE_PATH)
gemfile = LogStash::Gemfile.new(File.new(LogStash::Environment::GEMFILE_PATH, "r+")).load
# keep a copy of the gemset to revert on error
original_gemset = gemfile.gemset.copy
# force Rubygems sources to our Gemfile sources
Gem.sources = gemfile.gemset.sources
# install_list will be an array of [plugin name, version] tuples, version can be nil
install_list = []
install_gems_list!(gems)
remove_unused_locally_installed_gems!
end
private
def validate_cli_options!
if development?
specs = LogStash::PluginManager.all_installed_plugins_gem_specs(gemfile)
install_list = specs.inject([]) do |result, spec|
result = result + spec.dependencies.select{|dep| dep.type == :development}.map{|dep| [dep.name] + dep.requirement.as_list + [{:group => :development}]}
end
signal_usage_error("Cannot specify plugin(s) with --development, it will add the development dependencies of the currently installed plugins") unless plugin_list.empty?
else
# at this point we know that plugin_list is not empty and if the --version is specified there is only one plugin in plugin_list
signal_usage_error("No plugin specified") if plugin_list.empty? && verify?
# TODO: find right syntax to allow specifying list of plugins with optional version specification for each
signal_usage_error("Only 1 plugin name can be specified with --version") if version && plugin_list.size > 1
end
signal_error("File #{LogStash::Environment::GEMFILE_PATH} does not exist or is not writable, aborting") unless ::File.writable?(LogStash::Environment::GEMFILE_PATH)
end
install_list = version ? [plugin_list << version] : plugin_list.map{|plugin| [plugin, nil]}
install_list.each do |plugin, version|
# Check if the specified gems contains
# the logstash `metadata`
def verify!(gems)
if verify?
gems.each do |plugin, version|
puts("Validating #{[plugin, version].compact.join("-")}")
raise(LogStash::PluginManager::Error, "Installation aborted") unless LogStash::PluginManager.logstash_plugin?(plugin, version)
end if verify?
# at this point we know that we either have a valid gem name & version or a valid .gem file path
# if LogStash::PluginManager.plugin_file?(plugin)
# raise(LogStash::PluginManager::Error) unless cache_gem_file(plugin)
# spec = LogStash::PluginManager.plugin_file_spec(plugin)
# gemfile.update(spec.name, spec.version.to_s)
# else
# plugins.each{|tuple| gemfile.update(*tuple)}
# end
signal_error("Installation aborted, verification failed for #{plugin} #{version}") unless LogStash::PluginManager.logstash_plugin?(plugin, version)
end
end
end
def plugins_development_gems
# Get currently defined gems and their dev dependencies
specs = []
specs = LogStash::PluginManager.all_installed_plugins_gem_specs(gemfile)
# Construct the list of dependencies to add to the current gemfile
specs.each_with_object([]) do |spec, install_list|
dependencies = spec.dependencies
.select { |dep| dep.type == :development }
.map { |dep| [dep.name] + dep.requirement.as_list }
install_list.concat(dependencies)
end
end
def plugins_gems
version ? [plugin_list << version] : plugin_list.map { |plugin| [plugin, nil] }
end
# install_list will be an array of [plugin name, version, options] tuples, version it
# can be nil at this point we know that plugin_list is not empty and if the
# --version is specified there is only one plugin in plugin_list
#
def install_gems_list!(install_list)
# If something goes wrong during the installation `LogStash::Gemfile` will restore a backup version.
install_list = LogStash::PluginManager.merge_duplicates(install_list)
install_list.each{|plugin, version| gemfile.update(plugin, version)}
gemfile.save
puts("Installing" + (install_list.empty? ? "..." : " " + install_list.map{|plugin, version| plugin}.join(", ")))
# Add plugins/gems to the current gemfile
puts("Installing" + (install_list.empty? ? "..." : " " + install_list.collect(&:first).join(", ")))
install_list.each { |plugin, version, options| gemfile.update(plugin, version, options) }
# Sync gemfiles changes to disk to make them available to the `bundler install`'s API
gemfile.save
bundler_options = {:install => true}
bundler_options[:without] = [] if development?
bundler_options[:rubygems_source] = gemfile.gemset.sources
# any errors will be logged to $stderr by invoke_bundler!
output, exception = LogStash::Bundler.invoke_bundler!(bundler_options)
if ENV["DEBUG"]
$stderr.puts(output)
$stderr.puts("Error: #{exception.class}, #{exception.message}") if exception
end
if exception
# revert to original Gemfile content
gemfile.gemset = original_gemset
gemfile.save
raise(LogStash::PluginManager::Error, "Installation aborted")
end
output = LogStash::Bundler.invoke_bundler!(bundler_options)
puts("Installation successful")
rescue => exception
gemfile.restore!
report_exception("Installation Aborded", exception)
ensure
display_bundler_output(output)
end
# copy .gem file into bundler cache directory, log any error to $stderr
# @param path [String] the source .gem file to copy
# @return [Boolean] true if successful
def cache_gem_file(path)
dest = ::File.join(LogStash::Environment.logstash_gem_home, "cache")
begin
FileUtils.cp(path, dest)
rescue => e
$stderr.puts("Error copying #{plugin} to #{dest}, caused by #{e.class}")
return false
# Extract the specified local gems in a predefined local path
# Update the gemfile to use a relative path to this plugin and run
# Bundler, this will mark the gem not updatable by `bin/plugin update`
# This is the most reliable way to make it work in bundler without
# hacking with `how bundler works`
#
# Bundler 2.0, will have support for plugins source we could create a .gem source
# to support it.
def extract_local_gems_plugins
plugin_list.collect do |plugin|
package, path = LogStash::Bundler.unpack(plugin, LogStash::Environment::LOCAL_GEM_PATH)
[package.spec.name, package.spec.version, { :path => relative_path(path) }]
end
end
# We cannot install both .gem and normal plugin in one call of `plugin install`
def local_gems?
return false if plugin_list.empty?
local_gem = plugin_list.collect { |plugin| ::File.extname(plugin) == ".gem" }.uniq
if local_gem.size == 1
return local_gem.first
else
signal_usage_error("Mixed source of plugins, you can't mix local `.gem` and remote gems")
end
true
end
end # class Logstash::PluginManager

28
lib/logstash/pluginmanager/list.rb
View file

@ -18,24 +18,24 @@ class LogStash::PluginManager::List < Clamp::Command
require 'logstash/environment'
LogStash::Environment.bundler_setup!
Gem.configuration.verbose = false
signal_error("No plugins found") if filtered_specs.empty?
gemfile = LogStash::Gemfile.new(File.new(LogStash::Environment::GEMFILE_PATH, "r+")).load
# start with all locally installed plugin gems regardless of the Gemfile content
specs = LogStash::PluginManager.find_plugins_gem_specs
# apply filters
specs = specs.select{|spec| gemfile.find(spec.name)} if installed?
specs = specs.select{|spec| spec.name =~ /#{plugin}/i} if plugin
specs = specs.select{|spec| spec.metadata['logstash_group'] == group} if group
raise(LogStash::PluginManager::Error, "No plugins found") if specs.empty?
specs.sort_by{|spec| spec.name}.each do |spec|
filtered_specs.sort_by{|spec| spec.name}.each do |spec|
line = "#{spec.name}"
line += " (#{spec.version})" if verbose?
puts(line)
end
end
def filtered_specs
@filtered_specs ||= begin
# start with all locally installed plugin gems regardless of the Gemfile content
specs = LogStash::PluginManager.find_plugins_gem_specs
# apply filters
specs = specs.select{|spec| gemfile.find(spec.name)} if installed?
specs = specs.select{|spec| spec.name =~ /#{plugin}/i} if plugin
specs = specs.select{|spec| spec.metadata['logstash_group'] == group} if group
end
end
end # class Logstash::PluginManager

2
lib/logstash/pluginmanager/main.rb
View file

@ -5,7 +5,7 @@ require "logstash/pluginmanager/uninstall"
require "logstash/pluginmanager/list"
require "logstash/pluginmanager/update"
require "logstash/pluginmanager/util"
require "logstash/pluginmanager/maven_tools_patch"
require "logstash/patches/maven_tools_patch"
require "clamp"
module LogStash

34
lib/logstash/pluginmanager/uninstall.rb
View file

@ -3,25 +3,23 @@ require "logstash/logging"
require "logstash/errors"
require "logstash/environment"
require "logstash/pluginmanager/util"
require "logstash/pluginmanager/base"
require "clamp"
require "logstash/gemfile"
require "logstash/bundler"
class LogStash::PluginManager::Uninstall < Clamp::Command
class LogStash::PluginManager::Uninstall < LogStash::PluginManager::Base
parameter "PLUGIN", "plugin name"
def execute
raise(LogStash::PluginManager::Error, "File #{LogStash::Environment::GEMFILE_PATH} does not exist or is not writable, aborting") unless File.writable?(LogStash::Environment::GEMFILE_PATH)
LogStash::Environment.bundler_setup!
gemfile = LogStash::Gemfile.new(File.new(LogStash::Environment::GEMFILE_PATH, "r+")).load
# keep a copy of the gemset to revert on error
original_gemset = gemfile.gemset.copy
signal_error("File #{LogStash::Environment::GEMFILE_PATH} does not exist or is not writable, aborting") unless File.writable?(LogStash::Environment::GEMFILE_PATH)
# make sure this is an installed plugin and present in Gemfile.
# it is not possible to uninstall a dependency not listed in the Gemfile, for example a dependent codec
raise(LogStash::PluginManager::Error, "This plugin has not been previously installed, aborting") unless LogStash::PluginManager.installed_plugin?(plugin, gemfile)
signal_error("This plugin has not been previously installed, aborting") unless LogStash::PluginManager.installed_plugin?(plugin, gemfile)
# since we previously did a gemfile.find(plugin) there is no reason why
# remove would not work (return nil) here
@ -31,19 +29,15 @@ class LogStash::PluginManager::Uninstall < Clamp::Command
puts("Uninstalling #{plugin}")
# any errors will be logged to $stderr by invoke_bundler!
output, exception = LogStash::Bundler.invoke_bundler!(:install => true, :clean => true)
if ENV["DEBUG"]
$stderr.puts(output)
$stderr.puts("Error: #{exception.class}, #{exception.message}") if exception
end
if exception
# revert to original Gemfile content
gemfile.gemset = original_gemset
gemfile.save
raise(LogStash::PluginManager::Error, "Uninstall aborted")
end
# output, exception = LogStash::Bundler.invoke_bundler!(:install => true, :clean => true)
output = LogStash::Bundler.invoke_bundler!(:install => true)
remove_unused_locally_installed_gems!
end
rescue => exception
gemfile.restore!
report_exception("Uninstall aborded", exception)
ensure
display_bundler_output(output)
end
end

83
lib/logstash/pluginmanager/update.rb
View file

@ -1,51 +1,80 @@
require 'clamp'
require 'logstash/namespace'
require 'logstash/pluginmanager/util'
require 'logstash/pluginmanager/base'
require 'jar-dependencies'
require 'jar_install_post_install_hook'
require 'file-dependencies/gem'
require "logstash/gemfile"
require "logstash/bundler"
class LogStash::PluginManager::Update < Clamp::Command
class LogStash::PluginManager::Update < LogStash::PluginManager::Base
parameter "[PLUGIN] ...", "Plugin name(s) to upgrade to latest version"
def execute
gemfile = LogStash::Gemfile.new(File.new(LogStash::Environment::GEMFILE_PATH, "r+")).load
# keep a copy of the gemset to revert on error
original_gemset = gemfile.gemset.copy
local_gems = gemfile.locally_installed_gems
previous_gem_specs_map = find_latest_gem_specs
# create list of plugins to update
plugins = unless plugin_list.empty?
not_installed = plugin_list.select{|plugin| !previous_gem_specs_map.has_key?(plugin.downcase)}
raise(LogStash::PluginManager::Error, "Plugin #{not_installed.join(', ')} is not installed so it cannot be updated, aborting") unless not_installed.empty?
plugin_list
if update_all? || !local_gems.empty?
error_plugin_that_use_path!(local_gems)
else
previous_gem_specs_map.values.map{|spec| spec.name}
plugins_with_path = plugin_list & local_gems
error_plugin_that_use_path!(plugins_with_path) if plugins_with_path.size > 0
end
update_gems!
end
private
def error_plugin_that_use_path!(plugins)
signal_error("You have installed plugins from a .gem or you have manually defined a plugin in the Gemfile, we cannot update all or update this specific plugin, problematic plugins: #{plugins.collect(&:name).join(",")}")
end
def update_all?
plugin_list.size == 0
end
def update_gems!
# If any error is raise inside the block the Gemfile will restore a backup of the Gemfile
previous_gem_specs_map = find_latest_gem_specs
# remove any version constrain from the Gemfile so the plugin(s) can be updated to latest version
# calling update without requiremend will remove any previous requirements
plugins.select{|plugin| gemfile.find(plugin)}.each{|plugin| gemfile.update(plugin)}
plugins = plugins_to_update(previous_gem_specs_map)
plugins
.select { |plugin| gemfile.find(plugin) }
.each { |plugin| gemfile.update(plugin) }
# force a disk sync before running bundler
gemfile.save
puts("Updating " + plugins.join(", "))
# any errors will be logged to $stderr by invoke_bundler!
output, exception = LogStash::Bundler.invoke_bundler!(:update => plugins)
output, exception = LogStash::Bundler.invoke_bundler!(:clean => true) unless exception
# Bundler cannot update and clean gems in one operation so we have to call the CLI twice.
output = LogStash::Bundler.invoke_bundler!(:update => plugins)
output = LogStash::Bundler.invoke_bundler!(:clean => true)
if exception
# revert to original Gemfile content
gemfile.gemset = original_gemset
gemfile.save
display_updated_plugins(previous_gem_specs_map)
rescue => exception
gemfile.restore!
report_exception("Updated Aborded", exception)
ensure
display_bundler_output(output)
end
report_exception(output, exception)
# create list of plugins to update
def plugins_to_update(previous_gem_specs_map)
unless plugin_list.empty?
not_installed = plugin_list.select{|plugin| !previous_gem_specs_map.has_key?(plugin.downcase)}
signal_error("Plugin #{not_installed.join(', ')} is not installed so it cannot be updated, aborting") unless not_installed.empty?
plugin_list
else
previous_gem_specs_map.values.map{|spec| spec.name}
end
end
# We compare the before the update and after the update
def display_updated_plugins(previous_gem_specs_map)
update_count = 0
find_latest_gem_specs.values.each do |spec|
name = spec.name.downcase
@ -59,11 +88,10 @@ class LogStash::PluginManager::Update < Clamp::Command
update_count += 1
end
end
puts("No plugin updated") if update_count.zero?
end
private
# retrieve only the latest spec for all locally installed plugins
# @return [Hash] result hash {plugin_name.downcase => plugin_spec}
def find_latest_gem_specs
@ -73,13 +101,4 @@ class LogStash::PluginManager::Update < Clamp::Command
result
end
end
def report_exception(output, exception)
if ENV["DEBUG"]
$stderr.puts(output)
$stderr.puts("Error: #{exception.class}, #{exception.message}") if exception
end
raise(LogStash::PluginManager::Error, "Update aborted")
end
end

3
lib/logstash/pluginmanager/util.rb
View file

@ -1,5 +1,4 @@
module LogStash::PluginManager
# check for valid logstash plugin gem name & version or .gem file, logs errors to $stdout
# uses Rubygems API and will remotely validated agains the current Gem.sources
# @param plugin [String] plugin name or .gem file path
@ -85,4 +84,4 @@ module LogStash::PluginManager
# TODO: properly merge versions requirements
plugin_list.uniq(&:first)
end
end
end

1
lib/logstash/util.rb
View file

@ -148,5 +148,4 @@ module LogStash::Util
o
end
end
end # module LogStash::Util

40
spec/util/gemfile_spec.rb
View file

@ -134,6 +134,44 @@ describe "logstash Gemfile Manager" do
end
end
describe "Locally installed gems" do
subject { LogStash::Gemfile.new(StringIO.new(file)).load.locally_installed_gems }
context "has gems defined with a path" do
let(:file) {
%Q[
source "https://rubygems.org"
gemspec :a => "a", "b" => 1
gem "foo", "> 1.0", :path => "/tmp/foo"
gem "bar", :path => "/tmp/bar"
gem "no-fun"
]
}
it "returns the list of gems" do
expect(subject.collect(&:name)).to eq(["foo", "bar"])
end
end
context "no gems defined with a path" do
let(:file) {
%Q[
source "https://rubygems.org"
gemspec :a => "a", "b" => 1
gem "no-fun"
]
}
it "return an empty list" do
expect(subject.size).to eq(0)
end
end
context "keep a backup of the original file" do
end
end
context "save" do
it "should save" do
file = <<-END
@ -171,4 +209,4 @@ describe "logstash Gemfile Manager" do
end
end
end
end
end