add the concept of a jar manager to load jars within plugins

make the jar manager to have some of the checks already present in the old load_elasticsearch_jars

removed the load_elasticsearch_jars! method as it's a legacy code from when ES was delivered within LS, nowadays this comes within the differents plugins

clean up Environment::JAR_DIR as is another legacy variable not used anymore

move the jar loading code within the environment module

ammend previous commit adding more clear code

ammend previous commits

added a few simple unit test for the jar manager in the environment module

refactor spec naming

dry out common code for loading jars

clean up env_spec bein in the util dir

cleaning up logstash-core dependencies related to jar-dependencies

ammend stale maen patches created for the LS way of using maven plugins

applied some comments coming from the review by @ph

bring back the maven tools and jar-dependency code as we still need some code like this for the kafka plugins

Revert "bring back the maven tools and jar-dependency code as we still need some code like this for the kafka plugins"

This reverts commit c23f456362776c0a600d78369c2ece8789fa49c2.

Fixes #2980

lib/logstash/agent.rb

@@ -172,7 +172,6 @@ class LogStash::Agent < Clamp::Command
 
       if RUBY_PLATFORM == "java"
         show_version_java
-        show_version_elasticsearch
       end
 
       if [:debug].include?(verbosity?) || debug?
@@ -190,13 +189,6 @@ class LogStash::Agent < Clamp::Command
     puts RUBY_DESCRIPTION
   end # def show_version_ruby
 
-  def show_version_elasticsearch
-    LogStash::Environment.load_elasticsearch_jars!
-
-    $stdout.write("Elasticsearch: ");
-    org.elasticsearch.Version::main([])
-  end # def show_version_elasticsearch
-
   def show_version_java
     properties = java.lang.System.getProperties
     puts "java #{properties["java.version"]} (#{properties["java.vendor"]})"

lib/logstash/environment.rb

@@ -1,5 +1,5 @@
 require "logstash/errors"
-require 'logstash/version'
+require "logstash/version"
 
 # monkey patch RubyGems to silence ffi warnings:
 #
@@ -44,8 +44,6 @@ module LogStash
     extend self
 
     LOGSTASH_HOME = ::File.expand_path(::File.join(::File.dirname(__FILE__), "..", ".."))
-    JAR_DIR = ::File.join(LOGSTASH_HOME, "vendor", "jar")
-    ELASTICSEARCH_DIR = ::File.join(LOGSTASH_HOME, "vendor", "elasticsearch")
     BUNDLE_DIR = ::File.join(LOGSTASH_HOME, "vendor", "bundle")
     GEMFILE_PATH = ::File.join(LOGSTASH_HOME, "Gemfile")
     BUNDLE_CONFIG_PATH = ::File.join(LOGSTASH_HOME, ".bundle", "config")
@@ -54,23 +52,6 @@ module LogStash
 
     LOGSTASH_ENV = (ENV["LS_ENV"] || 'production').to_s.freeze
 
-    # loads currently embedded elasticsearch jars
-    # @raise LogStash::EnvironmentError if not running under JRuby or if no jar files are found
-    def load_elasticsearch_jars!
-      raise(LogStash::EnvironmentError, "JRuby is required") unless jruby?
-
-      require "java"
-      jars_path = ::File.join(ELASTICSEARCH_DIR, "**", "*.jar")
-      jar_files = Dir.glob(jars_path)
-
-      raise(LogStash::EnvironmentError, "Could not find Elasticsearch jar files under #{ELASTICSEARCH_DIR}") if jar_files.empty?
-
-      jar_files.each do |jar|
-        loaded = require jar
-        puts("Loaded #{jar}") if $DEBUG && loaded
-      end
-    end
-
     def logstash_gem_home
       ::File.join(BUNDLE_DIR, ruby_engine, gem_ruby_version)
     end
@@ -114,6 +95,43 @@ module LogStash
       ::Bundler.setup
     end
 
+    def runtime_jars_root(dir_name, package)
+      ::File.join(dir_name, package, "runtime-jars")
+    end
+
+    def test_jars_root(dir_name, package)
+      ::File.join(dir_name, package, "test-jars")
+    end
+
+    def load_runtime_jars!(dir_name="vendor", package="jar-dependencies")
+      load_jars!(::File.join(runtime_jars_root(dir_name, package), "*.jar"))
+    end
+
+    def load_test_jars!(dir_name="vendor", package="jar-dependencies")
+      load_jars!(::File.join(test_jars_root(dir_name, package), "*.jar"))
+    end
+
+    def load_jars!(pattern)
+      raise(LogStash::EnvironmentError, I18n.t("logstash.environment.jruby-required")) unless LogStash::Environment.jruby?
+
+      jar_files = find_jars(pattern)
+      require_jars! jar_files
+    end
+
+    def find_jars(pattern)
+      require 'java'
+      jar_files = Dir.glob(pattern)
+      raise(LogStash::EnvironmentError, I18n.t("logstash.environment.missing-jars", :pattern => pattern)) if jar_files.empty?
+      jar_files
+    end
+
+    def require_jars!(files)
+      files.each do |jar_file|
+        loaded = require jar_file
+        puts("Loaded #{jar_file}") if $DEBUG && loaded
+      end
+    end
+
     def ruby_bin
       ENV["USE_RUBY"] == "1" ? "ruby" : File.join("vendor", "jruby", "bin", "jruby")
     end

lib/logstash/patches/maven_tools_patch.rb

@@ -1,12 +0,0 @@
-# This adds the "repo" element to the jar-dependencies DSL
-# allowing a gemspec to require a jar that exists in a custom
-# maven repository
-# Example:
-#   gemspec.requirements << "repo http://localhosty/repo"
-require 'maven/tools/dsl/project_gemspec'
-class Maven::Tools::DSL::ProjectGemspec
-  def repo(url)
-    @parent.repository(:id => url, :url => url)
-  end
-end
-

lib/logstash/pluginmanager/main.rb

@@ -5,7 +5,6 @@ require "logstash/pluginmanager/uninstall"
 require "logstash/pluginmanager/list"
 require "logstash/pluginmanager/update"
 require "logstash/pluginmanager/util"
-require "logstash/patches/maven_tools_patch"
 require "clamp"
 
 module LogStash

locales/en.yml

@@ -7,6 +7,11 @@ en:
     The error reported is: 
       %{error}
   logstash:
+    environment:
+      jruby-required:  >-
+        JRuby is required
+      missing-jars: >-
+        Could not find jar files under %{pattern}
     pipeline:
       worker-error: |-
         A plugin had an unrecoverable error. Will restart this plugin.

logstash-core.gemspec

@@ -36,18 +36,6 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "sinatra"            #(MIT-style license)
 
   # Plugin manager dependencies
-
-  # Currently there is a blocking issue with the latest (3.1.1.0.9) version of
-  # `ruby-maven` # and installing jars dependencies. If you are declaring a gem
-  # in a gemfile # using the :github option it will make the bundle install crash,
-  # before upgrading this gem you need to test the version with any plugins
-  # that require jars.
-  #
-  # Ticket: https://github.com/elasticsearch/logstash/issues/2595
-  gem.add_runtime_dependency "jar-dependencies", '0.1.7'   #(MIT license)
-  gem.add_runtime_dependency "ruby-maven", '3.1.1.0.8'                       #(EPL license)
-  gem.add_runtime_dependency "maven-tools", '1.0.7'
-
   gem.add_runtime_dependency "minitar"
   gem.add_runtime_dependency "file-dependencies", '0.1.6'
 

spec/core/environment_spec.rb

@@ -0,0 +1,44 @@
+require "spec_helper"
+require "logstash/environment"
+
+describe LogStash::Environment do
+
+  context "when loading jars dependencies" do
+
+    let(:default_jars_location)    { File.join("vendor", "jar-dependencies") }
+    let(:default_runtime_location) { File.join(default_jars_location,"runtime-jars","*.jar") }
+    let(:default_test_location)    { File.join(default_jars_location,"test-jars","*.jar") }
+
+    it "raises an exception if jruby is not available" do
+      expect(subject).to receive(:jruby?).and_return(false)
+      expect { subject.load_runtime_jars! }.to raise_error
+    end
+
+    it "find runtime jars in the default location" do
+      expect(subject).to receive(:find_jars).with(default_runtime_location).and_return([])
+      subject.load_runtime_jars!
+    end
+
+    it "find test jars in the default location" do
+      expect(subject).to receive(:find_jars).with(default_test_location).and_return([])
+      subject.load_test_jars!
+    end
+
+    context "when loading a jar file" do
+
+      let(:dummy_jar_file) { File.join(default_jars_location,"runtime-jars","elasticsearch.jar") }
+
+      it "requires the jar files if there are jars to load" do
+        expect(subject).to receive(:find_jars).with(default_runtime_location).and_return([dummy_jar_file])
+        expect(subject).to receive(:require).with(dummy_jar_file)
+        subject.load_runtime_jars!
+      end
+
+      it "raises an exception if there are no jars to load" do
+        allow(Dir).to receive(:glob).and_return([])
+        expect { subject.load_runtime_jars! }.to raise_error
+      end
+
+    end
+  end
+end

spec/util/environment_spec.rb

@@ -1,6 +0,0 @@
-require "spec_helper"
-require "logstash/environment"
-
-describe LogStash::Environment do
-
-end