Update jruby version to 9.3.4.0 (#14114)

This commit updates the version of jruby used in Logstash to `9.3.4.0`.

* Updates the references of `jruby` from `9.2.20.1` to `9.3.4.0`
* Updates references/locations of ruby from `2.5.0` to `2.6.0`
* Updates java imports including `org.logstash.util` to be quoted
  * Without quoting the name of the import, the following error is observed in tests:
  * `java.lang.NoClassDefFoundError: org/logstash/Util (wrong name: org/logstash/util)`
  * Maybe an instance of https://github.com/jruby/jruby/issues/4861
* Adds a monkey patch to `require` to resolve compatibility issue between latest `jruby` and `polyglot` gem 
  * The addition of https://github.com/jruby/jruby/pull/7145 to disallow circular
causes, will throw when `polyglot` is thrown into the mix, and stop logstash from
starting and building - any gems that use an exception to determine whether or not
to load the native gem, will trigger the code added in that commit.
  * This commit adds a monkey patch of `require` to rollback the circular cause exception
back to the original cause.
* Removes the use of the deprecated `JavaClass`
* Adds additional `require time` in `generate_build_metadata`
* Rewrites a test helper to avoid potentially calling `~>` on `FalseClass`


Co-authored-by: Joao Duarte <jsvduarte@gmail.com>
Co-authored-by: João Duarte <jsvd@users.noreply.github.com>
This commit is contained in:
Rob Bavey 2022-05-19 10:29:59 -04:00 committed by GitHub
parent 379ebaf1bd
commit 4a2268a43f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
28 changed files with 81 additions and 64 deletions

View file

@ -35,7 +35,7 @@ ADD LICENSE.txt /opt/logstash/LICENSE.txt
ADD NOTICE.TXT /opt/logstash/NOTICE.TXT
ADD licenses /opt/logstash/licenses
ADD CONTRIBUTORS /opt/logstash/CONTRIBUTORS
ADD Gemfile.template Gemfile.jruby-2.5.lock.* /opt/logstash/
ADD Gemfile.template Gemfile.jruby-2.6.lock.* /opt/logstash/
ADD Rakefile /opt/logstash/Rakefile
ADD build.gradle /opt/logstash/build.gradle
ADD rubyUtils.gradle /opt/logstash/rubyUtils.gradle

View file

@ -163,7 +163,7 @@ setup_vendored_jruby() {
fi
if [ -z "$LS_GEM_HOME" ] ; then
export GEM_HOME="${LOGSTASH_HOME}/vendor/bundle/jruby/2.5.0"
export GEM_HOME="${LOGSTASH_HOME}/vendor/bundle/jruby/2.6.0"
else
export GEM_HOME=${LS_GEM_HOME}
fi

View file

@ -318,9 +318,9 @@ def bundlerVersion = "~> 2"
tasks.register("installBundler") {
dependsOn assemblyDeps
outputs.files file("${projectDir}/vendor/bundle/jruby/2.5.0/bin/bundle")
outputs.files file("${projectDir}/vendor/bundle/jruby/2.6.0/bin/bundle")
doLast {
gem(projectDir, buildDir, "bundler", bundlerVersion, "${projectDir}/vendor/bundle/jruby/2.5.0")
gem(projectDir, buildDir, "bundler", bundlerVersion, "${projectDir}/vendor/bundle/jruby/2.6.0")
}
}
@ -334,9 +334,9 @@ tasks.register("bootstrap"){
tasks.register("installDefaultGems") {
dependsOn bootstrap
doLast {
rake(projectDir, buildDir, 'plugin:install-default')
}
doLast {
rake(projectDir, buildDir, 'plugin:install-default')
}
}
tasks.register("installTestGems") {
@ -441,7 +441,7 @@ tasks.register("unpackTarDistribution", Copy) {
def qaBuildPath = "${buildDir}/qa/integration"
def qaVendorPath = "${qaBuildPath}/vendor"
def qaBundledGemPath = "${qaVendorPath}/jruby/2.5.0".toString()
def qaBundledGemPath = "${qaVendorPath}/jruby/2.6.0".toString()
def qaBundleBin = "${qaBundledGemPath}/bin/bundle"
tasks.register("installIntegrationTestBundler"){

View file

@ -119,7 +119,7 @@ module LogStash
)
end
# create Gemfile.jruby-1.9.lock from template iff a template exists it itself does not exist
lock_template = ::File.join(ENV["LOGSTASH_HOME"], "Gemfile.jruby-2.5.lock.release")
lock_template = ::File.join(ENV["LOGSTASH_HOME"], "Gemfile.jruby-2.6.lock.release")
if ::File.exists?(lock_template) && !::File.exists?(Environment::LOCKFILE)
FileUtils.copy(lock_template, Environment::LOCKFILE)
end

View file

@ -26,7 +26,7 @@ module LogStash module PluginManager
# - Generate the specifications
# - Copy the data in the right folders
class GemInstaller
GEM_HOME = Pathname.new(::File.join(LogStash::Environment::BUNDLE_DIR, "jruby", "2.5.0"))
GEM_HOME = Pathname.new(::File.join(LogStash::Environment::BUNDLE_DIR, "jruby", "2.6.0"))
SPECIFICATIONS_DIR = "specifications"
GEMS_DIR = "gems"
CACHE_DIR = "cache"

View file

@ -68,9 +68,9 @@ module LogStash::PluginManager
return false
end
else
dep = Gem::Dependency.new(plugin, version || Gem::Requirement.default)
Gem.sources = Gem::SourceList.from(options[:rubygems_source]) if options[:rubygems_source]
specs, errors = Gem::SpecFetcher.fetcher.spec_for_dependency(dep)
dep = ::Gem::Dependency.new(plugin, version || ::Gem::Requirement.default)
::Gem.sources = ::Gem::SourceList.from(options[:rubygems_source]) if options[:rubygems_source]
specs, errors = ::Gem::SpecFetcher.fetcher.spec_for_dependency(dep)
# dump errors
errors.each { |error| $stderr.puts(error.wordy) }
@ -112,7 +112,7 @@ module LogStash::PluginManager
# @return [Gem::Specification] .get file gem specification
# @raise [Exception] Gem::Package::FormatError will be raised on invalid .gem file format, might be other exceptions too
def self.plugin_file_spec(path)
Gem::Package.new(path).spec
::Gem::Package.new(path).spec
end
# @param plugin [String] the plugin name or the local path to a .gem file
@ -125,7 +125,7 @@ module LogStash::PluginManager
# @param name [String] specific plugin name to find or nil for all plugins
# @return [Array<Gem::Specification>] all local logstash plugin gem specs
def self.find_plugins_gem_specs(name = nil)
specs = name ? Gem::Specification.find_all_by_name(name) : Gem::Specification.find_all
specs = name ? ::Gem::Specification.find_all_by_name(name) : ::Gem::Specification.find_all
specs.select{|spec| logstash_plugin_gem_spec?(spec)}
end

View file

@ -60,8 +60,11 @@ module LogStash::Api::AppHelpers
end
def as_boolean(string)
return true if string == true || string =~ (/(true|t|yes|y|1)$/i)
return false if string == false || LogStash::Util.blank?(string) || string =~ (/(false|f|no|n|0)$/i)
return true if string == true
return false if string == false
return true if string =~ (/(true|t|yes|y|1)$/i)
return false if LogStash::Util.blank?(string) || string =~ (/(false|f|no|n|0)$/i)
raise ArgumentError.new("invalid value for Boolean: \"#{string}\"")
end

View file

@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
require "logstash/patches/polyglot"
require "treetop"
require "logstash/compiler/treetop_monkeypatches"
require "logstash/compiler/lscl/helpers"
@ -112,7 +113,7 @@ module LogStashCompilerLSCLGrammar; module LogStash; module Compiler; module LSC
def expr_attributes
# Turn attributes into a hash map
self.attributes.recursive_select(Attribute).map(&:expr).map {|k,v|
if v.java_kind_of?(Java::OrgLogstashConfigIrExpression::ValueExpression)
if v.kind_of?(Java::OrgLogstashConfigIrExpression::ValueExpression)
[k, v.get]
else
[k,v]
@ -347,7 +348,7 @@ module LogStashCompilerLSCLGrammar; module LogStash; module Compiler; module LSC
def jconvert(sexpr)
raise "jconvert cannot handle nils!" if sexpr.nil?
if sexpr.java_kind_of?(Java::OrgLogstashConfigIrExpression::Expression)
if sexpr.kind_of?(Java::OrgLogstashConfigIrExpression::Expression)
return sexpr
end

View file

@ -71,13 +71,13 @@ module LogStash;
end
def vertex_type(v)
if v.java_kind_of?(org.logstash.config.ir.graph.PluginVertex)
if v.kind_of?(org.logstash.config.ir.graph.PluginVertex)
:plugin
elsif v.java_kind_of?(org.logstash.config.ir.graph.IfVertex)
elsif v.kind_of?(org.logstash.config.ir.graph.IfVertex)
:if
elsif v.java_kind_of?(org.logstash.config.ir.graph.QueueVertex)
elsif v.kind_of?(org.logstash.config.ir.graph.QueueVertex)
:queue
elsif v.java_kind_of?(org.logstash.config.ir.graph.SeparatorVertex)
elsif v.kind_of?(org.logstash.config.ir.graph.SeparatorVertex)
:separator
else
raise "Unexpected vertex type! #{v}"
@ -121,7 +121,7 @@ module LogStash;
edges.each do |e|
if vertex_type(e.to) == :separator
e.to.getOutgoingEdges.each do |outgoing|
if e.java_kind_of?(org.logstash.config.ir.graph.BooleanEdge)
if e.kind_of?(org.logstash.config.ir.graph.BooleanEdge)
edges_with_separators_removed << edge(org.logstash.config.ir.graph.BooleanEdge.new(e.edgeType, e.from, outgoing.to))
else
edges_with_separators_removed << edge(org.logstash.config.ir.graph.PlainEdge.factory.make(e.from, outgoing.to))
@ -143,7 +143,7 @@ module LogStash;
"id" => e.id
}
if e.java_kind_of?(org.logstash.config.ir.graph.BooleanEdge)
if e.kind_of?(org.logstash.config.ir.graph.BooleanEdge)
e_json["when"] = e.edge_type
e_json["type"] = "boolean"
else

View file

@ -0,0 +1,19 @@
require 'polyglot'
module Kernel
alias original_require require
def require(*a, &b)
begin
original_require(*a, &b)
rescue RuntimeError => e
# https://github.com/jruby/jruby/pull/7145 introduced an exception check for circular causes, which
# breaks when the polyglot library is used and LoadErrors are emitted
if e.message == "circular causes"
raise e.cause
end
raise e
end
end
end

View file

@ -293,7 +293,6 @@ module LogStash module Plugins
# @return [Boolean] true if klass is a valid plugin for name
def is_a_plugin?(klass, name)
(klass.class == Java::JavaLang::Class && klass.simple_name.downcase == name.gsub('_','')) ||
(klass.class == Java::JavaClass && klass.simple_name.downcase == name.gsub('_','')) ||
(klass.ancestors.include?(LogStash::Plugin) && klass.respond_to?(:config_name) &&
klass.config_name == name)
end

View file

@ -32,7 +32,6 @@ if !LogStash::OSS
end
require "clamp"
require "logstash-core/logstash-core"
require "logstash/environment"
require "logstash/modules/cli_parser"

View file

@ -16,5 +16,5 @@
# under the License.
module LogStash; module Util
java_import org.logstash.util.CloudSettingAuth
java_import "org.logstash.util.CloudSettingAuth"
end; end

View file

@ -18,5 +18,5 @@
require "base64"
module LogStash; module Util
java_import org.logstash.util.CloudSettingId
java_import "org.logstash.util.CloudSettingId"
end; end

View file

@ -16,5 +16,5 @@
# under the License.
module LogStash; module Util
java_import org.logstash.util.ModulesSettingArray
java_import "org.logstash.util.ModulesSettingArray"
end; end

View file

@ -17,6 +17,6 @@
module LogStash
module Util
java_import org.logstash.util.TimeValue
java_import "org.logstash.util.TimeValue"
end
end

View file

@ -90,17 +90,5 @@ describe LogStash::Util do
expect( thread_id ).to be_a Integer
expect( thread_id ).to eq(java.lang.Thread.currentThread.getId)
end
context "when a (native) thread is collected" do
let(:dead_thread) { Thread.new { 42 }.tap { |t| sleep(0.01) while t.status } }
it "returns nil as id" do
thread = dead_thread
p thread if $VERBOSE
2.times { java.lang.System.gc || sleep(0.01) } # we're assuming a full-gc to clear all weak-refs
# NOTE: if you notice this spec failing - remote it (a java.lang.Thread weak-ref might stick around)
expect(LogStash::Util.get_thread_id(thread)).to be nil
end
end
end
end

View file

@ -110,7 +110,7 @@ end
RSpec::Matchers.define :ir_eql do |expected|
match do |actual|
next unless expected.java_kind_of?(org.logstash.config.ir.SourceComponent) && actual.java_kind_of?(org.logstash.config.ir.SourceComponent)
next unless expected.kind_of?(org.logstash.config.ir.SourceComponent) && actual.kind_of?(org.logstash.config.ir.SourceComponent)
expected.sourceComponentEquals(actual)
end

View file

@ -31,9 +31,11 @@ import java.util.stream.Stream;
import org.jruby.Ruby;
import org.jruby.RubyBasicObject;
import org.jruby.RubyString;
import org.jruby.RubyModule;
import org.jruby.anno.JRubyMethod;
import org.jruby.java.proxies.JavaProxy;
import org.jruby.java.proxies.MapJavaProxy;
import org.jruby.javasupport.Java;
import org.jruby.javasupport.JavaClass;
import org.jruby.javasupport.JavaUtil;
import org.jruby.runtime.Block;
@ -60,14 +62,14 @@ public final class RubyJavaIntegration {
ruby.getArray().defineAnnotatedMethods(RubyJavaIntegration.RubyArrayOverride.class);
ruby.getHash().defineAnnotatedMethods(RubyJavaIntegration.RubyHashOverride.class);
Stream.of(LinkedHashMap.class, HashMap.class).forEach(cls ->
JavaClass.get(ruby, cls).getProxyModule().defineAnnotatedMethods(
Java.getProxyClass(ruby, cls).defineAnnotatedMethods(
RubyJavaIntegration.RubyMapProxyOverride.class
)
);
JavaClass.get(ruby, Map.class).getProxyModule().defineAnnotatedMethods(
Java.getProxyClass(ruby, Map.class).defineAnnotatedMethods(
RubyJavaIntegration.JavaMapOverride.class
);
JavaClass.get(ruby, Collection.class).getProxyModule().defineAnnotatedMethods(
Java.getProxyClass(ruby, Collection.class).defineAnnotatedMethods(
RubyJavaIntegration.JavaCollectionOverride.class
);
}
@ -279,7 +281,8 @@ public final class RubyJavaIntegration {
@JRubyMethod
public static IRubyObject merge(final ThreadContext context, final IRubyObject self,
final IRubyObject other) {
return ((MapJavaProxy) self.dup()).merge_bang(context, other, Block.NULL_BLOCK);
IRubyObject[] other_array = { other };
return ((MapJavaProxy) self.dup()).merge_bang(context, other_array, Block.NULL_BLOCK);
}
}
}

View file

@ -23,6 +23,7 @@ package org.logstash.config.ir;
import java.nio.file.Path;
import org.jruby.RubyHash;
import org.jruby.runtime.load.LibrarySearcher;
import org.jruby.runtime.load.LoadService;
import org.junit.BeforeClass;
import org.logstash.RubyTestBase;
@ -41,9 +42,10 @@ public abstract class RubyEnvTestCase extends RubyTestBase {
*/
private static void ensureLoadpath() {
final LoadService loader = RubyUtil.RUBY.getLoadService();
if (loader.findFileForLoad("logstash/compiler").library == null) {
final LibrarySearcher librarySearcher = new LibrarySearcher(loader);
if (librarySearcher.findLibraryForLoad("logstash/compiler") == null) {
final String gems = LS_HOME.
resolve("vendor").resolve("bundle").resolve("jruby").resolve("2.5.0").
resolve("vendor").resolve("bundle").resolve("jruby").resolve("2.6.0").
toFile().getAbsolutePath();
final RubyHash environment = RubyUtil.RUBY.getENV();
environment.put("GEM_HOME", gems);

View file

@ -20,7 +20,7 @@ require 'rubygems'
::Gem.clear_paths
ENV['GEM_HOME'] = ENV['GEM_PATH'] = ::File.expand_path(
::File.join(__FILE__, "..", "..", "..", "build", "qa", "integration", "vendor", "jruby", "2.5.0")
::File.join(__FILE__, "..", "..", "..", "build", "qa", "integration", "vendor", "jruby", "2.6.0")
)
::Gem.paths = ENV

View file

@ -40,7 +40,7 @@ describe "CLI >" do
execute = @logstash.run
lines = execute.stderr_and_stdout.split("\n")
expect(lines.shift).to match(/^(Using system java)|(Using bundled JDK)|(Using LS_JAVA_HOME defined java):/)
while (up_line = lines.shift).match(/OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was deprecated|warning: ignoring JAVA_TOOL_OPTIONS/) do end
while (up_line = lines.shift).match(/OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was deprecated|warning: ignoring JAVA_TOOL_OPTIONS|warning: already initialized constant Socket::Constants/) do end
expect(up_line).to match(/^Sending Logstash logs to/)
end
end

View file

@ -346,6 +346,8 @@ namespace "artifact" do
end
task "generate_build_metadata" do
require 'time'
return if defined?(BUILD_METADATA_FILE)
BUILD_METADATA_FILE = Tempfile.new('build.rb')
BUILD_DATE=Time.now.iso8601

View file

@ -16,6 +16,7 @@
# under the License.
rule ".rb" => ".treetop" do |task, args|
require 'logstash/patches/polyglot'
require "treetop"
compiler = Treetop::Compiler::GrammarCompiler.new
compiler.compile(task.source, task.name)

View file

@ -25,7 +25,7 @@ buildscript {
dependencies {
classpath 'org.yaml:snakeyaml:1.29'
classpath "de.undercouch:gradle-download-task:4.0.4"
classpath "org.jruby:jruby-complete:9.2.20.1"
classpath "org.jruby:jruby-complete:9.3.4.0"
}
}
@ -139,7 +139,7 @@ void buildGem(File projectDir, File buildDir, String gemspec) {
void rake(File projectDir, File buildDir, String task) {
executeJruby projectDir, buildDir, { ScriptingContainer jruby ->
jruby.currentDirectory = projectDir
jruby.runScriptlet("require 'rake'")
jruby.runScriptlet("require 'rake'; require 'time'")
jruby.runScriptlet("""
rake = Rake.application
rake.init
@ -168,7 +168,7 @@ void setupJruby(File projectDir, File buildDir) {
Object executeJruby(File projectDir, File buildDir, Closure<?> /* Object*/ block) {
def jruby = new ScriptingContainer()
def env = jruby.environment
def gemDir = "${projectDir}/vendor/bundle/jruby/2.5.0".toString()
def gemDir = "${projectDir}/vendor/bundle/jruby/2.6.0".toString()
env.put "USE_RUBY", "1"
env.put "GEM_HOME", gemDir
env.put "GEM_SPEC_CACHE", "${buildDir}/cache".toString()

View file

@ -66,14 +66,14 @@ puts "Computing #{allow_bump_for} plugin dependency bump from #{base_logstash_ve
if base_logstash_version == "LOCAL"
puts "Using local lockfile..."
begin
result = File.read("Gemfile.jruby-2.5.lock.release")
result = File.read("Gemfile.jruby-2.6.lock.release")
rescue => e
puts "Failed to read local lockfile #{e}"
exit(1)
end
else
puts "Fetching lock file for #{base_logstash_version}.."
uri = URI.parse("https://raw.githubusercontent.com/elastic/logstash/v#{base_logstash_version}/Gemfile.jruby-2.5.lock.release")
uri = URI.parse("https://raw.githubusercontent.com/elastic/logstash/v#{base_logstash_version}/Gemfile.jruby-2.6.lock.release")
result = Net::HTTP.get(uri)
if result.match(/404/)
puts "Lock file or git tag for #{base_logstash_version} not found. Aborting"
@ -104,7 +104,7 @@ end
IO.write("Gemfile.template", gemfile)
puts "Cleaning up before running computing dependencies"
FileUtils.rm_f("Gemfile.jruby-2.5.lock.release")
FileUtils.rm_f("Gemfile.jruby-2.6.lock.release")
# compute new lock file
puts "Running: ./gradlew clean installDefaultGems"
@ -121,18 +121,18 @@ IO.write("Gemfile.lock", new_lock.join("\n"))
# rename file
puts "Finishing up.."
FileUtils.mv("Gemfile.lock", "Gemfile.jruby-2.5.lock.release")
FileUtils.mv("Gemfile.lock", "Gemfile.jruby-2.6.lock.release")
`git checkout -- Gemfile.template`
puts `git diff Gemfile.jruby-2.5.lock.release`
puts `git diff Gemfile.jruby-2.6.lock.release`
exit(0) unless options[:pr]
puts "Creating commit.."
branch_name = "update_lock_#{Time.now.to_i}"
`git checkout -b #{branch_name}`
`git commit Gemfile.jruby-2.5.lock.release -m "Update #{allow_bump_for} plugin versions in gemfile lock"`
`git commit Gemfile.jruby-2.6.lock.release -m "Update #{allow_bump_for} plugin versions in gemfile lock"`
puts "Pushing commit.."
`git remote add upstream git@github.com:elastic/logstash.git`

View file

@ -13,9 +13,8 @@ bundled_jdk:
# jruby must reference a *released* version of jruby which can be downloaded from the official download url
# *and* for which jars artifacts are published for compile-time
jruby:
version: 9.2.20.1
sha1: 7f45f8a3d53d5f19f69952014abcb542b2670846
version: 9.3.4.0
sha1: 91e54b1c8962dd7a7fbcbab012f5d8ba1f15e5a4
# jruby-runtime-override, if specified, will override the jruby version installed in vendor/jruby
#jruby-runtime-override:
# url: https://oss.sonatype.org/content/repositories/snapshots/org/jruby/jruby-dist/9.3.0.0-SNAPSHOT/jruby-dist-9.3.0.0-20210723.214927-259-bin.tar.gz

View file

@ -7,5 +7,6 @@ MONITORING_INDEXES = ".monitoring-logstash-*"
require_relative "support/helpers"
require_relative "support/shared_examples"
require_relative "support/elasticsearch/api/actions/update_password"
require "logstash/patches/polyglot"
require "json"
require "json-schema"