diff --git a/ci/unit_tests.sh b/ci/unit_tests.sh index 2a0ad0944..0284d31ea 100755 --- a/ci/unit_tests.sh +++ b/ci/unit_tests.sh @@ -19,14 +19,14 @@ if [[ $SELECTED_TEST_SUITE == $"core-fail-fast" ]]; then rake test:core-fail-fast elif [[ $SELECTED_TEST_SUITE == $"java" ]]; then echo "Running Java unit tests" - echo "Running test:core-java" - rake test:core-java + echo "Running Java Tests" + ./gradlew javaTests elif [[ $SELECTED_TEST_SUITE == $"ruby" ]]; then echo "Running Ruby unit tests" echo "Running test:install-core" rake test:install-core - echo "Running test:core-ruby" - rake test:core-ruby + echo "Running Ruby Tests" + ./gradlew rubyTests else echo "Running Java and Ruby unit tests" echo "Running test:install-core" diff --git a/logstash-core/build.gradle b/logstash-core/build.gradle index 8584dc4b8..36160e161 100644 --- a/logstash-core/build.gradle +++ b/logstash-core/build.gradle @@ -72,6 +72,13 @@ configurations.archives { task javaTests(type: Test) { exclude '/org/logstash/RSpecTests.class' + exclude 'org/logstash/config/ir/ConfigCompilerTest.class' +} + +task rubyTests(type: Test) { + systemProperty 'logstash.core.root.dir', projectDir.absolutePath + include '/org/logstash/RSpecTests.class' + include 'org/logstash/config/ir/ConfigCompilerTest.class' } artifacts { diff --git a/logstash-core/lib/logstash/compiler.rb b/logstash-core/lib/logstash/compiler.rb index 3f4a109e0..6c568c228 100644 --- a/logstash-core/lib/logstash/compiler.rb +++ b/logstash-core/lib/logstash/compiler.rb @@ -7,9 +7,9 @@ java_import org.logstash.config.ir.graph.Graph module LogStash; class Compiler include ::LogStash::Util::Loggable - def self.compile_sources(sources_with_metadata, settings) + def self.compile_sources(sources_with_metadata, support_escapes) graph_sections = sources_with_metadata.map do |swm| - self.compile_graph(swm, settings) + self.compile_graph(swm, support_escapes) end input_graph = Graph.combine(*graph_sections.map {|s| s[:input] }).graph @@ -30,7 +30,7 @@ module LogStash; class Compiler PipelineIR.new(input_graph, filter_graph, output_graph, original_source) end - def self.compile_ast(source_with_metadata, settings) + def self.compile_imperative(source_with_metadata, support_escapes) if !source_with_metadata.is_a?(org.logstash.common.SourceWithMetadata) raise ArgumentError, "Expected 'org.logstash.common.SourceWithMetadata', got #{source_with_metadata.class}" end @@ -42,15 +42,11 @@ module LogStash; class Compiler raise ConfigurationError, grammar.failure_reason end - config.process_escape_sequences = settings.get_value("config.support_escapes") + config.process_escape_sequences = support_escapes config.compile(source_with_metadata) end - def self.compile_imperative(source_with_metadata, settings) - compile_ast(source_with_metadata, settings) - end - - def self.compile_graph(source_with_metadata, settings) - Hash[compile_imperative(source_with_metadata, settings).map {|section,icompiled| [section, icompiled.toGraph]}] + def self.compile_graph(source_with_metadata, support_escapes) + Hash[compile_imperative(source_with_metadata, support_escapes).map {|section,icompiled| [section, icompiled.toGraph]}] end end; end diff --git a/logstash-core/lib/logstash/java_pipeline.rb b/logstash-core/lib/logstash/java_pipeline.rb index c96ad1387..8c722624b 100644 --- a/logstash-core/lib/logstash/java_pipeline.rb +++ b/logstash-core/lib/logstash/java_pipeline.rb @@ -29,6 +29,7 @@ java_import org.logstash.common.DeadLetterQueueFactory java_import org.logstash.common.SourceWithMetadata java_import org.logstash.common.io.DeadLetterQueueWriter java_import org.logstash.config.ir.CompiledPipeline +java_import org.logstash.config.ir.ConfigCompiler module LogStash; class JavaBasePipeline include org.logstash.config.ir.compiler.RubyIntegration::Pipeline @@ -47,7 +48,9 @@ module LogStash; class JavaBasePipeline @settings = pipeline_config.settings @config_hash = Digest::SHA1.hexdigest(@config_str) - @lir = compile_lir + @lir = ConfigCompiler.configToPipelineIR( + @config_str, @settings.get_value("config.support_escapes") + ) # Every time #plugin is invoked this is incremented to give each plugin # a unique id when auto-generating plugin ids @@ -85,13 +88,6 @@ module LogStash; class JavaBasePipeline end end - def compile_lir - sources_with_metadata = [ - SourceWithMetadata.new("str", "pipeline", 0, 0, self.config_str) - ] - LogStash::Compiler.compile_sources(sources_with_metadata, @settings) - end - def buildOutput(name, line, column, *args) plugin("output", name, line, column, *args) end diff --git a/logstash-core/lib/logstash/pipeline.rb b/logstash-core/lib/logstash/pipeline.rb index 37aeaeb3f..9d61065e2 100644 --- a/logstash-core/lib/logstash/pipeline.rb +++ b/logstash-core/lib/logstash/pipeline.rb @@ -29,6 +29,7 @@ require "securerandom" java_import org.logstash.common.DeadLetterQueueFactory java_import org.logstash.common.SourceWithMetadata java_import org.logstash.common.io.DeadLetterQueueWriter +java_import org.logstash.config.ir.ConfigCompiler module LogStash; class BasePipeline include LogStash::Util::Loggable @@ -46,7 +47,9 @@ module LogStash; class BasePipeline @settings = pipeline_config.settings @config_hash = Digest::SHA1.hexdigest(@config_str) - @lir = compile_lir + @lir = ConfigCompiler.configToPipelineIR( + @config_str, @settings.get_value("config.support_escapes") + ) # Every time #plugin is invoked this is incremented to give each plugin # a unique id when auto-generating plugin ids @@ -101,10 +104,9 @@ module LogStash; class BasePipeline end def compile_lir - sources_with_metadata = [ - SourceWithMetadata.new("str", "pipeline", 0, 0, self.config_str) - ] - LogStash::Compiler.compile_sources(sources_with_metadata, @settings) + org.logstash.config.ir.ConfigCompiler.configToPipelineIR( + self.config_str, @settings.get_value("config.support_escapes") + ) end def plugin(plugin_type, name, line, column, *args) diff --git a/logstash-core/spec/logstash/compiler/compiler_spec.rb b/logstash-core/spec/logstash/compiler/compiler_spec.rb index da0180620..16c81e854 100644 --- a/logstash-core/spec/logstash/compiler/compiler_spec.rb +++ b/logstash-core/spec/logstash/compiler/compiler_spec.rb @@ -47,7 +47,7 @@ describe LogStash::Compiler do end end - subject(:pipeline) { described_class.compile_sources(sources_with_metadata, settings) } + subject(:pipeline) { described_class.compile_sources(sources_with_metadata, false) } it "should generate a hash" do expect(pipeline.unique_hash).to be_a(String) @@ -100,7 +100,7 @@ describe LogStash::Compiler do describe "compiling imperative" do let(:source_id) { "fake_sourcefile" } let(:source_with_metadata) { org.logstash.common.SourceWithMetadata.new(source_protocol, source_id, 0, 0, source) } - subject(:compiled) { described_class.compile_imperative(source_with_metadata, settings) } + subject(:compiled) { described_class.compile_imperative(source_with_metadata, settings.get_value("config.support_escapes")) } context "when config.support_escapes" do let(:parser) { LogStashCompilerLSCLGrammarParser.new } diff --git a/logstash-core/src/main/java/org/logstash/config/ir/ConfigCompiler.java b/logstash-core/src/main/java/org/logstash/config/ir/ConfigCompiler.java new file mode 100644 index 000000000..98b48f75d --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/ConfigCompiler.java @@ -0,0 +1,69 @@ +package org.logstash.config.ir; + +import java.nio.file.Path; +import java.nio.file.Paths; +import org.jruby.RubyHash; +import org.jruby.javasupport.JavaUtil; +import org.jruby.runtime.builtin.IRubyObject; +import org.jruby.runtime.load.LoadService; +import org.logstash.RubyUtil; +import org.logstash.common.IncompleteSourceWithMetadataException; +import org.logstash.common.SourceWithMetadata; + +/** + * Java Implementation of the config compiler that is implemented by wrapping the Ruby + * {@code LogStash::Compiler}. + */ +public final class ConfigCompiler { + + private ConfigCompiler() { + // Utility Class + } + + /** + * @param config Logstash Config String + * @param supportEscapes The value of the setting {@code config.support_escapes} + * @return Compiled {@link PipelineIR} + * @throws IncompleteSourceWithMetadataException On Broken Configuration + */ + public static PipelineIR configToPipelineIR(final String config, final boolean supportEscapes) + throws IncompleteSourceWithMetadataException { + ensureLoadpath(); + final IRubyObject compiler = RubyUtil.RUBY.executeScript( + "require 'logstash/compiler'\nLogStash::Compiler", + "" + ); + final IRubyObject code = + compiler.callMethod(RubyUtil.RUBY.getCurrentContext(), "compile_sources", + new IRubyObject[]{ + RubyUtil.RUBY.newArray( + JavaUtil.convertJavaToRuby( + RubyUtil.RUBY, + new SourceWithMetadata("str", "pipeline", 0, 0, config) + ) + ), + RubyUtil.RUBY.newBoolean(supportEscapes) + } + ); + return (PipelineIR) code.toJava(PipelineIR.class); + } + + /** + * Loads the logstash-core/lib path if the load service can't find {@code logstash/compiler}. + */ + private static void ensureLoadpath() { + final LoadService loader = RubyUtil.RUBY.getLoadService(); + if (loader.findFileForLoad("logstash/compiler").library == null) { + final RubyHash environment = RubyUtil.RUBY.getENV(); + final Path root = Paths.get( + System.getProperty("logstash.core.root.dir", "") + ).toAbsolutePath(); + final String gems = root.getParent().resolve("vendor").resolve("bundle") + .resolve("jruby").resolve("2.3.0").toFile().getAbsolutePath(); + environment.put("GEM_HOME", gems); + environment.put("GEM_PATH", gems); + loader.addPaths(root.resolve("lib").toFile().getAbsolutePath() + ); + } + } +} diff --git a/logstash-core/src/test/java/org/logstash/config/ir/ConfigCompilerTest.java b/logstash-core/src/test/java/org/logstash/config/ir/ConfigCompilerTest.java new file mode 100644 index 000000000..6a2f89f33 --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/config/ir/ConfigCompilerTest.java @@ -0,0 +1,17 @@ +package org.logstash.config.ir; + +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +public class ConfigCompilerTest { + + @Test + public void testConfigToPipelineIR() throws Exception { + final PipelineIR pipelineIR = + ConfigCompiler.configToPipelineIR("input {stdin{}} output{stdout{}}", false); + assertThat(pipelineIR.getOutputPluginVertices().size(), is(1)); + assertThat(pipelineIR.getFilterPluginVertices().size(), is(0)); + } +}