Java Pipeline Compiler

Fixes #8569

ci/unit_tests.sh

@@ -19,14 +19,14 @@ if [[ $SELECTED_TEST_SUITE == $"core-fail-fast" ]]; then
   rake test:core-fail-fast
 elif [[ $SELECTED_TEST_SUITE == $"java" ]]; then
   echo "Running Java unit tests"
-  echo "Running test:core-java"
+  echo "Running Java Tests"
-  rake test:core-java
+  ./gradlew javaTests
 elif [[ $SELECTED_TEST_SUITE == $"ruby" ]]; then
   echo "Running Ruby unit tests"
   echo "Running test:install-core"
   rake test:install-core
-  echo "Running test:core-ruby"
+  echo "Running Ruby Tests"
-  rake test:core-ruby
+  ./gradlew rubyTests
 else
   echo "Running Java and Ruby unit tests"
   echo "Running test:install-core"

logstash-core/build.gradle

@@ -72,6 +72,13 @@ configurations.archives {
 
 task javaTests(type: Test) {
     exclude '/org/logstash/RSpecTests.class'
+    exclude 'org/logstash/config/ir/ConfigCompilerTest.class'
+}
+
+task rubyTests(type: Test) {
+    systemProperty 'logstash.core.root.dir', projectDir.absolutePath
+    include '/org/logstash/RSpecTests.class'
+    include 'org/logstash/config/ir/ConfigCompilerTest.class'
 }
 
 artifacts {

logstash-core/lib/logstash/compiler.rb

@@ -7,9 +7,9 @@ java_import org.logstash.config.ir.graph.Graph
 module LogStash; class Compiler
   include ::LogStash::Util::Loggable
 
-  def self.compile_sources(sources_with_metadata, settings)
+  def self.compile_sources(sources_with_metadata, support_escapes)
     graph_sections = sources_with_metadata.map do |swm|
-      self.compile_graph(swm, settings)
+      self.compile_graph(swm, support_escapes)
     end
 
     input_graph = Graph.combine(*graph_sections.map {|s| s[:input] }).graph
@@ -30,7 +30,7 @@ module LogStash; class Compiler
     PipelineIR.new(input_graph, filter_graph, output_graph, original_source)
   end
 
-  def self.compile_ast(source_with_metadata, settings)
+  def self.compile_imperative(source_with_metadata, support_escapes)
     if !source_with_metadata.is_a?(org.logstash.common.SourceWithMetadata)
       raise ArgumentError, "Expected 'org.logstash.common.SourceWithMetadata', got #{source_with_metadata.class}"
     end
@@ -42,15 +42,11 @@ module LogStash; class Compiler
       raise ConfigurationError, grammar.failure_reason
     end
 
-    config.process_escape_sequences = settings.get_value("config.support_escapes")
+    config.process_escape_sequences = support_escapes
     config.compile(source_with_metadata)
   end
 
-  def self.compile_imperative(source_with_metadata, settings)
+  def self.compile_graph(source_with_metadata, support_escapes)
-    compile_ast(source_with_metadata, settings)
+    Hash[compile_imperative(source_with_metadata, support_escapes).map {|section,icompiled| [section, icompiled.toGraph]}]
-  end
-
-  def self.compile_graph(source_with_metadata, settings)
-    Hash[compile_imperative(source_with_metadata, settings).map {|section,icompiled| [section, icompiled.toGraph]}]
   end
 end; end

logstash-core/lib/logstash/java_pipeline.rb

@@ -29,6 +29,7 @@ java_import org.logstash.common.DeadLetterQueueFactory
 java_import org.logstash.common.SourceWithMetadata
 java_import org.logstash.common.io.DeadLetterQueueWriter
 java_import org.logstash.config.ir.CompiledPipeline
+java_import org.logstash.config.ir.ConfigCompiler
 
 module LogStash; class JavaBasePipeline
   include org.logstash.config.ir.compiler.RubyIntegration::Pipeline 
@@ -47,7 +48,9 @@ module LogStash; class JavaBasePipeline
     @settings = pipeline_config.settings
     @config_hash = Digest::SHA1.hexdigest(@config_str)
 
-    @lir = compile_lir
+    @lir = ConfigCompiler.configToPipelineIR(
+      @config_str, @settings.get_value("config.support_escapes")
+    )
 
     # Every time #plugin is invoked this is incremented to give each plugin
     # a unique id when auto-generating plugin ids
@@ -85,13 +88,6 @@ module LogStash; class JavaBasePipeline
     end
   end
 
-  def compile_lir
-    sources_with_metadata = [
-      SourceWithMetadata.new("str", "pipeline", 0, 0, self.config_str)
-    ]
-    LogStash::Compiler.compile_sources(sources_with_metadata, @settings)
-  end
-
   def buildOutput(name, line, column, *args)
     plugin("output", name, line, column, *args)
   end

logstash-core/lib/logstash/pipeline.rb

@@ -29,6 +29,7 @@ require "securerandom"
 java_import org.logstash.common.DeadLetterQueueFactory
 java_import org.logstash.common.SourceWithMetadata
 java_import org.logstash.common.io.DeadLetterQueueWriter
+java_import org.logstash.config.ir.ConfigCompiler
 
 module LogStash; class BasePipeline
   include LogStash::Util::Loggable
@@ -46,7 +47,9 @@ module LogStash; class BasePipeline
     @settings = pipeline_config.settings
     @config_hash = Digest::SHA1.hexdigest(@config_str)
 
-    @lir = compile_lir
+    @lir = ConfigCompiler.configToPipelineIR(
+      @config_str, @settings.get_value("config.support_escapes")
+    )
 
     # Every time #plugin is invoked this is incremented to give each plugin
     # a unique id when auto-generating plugin ids
@@ -101,10 +104,9 @@ module LogStash; class BasePipeline
   end
 
   def compile_lir
-    sources_with_metadata = [
+    org.logstash.config.ir.ConfigCompiler.configToPipelineIR(
-      SourceWithMetadata.new("str", "pipeline", 0, 0, self.config_str)
+      self.config_str, @settings.get_value("config.support_escapes")
-    ]
+    )
-    LogStash::Compiler.compile_sources(sources_with_metadata, @settings)
   end
 
   def plugin(plugin_type, name, line, column, *args)

logstash-core/spec/logstash/compiler/compiler_spec.rb

@@ -47,7 +47,7 @@ describe LogStash::Compiler do
         end
       end
 
-      subject(:pipeline) { described_class.compile_sources(sources_with_metadata, settings) }
+      subject(:pipeline) { described_class.compile_sources(sources_with_metadata, false) }
 
       it "should generate a hash" do
         expect(pipeline.unique_hash).to be_a(String)
@@ -100,7 +100,7 @@ describe LogStash::Compiler do
   describe "compiling imperative" do
     let(:source_id) { "fake_sourcefile" }
     let(:source_with_metadata) { org.logstash.common.SourceWithMetadata.new(source_protocol, source_id, 0, 0, source) }
-    subject(:compiled) { described_class.compile_imperative(source_with_metadata, settings) }
+    subject(:compiled) { described_class.compile_imperative(source_with_metadata, settings.get_value("config.support_escapes")) }
 
     context "when config.support_escapes" do
       let(:parser) { LogStashCompilerLSCLGrammarParser.new }

logstash-core/src/main/java/org/logstash/config/ir/ConfigCompiler.java

@@ -0,0 +1,69 @@
+package org.logstash.config.ir;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import org.jruby.RubyHash;
+import org.jruby.javasupport.JavaUtil;
+import org.jruby.runtime.builtin.IRubyObject;
+import org.jruby.runtime.load.LoadService;
+import org.logstash.RubyUtil;
+import org.logstash.common.IncompleteSourceWithMetadataException;
+import org.logstash.common.SourceWithMetadata;
+
+/**
+ * Java Implementation of the config compiler that is implemented by wrapping the Ruby
+ * {@code LogStash::Compiler}.
+ */
+public final class ConfigCompiler {
+
+    private ConfigCompiler() {
+        // Utility Class
+    }
+
+    /**
+     * @param config Logstash Config String
+     * @param supportEscapes The value of the setting {@code config.support_escapes}
+     * @return Compiled {@link PipelineIR}
+     * @throws IncompleteSourceWithMetadataException On Broken Configuration
+     */
+    public static PipelineIR configToPipelineIR(final String config, final boolean supportEscapes)
+        throws IncompleteSourceWithMetadataException {
+        ensureLoadpath();
+        final IRubyObject compiler = RubyUtil.RUBY.executeScript(
+            "require 'logstash/compiler'\nLogStash::Compiler",
+            ""
+        );
+        final IRubyObject code =
+            compiler.callMethod(RubyUtil.RUBY.getCurrentContext(), "compile_sources",
+                new IRubyObject[]{
+                    RubyUtil.RUBY.newArray(
+                        JavaUtil.convertJavaToRuby(
+                            RubyUtil.RUBY,
+                            new SourceWithMetadata("str", "pipeline", 0, 0, config)
+                        )
+                    ),
+                    RubyUtil.RUBY.newBoolean(supportEscapes)
+                }
+            );
+        return (PipelineIR) code.toJava(PipelineIR.class);
+    }
+
+    /**
+     * Loads the logstash-core/lib path if the load service can't find {@code logstash/compiler}.
+     */
+    private static void ensureLoadpath() {
+        final LoadService loader = RubyUtil.RUBY.getLoadService();
+        if (loader.findFileForLoad("logstash/compiler").library == null) {
+            final RubyHash environment = RubyUtil.RUBY.getENV();
+            final Path root = Paths.get(
+                System.getProperty("logstash.core.root.dir", "")
+            ).toAbsolutePath();
+            final String gems = root.getParent().resolve("vendor").resolve("bundle")
+                .resolve("jruby").resolve("2.3.0").toFile().getAbsolutePath();
+            environment.put("GEM_HOME", gems);
+            environment.put("GEM_PATH", gems);
+            loader.addPaths(root.resolve("lib").toFile().getAbsolutePath()
+            );
+        }
+    }
+}

logstash-core/src/test/java/org/logstash/config/ir/ConfigCompilerTest.java

@@ -0,0 +1,17 @@
+package org.logstash.config.ir;
+
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+public class ConfigCompilerTest {
+
+    @Test
+    public void testConfigToPipelineIR() throws Exception {
+        final PipelineIR pipelineIR =
+            ConfigCompiler.configToPipelineIR("input {stdin{}} output{stdout{}}", false);
+        assertThat(pipelineIR.getOutputPluginVertices().size(), is(1));
+        assertThat(pipelineIR.getFilterPluginVertices().size(), is(0));
+    }
+}