Bug Fix: Log messages from Java code base

This change re-configures default instead of creating a new context.

Fixes #7526

Fixes #7528

logstash-core/build.gradle

@@ -116,5 +116,6 @@ dependencies {
     testCompile 'junit:junit:4.12'
     testCompile 'net.javacrumbs.json-unit:json-unit:1.9.0'
     testCompile 'org.elasticsearch:securemock:1.2'
+    testCompile 'org.assertj:assertj-core:3.8.0'
     provided "org.jruby:jruby-core:$jrubyVersion"
 }

logstash-core/lib/logstash/logging/logger.rb

@@ -8,10 +8,12 @@ module LogStash
     java_import org.apache.logging.log4j.core.config.Configurator
     java_import org.apache.logging.log4j.core.config.DefaultConfiguration
     java_import org.apache.logging.log4j.core.config.LoggerConfig
+    java_import org.logstash.log.LogstashLoggerContextFactory
+    java_import org.apache.logging.log4j.core.LoggerContext
+    java_import java.net.URI
 
     class Logger
       @@config_mutex = Mutex.new
-      @@logging_context = nil
 
       def initialize(name)
         @logger = LogManager.getLogger(name)
@@ -73,43 +75,45 @@ module LogStash
 
       def self.initialize(config_location)
         @@config_mutex.synchronize do
-          if @@logging_context.nil?
-            file_path = URI(config_location).path
-            if ::File.exists?(file_path)
-              logs_location = java.lang.System.getProperty("ls.logs")
-              puts "Sending Logstash's logs to #{logs_location} which is now configured via log4j2.properties"
-              @@logging_context = Configurator.initialize(nil, config_location)
-            else
-              # fall back to default config
-              puts "Could not find log4j2 configuration at path #{file_path}. Using default config which logs to console"
-              @@logging_context = Configurator.initialize(DefaultConfiguration.new)
-            end
+          config_location_uri = URI.create(config_location)
+          file_path = config_location_uri.path
+          if ::File.exists?(file_path)
+            logs_location = java.lang.System.getProperty("ls.logs")
+            puts "Sending Logstash's logs to #{logs_location} which is now configured via log4j2.properties"
+            #reconfigure the default context to use our log4j2.properties file
+            get_logging_context.setConfigLocation(URI.create(config_location))
+            #ensure everyone agrees which context to use for the LogManager
+            context_factory = LogstashLoggerContextFactory.new(get_logging_context)
+            LogManager.setFactory(context_factory)
+          else
+            # fall back to default config
+            puts "Could not find log4j2 configuration at path #{file_path}. Using default config which logs errors to the console"
           end
         end
       end
 
       def self.get_logging_context
-        return @@logging_context
+        return  LoggerContext.getContext(false)
       end
 
-      # Clone of org.apache.logging.log4j.core.config.Configurator.setLevel(), but using initialized @@logging_context
+      # Clone of org.apache.logging.log4j.core.config.Configurator.setLevel(), but ensure the proper context is used
       def self.set_level(_level, path)
-        configuration = @@logging_context.getConfiguration()
+        configuration =  get_logging_context.getConfiguration()
         level = Level.valueOf(_level)
         if path.nil? || path.strip.empty?
           root_logger = configuration.getRootLogger()
           if root_logger.getLevel() != level
             root_logger.setLevel(level)
-            @@logging_context.updateLoggers()
+            get_logging_context.updateLoggers()
           end
         else
           package_logger = configuration.getLoggerConfig(path)
           if package_logger.name != path #no package logger found
             configuration.addLogger(path, LoggerConfig.new(path, level, true))
-            @@logging_context.updateLoggers()
+            get_logging_context.updateLoggers()
           elsif package_logger.getLevel() != level
             package_logger.setLevel(level)
-            @@logging_context.updateLoggers()
+            get_logging_context.updateLoggers()
           end
         end
       end

logstash-core/src/main/java/org/logstash/log/LogstashLoggerContextFactory.java

@@ -0,0 +1,41 @@
+package org.logstash.log;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.spi.LoggerContext;
+import org.apache.logging.log4j.spi.LoggerContextFactory;
+
+import java.net.URI;
+
+/**
+ * Log4j context factory to enable injection of a pre-established context. This may be used in conjunction with
+ * {@link LogManager#setFactory(LoggerContextFactory)} to ensure that the injected pre-established context is used by the {@link LogManager}
+ */
+public class LogstashLoggerContextFactory implements LoggerContextFactory {
+
+    private final LoggerContext context;
+
+    /**
+     * Constructor
+     *
+     * @param context The {@link LoggerContext} that this factory will ALWAYS return.
+     */
+    public LogstashLoggerContextFactory(LoggerContext context) {
+        this.context = context;
+    }
+
+    @Override
+    public LoggerContext getContext(String fqcn, ClassLoader loader, Object externalContext, boolean currentContext) {
+        return context;
+    }
+
+    @Override
+    public LoggerContext getContext(String fqcn, ClassLoader loader, Object externalContext, boolean currentContext,
+                                    URI configLocation, String name) {
+        return context;
+    }
+
+    @Override
+    public void removeContext(LoggerContext context) {
+        //do nothing
+    }
+}

logstash-core/src/main/resources/log4j2.properties

@@ -0,0 +1,11 @@
+name=default
+appenders = console
+
+appender.console.type = Console
+appender.console.name = STDOUT
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = [%-5level] %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %c{1} - %msg%n
+
+rootLogger.level = error
+rootLogger.appenderRefs = stdout
+rootLogger.appenderRef.stdout.ref = STDOUT

logstash-core/src/test/java/org/logstash/log/LogstashLoggerContextFactoryTest.java

@@ -0,0 +1,43 @@
+package org.logstash.log;
+
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.spi.LoggerContext;
+import org.apache.logging.log4j.spi.LoggerContextFactory;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+import static org.mockito.Mockito.verifyZeroInteractions;
+import static org.mockito.Mockito.verify;
+
+import java.net.URI;
+
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Unit test for {@link LogstashLoggerContextFactory}
+ */
+@RunWith(MockitoJUnitRunner.class)
+public class LogstashLoggerContextFactoryTest {
+
+    @Mock
+    private LoggerContext preEstablishedContext;
+
+    private LogstashLoggerContextFactory contextFactory;
+
+    @Before
+    public void setup() {
+        contextFactory = new LogstashLoggerContextFactory(preEstablishedContext);
+    }
+
+    @Test
+    public void testGetContextAlwaysReturnsTheSameObject() {
+        assertThat(contextFactory.getContext("", ClassLoader.getSystemClassLoader(), null, false))
+                .isEqualTo(contextFactory.getContext("someRandomValue", null, null, false))
+                .isEqualTo(contextFactory.getContext("someOtherRandomValue", ClassLoader.getSystemClassLoader(), null, false, URI.create("foo"), "name"));
+    }
+}

qa/integration/specs/monitoring_api_spec.rb

@@ -128,8 +128,8 @@ describe "Test Monitoring API" do
 
       result = logstash_service.monitoring_api.logging_get
       result["loggers"].each do | k, v |
-        #since we explicitly set the logstash.agent logger above, the parent logger will not take precedence
-        if k.eql? "logstash.agent"
+        #since we explicitly set the logstash.agent logger above, the logger.logstash parent logger will not take precedence
+        if k.eql?("logstash.agent") || k.start_with?("org.logstash")
           expect(v).to eq("INFO")
         else
           expect(v).to eq("ERROR")
@@ -143,7 +143,7 @@ describe "Test Monitoring API" do
   def logging_get_assert(logstash_service, logstash_level, slowlog_level)
     result = logstash_service.monitoring_api.logging_get
     result["loggers"].each do | k, v |
-      if k.start_with? "logstash"
+      if k.start_with? "logstash", "org.logstash" #logstash is the ruby namespace, and org.logstash for java
         expect(v).to eq(logstash_level)
       elsif k.start_with? "slowlog"
         expect(v).to eq(slowlog_level)