Make logstash.yml optional

5.0.0 required Logstash to have a valid logstash.yml before it could start successfully. This
was mostly fine for users who installed Logstash via tar.gz, but many many folks who install
it via packages still start Logstash manually. Also, our documentation uses -e flag for
getting started on Logstash and sending their first event.logstash.yml has only defaults defined,
and there is no required parameter to start Logstash. We should be able to use the defaults if no
logstash.yml. Obviously, this is not ideal from a user point of view, so we should log a warning but
continue to bootstrap.

Fixes #6170

Fixes #6172

logstash-core/lib/logstash/logging/logger.rb

@@ -1,4 +1,5 @@
 require "logstash/java_integration"
+require "uri"
 
 module LogStash
   module Logging
@@ -6,6 +7,7 @@ module LogStash
       java_import org.apache.logging.log4j.Level
       java_import org.apache.logging.log4j.LogManager
       java_import org.apache.logging.log4j.core.config.Configurator
+      java_import org.apache.logging.log4j.core.config.DefaultConfiguration
       @@config_mutex = Mutex.new
       @@logging_context = nil
 
@@ -70,7 +72,16 @@ module LogStash
       def self.initialize(config_location)
         @@config_mutex.synchronize do
           if @@logging_context.nil?
-            @@logging_context = Configurator.initialize(nil, config_location)
+            file_path = URI(config_location).path
+            if ::File.exists?(file_path)
+              logs_location = java.lang.System.getProperty("ls.logs")
+              puts "Sending Logstash's logs to #{logs_location} which is now configured via log4j2.properties"
+              @@logging_context = Configurator.initialize(nil, config_location)
+            else
+              # fall back to default config
+              puts "Could not find log4j2 configuration at path #{file_path}. Using default config which logs to console"
+              @@logging_context = Configurator.initialize(DefaultConfiguration.new)
+            end
           end
         end
       end

logstash-core/lib/logstash/runner.rb

@@ -164,11 +164,8 @@ class LogStash::Runner < Clamp::StrictCommand
 
     begin
       LogStash::SETTINGS.from_yaml(LogStash::SETTINGS.get("path.settings"))
-    rescue Errno::ENOENT 
-      unless cli_help?(args)
-        $stderr.puts "ERROR: Logstash requires a setting file which is typically located in $LS_HOME/config or /etc/logstash. If you installed Logstash through a package and are starting it manually, please specify the location to this settings file by passing --path.settings /etc/logstash"
-        return 1
-      end   
+    rescue Errno::ENOENT
+      $stderr.puts "WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults"
     rescue => e
       # abort unless we're just looking for the help
       unless cli_help?(args)
@@ -196,12 +193,6 @@ class LogStash::Runner < Clamp::StrictCommand
     end
     # override log level that may have been introduced from a custom log4j config file
     LogStash::Logging::Logger::configure_logging(setting("log.level"))
-    
-    # Adding this here because a ton of LS users install LS via packages and try to manually 
-    # start Logstash using bin/logstash. See #5986. I think not logging to console is good for 
-    # services, but until LS users re-learn that logs end up in path.logs, we should keep this 
-    # message. Otherwise we'll be answering the same question again and again.
-    puts "Sending Logstash logs to #{setting("path.logs")} which is now configured via log4j2.properties."
 
     if setting("config.debug") && logger.debug?
       logger.warn("--config.debug was specified, but log.level was not set to \'debug\'! No config info will be logged.")
@@ -288,7 +279,12 @@ class LogStash::Runner < Clamp::StrictCommand
     show_short_help
     return 1
   rescue => e
-    logger.fatal(I18n.t("oops"), :error => e, :backtrace => e.backtrace)
+    # if logger itself is not initialized
+    if LogStash::Logging::Logger.get_logging_context.nil?
+      puts e
+    else
+      logger.fatal(I18n.t("oops"), :error => e, :backtrace => e.backtrace)
+    end
     return 1
   ensure
     Stud::untrap("INT", sigint_id) unless sigint_id.nil?
@@ -421,6 +417,6 @@ class LogStash::Runner < Clamp::StrictCommand
   def cli_help?(args)
     # I know, double negative
     !(["--help", "-h"] & args).empty?
-  end  
+  end
 
 end

qa/integration/specs/settings_spec.rb

@@ -9,6 +9,7 @@ describe "Test Logstash instance whose default settings are overridden" do
   before(:all) {
     @fixture = Fixture.new(__FILE__)
     @logstash_service = @fixture.get_service("logstash")
+    @logstash_default_logs = File.join(@logstash_service.logstash_home, "logs", "logstash-plain.log")
   }
 
   after(:all) {
@@ -22,6 +23,7 @@ describe "Test Logstash instance whose default settings are overridden" do
   
   after(:each) {
     @logstash_service.teardown
+    FileUtils.rm(@logstash_default_logs) if File.exists?(@logstash_default_logs)
     # restore the application settings file -- logstash.yml
     FileUtils.mv("#{@logstash_service.application_settings_file}.original", @logstash_service.application_settings_file)
   }
@@ -39,7 +41,7 @@ describe "Test Logstash instance whose default settings are overridden" do
   
   def overwrite_settings(settings)
     IO.write(@logstash_service.application_settings_file, settings.to_yaml)
-  end  
+  end
   
   it "should start with a new data dir" do
     change_setting("path.data", temp_dir)
@@ -96,7 +98,7 @@ describe "Test Logstash instance whose default settings are overridden" do
     end
     expect(@logstash_service.exit_code).to eq(1)
   end
-  
+
   it "change pipeline settings" do
     s = {}
     workers = 31
@@ -110,13 +112,13 @@ describe "Test Logstash instance whose default settings are overridden" do
     try do
       expect(is_port_open?(test_port)).to be true
     end
-    
+
     # now check monitoring API to validate
     node_info = @logstash_service.monitoring_api.node_info
     expect(node_info["pipeline"]["workers"]).to eq(workers)
     expect(node_info["pipeline"]["batch_size"]).to eq(batch_size)
   end
-  
+
   it "start on a different HTTP port" do
     # default in 9600
     http_port = random_port
@@ -131,9 +133,30 @@ describe "Test Logstash instance whose default settings are overridden" do
       expect(is_port_open?(test_port)).to be true
     end
     
+    expect(File.exists?(@logstash_default_logs)).to be true
+
     resp = Manticore.get("http://localhost:#{http_port}/_node").body
     node_info = JSON.parse(resp)
     # should be default
     expect(node_info["http_address"]).to eq("127.0.0.1:#{http_port}")
   end
-end  
+
+  it "start even without a settings file specified" do
+    @logstash_service.spawn_logstash("-e", tcp_config, "--path.settings", "/tmp/fooooobbaaar")
+    http_port = 9600
+    try(num_retries) do
+      expect(is_port_open?(http_port)).to be true
+    end
+
+    try(num_retries) do
+      expect(is_port_open?(test_port)).to be true
+    end
+
+    resp = Manticore.get("http://localhost:#{http_port}/_node").body
+    node_info = JSON.parse(resp)
+    expect(node_info["http_address"]).to eq("127.0.0.1:#{http_port}")
+
+    # make sure we log to console and not to any file
+    expect(File.exists?(@logstash_default_logs)).to be false
+  end
+end