Allow Logstash to write its logs in JSON format

This is made available by a new `--log-in-json` flag. Default is false.
When false, the old behavior [1] is used. When true, JSON logs are
emitted.

[1] The old behavior is realy two things. First, using Object#inspect to
serialize. Second, to color the output if the IO is a tty.

Fixes #1569

Fixes #4820

logstash-core/lib/logstash/logging/json.rb

@@ -0,0 +1,21 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+require "logstash/json"
+
+module LogStash; class Logging; class JSON
+  def initialize(io)
+    raise ArgumentError, "Expected IO, got #{io.class.name}" unless io.is_a?(IO)
+
+    @io = io
+    @lock = Mutex.new
+  end
+
+  def <<(obj)
+    serialized = LogStash::Json.dump(obj)
+    @lock.synchronize do
+      @io.puts(serialized)
+      @io.flush
+    end
+  end
+end; end; end

logstash-core/lib/logstash/runner.rb

@@ -102,6 +102,10 @@ class LogStash::Runner < Clamp::Command
     I18n.t("logstash.runner.flag.allow-env"),
     :attribute_name => :allow_env, :default => false
 
+  option ["--[no-]log-in-json"], :flag,
+    I18n.t("logstash.runner.flag.log-in-json"),
+    :default => false
+
   def pipeline_workers=(pipeline_workers_value)
     @pipeline_settings[:pipeline_workers] = validate_positive_integer(pipeline_workers_value)
   end
@@ -136,7 +140,7 @@ class LogStash::Runner < Clamp::Command
     require "logstash/util/java_version"
     require "stud/task"
     require "cabin" # gem 'cabin'
-
+    require "logstash/logging/json"
 
     # Configure Logstash logging facility, this need to be done before everything else to
     # make sure the logger has the correct settings and the log level is correctly defined.
@@ -326,11 +330,20 @@ class LogStash::Runner < Clamp::Command
                     :path => path, :error => e))
       end
 
-      @logger.subscribe(STDOUT, :level => :fatal)
-      @logger.subscribe(@log_fd)
+      if log_in_json?
+        @logger.subscribe(LogStash::Logging::JSON.new(STDOUT), :level => :fatal)
+        @logger.subscribe(LogStash::Logging::JSON.new(@log_fd))
+      else
+        @logger.subscribe(STDOUT, :level => :fatal)
+        @logger.subscribe(@log_fd)
+      end
       @logger.terminal "Sending logstash logs to #{path}."
     else
-      @logger.subscribe(STDOUT)
+      if log_in_json?
+        @logger.subscribe(LogStash::Logging::JSON.new(STDOUT))
+      else
+        @logger.subscribe(STDOUT)
+      end
     end
 
     if debug_config? && @logger.level != :debug

logstash-core/locales/en.yml

@@ -246,4 +246,8 @@ en:
         debug_config: |+
           Print the compiled config ruby code out as a debug log (you must also have --debug enabled).
           WARNING: This will include any 'password' options passed to plugin configs as plaintext, and may result
-          in plaintext passwords appearing in your logs!
+          in plaintext passwords appearing in your logs!
+        log-in-json: |+
+          Specify that Logstash should write its own logs in JSON form - one
+          event per line. If false, Logstash will log using Ruby's
+          Object#inspect (not easy to machine-parse)

logstash-core/spec/logstash/runner_spec.rb

@@ -3,7 +3,10 @@ require "spec_helper"
 require "logstash/runner"
 require "stud/task"
 require "stud/trap"
+require "stud/temporary"
 require "logstash/util/java_version"
+require "logstash/logging/json"
+require "json"
 
 class NullRunner
   def run(args); end
@@ -16,7 +19,7 @@ describe LogStash::Runner do
 
   before :each do
     allow(Cabin::Channel).to receive(:get).with(LogStash).and_return(channel)
-    allow(channel).to receive(:subscribe).with(any_args)
+    allow(channel).to receive(:subscribe).with(any_args).and_call_original
   end
 
   describe "argument parsing" do
@@ -95,6 +98,32 @@ describe LogStash::Runner do
     end
   end
 
+  context "--log-in-json" do
+    subject { LogStash::Runner.new("") }
+    let(:logfile) { Stud::Temporary.file }
+    let(:args) { [ "--log-in-json", "-l", logfile.path, "-e", "input {} output{}" ] }
+
+    after do
+      logfile.close
+      File.unlink(logfile.path)
+    end
+
+    before do
+      expect(channel).to receive(:subscribe).with(kind_of(LogStash::Logging::JSON)).and_call_original
+      subject.run(args)
+
+      # Log file should have stuff in it.
+      expect(logfile.stat.size).to be > 0
+    end
+
+    it "should log in valid json. One object per line." do
+      logfile.each_line do |line|
+        expect(line).not_to be_empty
+        expect { JSON.parse(line) }.not_to raise_error
+      end
+    end
+  end
+
   describe "--config-test" do
     subject { LogStash::Runner.new("") }
     let(:args) { ["-t", "-e", pipeline_string] }