Setting --path.data on CLI should also change path.queue

This change was harder than it first appeared! Due to the complicated
interactions between our Setting class and our monkey-patched Clamp
classes this required adding some new hooks into various places to
properly intercept the settings at the right point and set this
dynamically.

Crucially, this only changes path.queue when the user has *not*
overriden it explicitly in the settings.yml file.

Fixes #6378 and #6387

Fixes #6731

logstash-core/lib/logstash/environment.rb

@@ -56,6 +56,15 @@ module LogStash
   # Compute the default queue path based on `path.data`
   default_queue_file_path = ::File.join(SETTINGS.get("path.data"), "queue")
   SETTINGS.register Setting::WritableDirectory.new("path.queue", default_queue_file_path)
+  
+  SETTINGS.on_post_process do |settings|
+    # If the data path is overriden but the queue path isn't recompute the queue path
+    # We need to do this at this stage because of the weird execution order
+    # our monkey-patched Clamp follows
+    if settings.set?("path.data") && !settings.set?("path.queue")
+      settings.set_value("path.queue", ::File.join(settings.get("path.data"), "queue"))
+    end
+  end
 
   module Environment
     extend self

logstash-core/lib/logstash/runner.rb

@@ -184,6 +184,11 @@ class LogStash::Runner < Clamp::StrictCommand
   end
 
   def execute
+    # Only when execute is have the CLI options been added to the @settings
+    # We invoke post_process to apply extra logic to them.
+    # The post_process callbacks have been added in environment.rb
+    @settings.post_process
+    
     require "logstash/util"
     require "logstash/util/java_version"
     require "stud/task"

logstash-core/lib/logstash/settings.rb

@@ -108,6 +108,20 @@ module LogStash
     def from_yaml(yaml_path)
       settings = read_yaml(::File.join(yaml_path, "logstash.yml"))
       self.merge(flatten_hash(settings), true)
+      self
+    end
+    
+    def post_process
+      if @post_process_callbacks
+        @post_process_callbacks.each do |callback|
+          callback.call(self)
+        end
+      end
+    end
+    
+    def on_post_process(&block)
+      @post_process_callbacks ||= []
+      @post_process_callbacks << block
     end
 
     def validate_all
@@ -232,7 +246,6 @@ module LogStash
           @default = default
         end
       end
-
       def set(value)
         coerced_value = coerce(value)
         validate(coerced_value)
@@ -520,4 +533,3 @@ module LogStash
 
   SETTINGS = Settings.new
 end
-

logstash-core/spec/logstash/runner_spec.rb

@@ -165,7 +165,38 @@ describe LogStash::Runner do
       allow(pipeline).to receive(:run).and_return(task)
       allow(pipeline).to receive(:shutdown)
     end
-
+    
+    context "when :path.data is defined by the user" do
+      let(:test_data_path) { "/tmp/ls-test-data" }
+      let(:test_queue_path) { test_data_path + "/" + "queue" }
+      
+      it "should set data paths" do
+        expect(LogStash::Agent).to receive(:new) do |settings|
+          expect(settings.get("path.data")).to eq(test_data_path)
+          expect(settings.get("path.queue")).to eq(test_queue_path)
+        end
+        
+        args = ["--path.data", test_data_path, "-e", pipeline_string]
+        subject.run("bin/logstash", args)
+      end
+      
+      context "and path.queue is manually set" do
+        let(:queue_override_path) { "/tmp/queue-override_path" }
+        
+        it "should set data paths" do
+          expect(LogStash::Agent).to receive(:new) do |settings|
+            expect(settings.get("path.data")).to eq(test_data_path)
+            expect(settings.get("path.queue")).to eq(queue_override_path)
+          end
+          
+          LogStash::SETTINGS.set("path.queue", queue_override_path)
+          
+          args = ["--path.data", test_data_path, "-e", pipeline_string]
+          subject.run("bin/logstash", args)
+        end
+      end
+    end
+    
     context "when :http.host is defined by the user" do
       it "should pass the value to the webserver" do
         expect(LogStash::Agent).to receive(:new) do |settings|

logstash-core/spec/logstash/settings_spec.rb

@@ -87,6 +87,27 @@ describe LogStash::Settings do
       end
     end
   end
+  
+  describe "post_process" do
+    subject(:settings) { described_class.new }
+    
+    before do
+      settings.on_post_process do
+        settings.set("baz", "bot")
+      end
+      settings.register(LogStash::Setting::String.new("foo", "bar"))
+      settings.register(LogStash::Setting::String.new("baz", "somedefault"))
+      settings.post_process
+    end
+    
+    it "should run the post process callbacks" do
+      expect(settings.get("baz")).to eq("bot")
+    end
+    
+    it "should preserve original settings" do
+      expect(settings.get("foo")).to eq("bar")
+    end
+  end
 
   context "transient settings" do
     subject do