#8004 fix load order to start webserver after pipeline

Fixes #9398

logstash-core/lib/logstash/agent.rb

@@ -84,12 +84,12 @@ class LogStash::Agent
     @thread = Thread.current # this var is implicitly used by Stud.stop?
     logger.debug("Starting agent")
 
-    start_webserver
-
     transition_to_running
 
     converge_state_and_update
 
+    start_webserver
+
     if auto_reload?
       # `sleep_then_run` instead of firing the interval right away
       Stud.interval(@reload_interval, :sleep_then_run => true) do

qa/integration/framework/helpers.rb

@@ -20,13 +20,11 @@ def wait_for_port(port, retry_attempts)
 end
 
 def is_port_open?(port)
-  begin
+  TCPSocket.open("localhost", port) do
-    s = TCPSocket.open("localhost", port)
-    s.close
     return true
-  rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH
-    return false
   end
+rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH
+  return false
 end
 
 def send_data(port, data)

qa/integration/services/logstash_service.rb

@@ -17,7 +17,7 @@ class LogstashService < Service
   SETTINGS_CLI_FLAG = "--path.settings"
 
   STDIN_CONFIG = "input {stdin {}} output { }"
-  RETRY_ATTEMPTS = 10
+  RETRY_ATTEMPTS = 60
 
   @process = nil
 
@@ -115,7 +115,6 @@ class LogstashService < Service
       @env_variables.map { |k, v|  @process.environment[k] = v} unless @env_variables.nil?
       @process.io.inherit!
       @process.start
-      wait_for_logstash
       puts "Logstash started with PID #{@process.pid}" if @process.alive?
     end
   end
@@ -164,12 +163,13 @@ class LogstashService < Service
     tries = RETRY_ATTEMPTS
     while tries > 0
       if is_port_open?
-        break
+        return
       else
         sleep 1
       end
       tries -= 1
     end
+    raise "Logstash REST API did not come up after #{RETRY_ATTEMPTS}s."
   end
 
   # this method only overwrites existing config with new config

qa/integration/specs/secret_store_spec.rb

@@ -69,7 +69,6 @@ describe "Test that Logstash" do
       test_env["LOGSTASH_KEYSTORE_PASS"] = "WRONG_PASSWRD"
       @logstash.env_variables = test_env
       @logstash.spawn_logstash("-e", "input {generator { count => 1 }} output { }", "--path.settings", settings_dir)
-      @logstash.wait_for_logstash
       try(num_retries) do
         expect(@logstash.exited?).to be(true)
       end
@@ -83,7 +82,6 @@ describe "Test that Logstash" do
       test_env["LOGSTASH_KEYSTORE_PASS"] = "WRONG_PASSWRD"
       @logstash.env_variables = test_env
       @logstash.spawn_logstash("-e", "input {generator { count => 1 }} output { }", "--path.settings", settings_dir)
-      @logstash.wait_for_logstash
       try(num_retries) do
         expect(@logstash.exited?).to be(true)
       end
@@ -97,11 +95,10 @@ describe "Test that Logstash" do
       test_env["LOGSTASH_KEYSTORE_PASS"] = "keystore_pa9454w3rd"
       @logstash.env_variables = test_env
       @logstash.spawn_logstash("-e", "input {stdin {}} output { }", "--path.settings", settings_dir)
-      @logstash.wait_for_logstash
       try(num_retries) do
         expect(@logstash.exited?).to be(true)
       end
       expect(@logstash.exit_code).to be(1)
     end
   end
-end
+end

qa/integration/specs/settings_spec.rb

@@ -15,13 +15,13 @@ describe "Test Logstash instance whose default settings are overridden" do
   after(:all) {
     @fixture.teardown
   }
-  
+
   before(:each) {
     FileUtils.rm(@logstash_default_logs) if File.exists?(@logstash_default_logs)
     # backup the application settings file -- logstash.yml
     FileUtils.cp(@logstash_service.application_settings_file, "#{@logstash_service.application_settings_file}.original")
   }
-  
+
   after(:each) {
     @logstash_service.teardown
     # restore the application settings file -- logstash.yml
@@ -32,17 +32,17 @@ describe "Test Logstash instance whose default settings are overridden" do
   let(:test_port) { random_port }
   let(:temp_dir) { Stud::Temporary.directory("logstash-settings-test") }
   let(:tcp_config) { @fixture.config("root", { :port => test_port }) }
-  
+
   def change_setting(name, value)
     settings = {}
     settings[name] = value
     overwrite_settings(settings)
   end
-  
+
   def overwrite_settings(settings)
     IO.write(@logstash_service.application_settings_file, settings.to_yaml)
   end
-  
+
   it "should start with a new data dir" do
     change_setting("path.data", temp_dir)
     @logstash_service.spawn_logstash("-e", tcp_config)
@@ -52,7 +52,7 @@ describe "Test Logstash instance whose default settings are overridden" do
       expect(is_port_open?(test_port)).to be true
     end
   end
-  
+
   it "should write logs to a new dir" do
     change_setting("path.logs", temp_dir)
     @logstash_service.spawn_logstash("-e", tcp_config)
@@ -63,7 +63,7 @@ describe "Test Logstash instance whose default settings are overridden" do
     end
     expect(File.exists?("#{temp_dir}/logstash-plain.log")).to be true
   end
-  
+
   it "should read config from the specified dir in logstash.yml" do
     change_setting("path.config", temp_dir)
     test_config_path = File.join(temp_dir, "test.config")
@@ -76,7 +76,7 @@ describe "Test Logstash instance whose default settings are overridden" do
       expect(is_port_open?(test_port)).to be true
     end
   end
-  
+
   it "should exit when config test_and_exit is set" do
     test_config_path = File.join(temp_dir, "test.config")
     IO.write(test_config_path, "#{tcp_config}")
@@ -91,7 +91,7 @@ describe "Test Logstash instance whose default settings are overridden" do
       expect(@logstash_service.exited?).to be true
     end
     expect(@logstash_service.exit_code).to eq(0)
-    
+
     # now with bad config
     IO.write(test_config_path, "#{tcp_config} filters {} ")
     expect(File.exists?(test_config_path)).to be true
@@ -127,16 +127,13 @@ describe "Test Logstash instance whose default settings are overridden" do
     http_port = random_port
     change_setting("http.port", http_port)
     @logstash_service.spawn_logstash("-e", tcp_config)
-    @logstash_service.wait_for_logstash
+    wait_for_port(http_port, 60)
-    
+    expect(is_port_open?(http_port)).to be true
-    try(num_retries) do
-      expect(is_port_open?(http_port)).to be true
-    end
     # check LS is up and running with new data path
     try(num_retries) do
       expect(is_port_open?(test_port)).to be true
     end
-    
+
     expect(File.exists?(@logstash_default_logs)).to be true
 
     resp = Manticore.get("http://localhost:#{http_port}/_node").body