make integration specs a bit more resilient

* wait for logstash starting in monitoring_api integration tests
* remove initialization code from monitoring api integration spec

qa/integration/specs/01_logstash_bin_smoke_spec.rb

@@ -29,7 +29,7 @@ describe "Test Logstash instance" do
   let(:file_config2) { Stud::Temporary.file.path }
   let(:file_config3) { Stud::Temporary.file.path }
 
-  let(:num_retries) { 10 }
+  let(:num_retries) { 50 }
   let(:config1) { config_to_temp_file(@fixture.config("root", { :port => port1, :random_file => file_config1 })) }
   let(:config2) { config_to_temp_file(@fixture.config("root", { :port => port2 , :random_file => file_config2 })) }
   let(:config3) { config_to_temp_file(@fixture.config("root", { :port => port3, :random_file => file_config3 })) }
@@ -42,7 +42,7 @@ describe "Test Logstash instance" do
   it "can start the embedded http server on default port 9600" do
     @ls1.start_with_stdin
     try(num_retries) do
-      expect(is_port_open?(9600)).to be true
+      expect(is_port_open?(9600)).to be(true)
     end
   end
 
@@ -56,7 +56,7 @@ describe "Test Logstash instance" do
       expect(is_port_open?(9600)).to be true
 
       @ls2.spawn_logstash("-f", config2, "--path.data", tmp_data_path)
-      try(20) do
+      try(num_retries) do
         expect(@ls2.exited?).to be(true)
       end
       expect(@ls2.exit_code).to be(1)

qa/integration/specs/env_variables_config_spec.rb

@@ -13,7 +13,7 @@ describe "Test Logstash configuration" do
     @fixture.teardown
   }
   
-  let(:num_retries) { 10 }
+  let(:num_retries) { 50 }
   let(:test_tcp_port) { random_port }
   let(:test_tag) { "environment_variables_are_evil" }
   let(:test_path) { Stud::Temporary.directory }

qa/integration/specs/monitoring_api_spec.rb

@@ -23,14 +23,9 @@ describe "Test Monitoring API" do
   it "can retrieve event stats" do
     logstash_service = @fixture.get_service("logstash")
     logstash_service.start_with_stdin
+    logstash_service.wait_for_logstash
     number_of_events.times { logstash_service.write_to_stdin("Hello world") }
 
-    begin
-      sleep(1) while (result = logstash_service.monitoring_api.event_stats).nil?
-    rescue
-      retry
-    end
-
     Stud.try(max_retry.times, RSpec::Expectations::ExpectationNotMetError) do
       # event_stats can fail if the stats subsystem isn't ready
       result = logstash_service.monitoring_api.event_stats rescue {}

qa/integration/specs/settings_spec.rb

@@ -28,7 +28,7 @@ describe "Test Logstash instance whose default settings are overridden" do
     FileUtils.mv("#{@logstash_service.application_settings_file}.original", @logstash_service.application_settings_file)
   }
 
-  let(:num_retries) { 15 }
+  let(:num_retries) { 50 }
   let(:test_port) { random_port }
   let(:temp_dir) { Stud::Temporary.directory("logstash-settings-test") }
   let(:tcp_config) { @fixture.config("root", { :port => test_port }) }
@@ -46,6 +46,7 @@ describe "Test Logstash instance whose default settings are overridden" do
   it "should start with a new data dir" do
     change_setting("path.data", temp_dir)
     @logstash_service.spawn_logstash("-e", tcp_config)
+    @logstash_service.wait_for_logstash
     # check LS is up and running with new data path
     try(num_retries) do
       expect(is_port_open?(test_port)).to be true
@@ -55,6 +56,7 @@ describe "Test Logstash instance whose default settings are overridden" do
   it "should write logs to a new dir" do
     change_setting("path.logs", temp_dir)
     @logstash_service.spawn_logstash("-e", tcp_config)
+    @logstash_service.wait_for_logstash
     # check LS is up and running with new data path
     try(num_retries) do
       expect(is_port_open?(test_port)).to be true
@@ -68,6 +70,7 @@ describe "Test Logstash instance whose default settings are overridden" do
     IO.write(test_config_path, tcp_config)
     expect(File.exists?(test_config_path)).to be true
     @logstash_service.spawn_logstash
+    @logstash_service.wait_for_logstash
     # check LS is up and running with new data path
     try(num_retries) do
       expect(is_port_open?(test_port)).to be true
@@ -102,14 +105,14 @@ describe "Test Logstash instance whose default settings are overridden" do
   it "change pipeline settings" do
     s = {}
     workers = 31
-    batch_size = 1250
+    batch_size = 1
     s["pipeline.workers"] = workers
     s["pipeline.batch.size"] = batch_size
     overwrite_settings(s)
     @logstash_service.spawn_logstash("-e", tcp_config)
     @logstash_service.wait_for_logstash
     # check LS is up and running with new data path
-    try do
+    try(num_retries) do
       expect(is_port_open?(test_port)).to be true
     end
 
@@ -124,6 +127,7 @@ describe "Test Logstash instance whose default settings are overridden" do
     http_port = random_port
     change_setting("http.port", http_port)
     @logstash_service.spawn_logstash("-e", tcp_config)
+    @logstash_service.wait_for_logstash
     
     try(num_retries) do
       expect(is_port_open?(http_port)).to be true
@@ -143,6 +147,7 @@ describe "Test Logstash instance whose default settings are overridden" do
 
   it "start even without a settings file specified" do
     @logstash_service.spawn_logstash("-e", tcp_config, "--path.settings", "/tmp/fooooobbaaar")
+    @logstash_service.wait_for_logstash
     http_port = 9600
     try(num_retries) do
       expect(is_port_open?(http_port)).to be true