Improvements to RATS test suite

* Reduce sleeps and output messages
* Cache installed services directory

Fixes #5968

.travis.yml

@@ -19,7 +19,6 @@ install:
   - ci/travis_integration_install.sh
 before_script:
   - echo "--order rand" > .rspec
-  - echo "--format documentation" >> .rspec
 script:
   - rake test:core
   - ci/travis_integration_run.sh

qa/integration/.gitignore

@@ -1,5 +1,2 @@
-/services/filebeat
-/fixtures/how.input
-/services/elasticsearch
-/services/kafka
+/services/installed
 /fixtures/certificates

qa/integration/fixtures/beats_input_spec.yml

@@ -9,6 +9,7 @@ config:
         port => 5044
       }
     }
+    output {}
   tls_server_auth: |-
     input {
       beats {
@@ -18,6 +19,7 @@ config:
         ssl_key => '<%=options[:ssl_key]%>'
       }
     }
+    output {}
   tls_mutual_auth: |-
     input {
       beats {
@@ -28,5 +30,6 @@ config:
         ssl_verify_mode => "peer"
       }
     }
+    output {}
 input: how_sample.input
 teardown_script:

qa/integration/services/elasticsearch_setup.sh

@@ -2,6 +2,8 @@
 set -e
 current_dir="$(dirname "$0")"
 
+source "$current_dir/helpers.sh"
+
 if [ -n "${ES_VERSION+1}" ]; then
   echo "Elasticsearch version is $ES_VERSION"
   version=$ES_VERSION
@@ -9,20 +11,22 @@ else
    version=5.0.0-beta1
 fi
 
+ES_HOME=$INSTALL_DIR/elasticsearch
+
 setup_es() {
-  if [ ! -d $current_dir/elasticsearch ]; then
+  if [ ! -d $ES_HOME ]; then
       local version=$1
       download_url=https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$version.tar.gz
-      curl -sL $download_url > $current_dir/elasticsearch.tar.gz
-      mkdir $current_dir/elasticsearch
-      tar -xzf $current_dir/elasticsearch.tar.gz --strip-components=1 -C $current_dir/elasticsearch/.
-      rm $current_dir/elasticsearch.tar.gz
+      curl -sL $download_url > $INSTALL_DIR/elasticsearch.tar.gz
+      mkdir $ES_HOME
+      tar -xzf $INSTALL_DIR/elasticsearch.tar.gz --strip-components=1 -C $ES_HOME/.
+      rm $INSTALL_DIR/elasticsearch.tar.gz
   fi
 }
 
 start_es() {
   es_args=$@
-  $current_dir/elasticsearch/bin/elasticsearch $es_args -p $current_dir/elasticsearch/elasticsearch.pid > /tmp/elasticsearch.log 2>/dev/null &
+  $ES_HOME/bin/elasticsearch $es_args -p $ES_HOME/elasticsearch.pid > /tmp/elasticsearch.log 2>/dev/null &
   count=120
   echo "Waiting for elasticsearch to respond..."
   while ! curl --silent localhost:9200 && [[ $count -ne 0 ]]; do
@@ -34,5 +38,6 @@ start_es() {
   return 0
 }
 
+setup_install_dir
 setup_es $version
 start_es

qa/integration/services/elasticsearch_teardown.sh

@@ -2,8 +2,12 @@
 set -e
 current_dir="$(dirname "$0")"
 
+source "$current_dir/helpers.sh"
+
+ES_HOME=$INSTALL_DIR/elasticsearch
+
 stop_es() {
-    pid=$(cat $current_dir/elasticsearch/elasticsearch.pid)
+    pid=$(cat $ES_HOME/elasticsearch.pid)
     [ "x$pid" != "x" ] && [ "$pid" -gt 0 ]
     kill -SIGTERM $pid
 }

qa/integration/services/filebeat_service.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
 class FilebeatService < Service
-  FILEBEAT_CMD = [File.join(File.dirname(__FILE__), "filebeat", "filebeat"), "-c"]
+  FILEBEAT_CMD = [File.join(File.dirname(__FILE__), "installed", "filebeat", "filebeat"), "-c"]
 
   class BackgroundProcess
     def initialize(cmd)

qa/integration/services/filebeat_setup.sh

@@ -2,6 +2,8 @@
 set -e
 current_dir="$(dirname "$0")"
 
+source "$current_dir/helpers.sh"
+
 if [ -n "${FILEBEAT_VERSION}" ]; then
   echo "Filebeat version is $FILEBEAT_VERSION"
   version=$FILEBEAT_VERSION
@@ -9,15 +11,17 @@ else
   version=5.0.0-beta1
 fi
 
+FB_HOME=$INSTALL_DIR/filebeat
+
 setup_fb() {
     local version=$1
     platform=`uname -s | tr '[:upper:]' '[:lower:]'`
     architecture=`uname -m | tr '[:upper:]' '[:lower:]'`
     download_url=https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-$version-$platform-$architecture.tar.gz
-    curl -sL $download_url > $current_dir/filebeat.tar.gz
-    mkdir $current_dir/filebeat
-    tar -xzf $current_dir/filebeat.tar.gz --strip-components=1 -C $current_dir/filebeat/.
-    rm $current_dir/filebeat.tar.gz
+    curl -sL $download_url > $INSTALL_DIR/filebeat.tar.gz
+    mkdir $FB_HOME
+    tar -xzf $INSTALL_DIR/filebeat.tar.gz --strip-components=1 -C $FB_HOME/.
+    rm $INSTALL_DIR/filebeat.tar.gz
 }
 
 generate_certificate() {
@@ -26,7 +30,9 @@ generate_certificate() {
     openssl req -subj '/CN=localhost/' -x509 -days $((100 * 365)) -batch -nodes -newkey rsa:2048 -keyout $target_directory/certificate.key -out $target_directory/certificate.crt
 }
 
-if [ ! -d $current_dir/filebeat ]; then
+setup_install_dir
+
+if [ ! -d $FB_HOME ]; then
     generate_certificate
     setup_fb $version
 fi

qa/integration/services/helpers.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+set -e
+current_dir="$(dirname "$0")"
+
+INSTALL_DIR=$current_dir/installed
+PORT_WAIT_COUNT=20
+
+setup_install_dir() {
+    if [[ ! -d "$INSTALL_DIR" ]]; then
+        mkdir $INSTALL_DIR
+    fi
+}
+
+wait_for_port() {
+    count=$PORT_WAIT_COUNT
+    port=$1
+    while ! nc -z localhost $port && [[ $count -ne 0 ]]; do
+        count=$(( $count - 1 ))
+        [[ $count -eq 0 ]] && return 1
+        sleep 0.5
+    done
+}
+
+clean_install_dir() {
+    if [[ -d "$INSTALL_DIR" ]]; then
+        rm -rf $INSTALL_DIR
+    fi
+}

qa/integration/services/kafka_setup.sh

@@ -2,6 +2,8 @@
 set -e
 current_dir="$(dirname "$0")"
 
+source "$current_dir/helpers.sh"
+
 if [ -n "${KAFKA_VERSION+1}" ]; then
     echo "KAFKA_VERSION is $KAFKA_VERSION"
     version=$KAFKA_VERSION
@@ -9,31 +11,47 @@ else
     version=0.10.0.1
 fi
 
+KAFKA_HOME=$INSTALL_DIR/kafka
+KAFKA_TOPIC=logstash_topic_plain
+KAFKA_MESSAGES=37
+
 setup_kafka() {
     local version=$1
-    if [ ! -d $current_dir/kafka ]; then
+    if [ ! -d $KAFKA_HOME ]; then
         echo "Downloading Kafka version $version"
-        curl -s -o $current_dir/kafka.tgz "http://ftp.wayne.edu/apache/kafka/$version/kafka_2.11-$version.tgz"
-        mkdir $current_dir/kafka && tar xzf $current_dir/kafka.tgz -C $current_dir/kafka --strip-components 1
-        rm $current_dir/kafka.tgz
+        curl -s -o $INSTALL_DIR/kafka.tgz "http://ftp.wayne.edu/apache/kafka/$version/kafka_2.11-$version.tgz"
+        mkdir $KAFKA_HOME && tar xzf $INSTALL_DIR/kafka.tgz -C $KAFKA_HOME --strip-components 1
+        rm $INSTALL_DIR/kafka.tgz
     fi
 }
 
 start_kafka() {
     echo "Starting ZooKeeper"
-    $current_dir/kafka/bin/zookeeper-server-start.sh -daemon $current_dir/kafka/config/zookeeper.properties
-    sleep 3
+    $KAFKA_HOME/bin/zookeeper-server-start.sh -daemon $KAFKA_HOME/config/zookeeper.properties
+    wait_for_port 2181
     echo "Starting Kafka broker"
-    $current_dir/kafka/bin/kafka-server-start.sh -daemon $current_dir/kafka/config/server.properties --override delete.topic.enable=true
-    sleep 3
+    $KAFKA_HOME/bin/kafka-server-start.sh -daemon $KAFKA_HOME/config/server.properties --override delete.topic.enable=true --override log.dirs=/tmp/ls_integration/kafka-logs
+    wait_for_port 9092
 }
 
+wait_for_messages() {
+    local count=10
+    local read_lines=0
+    while [[ $read_lines -ne $KAFKA_MESSAGES ]] && [[ $count -ne 0 ]]; do
+        read_lines=`$KAFKA_HOME/bin/kafka-console-consumer.sh --topic $KAFKA_TOPIC --new-consumer --bootstrap-server localhost:9092 --from-beginning --max-messages $KAFKA_MESSAGES | wc -l`
+        count=$(( $count - 1 ))
+        [[ $count -eq 0 ]] && return 1
+        sleep 0.5
+    done
+}
+
+setup_install_dir
 setup_kafka $version
 start_kafka
-sleep 3
 # Set up topics
-$current_dir/kafka/bin/kafka-topics.sh --create --partitions 1 --replication-factor 1 --topic logstash_topic_plain --zookeeper localhost:2181
-sleep 1
-cat $current_dir/../fixtures/how_sample.input | $current_dir/kafka/bin/kafka-console-producer.sh --topic logstash_topic_plain --broker-list localhost:9092
-sleep 1
+$KAFKA_HOME/bin/kafka-topics.sh --create --partitions 1 --replication-factor 1 --topic logstash_topic_plain --zookeeper localhost:2181
+# Add test messages to the newly created topic
+cat $current_dir/../fixtures/how_sample.input | $KAFKA_HOME/bin/kafka-console-producer.sh --topic $KAFKA_TOPIC --broker-list localhost:9092
+# Wait until broker has all messages
+wait_for_messages
 echo "Kafka Setup complete"

qa/integration/services/kafka_teardown.sh

@@ -2,16 +2,21 @@
 set -e
 current_dir="$(dirname "$0")"
 
+source "$current_dir/helpers.sh"
+
+KAFKA_HOME=$INSTALL_DIR/kafka
+
 stop_kafka() {
     echo "Stopping Kafka broker"
-    $current_dir/kafka/bin/kafka-server-stop.sh
-    sleep 2
+    $KAFKA_HOME/bin/kafka-server-stop.sh
     echo "Stopping zookeeper"
-    $current_dir/kafka/bin/zookeeper-server-stop.sh
-    sleep 2
+    $KAFKA_HOME/bin/zookeeper-server-stop.sh
 }
 
 # delete test topic
 echo "Deleting test topic in Kafka"
-$current_dir/kafka/bin/kafka-topics.sh --delete --topic logstash_topic_plain --zookeeper localhost:2181 --if-exists
+$KAFKA_HOME/bin/kafka-topics.sh --delete --topic logstash_topic_plain --zookeeper localhost:2181 --if-exists
 stop_kafka
+rm -rf /tmp/ls_integration/kafka-logs
+rm -rf /tmp/zookeeper
+