Enable RATS integration tests on travis

Fixes #5963

.travis.yml

@@ -8,9 +8,18 @@ rvm:
   - jruby-1.7.25
 jdk:
   - oraclejdk8
+env:
+  - INTEGRATION=true  
+before_install:
+  # Force bundler 1.12.5 because version 1.13 has issues, see https://github.com/fastlane/fastlane/issues/6065#issuecomment-246044617
+  - gem uninstall -i /home/travis/.rvm/gems/jruby-1.7.25@global bundler
+  - gem install bundler -v 1.12.5 --no-rdoc --no-ri --no-document --quiet   
 install:
   - rake test:install-core
+  - ci/travis_integration_install.sh
 before_script:
   - echo "--order rand" > .rspec
   - echo "--format documentation" >> .rspec
-script: rake test:core
+script:
+  - rake test:core
+  - ci/travis_integration_run.sh

ci/ci_integration.sh

@@ -6,9 +6,18 @@ set -e
 # installing gems. See https://github.com/elastic/logstash/issues/5179
 export JRUBY_OPTS="-J-Xmx1g"
 
-rm -rf build/*
+echo "Running integration tests from qa/integration"
+if [[ ! -d "build" ]]; then
+  mkdir build
+fi  
+rm -rf build/*  
+echo "Building logstash tar file in build/"
 rake artifact:tar
 cd build
-tar xvf *.tar.gz
+echo "Extracting logstash tar file in build/"
+tar xf *.tar.gz
 cd ../qa/integration
+# to install test dependencies
+bundle install
+# runs all tests
 rspec

ci/travis_integration_install.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -e
+
+# This file sets up the environment for travis integration tests
+
+
+if [[ "$INTEGRATION" != "true" ]]; then
+    exit
+fi
+  
+echo "Setting up integration tests"
+if [[ ! -d "build" ]]; then
+    mkdir build
+fi  
+rm -rf build/*  
+echo "Building logstash tar file in build/"
+rake artifact:tar
+cd build
+echo "Extracting logstash tar file in build/"
+tar xf *.tar.gz
+cd ../qa/integration
+pwd
+echo $BUNDLE_GEMFILE
+# to install test dependencies
+bundle install --gemfile="Gemfile"

ci/travis_integration_run.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -e
+
+if [[ "$INTEGRATION" != "true" ]]; then
+    exit
+fi
+
+echo "Running integration tests from qa/integration directory"
+cd qa/integration
+rspec

qa/integration/fixtures/beats_input_spec.yml

@@ -29,5 +29,4 @@ config:
       }
     }
 input: how_sample.input
-setup_script: download_input.sh
 teardown_script:

qa/integration/fixtures/download_input.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-set -ex
-current_dir="$(dirname "$0")"
-HOW_DATA_SET_URL=https://s3.amazonaws.com/data.elasticsearch.org/logstash/logs.gz
-
-if [ ! -f ${current_dir}/how.input ]; then
-  curl -sL $HOW_DATA_SET_URL > ${current_dir}/logs.gz
-  gunzip ${current_dir}/logs.gz
-  mv ${current_dir}/logs ${current_dir}/how.input
-fi

qa/integration/fixtures/es_output_how_spec.yml

@@ -30,6 +30,5 @@ config: |-
     elasticsearch {}
   }
 
-input: how.input
-setup_script: download_input.sh
+input: how_sample.input
 teardown_script:

qa/integration/services/elasticsearch_setup.sh

@@ -1,18 +1,18 @@
 #!/bin/bash
-set -ex
+set -e
 current_dir="$(dirname "$0")"
 
 if [ -n "${ES_VERSION+1}" ]; then
   echo "Elasticsearch version is $ES_VERSION"
   version=$ES_VERSION
 else
-   version=5.0.0-alpha5
+   version=5.0.0-beta1
 fi
 
 setup_es() {
   if [ ! -d $current_dir/elasticsearch ]; then
       local version=$1
-      download_url=https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/$version/elasticsearch-$version.tar.gz
+      download_url=https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$version.tar.gz
       curl -sL $download_url > $current_dir/elasticsearch.tar.gz
       mkdir $current_dir/elasticsearch
       tar -xzf $current_dir/elasticsearch.tar.gz --strip-components=1 -C $current_dir/elasticsearch/.

qa/integration/services/elasticsearch_teardown.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-set -ex
+set -e
 current_dir="$(dirname "$0")"
 
 stop_es() {

qa/integration/services/filebeat_setup.sh

@@ -1,19 +1,19 @@
 #!/bin/bash
-set -ex
+set -e
 current_dir="$(dirname "$0")"
 
 if [ -n "${FILEBEAT_VERSION}" ]; then
   echo "Filebeat version is $FILEBEAT_VERSION"
   version=$FILEBEAT_VERSION
 else
-   version=5.0.0-alpha5
+  version=5.0.0-beta1
 fi
 
 setup_fb() {
     local version=$1
     platform=`uname -s | tr '[:upper:]' '[:lower:]'`
     architecture=`uname -m | tr '[:upper:]' '[:lower:]'`
-    download_url=https://download.elastic.co/beats/filebeat/filebeat-$version-$platform-$architecture.tar.gz
+    download_url=https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-$version-$platform-$architecture.tar.gz
     curl -sL $download_url > $current_dir/filebeat.tar.gz
     mkdir $current_dir/filebeat
     tar -xzf $current_dir/filebeat.tar.gz --strip-components=1 -C $current_dir/filebeat/.

qa/integration/services/kafka_setup.sh

@@ -1,12 +1,12 @@
 #!/bin/bash
-set -ex
+set -e
 current_dir="$(dirname "$0")"
 
 if [ -n "${KAFKA_VERSION+1}" ]; then
     echo "KAFKA_VERSION is $KAFKA_VERSION"
     version=$KAFKA_VERSION
 else
-    version=0.10.0.0
+    version=0.10.0.1
 fi
 
 setup_kafka() {
@@ -30,8 +30,10 @@ start_kafka() {
 
 setup_kafka $version
 start_kafka
-
+sleep 3
 # Set up topics
 $current_dir/kafka/bin/kafka-topics.sh --create --partitions 1 --replication-factor 1 --topic logstash_topic_plain --zookeeper localhost:2181
+sleep 1
 cat $current_dir/../fixtures/how_sample.input | $current_dir/kafka/bin/kafka-console-producer.sh --topic logstash_topic_plain --broker-list localhost:9092
+sleep 1
 echo "Kafka Setup complete"

qa/integration/services/kafka_teardown.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-set -ex
+set -e
 current_dir="$(dirname "$0")"
 
 stop_kafka() {

qa/integration/specs/es_output_how_spec.rb

@@ -17,13 +17,13 @@ describe "a config which indexes data into Elasticsearch" do
   it "can ingest 300K log lines" do
     logstash_service = @fixture.get_service("logstash")
     es_service = @fixture.get_service("elasticsearch")
-    puts "Ingesting 300K lines of apache logs to ES. This may make your CPU sing.."
+    puts "Ingesting 37 apache log lines to ES."
     logstash_service.start_with_input(@fixture.config, @fixture.input)
     es_client = es_service.get_client
     # now we test if all data was indexed by ES, but first refresh manually
     es_client.indices.refresh
     result = es_client.search(index: 'logstash-*', size: 0, q: '*')
-    expect(result["hits"]["total"]).to eq(300000)
+    expect(result["hits"]["total"]).to eq(37)
   end
 
 end

qa/integration/specs/kafka_input_spec.rb

@@ -4,7 +4,7 @@ require_relative '../services/logstash_service'
 require "rspec/wait"
 
 describe "Kafka Input" do
-  let(:timeout_seconds) { 5 }
+  let(:timeout_seconds) { 30 }
   before(:all) {
     @fixture = Fixture.new(__FILE__)
   }