RATS: Remove Docker from integration tests

This is in prep for a Docker based test run approach, and by removing the dependent Docker containers we will avoid Docker in Docker requirements.

Fixes #8211

qa/integration/integration_tests.gemspec

@@ -21,5 +21,4 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'flores'
   s.add_development_dependency 'rubyzip'
-  s.add_development_dependency 'docker-api'
 end

qa/integration/services/dockerized/Dockerfile

@@ -1,32 +0,0 @@
-FROM debian:stretch
-##
-# Define a base image for all service images.
-##
-
-ENV _JAVA_OPTIONS "-Djava.net.preferIPv4Stack=true"
-ENV TERM=linux
-
-RUN apt-get update && apt-get install -y curl openjdk-8-jre-headless netcat
-
-RUN adduser --disabled-password --gecos '' tester
-USER tester
-
-# Define the work directory. Use this variable in derivated images and
-# shell scripts.
-ENV WORKDIR /home/tester
-WORKDIR $WORKDIR
-
-# Script with routines that can be used in derived images.
-ADD helpers.sh .
-
-# Expect this script in the context directory of a derived image. It should
-# contain the service setup instructions, and is going to be the first executed
-# command when a derived image is built.
-ONBUILD ADD setup.sh .
-ONBUILD RUN ./setup.sh
-
-# Expect this script in the context directory of a derived image. It should
-# contain the service start up instructions, and is going to be executed when
-# a container is started.
-ONBUILD ADD run.sh .
-CMD ["./run.sh"]

qa/integration/services/dockerized/elasticsearch/Dockerfile

@@ -1,3 +0,0 @@
-FROM logstash:ci_sandbox
-
-expose 9200 9300

qa/integration/services/dockerized/elasticsearch/run.sh

@@ -1,6 +0,0 @@
-#!/bin/bash
-
-ES_HOME=${WORKDIR}/elasticsearch
-
-# Set "http.host" to make the service visible from outside the container.
-${ES_HOME}/bin/elasticsearch -E http.host=0.0.0.0

qa/integration/services/dockerized/elasticsearch/setup.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-
-if [ -n "${ES_VERSION+1}" ]; then
-  echo "Elasticsearch version is $ES_VERSION"
-  version=$ES_VERSION
-else
-   version=5.0.1
-fi
-
-ES_HOME=${WORKDIR}/elasticsearch
-
-download_url=https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$version.tar.gz
-curl -s -o elasticsearch.tar.gz $download_url
-mkdir -p $ES_HOME
-tar -xzf elasticsearch.tar.gz --strip-components=1 -C $ES_HOME/.

qa/integration/services/dockerized/helpers.sh

@@ -1,23 +0,0 @@
-#!/bin/bash
-
-##
-# Add routines and/or variables that can be shared between the
-# service containers.
-##
-
-PORT_WAIT_COUNT=20
-
-# Check service responds on given port.
-# Parameters:
-#   - the port number.
-wait_for_port() {
-    count=$PORT_WAIT_COUNT
-    port=$1
-    while ! nc -z localhost $port && [[ $count -ne 0 ]]; do
-        count=$(( $count - 1 ))
-        [[ $count -eq 0 ]] && return 1
-        sleep 0.5
-    done
-    # just in case, one more time
-    nc -z localhost $port
-}

qa/integration/services/elasticsearch_service.rb

@@ -1,17 +1,8 @@
-require_relative "service_container"
 require 'elasticsearch'
-require 'docker'
 
-class ElasticsearchService < ServiceContainer
+class ElasticsearchService < Service
   def initialize(settings)
     super("elasticsearch", settings)
-
-    # Binding container to host ports.
-    @container_create_opts[:HostConfig] = {
-                                            :PortBindings => {
-                                              '9200/tcp' => [{ :HostPort => '9200' }],
-                                              '9300/tcp' => [{ :HostPort => '9300' }]
-                                          }}
   end
 
   def get_client

qa/integration/services/elasticsearch_setup.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+set -ex
+current_dir="$(dirname "$0")"
+
+source "$current_dir/helpers.sh"
+
+if [ -n "${ES_VERSION+1}" ]; then
+  echo "Elasticsearch version is $ES_VERSION"
+  version=$ES_VERSION
+else
+   version=5.0.1
+fi
+
+ES_HOME=$INSTALL_DIR/elasticsearch
+
+setup_es() {
+  if [ ! -d $ES_HOME ]; then
+      local version=$1
+      download_url=https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$version.tar.gz
+      curl -sL $download_url > $INSTALL_DIR/elasticsearch.tar.gz
+      mkdir $ES_HOME
+      tar -xzf $INSTALL_DIR/elasticsearch.tar.gz --strip-components=1 -C $ES_HOME/.
+      rm $INSTALL_DIR/elasticsearch.tar.gz
+  fi
+}
+
+start_es() {
+  es_args=$@
+  $ES_HOME/bin/elasticsearch $es_args -p $ES_HOME/elasticsearch.pid > /tmp/elasticsearch.log 2>/dev/null &
+  count=120
+  echo "Waiting for elasticsearch to respond..."
+  while ! curl --silent localhost:9200 && [[ $count -ne 0 ]]; do
+      count=$(( $count - 1 ))
+      [[ $count -eq 0 ]] && cat /tmp/elasticsearch.log && return 1
+      sleep 1
+  done
+  echo "Elasticsearch is Up !"
+  return 0
+}
+
+setup_install_dir
+setup_es $version
+export ES_JAVA_OPTS="-Xms512m -Xmx512m"
+start_es

qa/integration/services/elasticsearch_teardown.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -e
+current_dir="$(dirname "$0")"
+
+source "$current_dir/helpers.sh"
+
+ES_HOME=$INSTALL_DIR/elasticsearch
+
+stop_es() {
+    pid=$(cat $ES_HOME/elasticsearch.pid)
+    [ "x$pid" != "x" ] && [ "$pid" -gt 0 ]
+    kill -SIGTERM $pid
+}
+
+stop_es

qa/integration/services/kafka_dockerized/Dockerfile

@@ -1,18 +0,0 @@
-FROM debian:stretch
-
-ENV KAFKA_HOME /kafka
-ENV KAFKA_LOGS_DIR="/kafka-logs"
-ENV KAFKA_VERSION 0.10.2.1
-ENV _JAVA_OPTIONS "-Djava.net.preferIPv4Stack=true"
-ENV TERM=linux
-
-RUN apt-get update && apt-get install -y curl openjdk-8-jre-headless netcat
-
-RUN mkdir -p ${KAFKA_LOGS_DIR} && mkdir -p ${KAFKA_HOME} && curl -s -o $INSTALL_DIR/kafka.tgz \
-    "http://ftp.wayne.edu/apache/kafka/${KAFKA_VERSION}/kafka_2.11-${KAFKA_VERSION}.tgz" && \
-    tar xzf ${INSTALL_DIR}/kafka.tgz -C ${KAFKA_HOME} --strip-components 1
-    
-ADD run.sh /run.sh
-
-EXPOSE 9092
-EXPOSE 2181

qa/integration/services/kafka_dockerized/run.sh

@@ -1,34 +0,0 @@
-#!/bin/bash
-
-KAFKA_TOPIC=logstash_topic_plain
-
-wait_for_port() {
-    count=20
-    port=$1
-    while ! nc -z localhost $port && [[ $count -ne 0 ]]; do
-        count=$(( $count - 1 ))
-        [[ $count -eq 0 ]] && return 1
-        sleep 0.5
-    done
-    # just in case, one more time
-    nc -z localhost $port
-}
-
-echo "Starting ZooKeeper"
-${KAFKA_HOME}/bin/zookeeper-server-start.sh ${KAFKA_HOME}/config/zookeeper.properties &
-wait_for_port 2181
-echo "Starting Kafka broker"
-mkdir -p ${KAFKA_LOGS_DIR}
-${KAFKA_HOME}/bin/kafka-server-start.sh ${KAFKA_HOME}/config/server.properties \
-    --override delete.topic.enable=true --override advertised.host.name=127.0.0.1 \
-    --override logs.dir=${KAFKA_LOGS_DIR} --override log.flush.interval.ms=200 &
-
-wait_for_port 9092
-
-${KAFKA_HOME}/bin/kafka-topics.sh --create --partitions 1 --replication-factor 1 --topic ${KAFKA_TOPIC} --zookeeper 127.0.0.1:2181
-
-${KAFKA_HOME}/bin/kafka-console-producer.sh --topic ${KAFKA_TOPIC} --broker-list 127.0.0.1:9092 < /how_sample.input
-
-echo "Kafka load status code $?"
-
-tail -f /dev/null

qa/integration/services/kafka_service.rb

@@ -1,35 +1,7 @@
 require_relative "service"
-require "docker"
-require "logstash/devutils/rspec/logstash_helpers"
 
 class KafkaService < Service
-  include LogStashHelper
-
   def initialize(settings)
     super("kafka", settings)
   end
-
-  def setup
-    try(20) do
-      @kafka_image = Docker::Image.build_from_dir(File.expand_path("../kafka_dockerized", __FILE__))
-                       .insert_local(
-                         'localPath' => File.join(TestSettings::FIXTURES_DIR, "how_sample.input"),
-                         'outputPath' => '/')
-    end
-    @kafka_container = Docker::Container.create(:Image => @kafka_image.id,
-                                                :HostConfig => {
-                                                  :PortBindings => {
-                                                    '9092/tcp' => [{ :HostPort => '9092' }],
-                                                    '2181/tcp' => [{ :HostPort => '2181' }]
-                                                  }
-                                                }, :Cmd => ["/bin/bash", "-l", "/run.sh"])
-    @kafka_container.start
-    super()
-  end
-
-  def teardown
-    @kafka_container.kill(:signal => "SIGHUP")
-    @kafka_container.delete(:force => true, :volumes => true)
-    super()
-  end
 end

qa/integration/services/kafka_setup.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+set -ex
+current_dir="$(dirname "$0")"
+
+export _JAVA_OPTIONS="-Djava.net.preferIPv4Stack=true"
+
+source "$current_dir/helpers.sh"
+
+if [ -n "${KAFKA_VERSION+1}" ]; then
+    echo "KAFKA_VERSION is $KAFKA_VERSION"
+    version=$KAFKA_VERSION
+else
+    version=0.10.2.1
+fi
+
+KAFKA_HOME=$INSTALL_DIR/kafka
+KAFKA_TOPIC=logstash_topic_plain
+KAFKA_MESSAGES=37
+KAFKA_LOGS_DIR=/tmp/ls_integration/kafka-logs
+
+setup_kafka() {
+    local version=$1
+    if [ ! -d $KAFKA_HOME ]; then
+        echo "Downloading Kafka version $version"
+        curl -s -o $INSTALL_DIR/kafka.tgz "http://ftp.wayne.edu/apache/kafka/$version/kafka_2.11-$version.tgz"
+        mkdir $KAFKA_HOME && tar xzf $INSTALL_DIR/kafka.tgz -C $KAFKA_HOME --strip-components 1
+        rm $INSTALL_DIR/kafka.tgz
+    fi
+}
+
+start_kafka() {
+    echo "Starting ZooKeeper"
+    $KAFKA_HOME/bin/zookeeper-server-start.sh -daemon $KAFKA_HOME/config/zookeeper.properties
+    wait_for_port 2181
+    echo "Starting Kafka broker"
+    rm -rf ${KAFKA_LOGS_DIR}
+    mkdir -p ${KAFKA_LOGS_DIR}
+    $KAFKA_HOME/bin/kafka-server-start.sh -daemon $KAFKA_HOME/config/server.properties --override delete.topic.enable=true --override advertised.host.name=127.0.0.1 --override log.dir=${KAFKA_LOGS_DIR} --override log.flush.interval.ms=200
+    wait_for_port 9092
+}
+
+wait_for_messages() {
+    local count=10
+    local read_lines=0
+    
+    echo "Checking if Kafka topic has been populated with data"
+    while [[ $read_lines -ne $KAFKA_MESSAGES ]] && [[ $count -ne 0 ]]; do
+        read_lines=`$KAFKA_HOME/bin/kafka-console-consumer.sh --topic $KAFKA_TOPIC --new-consumer --bootstrap-server localhost:9092 --from-beginning --max-messages $KAFKA_MESSAGES --timeout-ms 10000 | wc -l`
+        count=$(( $count - 1 ))
+        [[ $count -eq 0 ]] && return 1
+        sleep 0.5
+        #ls -lrt $KAFKA_LOGS_DIR/$KAFKA_TOPIC-0/
+    done
+    echo "Kafka topic has been populated with test data"
+}
+
+setup_install_dir
+setup_kafka $version
+start_kafka
+# Set up topics
+$KAFKA_HOME/bin/kafka-topics.sh --create --partitions 1 --replication-factor 1 --topic $KAFKA_TOPIC --zookeeper localhost:2181
+# check topic got created
+num_topic=`$KAFKA_HOME/bin/kafka-topics.sh --list --zookeeper localhost:2181 | grep $KAFKA_TOPIC | wc -l`
+[[ $num_topic -eq 1 ]]
+# Add test messages to the newly created topic
+cp $current_dir/../fixtures/how_sample.input $KAFKA_HOME
+[[ ! -s  how_sample.input ]]
+$KAFKA_HOME/bin/kafka-console-producer.sh --topic $KAFKA_TOPIC --broker-list localhost:9092 < $KAFKA_HOME/how_sample.input
+echo "Kafka load status code $?"
+# Wait until broker has all messages
+wait_for_messages
+echo "Kafka Setup complete"

qa/integration/services/kafka_teardown.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -ex
+current_dir="$(dirname "$0")"
+
+source "$current_dir/helpers.sh"
+
+KAFKA_HOME=$INSTALL_DIR/kafka
+
+stop_kafka() {
+    echo "Stopping Kafka broker"
+    $KAFKA_HOME/bin/kafka-server-stop.sh
+    echo "Stopping zookeeper"
+    $KAFKA_HOME/bin/zookeeper-server-stop.sh
+}
+
+# delete test topic
+echo "Deleting test topic in Kafka"
+$KAFKA_HOME/bin/kafka-topics.sh --delete --topic logstash_topic_plain --zookeeper localhost:2181 --if-exists
+stop_kafka
+rm -rf /tmp/ls_integration/kafka-logs
+rm -rf /tmp/zookeeper
+

qa/integration/services/logstash_service.rb

@@ -37,7 +37,7 @@ class LogstashService < Service
       ls_file = "logstash-" + ls_version_file["logstash"]
       # First try without the snapshot if it's there
       @logstash_home = File.expand_path(File.join(LS_BUILD_DIR, ls_file), __FILE__)
-      @logstash_home += "-SNAPSHOT" unless Dir.exists?(@logstash_home)
+      @logstash_home += "-SNAPSHOT" unless Dir.exist?(@logstash_home)
 
       puts "Using #{@logstash_home} as LS_HOME"
       @logstash_bin = File.join("#{@logstash_home}", LS_BIN)

qa/integration/services/service_container.rb

@@ -1,67 +0,0 @@
-require_relative "service"
-require "docker"
-require "logstash/devutils/rspec/logstash_helpers"
-
-# Represents a service running within a container.
-class ServiceContainer < Service
-  include LogStashHelper
-
-  def initialize(name, settings)
-    super(name, settings)
-
-    @base_image_context = File.expand_path("../dockerized", __FILE__)
-
-    @image_context = File.join(@base_image_context, @name)
-
-    # Options to create the container.
-    @container_create_opts = {}
-  end
-
-  def setup
-    puts "Setting up #{@name} service."
-
-    puts "Building the base container image."
-    @base_image = Docker::Image.build_from_dir(@base_image_context)
-    # Tag the base image.
-    #Caution: change this tag can cause failure to build the service container.
-    @base_image.tag('repo' => 'logstash', 'tag' => 'ci_sandbox', force: true)
-    puts "Finished building the base image."
-
-    puts "Building the container image."
-    self.build_image
-    puts "Finished building the image."
-
-    puts "Starting the container."
-    self.start_container
-    puts "Finished starting the container."
-
-    puts "Finished setting up #{@name} service."
-  end
-
-  def teardown
-    puts "Tearing down #{@name} service."
-
-    puts "Stop the container."
-    self.stop_container
-    puts "Finished stopping the container."
-
-    puts "Finished tearing down of #{@name} service."
-  end
-
-  def build_image
-    try(20) do
-      @image = Docker::Image.build_from_dir(@image_context)
-    end
-  end
-
-  def start_container
-    @container_create_opts[:Image] = @image.id
-    @container = Docker::Container.create(@container_create_opts)
-    @container.start
-  end
-
-  def stop_container
-    @container.stop
-    @container.delete(:force => true, :volumes => true)
-  end
-end