github-mirrors/logstash
1
0
Fork 0
mirror of https://github.com/elastic/logstash.git synced 2025-04-24 14:47:19 -04:00
Code Issues Projects Releases Packages Wiki Activity

RATS: Remove Docker from integration tests

Browse source 
This is in prep for a Docker based test run approach, and by removing the dependent Docker containers we will avoid Docker in Docker requirements.

Fixes #8211
This commit is contained in:
Jake Landis 2017-09-11 17:40:38 -05:00
parent a25939771a
commit 1b33cbab67
16 changed files with 156 additions and 239 deletions
Download patch file Download diff file Expand all files Collapse all files

1
qa/integration/integration_tests.gemspec
View file

@ -21,5 +21,4 @@ Gem::Specification.new do |s|
s.add_development_dependency 'logstash-devutils'
s.add_development_dependency 'flores'
s.add_development_dependency 'rubyzip'
s.add_development_dependency 'docker-api'
end

32
qa/integration/services/dockerized/Dockerfile
View file

@ -1,32 +0,0 @@
FROM debian:stretch
##
# Define a base image for all service images.
##
ENV _JAVA_OPTIONS "-Djava.net.preferIPv4Stack=true"
ENV TERM=linux
RUN apt-get update && apt-get install -y curl openjdk-8-jre-headless netcat
RUN adduser --disabled-password --gecos '' tester
USER tester
# Define the work directory. Use this variable in derivated images and
# shell scripts.
ENV WORKDIR /home/tester
WORKDIR $WORKDIR
# Script with routines that can be used in derived images.
ADD helpers.sh .
# Expect this script in the context directory of a derived image. It should
# contain the service setup instructions, and is going to be the first executed
# command when a derived image is built.
ONBUILD ADD setup.sh .
ONBUILD RUN ./setup.sh
# Expect this script in the context directory of a derived image. It should
# contain the service start up instructions, and is going to be executed when
# a container is started.
ONBUILD ADD run.sh .
CMD ["./run.sh"]

3
qa/integration/services/dockerized/elasticsearch/Dockerfile
View file

@ -1,3 +0,0 @@
FROM logstash:ci_sandbox
expose 9200 9300

6
qa/integration/services/dockerized/elasticsearch/run.sh
View file

@ -1,6 +0,0 @@
#!/bin/bash
ES_HOME=${WORKDIR}/elasticsearch
# Set "http.host" to make the service visible from outside the container.
${ES_HOME}/bin/elasticsearch -E http.host=0.0.0.0

15
qa/integration/services/dockerized/elasticsearch/setup.sh
View file

@ -1,15 +0,0 @@
#!/bin/bash
if [ -n "${ES_VERSION+1}" ]; then
echo "Elasticsearch version is $ES_VERSION"
version=$ES_VERSION
else
version=5.0.1
fi
ES_HOME=${WORKDIR}/elasticsearch
download_url=https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$version.tar.gz
curl -s -o elasticsearch.tar.gz $download_url
mkdir -p $ES_HOME
tar -xzf elasticsearch.tar.gz --strip-components=1 -C $ES_HOME/.

23
qa/integration/services/dockerized/helpers.sh
View file

@ -1,23 +0,0 @@
#!/bin/bash
##
# Add routines and/or variables that can be shared between the
# service containers.
##
PORT_WAIT_COUNT=20
# Check service responds on given port.
# Parameters:
# - the port number.
wait_for_port() {
count=$PORT_WAIT_COUNT
port=$1
while ! nc -z localhost $port && [[ $count -ne 0 ]]; do
count=$(( $count - 1 ))
[[ $count -eq 0 ]] && return 1
sleep 0.5
done
# just in case, one more time
nc -z localhost $port
}

13
qa/integration/services/elasticsearch_service.rb
View file

@ -1,21 +1,12 @@
require_relative "service_container"
require 'elasticsearch'
require 'docker'
class ElasticsearchService < ServiceContainer
class ElasticsearchService < Service
def initialize(settings)
super("elasticsearch", settings)
# Binding container to host ports.
@container_create_opts[:HostConfig] = {
:PortBindings => {
'9200/tcp' => [{ :HostPort => '9200' }],
'9300/tcp' => [{ :HostPort => '9300' }]
}}
end
def get_client
Elasticsearch::Client.new(:hosts => "localhost:9200")
end
end
end

44
qa/integration/services/elasticsearch_setup.sh Executable file
View file

@ -0,0 +1,44 @@
#!/bin/bash
set -ex
current_dir="$(dirname "$0")"
source "$current_dir/helpers.sh"
if [ -n "${ES_VERSION+1}" ]; then
echo "Elasticsearch version is $ES_VERSION"
version=$ES_VERSION
else
version=5.0.1
fi
ES_HOME=$INSTALL_DIR/elasticsearch
setup_es() {
if [ ! -d $ES_HOME ]; then
local version=$1
download_url=https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$version.tar.gz
curl -sL $download_url > $INSTALL_DIR/elasticsearch.tar.gz
mkdir $ES_HOME
tar -xzf $INSTALL_DIR/elasticsearch.tar.gz --strip-components=1 -C $ES_HOME/.
rm $INSTALL_DIR/elasticsearch.tar.gz
fi
}
start_es() {
es_args=$@
$ES_HOME/bin/elasticsearch $es_args -p $ES_HOME/elasticsearch.pid > /tmp/elasticsearch.log 2>/dev/null &
count=120
echo "Waiting for elasticsearch to respond..."
while ! curl --silent localhost:9200 && [[ $count -ne 0 ]]; do
count=$(( $count - 1 ))
[[ $count -eq 0 ]] && cat /tmp/elasticsearch.log && return 1
sleep 1
done
echo "Elasticsearch is Up !"
return 0
}
setup_install_dir
setup_es $version
export ES_JAVA_OPTS="-Xms512m -Xmx512m"
start_es

15
qa/integration/services/elasticsearch_teardown.sh Executable file
View file

@ -0,0 +1,15 @@
#!/bin/bash
set -e
current_dir="$(dirname "$0")"
source "$current_dir/helpers.sh"
ES_HOME=$INSTALL_DIR/elasticsearch
stop_es() {
pid=$(cat $ES_HOME/elasticsearch.pid)
[ "x$pid" != "x" ] && [ "$pid" -gt 0 ]
kill -SIGTERM $pid
}
stop_es

18
qa/integration/services/kafka_dockerized/Dockerfile
View file

@ -1,18 +0,0 @@
FROM debian:stretch
ENV KAFKA_HOME /kafka
ENV KAFKA_LOGS_DIR="/kafka-logs"
ENV KAFKA_VERSION 0.10.2.1
ENV _JAVA_OPTIONS "-Djava.net.preferIPv4Stack=true"
ENV TERM=linux
RUN apt-get update && apt-get install -y curl openjdk-8-jre-headless netcat
RUN mkdir -p ${KAFKA_LOGS_DIR} && mkdir -p ${KAFKA_HOME} && curl -s -o $INSTALL_DIR/kafka.tgz \
"http://ftp.wayne.edu/apache/kafka/${KAFKA_VERSION}/kafka_2.11-${KAFKA_VERSION}.tgz" && \
tar xzf ${INSTALL_DIR}/kafka.tgz -C ${KAFKA_HOME} --strip-components 1
ADD run.sh /run.sh
EXPOSE 9092
EXPOSE 2181

34
qa/integration/services/kafka_dockerized/run.sh
View file

@ -1,34 +0,0 @@
#!/bin/bash
KAFKA_TOPIC=logstash_topic_plain
wait_for_port() {
count=20
port=$1
while ! nc -z localhost $port && [[ $count -ne 0 ]]; do
count=$(( $count - 1 ))
[[ $count -eq 0 ]] && return 1
sleep 0.5
done
# just in case, one more time
nc -z localhost $port
}
echo "Starting ZooKeeper"
${KAFKA_HOME}/bin/zookeeper-server-start.sh ${KAFKA_HOME}/config/zookeeper.properties &
wait_for_port 2181
echo "Starting Kafka broker"
mkdir -p ${KAFKA_LOGS_DIR}
${KAFKA_HOME}/bin/kafka-server-start.sh ${KAFKA_HOME}/config/server.properties \
--override delete.topic.enable=true --override advertised.host.name=127.0.0.1 \
--override logs.dir=${KAFKA_LOGS_DIR} --override log.flush.interval.ms=200 &
wait_for_port 9092
${KAFKA_HOME}/bin/kafka-topics.sh --create --partitions 1 --replication-factor 1 --topic ${KAFKA_TOPIC} --zookeeper 127.0.0.1:2181
${KAFKA_HOME}/bin/kafka-console-producer.sh --topic ${KAFKA_TOPIC} --broker-list 127.0.0.1:9092 < /how_sample.input
echo "Kafka load status code $?"
tail -f /dev/null

28
qa/integration/services/kafka_service.rb
View file

@ -1,35 +1,7 @@
require_relative "service"
require "docker"
require "logstash/devutils/rspec/logstash_helpers"
class KafkaService < Service
include LogStashHelper
def initialize(settings)
super("kafka", settings)
end
def setup
try(20) do
@kafka_image = Docker::Image.build_from_dir(File.expand_path("../kafka_dockerized", __FILE__))
.insert_local(
'localPath' => File.join(TestSettings::FIXTURES_DIR, "how_sample.input"),
'outputPath' => '/')
end
@kafka_container = Docker::Container.create(:Image => @kafka_image.id,
:HostConfig => {
:PortBindings => {
'9092/tcp' => [{ :HostPort => '9092' }],
'2181/tcp' => [{ :HostPort => '2181' }]
}
}, :Cmd => ["/bin/bash", "-l", "/run.sh"])
@kafka_container.start
super()
end
def teardown
@kafka_container.kill(:signal => "SIGHUP")
@kafka_container.delete(:force => true, :volumes => true)
super()
end
end

72
qa/integration/services/kafka_setup.sh Executable file
View file

@ -0,0 +1,72 @@
#!/bin/bash
set -ex
current_dir="$(dirname "$0")"
export _JAVA_OPTIONS="-Djava.net.preferIPv4Stack=true"
source "$current_dir/helpers.sh"
if [ -n "${KAFKA_VERSION+1}" ]; then
echo "KAFKA_VERSION is $KAFKA_VERSION"
version=$KAFKA_VERSION
else
version=0.10.2.1
fi
KAFKA_HOME=$INSTALL_DIR/kafka
KAFKA_TOPIC=logstash_topic_plain
KAFKA_MESSAGES=37
KAFKA_LOGS_DIR=/tmp/ls_integration/kafka-logs
setup_kafka() {
local version=$1
if [ ! -d $KAFKA_HOME ]; then
echo "Downloading Kafka version $version"
curl -s -o $INSTALL_DIR/kafka.tgz "http://ftp.wayne.edu/apache/kafka/$version/kafka_2.11-$version.tgz"
mkdir $KAFKA_HOME && tar xzf $INSTALL_DIR/kafka.tgz -C $KAFKA_HOME --strip-components 1
rm $INSTALL_DIR/kafka.tgz
fi
}
start_kafka() {
echo "Starting ZooKeeper"
$KAFKA_HOME/bin/zookeeper-server-start.sh -daemon $KAFKA_HOME/config/zookeeper.properties
wait_for_port 2181
echo "Starting Kafka broker"
rm -rf ${KAFKA_LOGS_DIR}
mkdir -p ${KAFKA_LOGS_DIR}
$KAFKA_HOME/bin/kafka-server-start.sh -daemon $KAFKA_HOME/config/server.properties --override delete.topic.enable=true --override advertised.host.name=127.0.0.1 --override log.dir=${KAFKA_LOGS_DIR} --override log.flush.interval.ms=200
wait_for_port 9092
}
wait_for_messages() {
local count=10
local read_lines=0
echo "Checking if Kafka topic has been populated with data"
while [[ $read_lines -ne $KAFKA_MESSAGES ]] && [[ $count -ne 0 ]]; do
read_lines=`$KAFKA_HOME/bin/kafka-console-consumer.sh --topic $KAFKA_TOPIC --new-consumer --bootstrap-server localhost:9092 --from-beginning --max-messages $KAFKA_MESSAGES --timeout-ms 10000 | wc -l`
count=$(( $count - 1 ))
[[ $count -eq 0 ]] && return 1
sleep 0.5
#ls -lrt $KAFKA_LOGS_DIR/$KAFKA_TOPIC-0/
done
echo "Kafka topic has been populated with test data"
}
setup_install_dir
setup_kafka $version
start_kafka
# Set up topics
$KAFKA_HOME/bin/kafka-topics.sh --create --partitions 1 --replication-factor 1 --topic $KAFKA_TOPIC --zookeeper localhost:2181
# check topic got created
num_topic=`$KAFKA_HOME/bin/kafka-topics.sh --list --zookeeper localhost:2181 | grep $KAFKA_TOPIC | wc -l`
[[ $num_topic -eq 1 ]]
# Add test messages to the newly created topic
cp $current_dir/../fixtures/how_sample.input $KAFKA_HOME
[[ ! -s how_sample.input ]]
$KAFKA_HOME/bin/kafka-console-producer.sh --topic $KAFKA_TOPIC --broker-list localhost:9092 < $KAFKA_HOME/how_sample.input
echo "Kafka load status code $?"
# Wait until broker has all messages
wait_for_messages
echo "Kafka Setup complete"

22
qa/integration/services/kafka_teardown.sh Executable file
View file

@ -0,0 +1,22 @@
#!/bin/bash
set -ex
current_dir="$(dirname "$0")"
source "$current_dir/helpers.sh"
KAFKA_HOME=$INSTALL_DIR/kafka
stop_kafka() {
echo "Stopping Kafka broker"
$KAFKA_HOME/bin/kafka-server-stop.sh
echo "Stopping zookeeper"
$KAFKA_HOME/bin/zookeeper-server-stop.sh
}
# delete test topic
echo "Deleting test topic in Kafka"
$KAFKA_HOME/bin/kafka-topics.sh --delete --topic logstash_topic_plain --zookeeper localhost:2181 --if-exists
stop_kafka
rm -rf /tmp/ls_integration/kafka-logs
rm -rf /tmp/zookeeper

2
qa/integration/services/logstash_service.rb
View file

@ -37,7 +37,7 @@ class LogstashService < Service
ls_file = "logstash-" + ls_version_file["logstash"]
# First try without the snapshot if it's there
@logstash_home = File.expand_path(File.join(LS_BUILD_DIR, ls_file), __FILE__)
@logstash_home += "-SNAPSHOT" unless Dir.exists?(@logstash_home)
@logstash_home += "-SNAPSHOT" unless Dir.exist?(@logstash_home)
puts "Using #{@logstash_home} as LS_HOME"
@logstash_bin = File.join("#{@logstash_home}", LS_BIN)

67
qa/integration/services/service_container.rb
View file

@ -1,67 +0,0 @@
require_relative "service"
require "docker"
require "logstash/devutils/rspec/logstash_helpers"
# Represents a service running within a container.
class ServiceContainer < Service
include LogStashHelper
def initialize(name, settings)
super(name, settings)
@base_image_context = File.expand_path("../dockerized", __FILE__)
@image_context = File.join(@base_image_context, @name)
# Options to create the container.
@container_create_opts = {}
end
def setup
puts "Setting up #{@name} service."
puts "Building the base container image."
@base_image = Docker::Image.build_from_dir(@base_image_context)
# Tag the base image.
#Caution: change this tag can cause failure to build the service container.
@base_image.tag('repo' => 'logstash', 'tag' => 'ci_sandbox', force: true)
puts "Finished building the base image."
puts "Building the container image."
self.build_image
puts "Finished building the image."
puts "Starting the container."
self.start_container
puts "Finished starting the container."
puts "Finished setting up #{@name} service."
end
def teardown
puts "Tearing down #{@name} service."
puts "Stop the container."
self.stop_container
puts "Finished stopping the container."
puts "Finished tearing down of #{@name} service."
end
def build_image
try(20) do
@image = Docker::Image.build_from_dir(@image_context)
end
end
def start_container
@container_create_opts[:Image] = @image.id
@container = Docker::Container.create(@container_create_opts)
@container.start
end
def stop_container
@container.stop
@container.delete(:force => true, :volumes => true)
end
end