github-mirrors/logstash
1
0
Fork 0
mirror of https://github.com/elastic/logstash.git synced 2025-04-24 14:47:19 -04:00
Code Issues Projects Releases Packages Wiki Activity

add more debuggability for kafka tests

Browse source 
Fixes #6072
This commit is contained in:
Suyog Rao 2016-10-17 19:44:16 -07:00
parent 82eb70ff05
commit 8cdd247070
2 changed files with 17 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

2
qa/integration/services/helpers.sh
View file

@ -19,6 +19,8 @@ wait_for_port() {
[[ $count -eq 0 ]] && return 1 [[ $count -eq 0 ]] && return 1
sleep 0.5 sleep 0.5
done done
# just in case, one more time
nc -z localhost $port
} }
clean_install_dir() { clean_install_dir() {

19
qa/integration/services/kafka_setup.sh
View file

@ -14,6 +14,7 @@ fi
KAFKA_HOME=$INSTALL_DIR/kafka KAFKA_HOME=$INSTALL_DIR/kafka
KAFKA_TOPIC=logstash_topic_plain KAFKA_TOPIC=logstash_topic_plain
KAFKA_MESSAGES=37 KAFKA_MESSAGES=37
KAFKA_LOGS_DIR=/tmp/ls_integration/kafka-logs
setup_kafka() { setup_kafka() {
local version=$1 local version=$1
@ -30,28 +31,38 @@ start_kafka() {
$KAFKA_HOME/bin/zookeeper-server-start.sh -daemon $KAFKA_HOME/config/zookeeper.properties $KAFKA_HOME/bin/zookeeper-server-start.sh -daemon $KAFKA_HOME/config/zookeeper.properties
wait_for_port 2181 wait_for_port 2181
echo "Starting Kafka broker" echo "Starting Kafka broker"
$KAFKA_HOME/bin/kafka-server-start.sh -daemon $KAFKA_HOME/config/server.properties --override delete.topic.enable=true --override log.dirs=/tmp/ls_integration/kafka-logs $KAFKA_HOME/bin/kafka-server-start.sh -daemon $KAFKA_HOME/config/server.properties --override delete.topic.enable=true --override log.dirs=$KAFKA_LOGS_DIR --override log.flush.interval.ms=200
wait_for_port 9092 wait_for_port 9092
} }
wait_for_messages() { wait_for_messages() {
local count=10 local count=10
local read_lines=0 local read_lines=0
echo "Checking if Kafka topic has been populated with data"
while [[ $read_lines -ne $KAFKA_MESSAGES ]] && [[ $count -ne 0 ]]; do while [[ $read_lines -ne $KAFKA_MESSAGES ]] && [[ $count -ne 0 ]]; do
read_lines=`$KAFKA_HOME/bin/kafka-console-consumer.sh --topic $KAFKA_TOPIC --new-consumer --bootstrap-server localhost:9092 --from-beginning --max-messages $KAFKA_MESSAGES | wc -l` read_lines=`$KAFKA_HOME/bin/kafka-console-consumer.sh --topic $KAFKA_TOPIC --new-consumer --bootstrap-server localhost:9092 --from-beginning --max-messages $KAFKA_MESSAGES --timeout-ms 10000 | wc -l`
count=$(( $count - 1 )) count=$(( $count - 1 ))
[[ $count -eq 0 ]] && return 1 [[ $count -eq 0 ]] && return 1
sleep 0.5 sleep 0.5
ls -lrt $KAFKA_LOGS_DIR/$KAFKA_TOPIC-0/
done done
echo "Kafka topic has been populated with test data"
} }
setup_install_dir setup_install_dir
setup_kafka $version setup_kafka $version
start_kafka start_kafka
# Set up topics # Set up topics
$KAFKA_HOME/bin/kafka-topics.sh --create --partitions 1 --replication-factor 1 --topic logstash_topic_plain --zookeeper localhost:2181 $KAFKA_HOME/bin/kafka-topics.sh --create --partitions 1 --replication-factor 1 --topic $KAFKA_TOPIC --zookeeper localhost:2181
# check topic got created
num_topic=`$KAFKA_HOME/bin/kafka-topics.sh --list --zookeeper localhost:2181 | grep $KAFKA_TOPIC | wc -l`
[[ $num_topic -eq 1 ]]
# Add test messages to the newly created topic # Add test messages to the newly created topic
cat $current_dir/../fixtures/how_sample.input | $KAFKA_HOME/bin/kafka-console-producer.sh --topic $KAFKA_TOPIC --broker-list localhost:9092 cp $current_dir/../fixtures/how_sample.input $KAFKA_HOME
[[ ! -s how_sample.input ]]
$KAFKA_HOME/bin/kafka-console-producer.sh --topic $KAFKA_TOPIC --broker-list localhost:9092 < $KAFKA_HOME/how_sample.input
echo "Kafka load status code $?"
# Wait until broker has all messages # Wait until broker has all messages
wait_for_messages wait_for_messages
echo "Kafka Setup complete" echo "Kafka Setup complete"