mirror of
https://github.com/elastic/logstash.git
synced 2025-04-24 14:47:19 -04:00
1b33cbab67
This is in prep for a Docker based test run approach, and by removing the dependent Docker containers we will avoid Docker in Docker requirements. Fixes #8211
72 lines
2.6 KiB
Bash
Executable file
72 lines
2.6 KiB
Bash
Executable file
#!/bin/bash
|
|
set -ex
|
|
current_dir="$(dirname "$0")"
|
|
|
|
export _JAVA_OPTIONS="-Djava.net.preferIPv4Stack=true"
|
|
|
|
source "$current_dir/helpers.sh"
|
|
|
|
if [ -n "${KAFKA_VERSION+1}" ]; then
|
|
echo "KAFKA_VERSION is $KAFKA_VERSION"
|
|
version=$KAFKA_VERSION
|
|
else
|
|
version=0.10.2.1
|
|
fi
|
|
|
|
KAFKA_HOME=$INSTALL_DIR/kafka
|
|
KAFKA_TOPIC=logstash_topic_plain
|
|
KAFKA_MESSAGES=37
|
|
KAFKA_LOGS_DIR=/tmp/ls_integration/kafka-logs
|
|
|
|
setup_kafka() {
|
|
local version=$1
|
|
if [ ! -d $KAFKA_HOME ]; then
|
|
echo "Downloading Kafka version $version"
|
|
curl -s -o $INSTALL_DIR/kafka.tgz "http://ftp.wayne.edu/apache/kafka/$version/kafka_2.11-$version.tgz"
|
|
mkdir $KAFKA_HOME && tar xzf $INSTALL_DIR/kafka.tgz -C $KAFKA_HOME --strip-components 1
|
|
rm $INSTALL_DIR/kafka.tgz
|
|
fi
|
|
}
|
|
|
|
start_kafka() {
|
|
echo "Starting ZooKeeper"
|
|
$KAFKA_HOME/bin/zookeeper-server-start.sh -daemon $KAFKA_HOME/config/zookeeper.properties
|
|
wait_for_port 2181
|
|
echo "Starting Kafka broker"
|
|
rm -rf ${KAFKA_LOGS_DIR}
|
|
mkdir -p ${KAFKA_LOGS_DIR}
|
|
$KAFKA_HOME/bin/kafka-server-start.sh -daemon $KAFKA_HOME/config/server.properties --override delete.topic.enable=true --override advertised.host.name=127.0.0.1 --override log.dir=${KAFKA_LOGS_DIR} --override log.flush.interval.ms=200
|
|
wait_for_port 9092
|
|
}
|
|
|
|
wait_for_messages() {
|
|
local count=10
|
|
local read_lines=0
|
|
|
|
echo "Checking if Kafka topic has been populated with data"
|
|
while [[ $read_lines -ne $KAFKA_MESSAGES ]] && [[ $count -ne 0 ]]; do
|
|
read_lines=`$KAFKA_HOME/bin/kafka-console-consumer.sh --topic $KAFKA_TOPIC --new-consumer --bootstrap-server localhost:9092 --from-beginning --max-messages $KAFKA_MESSAGES --timeout-ms 10000 | wc -l`
|
|
count=$(( $count - 1 ))
|
|
[[ $count -eq 0 ]] && return 1
|
|
sleep 0.5
|
|
#ls -lrt $KAFKA_LOGS_DIR/$KAFKA_TOPIC-0/
|
|
done
|
|
echo "Kafka topic has been populated with test data"
|
|
}
|
|
|
|
setup_install_dir
|
|
setup_kafka $version
|
|
start_kafka
|
|
# Set up topics
|
|
$KAFKA_HOME/bin/kafka-topics.sh --create --partitions 1 --replication-factor 1 --topic $KAFKA_TOPIC --zookeeper localhost:2181
|
|
# check topic got created
|
|
num_topic=`$KAFKA_HOME/bin/kafka-topics.sh --list --zookeeper localhost:2181 | grep $KAFKA_TOPIC | wc -l`
|
|
[[ $num_topic -eq 1 ]]
|
|
# Add test messages to the newly created topic
|
|
cp $current_dir/../fixtures/how_sample.input $KAFKA_HOME
|
|
[[ ! -s how_sample.input ]]
|
|
$KAFKA_HOME/bin/kafka-console-producer.sh --topic $KAFKA_TOPIC --broker-list localhost:9092 < $KAFKA_HOME/how_sample.input
|
|
echo "Kafka load status code $?"
|
|
# Wait until broker has all messages
|
|
wait_for_messages
|
|
echo "Kafka Setup complete"
|