mirror of
https://github.com/elastic/logstash.git
synced 2025-04-24 22:57:16 -04:00
f55513ba12
for an ack for that message before sending another. Work around this:
- batch up messages to sendmsg() and flush when there are more than 10 in the
queue or there has been more than 1 second since flushing and the queue is
non-empty
This increases the indexing rate by a factor of 6.
74 lines
1.6 KiB
Ruby
Executable file
74 lines
1.6 KiB
Ruby
Executable file
#!/usr/bin/env ruby
|
|
|
|
require 'rubygems'
|
|
require 'lib/net/client'
|
|
require 'lib/net/messages/indexevent'
|
|
require 'lib/file/tail/since'
|
|
require 'stomp'
|
|
require 'socket'
|
|
|
|
|
|
class Agent < LogStash::Net::MessageClient
|
|
def initialize(host, port)
|
|
super(username="", password="", host=host, port=port)
|
|
@hostname = Socket.gethostname
|
|
@msgs = []
|
|
end # def initialize
|
|
|
|
def start_log_watcher
|
|
#@t1 = Thread.new do
|
|
#File::Tail::Since.new("/var/log/messages").tail do |line|
|
|
#line.chomp!
|
|
#index("linux-syslog", line)
|
|
#end
|
|
##end
|
|
|
|
@t2 = Thread.new do
|
|
#File::Tail::Since.new("/b/access.10").tail do |line|
|
|
begin
|
|
count = 0
|
|
File.open("/b/access.1k").readlines.each do |line|
|
|
line.chomp!
|
|
index("httpd-access", line)
|
|
count += 1
|
|
#break if count >= 3
|
|
end
|
|
rescue => e
|
|
$stderr.puts e.inspect
|
|
$stderr.puts caller.join("\n")
|
|
raise e
|
|
end
|
|
#close
|
|
end
|
|
@t2.join
|
|
end # def start_log_watcher
|
|
|
|
def index(type, string)
|
|
ier = LogStash::Net::Messages::IndexEventRequest.new
|
|
ier.log_type = type
|
|
ier.log_data = string
|
|
ier.metadata["source_host"] = @hostname
|
|
|
|
puts "Sending: #{ier}"
|
|
sendmsg("/queue/logstash", ier)
|
|
end # def index
|
|
|
|
def IndexEventResponseHandler(msg)
|
|
end # def IndexEventResponseHandler
|
|
|
|
def run
|
|
start_log_watcher
|
|
super
|
|
end
|
|
end
|
|
|
|
|
|
if $0 == __FILE__
|
|
if ARGV.length == 0
|
|
puts "Usage: #{$0} host:port"
|
|
exit 1
|
|
end
|
|
host, port = ARGV[0].split(":")
|
|
agent = Agent.new(host, port)
|
|
agent.run
|
|
end
|