mirror of
https://github.com/elastic/logstash.git
synced 2025-04-24 22:57:16 -04:00
136 lines
3.6 KiB
Ruby
136 lines
3.6 KiB
Ruby
|
||
require 'rubygems'
|
||
require 'lib/net/server'
|
||
require 'lib/net/message'
|
||
require 'lib/net/messages/indexevent'
|
||
require 'lib/net/messages/search'
|
||
require 'lib/net/messages/ping'
|
||
|
||
require 'ferret'
|
||
require 'lib/log/text'
|
||
require 'config'
|
||
|
||
|
||
module LogStash; module Net; module Servers
|
||
class Indexer < LogStash::Net::MessageServer
|
||
SYNCDELAY = 10
|
||
|
||
def initialize(addr="0.0.0.0", port=3001)
|
||
# 'super' is not the same as 'super()', and we want super().
|
||
super(addr, port)
|
||
@indexes = Hash.new
|
||
@lines = Hash.new { |h,k| h[k] = 0 }
|
||
@indexcount = 0
|
||
end
|
||
|
||
def IndexEventRequestHandler(request)
|
||
response = LogStash::Net::Messages::IndexEventResponse.new
|
||
response.id = request.id
|
||
@indexcount += 1
|
||
|
||
print "\rK#{@indexcount} (vs #{request.id})"
|
||
#puts "#{@indexcount} (id: #{request.id})"
|
||
|
||
log_type = request.log_type
|
||
entry = $logs[log_type].parse_entry(request.log_data)
|
||
if !entry
|
||
response.code = 1
|
||
response.error = "Entry was #{entry.inspect} (log parsing failed)"
|
||
else
|
||
response.code = 0
|
||
if not @indexes.member?(log_type)
|
||
@indexes[log_type] = $logs[log_type].get_index
|
||
end
|
||
|
||
entry["@LOG_TYPE"] = log_type
|
||
@indexes[log_type] << entry
|
||
end
|
||
yield response
|
||
end
|
||
|
||
def PingRequestHandler(request)
|
||
response = LogStash::Net::Messages::PingResponse.new
|
||
response.id = request.id
|
||
response.pingdata = request.pingdata
|
||
yield response
|
||
end
|
||
|
||
def SearchRequestHandler(request)
|
||
puts "Search for #{request.query.inspect}"
|
||
|
||
reader = Ferret::Index::IndexReader.new($logs[request.log_type].index_dir)
|
||
search = Ferret::Search::Searcher.new(reader)
|
||
|
||
puts reader.fields.join("\n")
|
||
qp = Ferret::QueryParser.new(:fields => reader.fields,
|
||
:tokenized_fields => reader.tokenized_fields,
|
||
:or_default => false)
|
||
query = qp.parse(request.query)
|
||
results = []
|
||
offset = (request.offset or 0)
|
||
total = request.limit
|
||
limit = 50
|
||
|
||
done = false
|
||
while !done
|
||
done = true
|
||
puts "Searching..."
|
||
|
||
if total
|
||
limit = [total, limit].min
|
||
total -= limit
|
||
|
||
if limit <= 0
|
||
done = true
|
||
next
|
||
end
|
||
end
|
||
|
||
search.search_each(query, :limit => limit, :offset => offset,
|
||
:sort => "@DATE") do |docid, score|
|
||
done = false
|
||
result = reader[docid][:@LINE]
|
||
results << result
|
||
end
|
||
|
||
response = LogStash::Net::Messages::SearchResponse.new
|
||
response.id = request.id
|
||
response.results = results
|
||
yield response
|
||
results = []
|
||
offset += limit
|
||
end
|
||
response = LogStash::Net::Messages::SearchResponse.new
|
||
response.id = request.id
|
||
response.results = results
|
||
response.finished = true
|
||
yield response
|
||
end
|
||
|
||
# Special 'run' override because we want sync to disk once per minute.
|
||
def _run
|
||
synctime = Time.now + SYNCDELAY
|
||
sleeptime = 1
|
||
loop do
|
||
active = sendrecv(sleeptime)
|
||
if !active
|
||
sleeptime *= 2
|
||
if sleeptime > SYNCDELAY
|
||
sleeptime = SYNCDELAY
|
||
end
|
||
puts "No activity, sleeping for #{sleeptime}"
|
||
end
|
||
|
||
if Time.now > synctime
|
||
@indexes.each do |log_type,index|
|
||
puts "Time's up. Syncing #{log_type}"
|
||
index.commit
|
||
end
|
||
|
||
synctime = Time.now + 60
|
||
end
|
||
end
|
||
end # def run
|
||
|
||
end # Indexer
|
||
end; end; end # LogStash::Net::Server
|