- add ability to query what keys are available for a log type

- cache indexreaders, searchers, and queryparsers

bin/list_log_keys.rb

@@ -0,0 +1,46 @@
+#!/usr/bin/ruby
+#
+require 'rubygems'
+require "socket"
+require "lib/net/message"
+require "lib/net/client"
+require "lib/net/messages/logkeys"
+
+Thread::abort_on_exception = true
+
+class LogKeysClient < LogStash::Net::MessageClient
+  attr_reader :keys
+
+  def initialize(opts)
+    @log_type = opts[:log_type]
+    @keys = []
+    super(opts)
+    start
+  end
+
+  def start
+    msg = LogStash::Net::Messages::LogKeysRequest.new
+    msg.log_type = @log_type
+    sendmsg("logstash-index", msg)
+    run
+  end
+
+  def LogKeysResponseHandler(msg)
+    @keys = msg.keys
+    close
+  end
+end
+
+def main(args)
+  client = LogKeysClient.new(:log_type => args[0], :host => "localhost")
+
+  # Collate & print results.
+  puts "Log keys for #{args[0]}:"
+  client.keys.each { |t| puts " - #{t}" }
+
+  return 0
+end
+
+if $0 == __FILE__
+  exit main(ARGV)
+end

lib/net/messages/logkeys.rb

@@ -0,0 +1,17 @@
+require "lib/net/message"
+
+module LogStash; module Net; module Messages
+  class LogKeysRequest < RequestMessage
+    register
+
+    # Message attributes
+    hashbind :log_type, "/args/log_type"
+  end # class LogKeysRequest
+
+  class LogKeysResponse < ResponseMessage
+    register
+
+    # Message attributes
+    hashbind :keys, "/args/keys"
+  end # class LogKeysResponse
+end; end; end # module LogStash::Net::Messages

lib/net/servers/indexer.rb

@@ -5,6 +5,7 @@ require 'lib/net/message'
 require 'lib/net/messages/broadcast'
 require 'lib/net/messages/directory'
 require 'lib/net/messages/indexevent'
+require 'lib/net/messages/logkeys'
 require 'lib/net/messages/logtypes'
 require 'lib/net/messages/search'
 require 'lib/net/messages/searchhits'
@@ -29,6 +30,9 @@ module LogStash; module Net; module Servers
       @starttime = Time.now
       @indexers = Hash.new
       @indexers_mutex = Mutex.new
+      @readers = Hash.new
+      @searchers = Hash.new
+      @qps = Hash.new
     end
 
     def QuitRequestHandler(request)
@@ -70,6 +74,25 @@ module LogStash; module Net; module Servers
       yield response
     end
 
+    def LogKeysRequestHandler(request)
+      reader, search, qp = get_ferret(request.log_type)
+      response = LogStash::Net::Messages::LogKeysResponse.new
+      response.keys = reader.fields
+      yield response
+    end
+
+    def get_ferret(log_type)
+      @readers[log_type] ||= Ferret::Index::IndexReader.new(
+                               @config.logs[log_type].index_dir)
+      reader = @readers[log_type]
+      @searchers[log_type] ||= Ferret::Search::Searcher.new(reader)
+      @qps[log_type] ||= Ferret::QueryParser.new(
+                           :fields => reader.fields,
+                           :tokenized_fields => reader.tokenized_fields,
+                           :or_default => false)
+      return @readers[log_type], @searchers[log_type], @qps[log_type]
+    end
+
     def SearchRequestHandler(request)
       puts "Search for #{request.query.inspect} in #{request.log_type}"
       response = LogStash::Net::Messages::SearchResponse.new
@@ -84,11 +107,7 @@ module LogStash; module Net; module Servers
         return
       end
 
-      reader = Ferret::Index::IndexReader.new(@config.logs[request.log_type].index_dir)
-      search = Ferret::Search::Searcher.new(reader)
-      qp = Ferret::QueryParser.new(:fields => reader.fields,
-                                   :tokenized_fields => reader.tokenized_fields,
-                                   :or_default => false)
+      reader, search, qp = get_ferret(request.log_type)
       query = qp.parse(request.query)
       offset = (request.offset or 0)
       total = request.limit