- Add elasticsearch_host config option

- Make LogStash::Log index to elasticsearch

lib/config/agent.rb

@@ -34,6 +34,7 @@ module LogStash; module Config
                         :date_format => log_data["date"]["format"],
                         :logstash_dir => @logstash_dir,
                         :pattern_dir => @pattern_dir,
+                        :elasticsearch_host => @elasticsearch_host,
                        }
 
           log = nil

lib/config/base.rb

@@ -3,9 +3,13 @@ require 'yaml'
 
 module LogStash; module Config
   # Base config class. All configs need to know how to get to a broker.
+  attr_reader :elasticsearch_host
+
   class BaseConfig
     def initialize(file)
       obj = YAML::load(::File.open(file).read())
+      @elasticsearch_host = obj["elasticsearch_host"] || "localhost:9200"
+
       @mqhost = obj["mqhost"] || "localhost"
       @mqport = obj["mqport"] || 5672
       @mquser = obj["mquser"] || "guest"

lib/log.rb

@@ -1,6 +1,6 @@
 require 'date'
 require 'json'
-require 'ferret'
+#require 'ferret'
 
 module LogStash
   class LogException < StandardError
@@ -12,7 +12,7 @@ module LogStash
   class Log
     REQUIRED_KEYS = [:type, :encoding]
     OPTIONAL_KEYS = [:attrs, :date_key, :date_format, :logstash_dir,
-                     :pattern_dir]
+                     :pattern_dir, :elasticsearch_host]
     attr_accessor :attrs
 
     LogParseError = Class.new(StandardError)
@@ -70,8 +70,16 @@ module LogStash
     end
 
     def get_index
-      create_index unless File.exists?(index_dir)
-      return Ferret::Index::Index.new(:path => index_dir)
+      #create_index unless File.exists?(index_dir)
+      #return Ferret::Index::Index.new(:path => index_dir)
+      #http = EventMachine::HttpRequest.new("http://localhost:9200/logstash/#{@config[:type]}")
+      #req = http.post :body => entry.to_json
+    end
+
+    def index(entry)
+      #$logger.debug("Logging #{entry}")
+      http = EventMachine::HttpRequest.new("http://#{@config[:elasticsearch_host]}/logstash/#{@config[:type]}")
+      req = http.post :body => entry.to_json
     end
 
     def fix_date(res)

lib/net/servers/parser.rb

@@ -1,5 +1,4 @@
 require 'rubygems'
-require 'ferret'
 #require 'net/http'
 #require 'curb'
 require 'em-http'

lib/programs/agent.rb

@@ -33,8 +33,7 @@ module LogStash; module Programs;
       super(options)
       @config = LogStash::Config::AgentConfig.new(options[:config])
       @config.merge!(options)
-      #@indexes = Hash.new { |h,k| h[k] = @config.logs[k].get_index }
-      @index = LogStash::DB::Index.new(@config.logstash_dir + "/index.tct")
+      @indexes = Hash.new { |h,k| h[k] = @config.logs[k] }
 
       @hostname = Socket.gethostname
       @needs_flushing = Set.new
@@ -82,8 +81,9 @@ module LogStash; module Programs;
 
     private
     def index(name, entry)
+      @indexes[name].index(entry)
       #logstash_index(name, entry)
-      elastic_index(name, entry)
+      #elastic_index(name, entry)
     end
 
     def logstash_index(name, entry)
@@ -112,13 +112,13 @@ module LogStash; module Programs;
     def ferret_index(name, entry)
       @indexes[name] << entry
       @needs_flushing << name
-        @count += 1
-        if @count % PROGRESS_AMOUNT == 0
-          #flush_indexes
-          #puts "match #{name} in #{path}: #{line}"
-          puts "count: #{@count} #{AMOUNT / (Time.now - @start)}"
-          @start = Time.now
-        end
+      @count += 1
+      if @count % PROGRESS_AMOUNT == 0
+        #flush_indexes
+        #puts "match #{name} in #{path}: #{line}"
+        puts "count: #{@count} #{AMOUNT / (Time.now - @start)}"
+        @start = Time.now
+      end
     end
 
     private

logstash-agent.yaml

@@ -6,6 +6,7 @@ watch:
 
 logstash_dir: /c/logstash
 pattern_dir: /c/logstash/patterns
+elasticsearch_host: snack.home:9200
 
 log-types:
   linux-syslog:
@@ -25,21 +26,6 @@ log-types:
     patterns:
     - %{COMBINEDAPACHELOG}
 
-  glu:
-    type: json
-    date:
-      key: timestamp
-      format: %Y-%m-%dT%H:%M:%S
-    display_format: "<%= entry['timestamp'] %> | <%= entry['level'] %> | <%= entry['context/sessionKey'] %> | <%= entry['sourceHostName'] %> | <%= entry['context/componentName'] %> | <%= entry['message'] %>"
-
-  netscreen:
-    type: text
-    date:
-      key: date
-      format: %b %e %H:%M:%S
-    patterns:
-    - %{NETSCREENSESSIONLOG}
-
   haproxy:
     type: text
     date:

logstashd.yaml

@@ -1,6 +1,7 @@
 ---
 logstash_dir: /c/logstash
 pattern_dir: /c/logstash/patterns
+elasticsearch_host: localhost;9200
 
 log-types:
   linux-syslog:

web/config/init.rb

@@ -24,5 +24,7 @@ end
  
 Merb::BootLoader.after_app_loads do
   # This will get executed after your app's classes have been loaded.
-  #$search = LogStash::Net::Clients::Search.new("/opt/logstash/etc/logstashd.yaml")
+  $: << "../"
+  require "lib/net/clients/elasticsearch"
+  $search = LogStash::Net::Clients::ElasticSearch.new("/opt/logstash/etc/logstashd.yaml")
 end