From c785f1342858295009ca704244f78598e492b80e Mon Sep 17 00:00:00 2001 From: "michael.gibson" Date: Mon, 24 Dec 2012 09:29:16 -0700 Subject: [PATCH 1/3] adding ability to define the event.id in a custom filter for the elasticsearch output. LOGSTASH-256 --- lib/logstash/event.rb | 4 ++++ lib/logstash/outputs/elasticsearch.rb | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/logstash/event.rb b/lib/logstash/event.rb index ed057bed6..31e153182 100644 --- a/lib/logstash/event.rb +++ b/lib/logstash/event.rb @@ -124,6 +124,10 @@ class LogStash::Event def tags; @data["@tags"]; end # def tags def tags=(val); @data["@tags"] = val; end # def tags= + public + def id; @data["@id"]; end # def id + def id=(val); @data["@id"] = val; end # def id= + # field-related access public def [](key) diff --git a/lib/logstash/outputs/elasticsearch.rb b/lib/logstash/outputs/elasticsearch.rb index 787ce6ed5..51578e567 100644 --- a/lib/logstash/outputs/elasticsearch.rb +++ b/lib/logstash/outputs/elasticsearch.rb @@ -37,6 +37,9 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base # similar events to the same 'type'. String expansion '%{foo}' works here. config :index_type, :validate => :string, :default => "%{@type}" + # The document ID for the index. Overwrites any existing entry in elasticsearch with the same ID. + config :id, :validate => :string, :default => "%{@id}" + # The name of your cluster if you set it on the ElasticSearch side. Useful # for discovery. config :cluster, :validate => :string @@ -140,6 +143,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base index = event.sprintf(@index) type = event.sprintf(@index_type) + id = event.sprintf(@id) # TODO(sissel): allow specifying the ID? # The document ID is how elasticsearch determines sharding hash, so it can # help performance if we allow folks to specify a specific ID. @@ -160,7 +164,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base end end - req = @client.index(index, type, event.to_hash) + req = @client.index(index, type, id, event.to_hash) increment_inflight_request_count #timer = @logger.time("elasticsearch write") req.on(:success) do |response| From 3066ac7f9fcd04103211a6af5cc3e1d9459bd42e Mon Sep 17 00:00:00 2001 From: "michael.gibson" Date: Mon, 24 Dec 2012 18:16:35 -0700 Subject: [PATCH 2/3] bug-fix for when id is not specified for elasticsearch. making it not required. LOGSTASH-256 --- lib/logstash/outputs/elasticsearch.rb | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/logstash/outputs/elasticsearch.rb b/lib/logstash/outputs/elasticsearch.rb index 51578e567..bb6341841 100644 --- a/lib/logstash/outputs/elasticsearch.rb +++ b/lib/logstash/outputs/elasticsearch.rb @@ -38,7 +38,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base config :index_type, :validate => :string, :default => "%{@type}" # The document ID for the index. Overwrites any existing entry in elasticsearch with the same ID. - config :id, :validate => :string, :default => "%{@id}" + config :id, :validate => :string, :default => "" # The name of your cluster if you set it on the ElasticSearch side. Useful # for discovery. @@ -164,7 +164,12 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base end end - req = @client.index(index, type, id, event.to_hash) + if id == "%{@id}" + req = @client.index(index, type, event.to_hash) + else + req = @client.index(index, type, id, event.to_hash) + end + increment_inflight_request_count #timer = @logger.time("elasticsearch write") req.on(:success) do |response| From d7505972ae746fb080a0b33650588fd92e2f6108 Mon Sep 17 00:00:00 2001 From: "michael.gibson" Date: Thu, 27 Dec 2012 09:05:32 -0700 Subject: [PATCH 3/3] updating changes to pull 286 --- lib/logstash/event.rb | 1 - lib/logstash/outputs/elasticsearch.rb | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/logstash/event.rb b/lib/logstash/event.rb index 31e153182..e08f69a69 100644 --- a/lib/logstash/event.rb +++ b/lib/logstash/event.rb @@ -124,7 +124,6 @@ class LogStash::Event def tags; @data["@tags"]; end # def tags def tags=(val); @data["@tags"] = val; end # def tags= - public def id; @data["@id"]; end # def id def id=(val); @data["@id"] = val; end # def id= diff --git a/lib/logstash/outputs/elasticsearch.rb b/lib/logstash/outputs/elasticsearch.rb index bb6341841..f4d7deaf3 100644 --- a/lib/logstash/outputs/elasticsearch.rb +++ b/lib/logstash/outputs/elasticsearch.rb @@ -38,7 +38,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base config :index_type, :validate => :string, :default => "%{@type}" # The document ID for the index. Overwrites any existing entry in elasticsearch with the same ID. - config :id, :validate => :string, :default => "" + config :id, :validate => :string, :default => nil # The name of your cluster if you set it on the ElasticSearch side. Useful # for discovery. @@ -143,7 +143,6 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base index = event.sprintf(@index) type = event.sprintf(@index_type) - id = event.sprintf(@id) # TODO(sissel): allow specifying the ID? # The document ID is how elasticsearch determines sharding hash, so it can # help performance if we allow folks to specify a specific ID. @@ -164,9 +163,10 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base end end - if id == "%{@id}" + if id.nil? req = @client.index(index, type, event.to_hash) else + id = event.sprintf(@id) req = @client.index(index, type, id, event.to_hash) end