Unique filter and spec.

lib/logstash/filters/unique.rb

@@ -0,0 +1,28 @@
+require "logstash/filters/base"
+require "logstash/namespace"
+
+class LogStash::Filters::Unique < LogStash::Filters::Base
+
+  config_name "unique"
+  milestone 1
+
+  # The fields on which to run the unique filter.
+  config :fields, :validate => :array, :required => true
+
+  public
+  def register
+    # Nothing to do
+  end # def register
+
+  public
+  def filter(event)
+    return unless filter?(event)
+
+    @fields.each do |field|
+      return unless event[field].class == Array
+
+      event[field] = event[field].uniq
+    end
+  end # def filter
+
+end # class Logstash::Filters::Unique

spec/filters/unique.rb

@@ -0,0 +1,25 @@
+require "test_utils"
+require "logstash/filters/unique"
+
+describe LogStash::Filters::Unique do
+  extend LogStash::RSpec
+
+  describe "unique when field is array" do
+    config <<-CONFIG
+    filter {
+      unique {
+        fields => ["noisy_field"]
+      }
+    }
+    CONFIG
+
+    sample("noisy_field" => %w(cat dog cat cat)) do
+      insist { subject["noisy_field"] } == %w(cat dog)
+    end
+
+    sample("not_an_array" => "Hello, world!") do
+      insist { subject["not_an_array"] } == "Hello, world!"
+    end
+
+  end
+end