Additional csv option for quote_char

This will allow users to configure a different quoting char in cases where the data may not exactly fit the csv standard.

A common example may be to completely ignore the quote char, and to do this a null \x00 char can be used instead.
Another example may be csv with ' as the quote char instead of "
This commit is contained in:
Alex Markham 2013-10-16 15:47:04 +01:00
parent 6ae99fcec9
commit 44a9bca63e
2 changed files with 38 additions and 1 deletions

View file

@ -24,6 +24,11 @@ class LogStash::Filters::CSV < LogStash::Filters::Base
# Optional.
config :separator, :validate => :string, :default => ","
# Define the character used to quote CSV fields. If this is not specified
# the default is a double quote '"'
# Optional.
config :quote_char, :validate => :string, :default => '"'
# Define target for placing the data
# Defaults to writing to the root of the event.
config :target, :validate => :string
@ -57,7 +62,7 @@ class LogStash::Filters::CSV < LogStash::Filters::Base
raw = event[@source].first
begin
values = CSV.parse_line(raw, :col_sep => @separator)
values = CSV.parse_line(raw, :col_sep => @separator, :quote_char => @quote_char)
if @target.nil?
# Default is to write to the root of the event.

View file

@ -35,6 +35,38 @@ describe LogStash::Filters::CSV do
end
end
describe "custom quote char" do
config <<-CONFIG
filter {
csv {
quote_char => "'"
}
}
CONFIG
sample "big,bird,'sesame street'" do
insist { subject["column1"] } == "big"
insist { subject["column2"] } == "bird"
insist { subject["column3"] } == "sesame street"
end
end
describe "null quote char" do
config <<-CONFIG
filter {
csv {
quote_char => "\x00"
}
}
CONFIG
sample 'big,bird,"sesame" street' do
insist { subject["column1"] } == 'big'
insist { subject["column2"] } == 'bird'
insist { subject["column3"] } == '"sesame" street'
end
end
describe "given columns" do
# The logstash config goes here.
# At this time, only filters are supported.