From 64e1792515d29327a30929e06d872da68dca66fd Mon Sep 17 00:00:00 2001 From: Louis Zuckerman Date: Tue, 1 Jan 2013 23:27:08 -0500 Subject: [PATCH 1/9] LOGSTASH-805 --- docs/docgen.rb | 7 +++ lib/logstash/namespace.rb | 1 + lib/logstash/outputs/cloudwatch.rb | 52 +++++++----------- lib/logstash/plugin_mixins/aws_config.rb | 69 ++++++++++++++++++++++++ 4 files changed, 96 insertions(+), 33 deletions(-) create mode 100644 lib/logstash/plugin_mixins/aws_config.rb diff --git a/docs/docgen.rb b/docs/docgen.rb index 1fc75a32b..5ea5cd7e8 100644 --- a/docs/docgen.rb +++ b/docs/docgen.rb @@ -157,6 +157,13 @@ class LogStashConfigDocGenerator parse(File.new(File.join(File.dirname(file), "threadable.rb"), "r").read) end + if code =~ /include LogStash::PluginMixins/ + mixin = code.gsub(/.*include LogStash::PluginMixins::(\w+)\s.*/m, '\1') + mixin.gsub!(/(.)([A-Z])/, '\1_\2') + mixin.downcase! + parse(File.new(File.join(File.dirname(file), "..", "plugin_mixins", "#{mixin}.rb")).read) + end + parse(code) puts "Generating docs for #{file}" diff --git a/lib/logstash/namespace.rb b/lib/logstash/namespace.rb index 0d3b107b6..85d7f6af0 100644 --- a/lib/logstash/namespace.rb +++ b/lib/logstash/namespace.rb @@ -9,6 +9,7 @@ module LogStash module File; end module Web; end module Util; end + module PluginMixins; end SHUTDOWN = :shutdown end # module LogStash diff --git a/lib/logstash/outputs/cloudwatch.rb b/lib/logstash/outputs/cloudwatch.rb index be53c8ce3..cf88b553b 100644 --- a/lib/logstash/outputs/cloudwatch.rb +++ b/lib/logstash/outputs/cloudwatch.rb @@ -1,5 +1,6 @@ require "logstash/outputs/base" require "logstash/namespace" +require "logstash/plugin_mixins/aws_config" # This output lets you aggregate and send metric data to AWS CloudWatch # @@ -9,10 +10,10 @@ require "logstash/namespace" # output plugin is configured, on the logstash indexer node, with just AWS API # credentials, and possibly a region and/or a namespace. The output looks # for fields present in events, and when it finds them, it uses them to -# calculate aggregate statistics. If the "metricname" option is set in this +# calculate aggregate statistics. If the `metricname` option is set in this # output, then any events which pass through it will be aggregated & sent to # CloudWatch, but that is not recommended. The intended use is to NOT set the -# metricname option here, and instead to add a "CW_metricname" field (and other +# metricname option here, and instead to add a `CW_metricname` field (and other # fields) to only the events you want sent to CloudWatch. # # When events pass through this output they are queued for background @@ -20,7 +21,7 @@ require "logstash/namespace" # queue has a maximum size, and when it is full aggregated statistics will be # sent to CloudWatch ahead of schedule. Whenever this happens a warning # message is written to logstash's log. If you see this you should increase -# the queue_size configuration option to avoid the extra API calls. The queue +# the `queue_size` configuration option to avoid the extra API calls. The queue # is emptied every time we send data to CloudWatch. # # Note: when logstash is stopped the queue is destroyed before it can be processed. @@ -34,7 +35,7 @@ require "logstash/namespace" # Event Field configuration... # You add fields to your events in inputs & filters and this output reads # those fields to aggregate events. The names of the fields read are -# configurable via the field_* options. +# configurable via the `field_*` options. # # Per-output defaults... # You set universal defaults in this output plugin's configuration, and @@ -45,13 +46,13 @@ require "logstash/namespace" # # At a minimum events must have a "metric name" to be sent to CloudWatch. # This can be achieved either by providing a default here OR by adding a -# "CW_metricname" field. By default, if no other configuration is provided +# `CW_metricname` field. By default, if no other configuration is provided # besides a metric name, then events will be counted (Unit: Count, Value: 1) -# by their metric name (either a default or from their CW_metricname field) +# by their metric name (either a default or from their `CW_metricname` field) # # Other fields which can be added to events to modify the behavior of this -# plugin are, "CW_namespace", "CW_unit", "CW_value", and -# "CW_dimensions". All of these field names are configurable in +# plugin are, `CW_namespace`, `CW_unit`, `CW_value`, and +# `CW_dimensions`. All of these field names are configurable in # this output. You can also set per-output defaults for any of them. # See below for details. # @@ -59,6 +60,8 @@ require "logstash/namespace" # and the specific of API endpoint this output uses, # [PutMetricData](http://docs.amazonwebservices.com/AmazonCloudWatch/latest/APIReference/API_PutMetricData.html) class LogStash::Outputs::CloudWatch < LogStash::Outputs::Base + include LogStash::PluginMixins::AwsConfig + config_name "cloudwatch" plugin_status "experimental" @@ -76,18 +79,6 @@ class LogStash::Outputs::CloudWatch < LogStash::Outputs::Base COUNT_UNIT = "Count" NONE = "None" - US_EAST_1 = "us-east-1" - # The AWS Region to send logs to. - config :region, :validate => [US_EAST_1, "us-west-1", "us-west-2", - "eu-west-1", "ap-southeast-1", "ap-southeast-2", - "ap-northeast-1", "sa-east-1", "us-gov-west-1"], :default => US_EAST_1 - - # The AWS Access Key ID - config :access_key, :validate => :string, :required => true - - # The AWS Secret Access Key - config :secret_key, :validate => :string, :required => true - # How often to send data to CloudWatch # This does not affect the event timestamps, events will always have their # actual timestamp (to-the-minute) sent to CloudWatch. @@ -97,11 +88,11 @@ class LogStash::Outputs::CloudWatch < LogStash::Outputs::Base # See the Rufus Scheduler docs for an [explanation of allowed values](https://github.com/jmettraux/rufus-scheduler#the-time-strings-understood-by-rufus-scheduler) config :timeframe, :validate => :string, :default => "1m" - # How many events to queue before forcing a call to the CloudWatch API ahead of "timeframe" schedule + # How many events to queue before forcing a call to the CloudWatch API ahead of `timeframe` schedule # Set this to the number of events-per-timeframe you will be sending to CloudWatch to avoid extra API calls config :queue_size, :validate => :number, :default => 10000 - # The default namespace to use for events which do not have a "CW_namespace" field + # The default namespace to use for events which do not have a `CW_namespace` field config :namespace, :validate => :string, :default => "Logstash" # The name of the field used to set a different namespace per event @@ -110,7 +101,7 @@ class LogStash::Outputs::CloudWatch < LogStash::Outputs::Base # and those cost money. config :field_namespace, :validate => :string, :default => "CW_namespace" - # The default metric name to use for events which do not have a "CW_metricname" field. + # The default metric name to use for events which do not have a `CW_metricname` field. # Beware: If this is provided then all events which pass through this output will be aggregated and # sent to CloudWatch, so use this carefully. Furthermore, when providing this option, you # will probably want to also restrict events from passing through this output using event @@ -132,23 +123,23 @@ class LogStash::Outputs::CloudWatch < LogStash::Outputs::Base "Bits/Second", "Kilobits/Second", "Megabits/Second", "Gigabits/Second", "Terabits/Second", "Count/Second", NONE] - # The default unit to use for events which do not have a "CW_unit" field + # The default unit to use for events which do not have a `CW_unit` field # If you set this option you should probably set the "value" option along with it config :unit, :validate => VALID_UNITS, :default => COUNT_UNIT # The name of the field used to set the unit on an event metric config :field_unit, :validate => :string, :default => "CW_unit" - # The default value to use for events which do not have a "CW_value" field + # The default value to use for events which do not have a `CW_value` field # If provided, this must be a string which can be converted to a float, for example... # "1", "2.34", ".5", and "0.67" - # If you set this option you should probably set the "unit" option along with it + # If you set this option you should probably set the `unit` option along with it config :value, :validate => :string, :default => "1" # The name of the field used to set the value (float) on an event metric config :field_value, :validate => :string, :default => "CW_value" - # The default dimensions [ name, value, ... ] to use for events which do not have a "CW_dimensions" field + # The default dimensions [ name, value, ... ] to use for events which do not have a `CW_dimensions` field config :dimensions, :validate => :hash # The name of the field used to set the dimensions on an event metric @@ -166,12 +157,7 @@ class LogStash::Outputs::CloudWatch < LogStash::Outputs::Base require "rufus/scheduler" require "aws" - AWS.config( - :access_key_id => @access_key, - :secret_access_key => @secret_key, - :cloud_watch_endpoint => "monitoring.#{@region}.amazonaws.com" - ) - @cw = AWS::CloudWatch.new + @cw = AWS::CloudWatch.new(aws_options_hash) @event_queue = SizedQueue.new(@queue_size) @scheduler = Rufus::Scheduler.start_new diff --git a/lib/logstash/plugin_mixins/aws_config.rb b/lib/logstash/plugin_mixins/aws_config.rb new file mode 100644 index 000000000..df551ac57 --- /dev/null +++ b/lib/logstash/plugin_mixins/aws_config.rb @@ -0,0 +1,69 @@ +require "logstash/config/mixin" + +module LogStash::PluginMixins::AwsConfig + include LogStash::Config::Mixin + + @logger = LogStash::Logger.new(STDOUT) + @logger.level = $DEBUG ? :debug : :warn + + US_EAST_1 = "us-east-1" + # The AWS Region + config :region, :validate => [US_EAST_1, "us-west-1", "us-west-2", + "eu-west-1", "ap-southeast-1", "ap-southeast-2", + "ap-northeast-1", "sa-east-1", "us-gov-west-1"], :default => US_EAST_1 + + # This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order... + # 1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config + # 2. External credentials file specified by `aws_credentials_file` + # 3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` + # 4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY` + # 5. IAM Instance Profile (available when running inside EC2) + config :access_key_id, :validate => :string + + # The AWS Secret Access Key + config :secret_access_key, :validate => :string + + # Should we require (true) or disable (false) using SSL for communicating with the AWS API + # If this option is not provided, the AWS SDK for Ruby default will be used + config :use_ssl, :validate => :boolean + + # Path to YAML file containing a hash of AWS credentials. + # This file will only be loaded if `access_key_id` and + # `secret_access_key` aren't set. The contents of the + # file should look like this: + # + # --- + # :access_key_id: "12345" + # :secret_access_key: "54321" + # + config :aws_credentials_file, :validate => :string + + def aws_options_hash + if @access_key_id ^ @secret_access_key + @logger.warn("Only one of access_key_id or secret_access_key was provided but not both.") + end + + if (!(@access_key_id && @secret_access_key)) && @aws_credentials_file + access_creds = YAML.load_file(@aws_credentials_file) + + @access_key_id = access_creds[:access_key_id] + @secret_access_key = access_creds[:secret_access_key] + end + + opts = {} + + if (@access_key_id && @secret_access_key) + opts[:access_key_id] = @access_key_id + opts[:secret_access_key] = @secret_access_key + end + + if @use_ssl + opts[:use_ssl] = @use_ssl + end + + opts[:region] = @region + + return opts + end # def aws_options_hash + +end \ No newline at end of file From 733bf5f07a80dd8673814d713081a84c0cca1f35 Mon Sep 17 00:00:00 2001 From: Louis Zuckerman Date: Sun, 6 Jan 2013 01:35:36 -0500 Subject: [PATCH 2/9] Managed to hack together something that works (idk about docs though) --- lib/logstash/outputs/cloudwatch.rb | 3 + lib/logstash/plugin_mixins/aws_config.rb | 76 ++++++++++++++---------- 2 files changed, 46 insertions(+), 33 deletions(-) diff --git a/lib/logstash/outputs/cloudwatch.rb b/lib/logstash/outputs/cloudwatch.rb index cf88b553b..002e4eaea 100644 --- a/lib/logstash/outputs/cloudwatch.rb +++ b/lib/logstash/outputs/cloudwatch.rb @@ -79,6 +79,9 @@ class LogStash::Outputs::CloudWatch < LogStash::Outputs::Base COUNT_UNIT = "Count" NONE = "None" + # Set up common configuration from AwsConfig + setup_aws_config + # How often to send data to CloudWatch # This does not affect the event timestamps, events will always have their # actual timestamp (to-the-minute) sent to CloudWatch. diff --git a/lib/logstash/plugin_mixins/aws_config.rb b/lib/logstash/plugin_mixins/aws_config.rb index df551ac57..b0186b90c 100644 --- a/lib/logstash/plugin_mixins/aws_config.rb +++ b/lib/logstash/plugin_mixins/aws_config.rb @@ -1,57 +1,67 @@ require "logstash/config/mixin" module LogStash::PluginMixins::AwsConfig - include LogStash::Config::Mixin @logger = LogStash::Logger.new(STDOUT) @logger.level = $DEBUG ? :debug : :warn US_EAST_1 = "us-east-1" - # The AWS Region - config :region, :validate => [US_EAST_1, "us-west-1", "us-west-2", - "eu-west-1", "ap-southeast-1", "ap-southeast-2", - "ap-northeast-1", "sa-east-1", "us-gov-west-1"], :default => US_EAST_1 - # This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order... - # 1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config - # 2. External credentials file specified by `aws_credentials_file` - # 3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` - # 4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY` - # 5. IAM Instance Profile (available when running inside EC2) - config :access_key_id, :validate => :string + # This method is called when someone includes this module + def self.included(base) + # Add these methods to the 'base' given. + base.extend(self) + end - # The AWS Secret Access Key - config :secret_access_key, :validate => :string + public + def setup_aws_config + # The AWS Region + config :region, :validate => [US_EAST_1, "us-west-1", "us-west-2", + "eu-west-1", "ap-southeast-1", "ap-southeast-2", + "ap-northeast-1", "sa-east-1", "us-gov-west-1"], :default => US_EAST_1 - # Should we require (true) or disable (false) using SSL for communicating with the AWS API - # If this option is not provided, the AWS SDK for Ruby default will be used - config :use_ssl, :validate => :boolean + # This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order... + # 1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config + # 2. External credentials file specified by `aws_credentials_file` + # 3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` + # 4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY` + # 5. IAM Instance Profile (available when running inside EC2) + config :access_key_id, :validate => :string - # Path to YAML file containing a hash of AWS credentials. - # This file will only be loaded if `access_key_id` and - # `secret_access_key` aren't set. The contents of the - # file should look like this: - # - # --- - # :access_key_id: "12345" - # :secret_access_key: "54321" - # - config :aws_credentials_file, :validate => :string + # The AWS Secret Access Key + config :secret_access_key, :validate => :string + # Should we require (true) or disable (false) using SSL for communicating with the AWS API + # If this option is not provided, the AWS SDK for Ruby default will be used + config :use_ssl, :validate => :boolean + + # Path to YAML file containing a hash of AWS credentials. + # This file will only be loaded if `access_key_id` and + # `secret_access_key` aren't set. The contents of the + # file should look like this: + # + # --- + # :access_key_id: "12345" + # :secret_access_key: "54321" + # + config :aws_credentials_file, :validate => :string + end + + public def aws_options_hash - if @access_key_id ^ @secret_access_key - @logger.warn("Only one of access_key_id or secret_access_key was provided but not both.") + if @access_key_id.is_a?(NilClass) ^ @secret_access_key.is_a?(NilClass) + @logger.warn("Likely config error: Only one of access_key_id or secret_access_key was provided but not both.") end - if (!(@access_key_id && @secret_access_key)) && @aws_credentials_file + if ((!@access_key_id || !@secret_access_key)) && @aws_credentials_file access_creds = YAML.load_file(@aws_credentials_file) @access_key_id = access_creds[:access_key_id] @secret_access_key = access_creds[:secret_access_key] end - + opts = {} - + if (@access_key_id && @secret_access_key) opts[:access_key_id] = @access_key_id opts[:secret_access_key] = @secret_access_key @@ -62,7 +72,7 @@ module LogStash::PluginMixins::AwsConfig end opts[:region] = @region - + return opts end # def aws_options_hash From 174ec361f843052ecab88dd1a82798cbb92bad87 Mon Sep 17 00:00:00 2001 From: Louis Zuckerman Date: Sun, 6 Jan 2013 12:13:05 -0500 Subject: [PATCH 3/9] removed unnecessary docgen code added earlier --- docs/docgen.rb | 7 ------- lib/logstash/plugin_mixins/aws_config.rb | 4 ++-- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/docs/docgen.rb b/docs/docgen.rb index accb49606..26b057e49 100644 --- a/docs/docgen.rb +++ b/docs/docgen.rb @@ -162,13 +162,6 @@ class LogStashConfigDocGenerator parse(File.new(File.join(File.dirname(file), "threadable.rb"), "r").read) end - if code =~ /include LogStash::PluginMixins/ - mixin = code.gsub(/.*include LogStash::PluginMixins::(\w+)\s.*/m, '\1') - mixin.gsub!(/(.)([A-Z])/, '\1_\2') - mixin.downcase! - parse(File.new(File.join(File.dirname(file), "..", "plugin_mixins", "#{mixin}.rb")).read) - end - parse(code) puts "Generating docs for #{file}" diff --git a/lib/logstash/plugin_mixins/aws_config.rb b/lib/logstash/plugin_mixins/aws_config.rb index b0186b90c..550e37cf5 100644 --- a/lib/logstash/plugin_mixins/aws_config.rb +++ b/lib/logstash/plugin_mixins/aws_config.rb @@ -5,14 +5,14 @@ module LogStash::PluginMixins::AwsConfig @logger = LogStash::Logger.new(STDOUT) @logger.level = $DEBUG ? :debug : :warn - US_EAST_1 = "us-east-1" - # This method is called when someone includes this module def self.included(base) # Add these methods to the 'base' given. base.extend(self) end + US_EAST_1 = "us-east-1" + public def setup_aws_config # The AWS Region From 1f5febc427eb9a5155dc54291a80333b958752d7 Mon Sep 17 00:00:00 2001 From: Louis Zuckerman Date: Sun, 6 Jan 2013 12:26:35 -0500 Subject: [PATCH 4/9] put docgen code back in, fixed region & ssl configs --- docs/docgen.rb | 7 +++++++ lib/logstash/outputs/cloudwatch.rb | 7 +++++++ lib/logstash/plugin_mixins/aws_config.rb | 22 ++++++++++++++-------- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/docs/docgen.rb b/docs/docgen.rb index 26b057e49..accb49606 100644 --- a/docs/docgen.rb +++ b/docs/docgen.rb @@ -162,6 +162,13 @@ class LogStashConfigDocGenerator parse(File.new(File.join(File.dirname(file), "threadable.rb"), "r").read) end + if code =~ /include LogStash::PluginMixins/ + mixin = code.gsub(/.*include LogStash::PluginMixins::(\w+)\s.*/m, '\1') + mixin.gsub!(/(.)([A-Z])/, '\1_\2') + mixin.downcase! + parse(File.new(File.join(File.dirname(file), "..", "plugin_mixins", "#{mixin}.rb")).read) + end + parse(code) puts "Generating docs for #{file}" diff --git a/lib/logstash/outputs/cloudwatch.rb b/lib/logstash/outputs/cloudwatch.rb index 002e4eaea..1beb15f4c 100644 --- a/lib/logstash/outputs/cloudwatch.rb +++ b/lib/logstash/outputs/cloudwatch.rb @@ -154,6 +154,13 @@ class LogStash::Outputs::CloudWatch < LogStash::Outputs::Base # add_field => [ "CW_dimensions", "prod" ] config :field_dimensions, :validate => :string, :default => "CW_dimensions" + public + def aws_service_endpoint(region) + return { + :cloud_watch_endpoint => "monitoring.#{region}.amazonaws.com" + } + end + public def register require "thread" diff --git a/lib/logstash/plugin_mixins/aws_config.rb b/lib/logstash/plugin_mixins/aws_config.rb index 550e37cf5..c8f22a87a 100644 --- a/lib/logstash/plugin_mixins/aws_config.rb +++ b/lib/logstash/plugin_mixins/aws_config.rb @@ -32,15 +32,14 @@ module LogStash::PluginMixins::AwsConfig config :secret_access_key, :validate => :string # Should we require (true) or disable (false) using SSL for communicating with the AWS API - # If this option is not provided, the AWS SDK for Ruby default will be used - config :use_ssl, :validate => :boolean + # The AWS SDK for Ruby defaults to SSL so we preserve that + config :use_ssl, :validate => :boolean, :default => true # Path to YAML file containing a hash of AWS credentials. # This file will only be loaded if `access_key_id` and # `secret_access_key` aren't set. The contents of the # file should look like this: # - # --- # :access_key_id: "12345" # :secret_access_key: "54321" # @@ -67,12 +66,19 @@ module LogStash::PluginMixins::AwsConfig opts[:secret_access_key] = @secret_access_key end - if @use_ssl - opts[:use_ssl] = @use_ssl - end - - opts[:region] = @region + opts[:use_ssl] = @use_ssl + # The AWS SDK for Ruby doesn't know how to make an endpoint hostname from a region + # for example us-west-1 -> foosvc.us-west-1.amazonaws.com + # So our plugins need to know how to generate their endpoints from a region + # Furthermore, they need to know the symbol required to set that value in the AWS SDK + # Classes using this module must implement aws_service_endpoint(region:string) + # which must return a hash with one key, the aws sdk for ruby config symbol of the service + # endpoint, which has a string value of the service endpoint hostname + # for example, CloudWatch, { :cloud_watch_endpoint => "monitoring.#{region}.amazonaws.com" } + # For a list, see https://github.com/aws/aws-sdk-ruby/blob/master/lib/aws/core/configuration.rb + opts.merge!(self.aws_service_endpoint(@region)) + return opts end # def aws_options_hash From 7ee52a727f77443a9d53a014bfbc0c101381083a Mon Sep 17 00:00:00 2001 From: Louis Zuckerman Date: Sun, 6 Jan 2013 13:51:53 -0500 Subject: [PATCH 5/9] applied AwsConfig mixin to all AWS plugins --- lib/logstash/inputs/sqs.rb | 23 ++++++++++++---------- lib/logstash/outputs/sns.rb | 38 +++++++++++++------------------------ lib/logstash/outputs/sqs.rb | 24 ++++++++++++++--------- 3 files changed, 41 insertions(+), 44 deletions(-) diff --git a/lib/logstash/inputs/sqs.rb b/lib/logstash/inputs/sqs.rb index 32a653565..2af019e8e 100644 --- a/lib/logstash/inputs/sqs.rb +++ b/lib/logstash/inputs/sqs.rb @@ -1,5 +1,6 @@ require "logstash/inputs/threadable" require "logstash/namespace" +require "logstash/plugin_mixins/aws_config" # Pull events from an Amazon Web Services Simple Queue Service (SQS) queue. # @@ -54,17 +55,23 @@ require "logstash/namespace" # See http://aws.amazon.com/iam/ for more details on setting up AWS identities. # class LogStash::Inputs::SQS < LogStash::Inputs::Threadable + include LogStash::PluginMixins::AwsConfig + config_name "sqs" plugin_status "experimental" + # Set up common configuration from AwsConfig + setup_aws_config + # Name of the SQS Queue name to pull messages from. Note that this is just the name of the queue, not the URL or ARN. config :queue, :validate => :string, :required => true - # AWS access key. Must have the appropriate permissions. - config :access_key, :validate => :string, :required => true - - # AWS secret key. Must have the appropriate permissions. - config :secret_key, :validate => :string, :required => true + public + def aws_service_endpoint(region) + return { + :sqs_endpoint => "sqs.#{region}.amazonaws.com" + } + end def initialize(params) super @@ -76,11 +83,7 @@ class LogStash::Inputs::SQS < LogStash::Inputs::Threadable @logger.info("Registering SQS input", :queue => @queue) require "aws-sdk" - # Connec to SQS - @sqs = AWS::SQS.new( - :access_key_id => @access_key, - :secret_access_key => @secret_key - ) + @sqs = AWS::SQS.new(aws_options_hash) begin @logger.debug("Connecting to AWS SQS queue", :queue => @queue) diff --git a/lib/logstash/outputs/sns.rb b/lib/logstash/outputs/sns.rb index 812eb4a48..a0fcad90b 100644 --- a/lib/logstash/outputs/sns.rb +++ b/lib/logstash/outputs/sns.rb @@ -1,5 +1,6 @@ require "logstash/outputs/base" require "logstash/namespace" +require "logstash/plugin_mixins/aws_config" # SNS output. # @@ -23,25 +24,16 @@ require "logstash/namespace" # MAX_MESSAGE_SIZE_IN_BYTES. # class LogStash::Outputs::Sns < LogStash::Outputs::Base + include LogStash::PluginMixins::AwsConfig + MAX_SUBJECT_SIZE_IN_CHARACTERS = 100 MAX_MESSAGE_SIZE_IN_BYTES = 32768 config_name "sns" plugin_status "experimental" - # Amazon API credentials. - config :access_key_id, :validate => :string - config :secret_access_key, :validate => :string - - # Path to YAML file containing a hash of AWS credentials. This file - # will be loaded if `access_key_id` and `secret_access_key` aren't - # set. The contents of the file should look like this: - # - # --- - # :access_key_id: "12345" - # :secret_access_key: "54321" - # - config :credentials, :validate => :string + # Set up common configuration from AwsConfig + setup_aws_config # Message format. Defaults to plain text. config :format, :validate => [ "json", "plain" ], :default => "plain" @@ -57,22 +49,18 @@ class LogStash::Outputs::Sns < LogStash::Outputs::Base # config :publish_boot_message_arn, :validate => :string + public + def aws_service_endpoint(region) + return { + :sns_endpoint => "sns.#{region}.amazonaws.com" + } + end + public def register require "aws-sdk" - # Credentials weren't specified in the configuration. - unless @access_key_id && @secret_access_key - access_creds = YAML.load_file(@credentials) - - @access_key_id = access_creds[:access_key_id] - @secret_access_key = access_creds[:secret_access_key] - end - - @sns = AWS::SNS.new( - :access_key_id => @access_key_id, - :secret_access_key => @secret_access_key - ) + @sns = AWS::SNS.new(aws_options_hash) # Try to publish a "Logstash booted" message to the ARN provided to # cause an error ASAP if the credentials are bad. diff --git a/lib/logstash/outputs/sqs.rb b/lib/logstash/outputs/sqs.rb index 0b6202c4e..4d9788ced 100644 --- a/lib/logstash/outputs/sqs.rb +++ b/lib/logstash/outputs/sqs.rb @@ -1,5 +1,6 @@ require "logstash/outputs/base" require "logstash/namespace" +require "logstash/plugin_mixins/aws_config" # Push events to an Amazon Web Services Simple Queue Service (SQS) queue. # @@ -55,25 +56,30 @@ require "logstash/namespace" # See http://aws.amazon.com/iam/ for more details on setting up AWS identities. # class LogStash::Outputs::SQS < LogStash::Outputs::Base + include LogStash::PluginMixins::AwsConfig + config_name "sqs" plugin_status "experimental" + # Set up common configuration from AwsConfig + setup_aws_config + # Name of SQS queue to push messages into. Note that this is just the name of the queue, not the URL or ARN. config :queue, :validate => :string, :required => true - # AWS access key. Must have the appropriate permissions. - config :access_key, :validate => :string, :required => true - - # AWS secret key. Must have the appropriate permissions. - config :secret_key, :validate => :string, :required => true + public + def aws_service_endpoint(region) + return { + :sqs_endpoint => "sqs.#{region}.amazonaws.com" + } + end public def register require "aws-sdk" - @sqs = AWS::SQS.new( - :access_key_id => @access_key, - :secret_access_key => @secret_key - ) + + @sqs = AWS::SQS.new(aws_options_hash) + begin @logger.debug("Connecting to AWS SQS queue '#{@queue}'...") @sqs_queue = @sqs.queues.named(@queue) From 29efac6904e733b30d5d87a60ce1383c0f837d4c Mon Sep 17 00:00:00 2001 From: Louis Zuckerman Date: Wed, 9 Jan 2013 22:04:35 -0500 Subject: [PATCH 6/9] added old config options for backward compatibility with deprecation warnings & info --- lib/logstash/inputs/sqs.rb | 10 ++++++++++ lib/logstash/outputs/cloudwatch.rb | 12 +++++++++++- lib/logstash/outputs/sns.rb | 6 ++++++ lib/logstash/outputs/sqs.rb | 10 ++++++++++ 4 files changed, 37 insertions(+), 1 deletion(-) diff --git a/lib/logstash/inputs/sqs.rb b/lib/logstash/inputs/sqs.rb index 2af019e8e..f5fd5cfa3 100644 --- a/lib/logstash/inputs/sqs.rb +++ b/lib/logstash/inputs/sqs.rb @@ -63,6 +63,12 @@ class LogStash::Inputs::SQS < LogStash::Inputs::Threadable # Set up common configuration from AwsConfig setup_aws_config + # The `access_key` option is deprecated, please update your configuration to use `access_key_id` instead + config :access_key, :validate => :string, :deprecated => true + + # The `secret_key` option is deprecated, please update your configuration to use `secret_access_key` instead + config :secret_key, :validate => :string, :deprecated => true + # Name of the SQS Queue name to pull messages from. Note that this is just the name of the queue, not the URL or ARN. config :queue, :validate => :string, :required => true @@ -83,6 +89,10 @@ class LogStash::Inputs::SQS < LogStash::Inputs::Threadable @logger.info("Registering SQS input", :queue => @queue) require "aws-sdk" + # These should be removed when the deprecated aws credential options are removed + @access_key_id = @access_key + @secret_access_key = @secret_key + @sqs = AWS::SQS.new(aws_options_hash) begin diff --git a/lib/logstash/outputs/cloudwatch.rb b/lib/logstash/outputs/cloudwatch.rb index 1beb15f4c..486181458 100644 --- a/lib/logstash/outputs/cloudwatch.rb +++ b/lib/logstash/outputs/cloudwatch.rb @@ -81,7 +81,13 @@ class LogStash::Outputs::CloudWatch < LogStash::Outputs::Base # Set up common configuration from AwsConfig setup_aws_config - + + # The `access_key` option is deprecated, please update your configuration to use `access_key_id` instead + config :access_key, :validate => :string, :deprecated => true + + # The `secret_key` option is deprecated, please update your configuration to use `secret_access_key` instead + config :secret_key, :validate => :string, :deprecated => true + # How often to send data to CloudWatch # This does not affect the event timestamps, events will always have their # actual timestamp (to-the-minute) sent to CloudWatch. @@ -167,6 +173,10 @@ class LogStash::Outputs::CloudWatch < LogStash::Outputs::Base require "rufus/scheduler" require "aws" + # These should be removed when the deprecated aws credential options are removed + @access_key_id = @access_key + @secret_access_key = @secret_key + @cw = AWS::CloudWatch.new(aws_options_hash) @event_queue = SizedQueue.new(@queue_size) diff --git a/lib/logstash/outputs/sns.rb b/lib/logstash/outputs/sns.rb index a0fcad90b..613e34970 100644 --- a/lib/logstash/outputs/sns.rb +++ b/lib/logstash/outputs/sns.rb @@ -35,6 +35,9 @@ class LogStash::Outputs::Sns < LogStash::Outputs::Base # Set up common configuration from AwsConfig setup_aws_config + # The `credentials` option is deprecated, please update your config to use `aws_credentials_file` instead + config :credentials, :validate => :string + # Message format. Defaults to plain text. config :format, :validate => [ "json", "plain" ], :default => "plain" @@ -60,6 +63,9 @@ class LogStash::Outputs::Sns < LogStash::Outputs::Base def register require "aws-sdk" + # This should be removed when the deprecated aws credentials option is removed + @aws_credentials_file = @credentials + @sns = AWS::SNS.new(aws_options_hash) # Try to publish a "Logstash booted" message to the ARN provided to diff --git a/lib/logstash/outputs/sqs.rb b/lib/logstash/outputs/sqs.rb index 4d9788ced..3078f233e 100644 --- a/lib/logstash/outputs/sqs.rb +++ b/lib/logstash/outputs/sqs.rb @@ -64,6 +64,12 @@ class LogStash::Outputs::SQS < LogStash::Outputs::Base # Set up common configuration from AwsConfig setup_aws_config + # The `access_key` option is deprecated, please update your configuration to use `access_key_id` instead + config :access_key, :validate => :string, :deprecated => true + + # The `secret_key` option is deprecated, please update your configuration to use `secret_access_key` instead + config :secret_key, :validate => :string, :deprecated => true + # Name of SQS queue to push messages into. Note that this is just the name of the queue, not the URL or ARN. config :queue, :validate => :string, :required => true @@ -78,6 +84,10 @@ class LogStash::Outputs::SQS < LogStash::Outputs::Base def register require "aws-sdk" + # These should be removed when the deprecated aws credential options are removed + @access_key_id = @access_key + @secret_access_key = @secret_key + @sqs = AWS::SQS.new(aws_options_hash) begin From 0e016c7a20aa92fe766c48225911950d278c1bed Mon Sep 17 00:00:00 2001 From: Louis Zuckerman Date: Wed, 9 Jan 2013 22:10:57 -0500 Subject: [PATCH 7/9] added conditional around compatibility code --- lib/logstash/inputs/sqs.rb | 8 +++++--- lib/logstash/outputs/cloudwatch.rb | 8 +++++--- lib/logstash/outputs/sns.rb | 4 +++- lib/logstash/outputs/sqs.rb | 8 +++++--- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/lib/logstash/inputs/sqs.rb b/lib/logstash/inputs/sqs.rb index f5fd5cfa3..2c0ee4d62 100644 --- a/lib/logstash/inputs/sqs.rb +++ b/lib/logstash/inputs/sqs.rb @@ -89,9 +89,11 @@ class LogStash::Inputs::SQS < LogStash::Inputs::Threadable @logger.info("Registering SQS input", :queue => @queue) require "aws-sdk" - # These should be removed when the deprecated aws credential options are removed - @access_key_id = @access_key - @secret_access_key = @secret_key + # This should be removed when the deprecated aws credential options are removed + if (@access_key && @secret_key) + @access_key_id = @access_key + @secret_access_key = @secret_key + end @sqs = AWS::SQS.new(aws_options_hash) diff --git a/lib/logstash/outputs/cloudwatch.rb b/lib/logstash/outputs/cloudwatch.rb index 486181458..75c3fe16e 100644 --- a/lib/logstash/outputs/cloudwatch.rb +++ b/lib/logstash/outputs/cloudwatch.rb @@ -173,9 +173,11 @@ class LogStash::Outputs::CloudWatch < LogStash::Outputs::Base require "rufus/scheduler" require "aws" - # These should be removed when the deprecated aws credential options are removed - @access_key_id = @access_key - @secret_access_key = @secret_key + # This should be removed when the deprecated aws credential options are removed + if (@access_key && @secret_key) + @access_key_id = @access_key + @secret_access_key = @secret_key + end @cw = AWS::CloudWatch.new(aws_options_hash) diff --git a/lib/logstash/outputs/sns.rb b/lib/logstash/outputs/sns.rb index 613e34970..796346ac8 100644 --- a/lib/logstash/outputs/sns.rb +++ b/lib/logstash/outputs/sns.rb @@ -64,7 +64,9 @@ class LogStash::Outputs::Sns < LogStash::Outputs::Base require "aws-sdk" # This should be removed when the deprecated aws credentials option is removed - @aws_credentials_file = @credentials + if (@credentials) + @aws_credentials_file = @credentials + end @sns = AWS::SNS.new(aws_options_hash) diff --git a/lib/logstash/outputs/sqs.rb b/lib/logstash/outputs/sqs.rb index 3078f233e..b7aa80621 100644 --- a/lib/logstash/outputs/sqs.rb +++ b/lib/logstash/outputs/sqs.rb @@ -84,9 +84,11 @@ class LogStash::Outputs::SQS < LogStash::Outputs::Base def register require "aws-sdk" - # These should be removed when the deprecated aws credential options are removed - @access_key_id = @access_key - @secret_access_key = @secret_key + # This should be removed when the deprecated aws credential options are removed + if (@access_key && @secret_key) + @access_key_id = @access_key + @secret_access_key = @secret_key + end @sqs = AWS::SQS.new(aws_options_hash) From 5cfac53d9d8d1246948d88ca179252207e579a5b Mon Sep 17 00:00:00 2001 From: Louis Zuckerman Date: Wed, 9 Jan 2013 22:12:35 -0500 Subject: [PATCH 8/9] added a missing deprecated flag --- lib/logstash/outputs/sns.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logstash/outputs/sns.rb b/lib/logstash/outputs/sns.rb index 796346ac8..650cb1885 100644 --- a/lib/logstash/outputs/sns.rb +++ b/lib/logstash/outputs/sns.rb @@ -36,7 +36,7 @@ class LogStash::Outputs::Sns < LogStash::Outputs::Base setup_aws_config # The `credentials` option is deprecated, please update your config to use `aws_credentials_file` instead - config :credentials, :validate => :string + config :credentials, :validate => :string, :deprecated => true # Message format. Defaults to plain text. config :format, :validate => [ "json", "plain" ], :default => "plain" From ed16099e538f9457b4628ff21efbe2f12511f3ca Mon Sep 17 00:00:00 2001 From: Louis Zuckerman Date: Thu, 10 Jan 2013 09:10:25 -0500 Subject: [PATCH 9/9] moved call to setup_aws_config to mixin --- lib/logstash/inputs/sqs.rb | 3 --- lib/logstash/outputs/cloudwatch.rb | 3 --- lib/logstash/outputs/sns.rb | 3 --- lib/logstash/outputs/sqs.rb | 3 --- lib/logstash/plugin_mixins/aws_config.rb | 1 + 5 files changed, 1 insertion(+), 12 deletions(-) diff --git a/lib/logstash/inputs/sqs.rb b/lib/logstash/inputs/sqs.rb index 2c0ee4d62..06edeb770 100644 --- a/lib/logstash/inputs/sqs.rb +++ b/lib/logstash/inputs/sqs.rb @@ -60,9 +60,6 @@ class LogStash::Inputs::SQS < LogStash::Inputs::Threadable config_name "sqs" plugin_status "experimental" - # Set up common configuration from AwsConfig - setup_aws_config - # The `access_key` option is deprecated, please update your configuration to use `access_key_id` instead config :access_key, :validate => :string, :deprecated => true diff --git a/lib/logstash/outputs/cloudwatch.rb b/lib/logstash/outputs/cloudwatch.rb index 75c3fe16e..5a6c34df1 100644 --- a/lib/logstash/outputs/cloudwatch.rb +++ b/lib/logstash/outputs/cloudwatch.rb @@ -79,9 +79,6 @@ class LogStash::Outputs::CloudWatch < LogStash::Outputs::Base COUNT_UNIT = "Count" NONE = "None" - # Set up common configuration from AwsConfig - setup_aws_config - # The `access_key` option is deprecated, please update your configuration to use `access_key_id` instead config :access_key, :validate => :string, :deprecated => true diff --git a/lib/logstash/outputs/sns.rb b/lib/logstash/outputs/sns.rb index 650cb1885..08c5e8c31 100644 --- a/lib/logstash/outputs/sns.rb +++ b/lib/logstash/outputs/sns.rb @@ -32,9 +32,6 @@ class LogStash::Outputs::Sns < LogStash::Outputs::Base config_name "sns" plugin_status "experimental" - # Set up common configuration from AwsConfig - setup_aws_config - # The `credentials` option is deprecated, please update your config to use `aws_credentials_file` instead config :credentials, :validate => :string, :deprecated => true diff --git a/lib/logstash/outputs/sqs.rb b/lib/logstash/outputs/sqs.rb index b7aa80621..14b3cf5c8 100644 --- a/lib/logstash/outputs/sqs.rb +++ b/lib/logstash/outputs/sqs.rb @@ -61,9 +61,6 @@ class LogStash::Outputs::SQS < LogStash::Outputs::Base config_name "sqs" plugin_status "experimental" - # Set up common configuration from AwsConfig - setup_aws_config - # The `access_key` option is deprecated, please update your configuration to use `access_key_id` instead config :access_key, :validate => :string, :deprecated => true diff --git a/lib/logstash/plugin_mixins/aws_config.rb b/lib/logstash/plugin_mixins/aws_config.rb index c8f22a87a..bbbd69362 100644 --- a/lib/logstash/plugin_mixins/aws_config.rb +++ b/lib/logstash/plugin_mixins/aws_config.rb @@ -9,6 +9,7 @@ module LogStash::PluginMixins::AwsConfig def self.included(base) # Add these methods to the 'base' given. base.extend(self) + base.setup_aws_config end US_EAST_1 = "us-east-1"