Merge pull request #307 from semiosis/aws-iam-roles

Common AWS plugin configuration mixin, supporting IAM Roles, YAML, &c (LOGSTASH-805)

docs/docgen.rb

@@ -162,6 +162,13 @@ class LogStashConfigDocGenerator
       parse(File.new(File.join(File.dirname(file), "threadable.rb"), "r").read)
     end
 
+    if code =~ /include LogStash::PluginMixins/
+      mixin = code.gsub(/.*include LogStash::PluginMixins::(\w+)\s.*/m, '\1')
+      mixin.gsub!(/(.)([A-Z])/, '\1_\2')
+      mixin.downcase!
+      parse(File.new(File.join(File.dirname(file), "..", "plugin_mixins", "#{mixin}.rb")).read)
+    end
+    
     parse(code)
 
     puts "Generating docs for #{file}"

lib/logstash/inputs/sqs.rb

@@ -1,5 +1,6 @@
 require "logstash/inputs/threadable"
 require "logstash/namespace"
+require "logstash/plugin_mixins/aws_config"
 
 # Pull events from an Amazon Web Services Simple Queue Service (SQS) queue.
 #
@@ -54,17 +55,26 @@ require "logstash/namespace"
 # See http://aws.amazon.com/iam/ for more details on setting up AWS identities.
 #
 class LogStash::Inputs::SQS < LogStash::Inputs::Threadable
+  include LogStash::PluginMixins::AwsConfig
+  
   config_name "sqs"
   plugin_status "experimental"
 
+  # The `access_key` option is deprecated, please update your configuration to use `access_key_id` instead
+  config :access_key, :validate => :string, :deprecated => true
+
+  # The `secret_key` option is deprecated, please update your configuration to use `secret_access_key` instead
+  config :secret_key, :validate => :string, :deprecated => true
+
   # Name of the SQS Queue name to pull messages from. Note that this is just the name of the queue, not the URL or ARN.
   config :queue, :validate => :string, :required => true
 
-  # AWS access key. Must have the appropriate permissions.
+  public
-  config :access_key, :validate => :string, :required => true
+  def aws_service_endpoint(region)
-
+    return {
-  # AWS secret key. Must have the appropriate permissions.
+        :sqs_endpoint => "sqs.#{region}.amazonaws.com"
-  config :secret_key, :validate => :string, :required => true
+    }
+  end
 
   def initialize(params)
     super
@@ -76,11 +86,13 @@ class LogStash::Inputs::SQS < LogStash::Inputs::Threadable
     @logger.info("Registering SQS input", :queue => @queue)
     require "aws-sdk"
 
-    # Connec to SQS
+    # This should be removed when the deprecated aws credential options are removed
-    @sqs = AWS::SQS.new(
+    if (@access_key && @secret_key)
-      :access_key_id => @access_key,
+      @access_key_id = @access_key
-      :secret_access_key => @secret_key
+      @secret_access_key = @secret_key
-    )
+    end
+    
+    @sqs = AWS::SQS.new(aws_options_hash)
 
     begin
       @logger.debug("Connecting to AWS SQS queue", :queue => @queue)

lib/logstash/namespace.rb

@@ -9,6 +9,7 @@ module LogStash
   module File; end
   module Web; end
   module Util; end
+  module PluginMixins; end
 
   SHUTDOWN = :shutdown
 end # module LogStash

lib/logstash/outputs/cloudwatch.rb

@@ -1,5 +1,6 @@
 require "logstash/outputs/base"
 require "logstash/namespace"
+require "logstash/plugin_mixins/aws_config"
 
 # This output lets you aggregate and send metric data to AWS CloudWatch
 #
@@ -9,10 +10,10 @@ require "logstash/namespace"
 # output plugin is configured, on the logstash indexer node, with just AWS API
 # credentials, and possibly a region and/or a namespace.  The output looks
 # for fields present in events, and when it finds them, it uses them to
-# calculate aggregate statistics.  If the "metricname" option is set in this
+# calculate aggregate statistics.  If the `metricname` option is set in this
 # output, then any events which pass through it will be aggregated & sent to
 # CloudWatch, but that is not recommended.  The intended use is to NOT set the
-# metricname option here, and instead to add a "CW_metricname" field (and other
+# metricname option here, and instead to add a `CW_metricname` field (and other
 # fields) to only the events you want sent to CloudWatch.
 #
 # When events pass through this output they are queued for background
@@ -20,7 +21,7 @@ require "logstash/namespace"
 # queue has a maximum size, and when it is full aggregated statistics will be
 # sent to CloudWatch ahead of schedule. Whenever this happens a warning
 # message is written to logstash's log.  If you see this you should increase
-# the queue_size configuration option to avoid the extra API calls.  The queue
+# the `queue_size` configuration option to avoid the extra API calls.  The queue
 # is emptied every time we send data to CloudWatch.
 #
 # Note: when logstash is stopped the queue is destroyed before it can be processed.
@@ -34,7 +35,7 @@ require "logstash/namespace"
 # Event Field configuration...
 # You add fields to your events in inputs & filters and this output reads
 # those fields to aggregate events.  The names of the fields read are
-# configurable via the field_* options.
+# configurable via the `field_*` options.
 #
 # Per-output defaults...
 # You set universal defaults in this output plugin's configuration, and
@@ -45,13 +46,13 @@ require "logstash/namespace"
 #
 # At a minimum events must have a "metric name" to be sent to CloudWatch.
 # This can be achieved either by providing a default here OR by adding a
-# "CW_metricname" field. By default, if no other configuration is provided
+# `CW_metricname` field. By default, if no other configuration is provided
 # besides a metric name, then events will be counted (Unit: Count, Value: 1)
-# by their metric name (either a default or from their CW_metricname field)
+# by their metric name (either a default or from their `CW_metricname` field)
 #
 # Other fields which can be added to events to modify the behavior of this
-# plugin are, "CW_namespace", "CW_unit", "CW_value", and 
+# plugin are, `CW_namespace`, `CW_unit`, `CW_value`, and 
-# "CW_dimensions".  All of these field names are configurable in
+# `CW_dimensions`.  All of these field names are configurable in
 # this output.  You can also set per-output defaults for any of them.
 # See below for details.
 #
@@ -59,6 +60,8 @@ require "logstash/namespace"
 # and the specific of API endpoint this output uses,
 # [PutMetricData](http://docs.amazonwebservices.com/AmazonCloudWatch/latest/APIReference/API_PutMetricData.html)
 class LogStash::Outputs::CloudWatch < LogStash::Outputs::Base
+  include LogStash::PluginMixins::AwsConfig
+  
   config_name "cloudwatch"
   plugin_status "experimental"
 
@@ -76,17 +79,11 @@ class LogStash::Outputs::CloudWatch < LogStash::Outputs::Base
   COUNT_UNIT = "Count"
   NONE = "None"
 
-  US_EAST_1 = "us-east-1"
+  # The `access_key` option is deprecated, please update your configuration to use `access_key_id` instead
-  # The AWS Region to send logs to.
+  config :access_key, :validate => :string, :deprecated => true
-  config :region, :validate => [US_EAST_1, "us-west-1", "us-west-2",
-                                "eu-west-1", "ap-southeast-1", "ap-southeast-2",
-                                "ap-northeast-1", "sa-east-1", "us-gov-west-1"], :default => US_EAST_1
 
-  # The AWS Access Key ID
+  # The `secret_key` option is deprecated, please update your configuration to use `secret_access_key` instead
-  config :access_key, :validate => :string, :required => true
+  config :secret_key, :validate => :string, :deprecated => true
-
-  # The AWS Secret Access Key
-  config :secret_key, :validate => :string, :required => true
 
   # How often to send data to CloudWatch   
   # This does not affect the event timestamps, events will always have their
@@ -97,11 +94,11 @@ class LogStash::Outputs::CloudWatch < LogStash::Outputs::Base
   # See the Rufus Scheduler docs for an [explanation of allowed values](https://github.com/jmettraux/rufus-scheduler#the-time-strings-understood-by-rufus-scheduler)
   config :timeframe, :validate => :string, :default => "1m"
 
-  # How many events to queue before forcing a call to the CloudWatch API ahead of "timeframe" schedule   
+  # How many events to queue before forcing a call to the CloudWatch API ahead of `timeframe` schedule   
   # Set this to the number of events-per-timeframe you will be sending to CloudWatch to avoid extra API calls
   config :queue_size, :validate => :number, :default => 10000
 
-  # The default namespace to use for events which do not have a "CW_namespace" field
+  # The default namespace to use for events which do not have a `CW_namespace` field
   config :namespace, :validate => :string, :default => "Logstash"
 
   # The name of the field used to set a different namespace per event   
@@ -110,7 +107,7 @@ class LogStash::Outputs::CloudWatch < LogStash::Outputs::Base
   # and those cost money.
   config :field_namespace, :validate => :string, :default => "CW_namespace"
 
-  # The default metric name to use for events which do not have a "CW_metricname" field.   
+  # The default metric name to use for events which do not have a `CW_metricname` field.   
   # Beware: If this is provided then all events which pass through this output will be aggregated and
   # sent to CloudWatch, so use this carefully.  Furthermore, when providing this option, you
   # will probably want to also restrict events from passing through this output using event
@@ -132,23 +129,23 @@ class LogStash::Outputs::CloudWatch < LogStash::Outputs::Base
                  "Bits/Second", "Kilobits/Second", "Megabits/Second",
                  "Gigabits/Second", "Terabits/Second", "Count/Second", NONE]
 
-  # The default unit to use for events which do not have a "CW_unit" field   
+  # The default unit to use for events which do not have a `CW_unit` field   
   # If you set this option you should probably set the "value" option along with it
   config :unit, :validate => VALID_UNITS, :default => COUNT_UNIT
 
   # The name of the field used to set the unit on an event metric   
   config :field_unit, :validate => :string, :default => "CW_unit"
 
-  # The default value to use for events which do not have a "CW_value" field   
+  # The default value to use for events which do not have a `CW_value` field   
   # If provided, this must be a string which can be converted to a float, for example...
   #     "1", "2.34", ".5", and "0.67"
-  # If you set this option you should probably set the "unit" option along with it
+  # If you set this option you should probably set the `unit` option along with it
   config :value, :validate => :string, :default => "1"
 
   # The name of the field used to set the value (float) on an event metric   
   config :field_value, :validate => :string, :default => "CW_value"
 
-  # The default dimensions [ name, value, ... ] to use for events which do not have a "CW_dimensions" field   
+  # The default dimensions [ name, value, ... ] to use for events which do not have a `CW_dimensions` field   
   config :dimensions, :validate => :hash
 
   # The name of the field used to set the dimensions on an event metric   
@@ -160,18 +157,26 @@ class LogStash::Outputs::CloudWatch < LogStash::Outputs::Base
   #     add_field => [ "CW_dimensions", "prod" ]
   config :field_dimensions, :validate => :string, :default => "CW_dimensions"
 
+  public
+  def aws_service_endpoint(region)
+    return {
+        :cloud_watch_endpoint => "monitoring.#{region}.amazonaws.com"
+    }
+  end
+  
   public
   def register
     require "thread"
     require "rufus/scheduler"
     require "aws"
 
-    AWS.config(
+    # This should be removed when the deprecated aws credential options are removed
-        :access_key_id => @access_key,
+    if (@access_key && @secret_key) 
-        :secret_access_key => @secret_key,
+      @access_key_id = @access_key
-        :cloud_watch_endpoint => "monitoring.#{@region}.amazonaws.com"
+      @secret_access_key = @secret_key
-    )
+    end
-    @cw = AWS::CloudWatch.new
+
+    @cw = AWS::CloudWatch.new(aws_options_hash)
 
     @event_queue = SizedQueue.new(@queue_size)
     @scheduler = Rufus::Scheduler.start_new

lib/logstash/outputs/sns.rb

@@ -1,5 +1,6 @@
 require "logstash/outputs/base"
 require "logstash/namespace"
+require "logstash/plugin_mixins/aws_config"
 
 # SNS output.
 #
@@ -23,25 +24,16 @@ require "logstash/namespace"
 #  MAX_MESSAGE_SIZE_IN_BYTES.
 #
 class LogStash::Outputs::Sns < LogStash::Outputs::Base
+  include LogStash::PluginMixins::AwsConfig
+
   MAX_SUBJECT_SIZE_IN_CHARACTERS  = 100
   MAX_MESSAGE_SIZE_IN_BYTES       = 32768
 
   config_name "sns"
   plugin_status "experimental"
 
-  # Amazon API credentials.
+  # The `credentials` option is deprecated, please update your config to use `aws_credentials_file` instead
-  config :access_key_id, :validate => :string
+  config :credentials, :validate => :string, :deprecated => true
-  config :secret_access_key, :validate => :string
-
-  # Path to YAML file containing a hash of AWS credentials.  This file
-  # will be loaded if `access_key_id` and `secret_access_key` aren't
-  # set. The contents of the file should look like this:
-  #
-  #     ---
-  #     :access_key_id: "12345"
-  #     :secret_access_key: "54321"
-  #
-  config :credentials, :validate => :string
 
   # Message format.  Defaults to plain text.
   config :format, :validate => [ "json", "plain" ], :default => "plain"
@@ -57,22 +49,23 @@ class LogStash::Outputs::Sns < LogStash::Outputs::Base
   #
   config :publish_boot_message_arn, :validate => :string
 
+  public
+  def aws_service_endpoint(region)
+    return {
+        :sns_endpoint => "sns.#{region}.amazonaws.com"
+    }
+  end
+
   public
   def register
     require "aws-sdk"
 
-    # Credentials weren't specified in the configuration.
+    # This should be removed when the deprecated aws credentials option is removed
-    unless @access_key_id && @secret_access_key
+    if (@credentials)
-      access_creds = YAML.load_file(@credentials)
+      @aws_credentials_file = @credentials
-
-      @access_key_id      = access_creds[:access_key_id]
-      @secret_access_key  = access_creds[:secret_access_key]
     end
 
-    @sns = AWS::SNS.new(
+    @sns = AWS::SNS.new(aws_options_hash)
-      :access_key_id      => @access_key_id,
-      :secret_access_key  => @secret_access_key
-    )
 
     # Try to publish a "Logstash booted" message to the ARN provided to
     # cause an error ASAP if the credentials are bad.

lib/logstash/outputs/sqs.rb

@@ -1,5 +1,6 @@
 require "logstash/outputs/base"
 require "logstash/namespace"
+require "logstash/plugin_mixins/aws_config"
 
 # Push events to an Amazon Web Services Simple Queue Service (SQS) queue.
 #
@@ -55,25 +56,39 @@ require "logstash/namespace"
 # See http://aws.amazon.com/iam/ for more details on setting up AWS identities.
 #
 class LogStash::Outputs::SQS < LogStash::Outputs::Base
+  include LogStash::PluginMixins::AwsConfig
+
   config_name "sqs"
   plugin_status "experimental"
 
+  # The `access_key` option is deprecated, please update your configuration to use `access_key_id` instead
+  config :access_key, :validate => :string, :deprecated => true
+
+  # The `secret_key` option is deprecated, please update your configuration to use `secret_access_key` instead
+  config :secret_key, :validate => :string, :deprecated => true
+
   # Name of SQS queue to push messages into. Note that this is just the name of the queue, not the URL or ARN.
   config :queue, :validate => :string, :required => true
 
-  # AWS access key. Must have the appropriate permissions.
+  public
-  config :access_key, :validate => :string, :required => true
+  def aws_service_endpoint(region)
-
+    return {
-  # AWS secret key. Must have the appropriate permissions.
+        :sqs_endpoint => "sqs.#{region}.amazonaws.com"
-  config :secret_key, :validate => :string, :required => true
+    }
+  end
 
   public 
   def register
     require "aws-sdk"
-    @sqs = AWS::SQS.new(
+
-      :access_key_id => @access_key,
+    # This should be removed when the deprecated aws credential options are removed
-      :secret_access_key => @secret_key
+    if (@access_key && @secret_key)
-    )
+      @access_key_id = @access_key
+      @secret_access_key = @secret_key
+    end
+
+    @sqs = AWS::SQS.new(aws_options_hash)
+
     begin
       @logger.debug("Connecting to AWS SQS queue '#{@queue}'...")
       @sqs_queue = @sqs.queues.named(@queue)

lib/logstash/plugin_mixins/aws_config.rb

@@ -0,0 +1,86 @@
+require "logstash/config/mixin"
+
+module LogStash::PluginMixins::AwsConfig
+
+  @logger = LogStash::Logger.new(STDOUT)
+  @logger.level = $DEBUG ? :debug : :warn
+
+  # This method is called when someone includes this module
+  def self.included(base)
+    # Add these methods to the 'base' given.
+    base.extend(self)
+    base.setup_aws_config
+  end
+
+  US_EAST_1 = "us-east-1"
+  
+  public
+  def setup_aws_config
+    # The AWS Region
+    config :region, :validate => [US_EAST_1, "us-west-1", "us-west-2",
+                                  "eu-west-1", "ap-southeast-1", "ap-southeast-2",
+                                  "ap-northeast-1", "sa-east-1", "us-gov-west-1"], :default => US_EAST_1
+
+    # This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order...   
+    # 1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config   
+    # 2. External credentials file specified by `aws_credentials_file`   
+    # 3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`   
+    # 4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY`   
+    # 5. IAM Instance Profile (available when running inside EC2)   
+    config :access_key_id, :validate => :string
+
+    # The AWS Secret Access Key
+    config :secret_access_key, :validate => :string
+
+    # Should we require (true) or disable (false) using SSL for communicating with the AWS API   
+    # The AWS SDK for Ruby defaults to SSL so we preserve that
+    config :use_ssl, :validate => :boolean, :default => true
+
+    # Path to YAML file containing a hash of AWS credentials.   
+    # This file will only be loaded if `access_key_id` and
+    # `secret_access_key` aren't set. The contents of the
+    # file should look like this:
+    #
+    #     :access_key_id: "12345"
+    #     :secret_access_key: "54321"
+    #
+    config :aws_credentials_file, :validate => :string
+  end
+
+  public
+  def aws_options_hash
+    if @access_key_id.is_a?(NilClass) ^ @secret_access_key.is_a?(NilClass)
+      @logger.warn("Likely config error: Only one of access_key_id or secret_access_key was provided but not both.")
+    end
+
+    if ((!@access_key_id || !@secret_access_key)) && @aws_credentials_file
+      access_creds = YAML.load_file(@aws_credentials_file)
+
+      @access_key_id = access_creds[:access_key_id]
+      @secret_access_key = access_creds[:secret_access_key]
+    end
+
+    opts = {}
+
+    if (@access_key_id && @secret_access_key)
+      opts[:access_key_id] = @access_key_id
+      opts[:secret_access_key] = @secret_access_key
+    end
+
+    opts[:use_ssl] = @use_ssl
+
+    # The AWS SDK for Ruby doesn't know how to make an endpoint hostname from a region
+    # for example us-west-1 -> foosvc.us-west-1.amazonaws.com
+    # So our plugins need to know how to generate their endpoints from a region
+    # Furthermore, they need to know the symbol required to set that value in the AWS SDK
+    # Classes using this module must implement aws_service_endpoint(region:string)
+    # which must return a hash with one key, the aws sdk for ruby config symbol of the service
+    # endpoint, which has a string value of the service endpoint hostname
+    # for example, CloudWatch, { :cloud_watch_endpoint => "monitoring.#{region}.amazonaws.com" }
+    # For a list, see https://github.com/aws/aws-sdk-ruby/blob/master/lib/aws/core/configuration.rb
+    opts.merge!(self.aws_service_endpoint(@region))
+    
+    return opts
+  end # def aws_options_hash
+
+end