diff --git a/docs/docgen.rb b/docs/docgen.rb index 26b057e49..accb49606 100644 --- a/docs/docgen.rb +++ b/docs/docgen.rb @@ -162,6 +162,13 @@ class LogStashConfigDocGenerator parse(File.new(File.join(File.dirname(file), "threadable.rb"), "r").read) end + if code =~ /include LogStash::PluginMixins/ + mixin = code.gsub(/.*include LogStash::PluginMixins::(\w+)\s.*/m, '\1') + mixin.gsub!(/(.)([A-Z])/, '\1_\2') + mixin.downcase! + parse(File.new(File.join(File.dirname(file), "..", "plugin_mixins", "#{mixin}.rb")).read) + end + parse(code) puts "Generating docs for #{file}" diff --git a/lib/logstash/inputs/sqs.rb b/lib/logstash/inputs/sqs.rb index 32a653565..06edeb770 100644 --- a/lib/logstash/inputs/sqs.rb +++ b/lib/logstash/inputs/sqs.rb @@ -1,5 +1,6 @@ require "logstash/inputs/threadable" require "logstash/namespace" +require "logstash/plugin_mixins/aws_config" # Pull events from an Amazon Web Services Simple Queue Service (SQS) queue. # @@ -54,17 +55,26 @@ require "logstash/namespace" # See http://aws.amazon.com/iam/ for more details on setting up AWS identities. # class LogStash::Inputs::SQS < LogStash::Inputs::Threadable + include LogStash::PluginMixins::AwsConfig + config_name "sqs" plugin_status "experimental" + # The `access_key` option is deprecated, please update your configuration to use `access_key_id` instead + config :access_key, :validate => :string, :deprecated => true + + # The `secret_key` option is deprecated, please update your configuration to use `secret_access_key` instead + config :secret_key, :validate => :string, :deprecated => true + # Name of the SQS Queue name to pull messages from. Note that this is just the name of the queue, not the URL or ARN. config :queue, :validate => :string, :required => true - # AWS access key. Must have the appropriate permissions. - config :access_key, :validate => :string, :required => true - - # AWS secret key. Must have the appropriate permissions. - config :secret_key, :validate => :string, :required => true + public + def aws_service_endpoint(region) + return { + :sqs_endpoint => "sqs.#{region}.amazonaws.com" + } + end def initialize(params) super @@ -76,11 +86,13 @@ class LogStash::Inputs::SQS < LogStash::Inputs::Threadable @logger.info("Registering SQS input", :queue => @queue) require "aws-sdk" - # Connec to SQS - @sqs = AWS::SQS.new( - :access_key_id => @access_key, - :secret_access_key => @secret_key - ) + # This should be removed when the deprecated aws credential options are removed + if (@access_key && @secret_key) + @access_key_id = @access_key + @secret_access_key = @secret_key + end + + @sqs = AWS::SQS.new(aws_options_hash) begin @logger.debug("Connecting to AWS SQS queue", :queue => @queue) diff --git a/lib/logstash/namespace.rb b/lib/logstash/namespace.rb index 0d3b107b6..85d7f6af0 100644 --- a/lib/logstash/namespace.rb +++ b/lib/logstash/namespace.rb @@ -9,6 +9,7 @@ module LogStash module File; end module Web; end module Util; end + module PluginMixins; end SHUTDOWN = :shutdown end # module LogStash diff --git a/lib/logstash/outputs/cloudwatch.rb b/lib/logstash/outputs/cloudwatch.rb index be53c8ce3..5a6c34df1 100644 --- a/lib/logstash/outputs/cloudwatch.rb +++ b/lib/logstash/outputs/cloudwatch.rb @@ -1,5 +1,6 @@ require "logstash/outputs/base" require "logstash/namespace" +require "logstash/plugin_mixins/aws_config" # This output lets you aggregate and send metric data to AWS CloudWatch # @@ -9,10 +10,10 @@ require "logstash/namespace" # output plugin is configured, on the logstash indexer node, with just AWS API # credentials, and possibly a region and/or a namespace. The output looks # for fields present in events, and when it finds them, it uses them to -# calculate aggregate statistics. If the "metricname" option is set in this +# calculate aggregate statistics. If the `metricname` option is set in this # output, then any events which pass through it will be aggregated & sent to # CloudWatch, but that is not recommended. The intended use is to NOT set the -# metricname option here, and instead to add a "CW_metricname" field (and other +# metricname option here, and instead to add a `CW_metricname` field (and other # fields) to only the events you want sent to CloudWatch. # # When events pass through this output they are queued for background @@ -20,7 +21,7 @@ require "logstash/namespace" # queue has a maximum size, and when it is full aggregated statistics will be # sent to CloudWatch ahead of schedule. Whenever this happens a warning # message is written to logstash's log. If you see this you should increase -# the queue_size configuration option to avoid the extra API calls. The queue +# the `queue_size` configuration option to avoid the extra API calls. The queue # is emptied every time we send data to CloudWatch. # # Note: when logstash is stopped the queue is destroyed before it can be processed. @@ -34,7 +35,7 @@ require "logstash/namespace" # Event Field configuration... # You add fields to your events in inputs & filters and this output reads # those fields to aggregate events. The names of the fields read are -# configurable via the field_* options. +# configurable via the `field_*` options. # # Per-output defaults... # You set universal defaults in this output plugin's configuration, and @@ -45,13 +46,13 @@ require "logstash/namespace" # # At a minimum events must have a "metric name" to be sent to CloudWatch. # This can be achieved either by providing a default here OR by adding a -# "CW_metricname" field. By default, if no other configuration is provided +# `CW_metricname` field. By default, if no other configuration is provided # besides a metric name, then events will be counted (Unit: Count, Value: 1) -# by their metric name (either a default or from their CW_metricname field) +# by their metric name (either a default or from their `CW_metricname` field) # # Other fields which can be added to events to modify the behavior of this -# plugin are, "CW_namespace", "CW_unit", "CW_value", and -# "CW_dimensions". All of these field names are configurable in +# plugin are, `CW_namespace`, `CW_unit`, `CW_value`, and +# `CW_dimensions`. All of these field names are configurable in # this output. You can also set per-output defaults for any of them. # See below for details. # @@ -59,6 +60,8 @@ require "logstash/namespace" # and the specific of API endpoint this output uses, # [PutMetricData](http://docs.amazonwebservices.com/AmazonCloudWatch/latest/APIReference/API_PutMetricData.html) class LogStash::Outputs::CloudWatch < LogStash::Outputs::Base + include LogStash::PluginMixins::AwsConfig + config_name "cloudwatch" plugin_status "experimental" @@ -76,17 +79,11 @@ class LogStash::Outputs::CloudWatch < LogStash::Outputs::Base COUNT_UNIT = "Count" NONE = "None" - US_EAST_1 = "us-east-1" - # The AWS Region to send logs to. - config :region, :validate => [US_EAST_1, "us-west-1", "us-west-2", - "eu-west-1", "ap-southeast-1", "ap-southeast-2", - "ap-northeast-1", "sa-east-1", "us-gov-west-1"], :default => US_EAST_1 + # The `access_key` option is deprecated, please update your configuration to use `access_key_id` instead + config :access_key, :validate => :string, :deprecated => true - # The AWS Access Key ID - config :access_key, :validate => :string, :required => true - - # The AWS Secret Access Key - config :secret_key, :validate => :string, :required => true + # The `secret_key` option is deprecated, please update your configuration to use `secret_access_key` instead + config :secret_key, :validate => :string, :deprecated => true # How often to send data to CloudWatch # This does not affect the event timestamps, events will always have their @@ -97,11 +94,11 @@ class LogStash::Outputs::CloudWatch < LogStash::Outputs::Base # See the Rufus Scheduler docs for an [explanation of allowed values](https://github.com/jmettraux/rufus-scheduler#the-time-strings-understood-by-rufus-scheduler) config :timeframe, :validate => :string, :default => "1m" - # How many events to queue before forcing a call to the CloudWatch API ahead of "timeframe" schedule + # How many events to queue before forcing a call to the CloudWatch API ahead of `timeframe` schedule # Set this to the number of events-per-timeframe you will be sending to CloudWatch to avoid extra API calls config :queue_size, :validate => :number, :default => 10000 - # The default namespace to use for events which do not have a "CW_namespace" field + # The default namespace to use for events which do not have a `CW_namespace` field config :namespace, :validate => :string, :default => "Logstash" # The name of the field used to set a different namespace per event @@ -110,7 +107,7 @@ class LogStash::Outputs::CloudWatch < LogStash::Outputs::Base # and those cost money. config :field_namespace, :validate => :string, :default => "CW_namespace" - # The default metric name to use for events which do not have a "CW_metricname" field. + # The default metric name to use for events which do not have a `CW_metricname` field. # Beware: If this is provided then all events which pass through this output will be aggregated and # sent to CloudWatch, so use this carefully. Furthermore, when providing this option, you # will probably want to also restrict events from passing through this output using event @@ -132,23 +129,23 @@ class LogStash::Outputs::CloudWatch < LogStash::Outputs::Base "Bits/Second", "Kilobits/Second", "Megabits/Second", "Gigabits/Second", "Terabits/Second", "Count/Second", NONE] - # The default unit to use for events which do not have a "CW_unit" field + # The default unit to use for events which do not have a `CW_unit` field # If you set this option you should probably set the "value" option along with it config :unit, :validate => VALID_UNITS, :default => COUNT_UNIT # The name of the field used to set the unit on an event metric config :field_unit, :validate => :string, :default => "CW_unit" - # The default value to use for events which do not have a "CW_value" field + # The default value to use for events which do not have a `CW_value` field # If provided, this must be a string which can be converted to a float, for example... # "1", "2.34", ".5", and "0.67" - # If you set this option you should probably set the "unit" option along with it + # If you set this option you should probably set the `unit` option along with it config :value, :validate => :string, :default => "1" # The name of the field used to set the value (float) on an event metric config :field_value, :validate => :string, :default => "CW_value" - # The default dimensions [ name, value, ... ] to use for events which do not have a "CW_dimensions" field + # The default dimensions [ name, value, ... ] to use for events which do not have a `CW_dimensions` field config :dimensions, :validate => :hash # The name of the field used to set the dimensions on an event metric @@ -160,18 +157,26 @@ class LogStash::Outputs::CloudWatch < LogStash::Outputs::Base # add_field => [ "CW_dimensions", "prod" ] config :field_dimensions, :validate => :string, :default => "CW_dimensions" + public + def aws_service_endpoint(region) + return { + :cloud_watch_endpoint => "monitoring.#{region}.amazonaws.com" + } + end + public def register require "thread" require "rufus/scheduler" require "aws" - AWS.config( - :access_key_id => @access_key, - :secret_access_key => @secret_key, - :cloud_watch_endpoint => "monitoring.#{@region}.amazonaws.com" - ) - @cw = AWS::CloudWatch.new + # This should be removed when the deprecated aws credential options are removed + if (@access_key && @secret_key) + @access_key_id = @access_key + @secret_access_key = @secret_key + end + + @cw = AWS::CloudWatch.new(aws_options_hash) @event_queue = SizedQueue.new(@queue_size) @scheduler = Rufus::Scheduler.start_new diff --git a/lib/logstash/outputs/sns.rb b/lib/logstash/outputs/sns.rb index 812eb4a48..08c5e8c31 100644 --- a/lib/logstash/outputs/sns.rb +++ b/lib/logstash/outputs/sns.rb @@ -1,5 +1,6 @@ require "logstash/outputs/base" require "logstash/namespace" +require "logstash/plugin_mixins/aws_config" # SNS output. # @@ -23,25 +24,16 @@ require "logstash/namespace" # MAX_MESSAGE_SIZE_IN_BYTES. # class LogStash::Outputs::Sns < LogStash::Outputs::Base + include LogStash::PluginMixins::AwsConfig + MAX_SUBJECT_SIZE_IN_CHARACTERS = 100 MAX_MESSAGE_SIZE_IN_BYTES = 32768 config_name "sns" plugin_status "experimental" - # Amazon API credentials. - config :access_key_id, :validate => :string - config :secret_access_key, :validate => :string - - # Path to YAML file containing a hash of AWS credentials. This file - # will be loaded if `access_key_id` and `secret_access_key` aren't - # set. The contents of the file should look like this: - # - # --- - # :access_key_id: "12345" - # :secret_access_key: "54321" - # - config :credentials, :validate => :string + # The `credentials` option is deprecated, please update your config to use `aws_credentials_file` instead + config :credentials, :validate => :string, :deprecated => true # Message format. Defaults to plain text. config :format, :validate => [ "json", "plain" ], :default => "plain" @@ -57,22 +49,23 @@ class LogStash::Outputs::Sns < LogStash::Outputs::Base # config :publish_boot_message_arn, :validate => :string + public + def aws_service_endpoint(region) + return { + :sns_endpoint => "sns.#{region}.amazonaws.com" + } + end + public def register require "aws-sdk" - # Credentials weren't specified in the configuration. - unless @access_key_id && @secret_access_key - access_creds = YAML.load_file(@credentials) - - @access_key_id = access_creds[:access_key_id] - @secret_access_key = access_creds[:secret_access_key] + # This should be removed when the deprecated aws credentials option is removed + if (@credentials) + @aws_credentials_file = @credentials end - @sns = AWS::SNS.new( - :access_key_id => @access_key_id, - :secret_access_key => @secret_access_key - ) + @sns = AWS::SNS.new(aws_options_hash) # Try to publish a "Logstash booted" message to the ARN provided to # cause an error ASAP if the credentials are bad. diff --git a/lib/logstash/outputs/sqs.rb b/lib/logstash/outputs/sqs.rb index 0b6202c4e..14b3cf5c8 100644 --- a/lib/logstash/outputs/sqs.rb +++ b/lib/logstash/outputs/sqs.rb @@ -1,5 +1,6 @@ require "logstash/outputs/base" require "logstash/namespace" +require "logstash/plugin_mixins/aws_config" # Push events to an Amazon Web Services Simple Queue Service (SQS) queue. # @@ -55,25 +56,39 @@ require "logstash/namespace" # See http://aws.amazon.com/iam/ for more details on setting up AWS identities. # class LogStash::Outputs::SQS < LogStash::Outputs::Base + include LogStash::PluginMixins::AwsConfig + config_name "sqs" plugin_status "experimental" + # The `access_key` option is deprecated, please update your configuration to use `access_key_id` instead + config :access_key, :validate => :string, :deprecated => true + + # The `secret_key` option is deprecated, please update your configuration to use `secret_access_key` instead + config :secret_key, :validate => :string, :deprecated => true + # Name of SQS queue to push messages into. Note that this is just the name of the queue, not the URL or ARN. config :queue, :validate => :string, :required => true - # AWS access key. Must have the appropriate permissions. - config :access_key, :validate => :string, :required => true - - # AWS secret key. Must have the appropriate permissions. - config :secret_key, :validate => :string, :required => true + public + def aws_service_endpoint(region) + return { + :sqs_endpoint => "sqs.#{region}.amazonaws.com" + } + end public def register require "aws-sdk" - @sqs = AWS::SQS.new( - :access_key_id => @access_key, - :secret_access_key => @secret_key - ) + + # This should be removed when the deprecated aws credential options are removed + if (@access_key && @secret_key) + @access_key_id = @access_key + @secret_access_key = @secret_key + end + + @sqs = AWS::SQS.new(aws_options_hash) + begin @logger.debug("Connecting to AWS SQS queue '#{@queue}'...") @sqs_queue = @sqs.queues.named(@queue) diff --git a/lib/logstash/plugin_mixins/aws_config.rb b/lib/logstash/plugin_mixins/aws_config.rb new file mode 100644 index 000000000..bbbd69362 --- /dev/null +++ b/lib/logstash/plugin_mixins/aws_config.rb @@ -0,0 +1,86 @@ +require "logstash/config/mixin" + +module LogStash::PluginMixins::AwsConfig + + @logger = LogStash::Logger.new(STDOUT) + @logger.level = $DEBUG ? :debug : :warn + + # This method is called when someone includes this module + def self.included(base) + # Add these methods to the 'base' given. + base.extend(self) + base.setup_aws_config + end + + US_EAST_1 = "us-east-1" + + public + def setup_aws_config + # The AWS Region + config :region, :validate => [US_EAST_1, "us-west-1", "us-west-2", + "eu-west-1", "ap-southeast-1", "ap-southeast-2", + "ap-northeast-1", "sa-east-1", "us-gov-west-1"], :default => US_EAST_1 + + # This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order... + # 1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config + # 2. External credentials file specified by `aws_credentials_file` + # 3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` + # 4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY` + # 5. IAM Instance Profile (available when running inside EC2) + config :access_key_id, :validate => :string + + # The AWS Secret Access Key + config :secret_access_key, :validate => :string + + # Should we require (true) or disable (false) using SSL for communicating with the AWS API + # The AWS SDK for Ruby defaults to SSL so we preserve that + config :use_ssl, :validate => :boolean, :default => true + + # Path to YAML file containing a hash of AWS credentials. + # This file will only be loaded if `access_key_id` and + # `secret_access_key` aren't set. The contents of the + # file should look like this: + # + # :access_key_id: "12345" + # :secret_access_key: "54321" + # + config :aws_credentials_file, :validate => :string + end + + public + def aws_options_hash + if @access_key_id.is_a?(NilClass) ^ @secret_access_key.is_a?(NilClass) + @logger.warn("Likely config error: Only one of access_key_id or secret_access_key was provided but not both.") + end + + if ((!@access_key_id || !@secret_access_key)) && @aws_credentials_file + access_creds = YAML.load_file(@aws_credentials_file) + + @access_key_id = access_creds[:access_key_id] + @secret_access_key = access_creds[:secret_access_key] + end + + opts = {} + + if (@access_key_id && @secret_access_key) + opts[:access_key_id] = @access_key_id + opts[:secret_access_key] = @secret_access_key + end + + opts[:use_ssl] = @use_ssl + + # The AWS SDK for Ruby doesn't know how to make an endpoint hostname from a region + # for example us-west-1 -> foosvc.us-west-1.amazonaws.com + # So our plugins need to know how to generate their endpoints from a region + # Furthermore, they need to know the symbol required to set that value in the AWS SDK + # Classes using this module must implement aws_service_endpoint(region:string) + # which must return a hash with one key, the aws sdk for ruby config symbol of the service + # endpoint, which has a string value of the service endpoint hostname + # for example, CloudWatch, { :cloud_watch_endpoint => "monitoring.#{region}.amazonaws.com" } + # For a list, see https://github.com/aws/aws-sdk-ruby/blob/master/lib/aws/core/configuration.rb + opts.merge!(self.aws_service_endpoint(@region)) + + return opts + end # def aws_options_hash + +end \ No newline at end of file