add experimental flag for environment variables in config

Fixes #4963

docs/static/command-line-flags.asciidoc

@@ -64,6 +64,9 @@ Logstash has the following flags. You can use the `--help` flag to display this
   
 -r, --[no-]auto-reload
   Monitor configuration changes and reload the configuration whenever it is changed.
+
+--allow-env
+  EXPERIMENTAL: Enable environment variable templating within configuration parameters.
   
 --reload-interval RELOAD_INTERVAL
   Specifies how often Logstash checks the config files for changes. The default is every 3 seconds.

logstash-core/lib/logstash/config/mixin.rb

@@ -38,7 +38,8 @@ module LogStash::Config::Mixin
   PLUGIN_VERSION_1_0_0 = LogStash::Util::PluginVersion.new(1, 0, 0)
   PLUGIN_VERSION_0_9_0 = LogStash::Util::PluginVersion.new(0, 9, 0)
 
-  ENV_PLACEHOLDER_REGEX = /\$(?<name>\w+)|\$\{(?<name>\w+)(\:(?<default>[^}]*))?\}/
+  ALLOW_ENV_FLAG = "__ALLOW_ENV__"
+  ENV_PLACEHOLDER_REGEX = /\$\{(?<name>\w+)(\:(?<default>[^}]*))?\}/
 
   # This method is called when someone does 'include LogStash::Config'
   def self.included(base)
@@ -47,6 +48,14 @@ module LogStash::Config::Mixin
   end
 
   def config_init(params)
+    # HACK(talevy): https://github.com/elastic/logstash/issues/4958
+    # Currently, the regular plugins params argument is hijacked
+    # to pass along the `allow_env` configuration variable. This was done as to 
+    # not change the method signature of Plugin. This also makes it difficul to 
+    # reason about at the same time. ALLOW_ENV_FLAG is a special param that users 
+    # are now forbidden to set in their configuration definitions.
+    allow_env = params.delete(LogStash::Config::Mixin::ALLOW_ENV_FLAG) { false }
+
     # Validation will modify the values inside params if necessary.
     # For example: converting a string to a number, etc.
     
@@ -102,22 +111,25 @@ module LogStash::Config::Mixin
     end
 
     # Resolve environment variables references
-    params.each do |name, value|
-      if (value.is_a?(Hash))
-        value.each do |valueHashKey, valueHashValue|
-          value[valueHashKey.to_s] = replace_env_placeholders(valueHashValue)
-        end
-      else
-        if (value.is_a?(Array))
-          value.each_index do |valueArrayIndex|
-            value[valueArrayIndex] = replace_env_placeholders(value[valueArrayIndex])
+    if allow_env
+        params.each do |name, value|
+        if (value.is_a?(Hash))
+          value.each do |valueHashKey, valueHashValue|
+            value[valueHashKey.to_s] = replace_env_placeholders(valueHashValue)
           end
         else
-          params[name.to_s] = replace_env_placeholders(value)
+          if (value.is_a?(Array))
+            value.each_index do |valueArrayIndex|
+              value[valueArrayIndex] = replace_env_placeholders(value[valueArrayIndex])
+            end
+          else
+            params[name.to_s] = replace_env_placeholders(value)
+          end
         end
       end
     end
 
+
     if !self.class.validate(params)
       raise LogStash::ConfigurationError,
         I18n.t("logstash.runner.configuration.invalid_plugin_settings")
@@ -154,7 +166,6 @@ module LogStash::Config::Mixin
   # Process following patterns : $VAR, ${VAR}, ${VAR:defaultValue}
   def replace_env_placeholders(value)
     return value unless value.is_a?(String)
-    #raise ArgumentError, "Cannot replace ENV placeholders on non-strings. Got #{value.class}" if !value.is_a?(String)
 
     value.gsub(ENV_PLACEHOLDER_REGEX) do |placeholder|
       # Note: Ruby docs claim[1] Regexp.last_match is thread-local and scoped to
@@ -168,7 +179,6 @@ module LogStash::Config::Mixin
       if replacement.nil?
         raise LogStash::ConfigurationError, "Cannot evaluate `#{placeholder}`. Environment variable `#{name}` is not set and there is no default value given."
       end
-      @logger.info? && @logger.info("Evaluating environment variable placeholder", :placeholder => placeholder, :replacement => replacement)
       replacement
     end
   end # def replace_env_placeholders

logstash-core/lib/logstash/pipeline.rb

@@ -45,7 +45,8 @@ module LogStash; class Pipeline
     :pipeline_batch_delay => 5, # in milliseconds
     :flush_interval => 5, # in seconds
     :flush_timeout_interval => 60, # in seconds
-    :debug_config => false
+    :debug_config => false,
+    :allow_env => false
   }
   MAX_INFLIGHT_WARN_THRESHOLD = 10_000
 
@@ -61,6 +62,7 @@ module LogStash; class Pipeline
     @settings = DEFAULT_SETTINGS.clone
     settings.each {|setting, value| configure(setting, value) }
     @reporter = LogStash::PipelineReporter.new(@logger, self)
+    @allow_env = settings[:allow_env]
 
     @inputs = nil
     @filters = nil
@@ -451,6 +453,7 @@ module LogStash; class Pipeline
 
   def plugin(plugin_type, name, *args)
     args << {} if args.empty?
+    args.first.merge!(LogStash::Config::Mixin::ALLOW_ENV_FLAG => @allow_env)
 
     pipeline_scoped_metric = metric.namespace([:stats, :pipelines, pipeline_id.to_s.to_sym, :plugins])
 

logstash-core/lib/logstash/runner.rb

@@ -98,6 +98,10 @@ class LogStash::Runner < Clamp::Command
     I18n.t("logstash.web_api.flag.http_port"),
     :attribute_name => :web_api_http_port, :default => 9600
 
+  option ["--allow-env"], :flag,
+    I18n.t("logstash.runner.flag.allow_env"),
+    :attribute_name => :allow_env, :default => false
+
   def pipeline_workers=(pipeline_workers_value)
     @pipeline_settings[:pipeline_workers] = validate_positive_integer(pipeline_workers_value)
   end
@@ -195,7 +199,8 @@ class LogStash::Runner < Clamp::Command
     @agent.register_pipeline("main", @pipeline_settings.merge({
                           :config_string => config_string,
                           :config_path => config_path,
-                          :debug_config => debug_config?
+                          :debug_config => debug_config?,
+                          :allow_env => allow_env?
                           }))
 
     # enable sigint/sigterm before starting the agent

logstash-core/locales/en.yml

@@ -182,6 +182,10 @@ en:
           the empty string for the '-e' flag.
         configtest: |+
           Check configuration for valid syntax and then exit.
+        allow-env: |+
+          EXPERIMENTAL. Enables templating of environment variable
+          values. Instances of "${VAR}" in strings will be replaced
+          with the respective environment variable value named "VAR".
         pipeline-workers: |+
           Sets the number of pipeline workers to run.
         pipeline-batch-size: |+

logstash-core/spec/logstash/agent_spec.rb

@@ -191,6 +191,45 @@ describe LogStash::Agent do
     end
   end
 
+  describe "Environment Variables In Configs" do
+    let(:agent_args) { {
+      :logger => logger,
+      :auto_reload => false,
+      :reload_interval => 0.01
+    } }
+    let(:pipeline_id) { "main" }
+    let(:pipeline_config) { "input { generator { message => '${FOO}-bar' } } filter { } output { }" }
+    let(:pipeline_settings) { {
+      :config_string => pipeline_config,
+    } }
+
+    context "when allow_env is false" do
+      it "does not interpolate environment variables" do
+        expect(subject).to receive(:fetch_config).and_return(pipeline_config)
+        subject.register_pipeline(pipeline_id, pipeline_settings)
+        expect(subject.pipelines[pipeline_id].inputs.first.message).to eq("${FOO}-bar")
+      end
+    end
+
+    context "when allow_env is true" do
+      before :each do
+        @foo_content = ENV["FOO"]
+        ENV["FOO"] = "foo"
+        pipeline_settings.merge!(:allow_env => true)
+      end
+
+      after :each do
+        ENV["FOO"] = @foo_content
+      end
+
+      it "doesn't upgrade the state" do
+        expect(subject).to receive(:fetch_config).and_return(pipeline_config)
+        subject.register_pipeline(pipeline_id, pipeline_settings)
+        expect(subject.pipelines[pipeline_id].inputs.first.message).to eq("foo-bar")
+      end
+    end
+  end
+
   describe "#upgrade_pipeline" do
     let(:pipeline_id) { "main" }
     let(:pipeline_config) { "input { } filter { } output { }" }

logstash-core/spec/logstash/config/mixin_spec.rb

@@ -165,6 +165,10 @@ describe LogStash::Config::Mixin do
         config :oneNumber, :validate => :number
         config :oneArray, :validate => :array
         config :oneHash, :validate => :hash
+
+        def initialize(params)
+          super(params.merge(LogStash::Config::Mixin::ALLOW_ENV_FLAG => true))
+        end
       end
     end
 
@@ -217,7 +221,7 @@ describe LogStash::Config::Mixin do
           "oneString" => "${FunString:foo}",
           "oneBoolean" => "${FunBool:false}",
           "oneArray" => [ "first array value", "${FunString:foo}" ],
-          "oneHash" => { "key1" => "${FunString:foo}", "key2" => "$FunString is ${FunBool}", "key3" => "${FunBool:false} or ${funbool:false}" }
+          "oneHash" => { "key1" => "${FunString:foo}", "key2" => "${FunString} is ${FunBool}", "key3" => "${FunBool:false} or ${funbool:false}" }
         )
       end
 

logstash-core/spec/logstash/runner_spec.rb

@@ -162,5 +162,19 @@ describe LogStash::Runner do
         subject.run("bin/logstash", args)
       end
     end
+
+    context "when configuring environment variable support" do
+      it "should set 'allow_env' to false by default" do
+        args = ["-e", pipeline_string]
+        expect(LogStash::Pipeline).to receive(:new).with(pipeline_string, hash_including(:allow_env => false)).and_return(pipeline)
+        subject.run("bin/logstash", args)
+      end
+
+      it "should support templating environment variables" do
+        args = ["-e", pipeline_string, "--allow-env"]
+        expect(LogStash::Pipeline).to receive(:new).with(pipeline_string, hash_including(:allow_env => true)).and_return(pipeline)
+        subject.run("bin/logstash", args)
+      end
+    end
   end
 end