Merge CLI into YML settings + limit user specified modules to 1 for 5.x (#7756)

logstash-core/lib/logstash/config/modules_common.rb

@@ -4,17 +4,30 @@ require "logstash/elasticsearch_client"
 require "logstash/modules/kibana_client"
 require "logstash/modules/elasticsearch_importer"
 require "logstash/modules/kibana_importer"
+require "logstash/modules/settings_merger"
 require "logstash/errors"
 
 module LogStash module Config
   class ModulesCommon # extracted here for bwc with 5.x
     include LogStash::Util::Loggable
 
+    MODULES_MAX_PIPELINES = 1
+
     def self.pipeline_configs(settings)
       pipelines = []
       plugin_modules = LogStash::PLUGIN_REGISTRY.plugins_with_type(:modules)
 
-      modules_array = settings.get("modules.cli").empty? ? settings.get("modules") : settings.get("modules.cli")
+      cli_settings = settings.get("modules.cli")
+      yml_settings = settings.get("modules")
+
+      modules_array = if !(cli_settings.empty? && yml_settings.empty?)
+            LogStash::Modules::SettingsMerger.merge(cli_settings, yml_settings)
+          elsif cli_settings.empty?
+             yml_settings
+          else
+            cli_settings
+          end
+
       if modules_array.empty?
         # no specifed modules
         return pipelines
@@ -22,6 +35,11 @@ module LogStash module Config
       logger.debug("Specified modules", :modules_array => modules_array.to_s)
 
       module_names = modules_array.collect {|module_hash| module_hash["name"]}
+      if module_names.size > MODULES_MAX_PIPELINES
+        error_message = I18n.t("logstash.modules.configuration.modules-too-many-specified", :max => MODULES_MAX_PIPELINES, :specified_modules => module_names.join(', '))
+        raise LogStash::ConfigLoadingError, error_message
+      end
+
       if module_names.length > module_names.uniq.length
         duplicate_modules = module_names.group_by(&:to_s).select { |_,v| v.size > 1 }.keys
         raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.modules-must-be-unique", :duplicate_modules => duplicate_modules)

logstash-core/lib/logstash/config/source/modules.rb

@@ -29,7 +29,7 @@ module LogStash module Config module Source
     def config_conflict?
       @conflict_messages.clear
       # Make note that if modules are configured in both cli and logstash.yml that cli module
-      # settings will be used, and logstash.yml modules settings ignored
+      # settings will overwrite the logstash.yml modules settings
       if modules_cli? && modules?
         logger.info(I18n.t("logstash.runner.cli-module-override"))
       end

logstash-core/lib/logstash/modules/settings_merger.rb

@@ -0,0 +1,23 @@
+# encoding: utf-8
+require "logstash/namespace"
+
+module LogStash module Modules class SettingsMerger
+  def self.merge(cli_settings, yml_settings)
+    # both args are arrays of hashes, e.g.
+    # [{"name"=>"mod1", "var.input.tcp.port"=>"3333"}, {"name"=>"mod2"}]
+    # [{"name"=>"mod1", "var.input.tcp.port"=>2222, "var.kibana.username"=>"rupert", "var.kibana.password"=>"fotherington"}, {"name"=>"mod3", "var.input.tcp.port"=>4445}]
+    merged = []
+    # union and group_by preserves order
+    # union will also coalesce identical hashes
+    union_of_settings = (cli_settings | yml_settings)
+    grouped_by_name = union_of_settings.group_by{|e| e["name"]}
+    grouped_by_name.each do |name, array|
+      if array.size == 2
+        merged << array.first.merge(array.last)
+      else
+        merged.concat(array)
+      end
+    end
+    merged
+  end
+end end end

logstash-core/locales/en.yml

@@ -104,6 +104,8 @@ en:
         elasticsearch_connection_failed: >-
           Failed to import module configurations to Elasticsearch and/or Kibana.
           Module: %{module_name} has Elasticsearch hosts: %{elasticsearch_hosts} and Kibana hosts: %{kibana_hosts}
+        modules-too-many-specified: >-
+          Too many modules specified. Maximum allowed: %{max}, specified: %{specified_modules}
 
     runner:
       short-help: |-
@@ -128,8 +130,7 @@ en:
         Configuration reloading can't be used with command-line or logstash.yml specified modules.
       cli-module-override: >-
         Both command-line and logstash.yml modules configurations detected.
-        Using command-line module configuration and ignoring logstash.yml module
-        configuration.
+        Using command-line module configuration to override logstash.yml module configuration.
       config-pipelines-failed-read: >-
         Failed to read pipelines yaml file. Location: %{path}
       config-pipelines-empty: >-