Use multiple pipelines feature by default in RPM and Debian packages.

This change keeps the previous behavior but implements it using multiple
pipelines (pipelines.yml) instead of with `path.config`.

* The default pipeline name is still "main"
* Default config path is still /etc/logstash/conf.d/*.conf

Other small changes included:
* Specify fpm workdir. Otherwise rpmbuild is given an empty value for
  its temporary directory. It's unclear if this is a bug in fpm or in my
  environment, but I like the change anyway to put fpm's build activity
  in the build directory.
* DRY's up the common files going in /etc/logstash for both rpm and deb
  packages.

For #8893

Fixes #8894

docs/static/setting-up-logstash.asciidoc

@@ -95,8 +95,8 @@ locations for the system:
 
 | conf
   | Logstash pipeline configuration files
-  | `/etc/logstash/conf.d`
-  | `path.config`
+  | `/etc/logstash/conf.d/*.conf`
+  | See `/etc/logstash/pipelines.yml`
 
 | logs
   | Log files

pkg/centos/after-install.sh

@@ -2,7 +2,6 @@ chown -R logstash:logstash /usr/share/logstash
 chown -R logstash /var/log/logstash
 chown logstash:logstash /var/lib/logstash
 sed -i \
-  -e 's|# path.config:|path.config: /etc/logstash/conf.d/*.conf|' \
   -e 's|# path.logs:|path.logs: /var/log/logstash|' \
   -e 's|# path.data:|path.data: /var/lib/logstash|' \
   /etc/logstash/logstash.yml

pkg/debian/after-install.sh

@@ -5,7 +5,6 @@ chown -R logstash /var/log/logstash
 chown logstash:logstash /var/lib/logstash
 chmod 755 /etc/logstash
 sed -i \
-  -e 's|# path.config:|path.config: /etc/logstash/conf.d/*.conf|' \
   -e 's|# path.logs:|path.logs: /var/log/logstash|' \
   -e 's|# path.data:|path.data: /var/lib/logstash|' \
   /etc/logstash/logstash.yml

pkg/pipelines.yml

@@ -0,0 +1,6 @@
+# This file is where you define your pipelines. You can define multiple.
+# For more information on multiple pipelines, see the documentation:
+#   https://www.elastic.co/guide/en/logstash/current/multiple-pipelines.html
+
+- pipeline.id: main
+  path.config: "/etc/logstash/conf.d/*.conf"

pkg/ubuntu/after-install.sh

@@ -4,7 +4,6 @@ chown -R logstash:logstash /usr/share/logstash
 chown -R logstash /var/log/logstash
 chown logstash:logstash /var/lib/logstash
 sed -i \
-  -e 's|# path.config:|path.config: /etc/logstash/conf.d/*.conf|' \
   -e 's|# path.logs:|path.logs: /var/log/logstash|' \
   -e 's|# path.data:|path.data: /var/lib/logstash|' \
   /etc/logstash/logstash.yml

rakelib/artifacts.rake

@@ -280,7 +280,9 @@ namespace "artifact" do
     # This will make a the thread dies, in 1.7.25 we had a Thread Death
     require_relative "childprocess_patch"
 
+    basedir = File.join(File.dirname(__FILE__), "..")
     dir = FPM::Package::Dir.new
+    dir.attributes[:workdir] = File.join(basedir, "build", "fpm")
 
     metadata_file_path = File.join("logstash-core", "lib", "logstash", "build.rb")
     metadata_source_file_path = BUILD_METADATA_FILE.path
@@ -294,8 +296,6 @@ namespace "artifact" do
       dir.input("#{path}=/usr/share/logstash/#{path}")
     end
 
-    basedir = File.join(File.dirname(__FILE__), "..")
-
     # Create an empty /var/log/logstash/ directory in the package
     # This is a bit obtuse, I suppose, but it is necessary until
     # we find a better way to do this with fpm.
@@ -313,17 +313,21 @@ namespace "artifact" do
     ensure_logstash_version_constant_defined
     package_filename = "logstash-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}.TYPE"
 
+    File.join(basedir, "pkg", "startup.options").tap do |path|
+      dir.input("#{path}=/etc/logstash")
+    end
+    File.join(basedir, "pkg", "jvm.options").tap do |path|
+      dir.input("#{path}=/etc/logstash")
+    end
+    File.join(basedir, "config", "logstash.yml").tap do |path|
+      dir.input("#{path}=/etc/logstash")
+    end
+    File.join(basedir, "pkg", "pipelines.yml").tap do |path|
+      dir.input("#{path}=/etc/logstash")
+    end
+
     case platform
       when "redhat", "centos"
-        File.join(basedir, "pkg", "startup.options").tap do |path|
-          dir.input("#{path}=/etc/logstash")
-        end
-        File.join(basedir, "pkg", "jvm.options").tap do |path|
-          dir.input("#{path}=/etc/logstash")
-        end
-        File.join(basedir, "config", "logstash.yml").tap do |path|
-          dir.input("#{path}=/etc/logstash")
-        end
         require "fpm/package/rpm"
         out = dir.convert(FPM::Package::RPM)
         out.license = "ASL 2.0" # Red Hat calls 'Apache Software License' == ASL
@@ -336,15 +340,6 @@ namespace "artifact" do
         out.config_files << "/etc/logstash/log4j2.properties"
         out.config_files << "/etc/logstash/logstash.yml"
       when "debian", "ubuntu"
-        File.join(basedir, "pkg", "startup.options").tap do |path|
-          dir.input("#{path}=/etc/logstash")
-        end
-        File.join(basedir, "pkg", "jvm.options").tap do |path|
-          dir.input("#{path}=/etc/logstash")
-        end
-        File.join(basedir, "config", "logstash.yml").tap do |path|
-          dir.input("#{path}=/etc/logstash")
-        end
         require "fpm/package/deb"
         out = dir.convert(FPM::Package::Deb)
         out.license = "Apache 2.0"