diff --git a/Gemfile b/Gemfile index eec32acf6..d2be4f22d 100644 --- a/Gemfile +++ b/Gemfile @@ -2,9 +2,9 @@ # If you modify this file manually all comments and formatting will be lost. source "https://rubygems.org" -gem "logstash-core", "5.0.0.alpha4.snapshot3" -gem "logstash-core-event-java", "5.0.0.alpha4.snapshot3" -gem "logstash-core-plugin-api", "2.1.9" +gem "logstash-core", :path => "./logstash-core" +gem "logstash-core-event-java", :path => "./logstash-core-event-java" +gem "logstash-core-plugin-api", :path => "./logstash-core-plugin-api" gem "file-dependencies", "0.1.6" gem "ci_reporter_rspec", "1.0.0", :group => :development gem "simplecov", :group => :development diff --git a/Gemfile.jruby-1.9.lock b/Gemfile.jruby-1.9.lock index 6ca574dc7..124cfe663 100644 --- a/Gemfile.jruby-1.9.lock +++ b/Gemfile.jruby-1.9.lock @@ -1,3 +1,40 @@ +PATH + remote: ./logstash-core + specs: + logstash-core (5.0.0.pre.alpha4-java) + cabin (~> 0.8.0) + chronic_duration (= 0.10.6) + clamp (~> 0.6.5) + concurrent-ruby (= 1.0.0) + filesize (= 0.0.4) + gems (~> 0.8.3) + i18n (= 0.6.9) + jrjackson (~> 0.3.7) + jrmonitor (~> 0.4.2) + jruby-openssl (= 0.9.16) + logstash-core-event-java (= 5.0.0.pre.alpha4) + minitar (~> 0.5.4) + pry (~> 0.10.1) + puma (~> 2.16) + rubyzip (~> 1.1.7) + sinatra (~> 1.4, >= 1.4.6) + stud (~> 0.0.19) + thread_safe (~> 0.3.5) + treetop (< 1.5.0) + +PATH + remote: ./logstash-core-event-java + specs: + logstash-core-event-java (5.0.0.pre.alpha4-java) + jar-dependencies + ruby-maven (~> 3.3.9) + +PATH + remote: ./logstash-core-plugin-api + specs: + logstash-core-plugin-api (2.1.9-java) + logstash-core (= 5.0.0.pre.alpha4) + GEM remote: https://rubygems.org/ specs: @@ -47,12 +84,12 @@ GEM domain_name (0.5.20160615) unf (>= 0.0.5, < 1.0.0) edn (1.1.1) - elasticsearch (1.0.17) - elasticsearch-api (= 1.0.17) - elasticsearch-transport (= 1.0.17) - elasticsearch-api (1.0.17) + elasticsearch (1.0.18) + elasticsearch-api (= 1.0.18) + elasticsearch-transport (= 1.0.18) + elasticsearch-api (1.0.18) multi_json - elasticsearch-transport (1.0.17) + elasticsearch-transport (1.0.18) faraday multi_json equalizer (0.0.10) @@ -144,31 +181,6 @@ GEM logstash-codec-rubydebug (3.0.1) awesome_print logstash-core-plugin-api (~> 2.0) - logstash-core (5.0.0.alpha4.snapshot3-java) - cabin (~> 0.8.0) - chronic_duration (= 0.10.6) - clamp (~> 0.6.5) - concurrent-ruby (= 1.0.0) - filesize (= 0.0.4) - gems (~> 0.8.3) - i18n (= 0.6.9) - jrjackson (~> 0.3.7) - jrmonitor (~> 0.4.2) - jruby-openssl (= 0.9.16) - logstash-core-event-java (= 5.0.0.alpha4.snapshot3) - minitar (~> 0.5.4) - pry (~> 0.10.1) - puma (~> 2.16) - rubyzip (~> 1.1.7) - sinatra (~> 1.4, >= 1.4.6) - stud (~> 0.0.19) - thread_safe (~> 0.3.5) - treetop (< 1.5.0) - logstash-core-event-java (5.0.0.alpha4.snapshot3-java) - jar-dependencies - ruby-maven (~> 3.3.9) - logstash-core-plugin-api (2.1.9-java) - logstash-core (= 5.0.0.alpha4.snapshot3) logstash-devutils (1.0.2-java) fivemat gem_publisher @@ -621,9 +633,9 @@ DEPENDENCIES logstash-codec-netflow logstash-codec-plain logstash-codec-rubydebug - logstash-core (= 5.0.0.alpha4.snapshot3) - logstash-core-event-java (= 5.0.0.alpha4.snapshot3) - logstash-core-plugin-api (= 2.1.9) + logstash-core! + logstash-core-event-java! + logstash-core-plugin-api! logstash-devutils logstash-filter-clone logstash-filter-csv diff --git a/lib/bootstrap/bundler.rb b/lib/bootstrap/bundler.rb index 1a7bb0572..13c04b58a 100644 --- a/lib/bootstrap/bundler.rb +++ b/lib/bootstrap/bundler.rb @@ -78,7 +78,7 @@ module LogStash # @return [String, Exception] the installation captured output and any raised exception or nil if none def invoke!(options = {}) options = {:max_tries => 10, :clean => false, :install => false, :update => false, :local => false, - :all => false, :package => false, :without => [:development]}.merge(options) + :jobs => 12, :all => false, :package => false, :without => [:development]}.merge(options) options[:without] = Array(options[:without]) options[:update] = Array(options[:update]) if options[:update] diff --git a/logstash-core-event-java/lib/logstash-core-event-java_jars.rb b/logstash-core-event-java/lib/logstash-core-event-java_jars.rb index 143d7a3e0..d45036714 100644 --- a/logstash-core-event-java/lib/logstash-core-event-java_jars.rb +++ b/logstash-core-event-java/lib/logstash-core-event-java_jars.rb @@ -1,6 +1,14 @@ # this is a generated file, to avoid over-writing it just delete this comment -require 'jar_dependencies' +begin + require 'jar_dependencies' +rescue LoadError + require 'com/fasterxml/jackson/core/jackson-core/2.7.1/jackson-core-2.7.1.jar' + require 'com/fasterxml/jackson/core/jackson-annotations/2.7.0/jackson-annotations-2.7.0.jar' + require 'com/fasterxml/jackson/core/jackson-databind/2.7.1-1/jackson-databind-2.7.1-1.jar' +end -require_jar( 'com.fasterxml.jackson.core', 'jackson-core', '2.7.1' ) -require_jar( 'com.fasterxml.jackson.core', 'jackson-annotations', '2.7.0' ) -require_jar( 'com.fasterxml.jackson.core', 'jackson-databind', '2.7.1-1' ) +if defined? Jars + require_jar( 'com.fasterxml.jackson.core', 'jackson-core', '2.7.1' ) + require_jar( 'com.fasterxml.jackson.core', 'jackson-annotations', '2.7.0' ) + require_jar( 'com.fasterxml.jackson.core', 'jackson-databind', '2.7.1-1' ) +end diff --git a/logstash-core-event-java/logstash-core-event-java.gemspec b/logstash-core-event-java/logstash-core-event-java.gemspec index c59c45320..ef8d2a2ba 100644 --- a/logstash-core-event-java/logstash-core-event-java.gemspec +++ b/logstash-core-event-java/logstash-core-event-java.gemspec @@ -15,7 +15,7 @@ Gem::Specification.new do |gem| gem.test_files = gem.files.grep(%r{^(test|spec|features)/}) gem.name = "logstash-core-event-java" gem.require_paths = ["lib"] - gem.version = LOGSTASH_CORE_EVENT_JAVA_VERSION.gsub(/-/, '.') + gem.version = LOGSTASH_CORE_EVENT_JAVA_VERSION gem.platform = "java" diff --git a/logstash-core-event/lib/logstash-core-event/version.rb b/logstash-core-event/lib/logstash-core-event/version.rb index e8c7b7d4f..04253b622 100644 --- a/logstash-core-event/lib/logstash-core-event/version.rb +++ b/logstash-core-event/lib/logstash-core-event/version.rb @@ -5,4 +5,4 @@ # Note to authors: this should not include dashes because 'gem' barfs if # you include a dash in the version string. -LOGSTASH_CORE_EVENT_VERSION = "5.0.0-alpha1" +LOGSTASH_CORE_EVENT_VERSION = "5.0.0" diff --git a/logstash-core-event/logstash-core-event.gemspec b/logstash-core-event/logstash-core-event.gemspec index 88ac2ede4..9e0a757a8 100644 --- a/logstash-core-event/logstash-core-event.gemspec +++ b/logstash-core-event/logstash-core-event.gemspec @@ -15,7 +15,7 @@ Gem::Specification.new do |gem| gem.test_files = gem.files.grep(%r{^(test|spec|features)/}) gem.name = "logstash-core-event" gem.require_paths = ["lib"] - gem.version = LOGSTASH_CORE_EVENT_VERSION.gsub(/-/, '.') + gem.version = LOGSTASH_CORE_EVENT_VERSION if RUBY_PLATFORM == 'java' gem.platform = RUBY_PLATFORM diff --git a/logstash-core-plugin-api/lib/logstash-core-plugin-api/version.rb b/logstash-core-plugin-api/lib/logstash-core-plugin-api/version.rb index 7e2dd6732..ca61727b5 100644 --- a/logstash-core-plugin-api/lib/logstash-core-plugin-api/version.rb +++ b/logstash-core-plugin-api/lib/logstash-core-plugin-api/version.rb @@ -1,3 +1,2 @@ # encoding: utf-8 -# 2.1.9 tracks 5.0.0-alpha4.snapshot3 LOGSTASH_CORE_PLUGIN_API = "2.1.9" diff --git a/logstash-core-plugin-api/logstash-core-plugin-api.gemspec b/logstash-core-plugin-api/logstash-core-plugin-api.gemspec index 18c85c91e..27da82969 100644 --- a/logstash-core-plugin-api/logstash-core-plugin-api.gemspec +++ b/logstash-core-plugin-api/logstash-core-plugin-api.gemspec @@ -11,7 +11,7 @@ Gem::Specification.new do |gem| gem.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html" gem.license = "Apache License (2.0)" - gem.files = Dir.glob(["logstash-core-event.gemspec", "lib/**/*.rb", "spec/**/*.rb"]) + gem.files = Dir.glob(["logstash-core-plugin-api.gemspec", "lib/**/*.rb", "spec/**/*.rb"]) gem.test_files = gem.files.grep(%r{^(test|spec|features)/}) gem.name = "logstash-core-plugin-api" gem.require_paths = ["lib"] diff --git a/logstash-core/lib/logstash/api/commands/system/basicinfo_command.rb b/logstash-core/lib/logstash/api/commands/system/basicinfo_command.rb index 36edc3d1c..6eacdbd5b 100644 --- a/logstash-core/lib/logstash/api/commands/system/basicinfo_command.rb +++ b/logstash-core/lib/logstash/api/commands/system/basicinfo_command.rb @@ -1,6 +1,7 @@ # encoding: utf-8 require 'logstash/api/commands/base' require "logstash/util/duration_formatter" +require 'logstash/build' module LogStash module Api @@ -9,7 +10,8 @@ module LogStash class BasicInfo < Commands::Base def run - {} # Just return the defaults + # Just merge this stuff with the defaults + BUILD_INFO end end end diff --git a/logstash-core/lib/logstash/build.rb b/logstash-core/lib/logstash/build.rb new file mode 100644 index 000000000..fb2132c22 --- /dev/null +++ b/logstash-core/lib/logstash/build.rb @@ -0,0 +1,6 @@ +# encoding: utf-8 + +# DO NOT EDIT +# this file acts as a placeholder for build information when executing +# logstash in dev mode (outside of a package build) +BUILD_INFO = {} diff --git a/logstash-core/logstash-core.gemspec b/logstash-core/logstash-core.gemspec index 8a8a3445f..afb8b7cd1 100644 --- a/logstash-core/logstash-core.gemspec +++ b/logstash-core/logstash-core.gemspec @@ -15,7 +15,7 @@ Gem::Specification.new do |gem| gem.test_files = gem.files.grep(%r{^(test|spec|features)/}) gem.name = "logstash-core" gem.require_paths = ["lib"] - gem.version = LOGSTASH_CORE_VERSION.gsub(/-/, '.') + gem.version = LOGSTASH_CORE_VERSION gem.add_runtime_dependency "logstash-core-event-java", "5.0.0.alpha4.snapshot3" diff --git a/qa/acceptance/spec/shared_examples/cli/logstash-plugin/install.rb b/qa/acceptance/spec/shared_examples/cli/logstash-plugin/install.rb index 13cb52036..afa8381bc 100644 --- a/qa/acceptance/spec/shared_examples/cli/logstash-plugin/install.rb +++ b/qa/acceptance/spec/shared_examples/cli/logstash-plugin/install.rb @@ -5,7 +5,7 @@ require "fileutils" shared_examples "logstash install" do |logstash| before(:each) do - logstash.install(LOGSTASH_VERSION) + logstash.install({:version => LOGSTASH_VERSION}) end after(:each) do diff --git a/qa/acceptance/spec/shared_examples/cli/logstash-plugin/list.rb b/qa/acceptance/spec/shared_examples/cli/logstash-plugin/list.rb index 64e6cd1be..89336dc31 100644 --- a/qa/acceptance/spec/shared_examples/cli/logstash-plugin/list.rb +++ b/qa/acceptance/spec/shared_examples/cli/logstash-plugin/list.rb @@ -6,7 +6,7 @@ require "fileutils" shared_examples "logstash list" do |logstash| describe "logstash-plugin list on #{logstash.hostname}" do before(:all) do - logstash.install(LOGSTASH_VERSION) + logstash.install({:version => LOGSTASH_VERSION}) end after(:all) do diff --git a/qa/acceptance/spec/shared_examples/cli/logstash-plugin/uninstall.rb b/qa/acceptance/spec/shared_examples/cli/logstash-plugin/uninstall.rb index d12bbb954..2fe6f9497 100644 --- a/qa/acceptance/spec/shared_examples/cli/logstash-plugin/uninstall.rb +++ b/qa/acceptance/spec/shared_examples/cli/logstash-plugin/uninstall.rb @@ -6,7 +6,7 @@ require "fileutils" shared_examples "logstash uninstall" do |logstash| describe "logstash uninstall on #{logstash.hostname}" do before :each do - logstash.install(LOGSTASH_VERSION) + logstash.install({:version => LOGSTASH_VERSION}) end after :each do diff --git a/qa/acceptance/spec/shared_examples/cli/logstash-plugin/update.rb b/qa/acceptance/spec/shared_examples/cli/logstash-plugin/update.rb index 3aaaa3052..9506654a3 100644 --- a/qa/acceptance/spec/shared_examples/cli/logstash-plugin/update.rb +++ b/qa/acceptance/spec/shared_examples/cli/logstash-plugin/update.rb @@ -5,7 +5,7 @@ require "logstash/version" shared_examples "logstash update" do |logstash| describe "logstash update on #{logstash.hostname}" do before :each do - logstash.install(LOGSTASH_VERSION) + logstash.install({:version => LOGSTASH_VERSION}) end after :each do diff --git a/qa/acceptance/spec/shared_examples/cli/logstash/version.rb b/qa/acceptance/spec/shared_examples/cli/logstash/version.rb index 97a202706..0980e15e8 100644 --- a/qa/acceptance/spec/shared_examples/cli/logstash/version.rb +++ b/qa/acceptance/spec/shared_examples/cli/logstash/version.rb @@ -5,7 +5,7 @@ require "logstash/version" shared_examples "logstash version" do |logstash| describe "logstash --version" do before :all do - logstash.install(LOGSTASH_VERSION) + logstash.install({:version => LOGSTASH_VERSION}) end after :all do diff --git a/qa/acceptance/spec/shared_examples/installed.rb b/qa/acceptance/spec/shared_examples/installed.rb index 940892321..8773dfa4d 100644 --- a/qa/acceptance/spec/shared_examples/installed.rb +++ b/qa/acceptance/spec/shared_examples/installed.rb @@ -5,7 +5,7 @@ require 'logstash/version' RSpec.shared_examples "installable" do |logstash| before(:each) do - logstash.install(LOGSTASH_VERSION) + logstash.install({:version => LOGSTASH_VERSION}) end it "is installed on #{logstash.hostname}" do diff --git a/qa/acceptance/spec/shared_examples/running.rb b/qa/acceptance/spec/shared_examples/running.rb index 4c051dc8f..55d18507c 100644 --- a/qa/acceptance/spec/shared_examples/running.rb +++ b/qa/acceptance/spec/shared_examples/running.rb @@ -5,7 +5,7 @@ require 'logstash/version' RSpec.shared_examples "runnable" do |logstash| before(:each) do - logstash.install(LOGSTASH_VERSION) + logstash.install({:version => LOGSTASH_VERSION}) end it "is running on #{logstash.hostname}" do diff --git a/qa/acceptance/spec/shared_examples/updated.rb b/qa/acceptance/spec/shared_examples/updated.rb index 689a9d190..f6c9e8131 100644 --- a/qa/acceptance/spec/shared_examples/updated.rb +++ b/qa/acceptance/spec/shared_examples/updated.rb @@ -11,12 +11,14 @@ RSpec.shared_examples "updated" do |logstash| end before(:each) do - logstash.install(LOGSTASH_LATEST_VERSION, "./") # make sure latest version is installed + options={:version => LOGSTASH_LATEST_VERSION, :snapshot => false, :base => "./" } + logstash.install(options) # make sure latest version is installed end it "can be updated an run on #{logstash.hostname}" do + expect(logstash).to be_installed # Performing the update - logstash.install(LOGSTASH_VERSION) + logstash.install({:version => LOGSTASH_VERSION}) expect(logstash).to be_installed # starts the service to be sure it runs after the upgrade logstash.start_service diff --git a/qa/rspec/commands.rb b/qa/rspec/commands.rb index c4e739198..f90bc74cc 100644 --- a/qa/rspec/commands.rb +++ b/qa/rspec/commands.rb @@ -56,8 +56,9 @@ module ServiceTester client.stop_service(name, host) end - def install(version, base=ServiceTester::Base::LOCATION) - package = client.package_for(version, base) + def install(options={}) + base = options.fetch(:base, ServiceTester::Base::LOCATION) + package = client.package_for(filename(options), base) client.install(package, host) end @@ -92,6 +93,13 @@ module ServiceTester def to_s "Artifact #{name}@#{host}" end + + private + + def filename(options={}) + snapshot = options.fetch(:snapshot, true) + "logstash-#{options[:version]}#{(snapshot ? "-SNAPSHOT" : "")}" + end end # Factory of commands used to select the right clients for a given type of OS and host name, diff --git a/qa/rspec/commands/debian.rb b/qa/rspec/commands/debian.rb index 710ffc48d..aa722d558 100644 --- a/qa/rspec/commands/debian.rb +++ b/qa/rspec/commands/debian.rb @@ -16,14 +16,14 @@ module ServiceTester stdout.match(/^Status: install ok installed$/) end - def package_for(version, base=ServiceTester::Base::LOCATION) - File.join(base, "logstash-#{version}.deb") + def package_for(filename, base=ServiceTester::Base::LOCATION) + File.join(base, "#{filename}.deb") end def install(package, host=nil) hosts = (host.nil? ? servers : Array(host)) at(hosts, {in: :serial}) do |_| - sudo_exec!("dpkg -i #{package}") + cmd = sudo_exec!("dpkg -i #{package}") end end diff --git a/qa/rspec/commands/redhat.rb b/qa/rspec/commands/redhat.rb index 8f0cf753c..67ec8143a 100644 --- a/qa/rspec/commands/redhat.rb +++ b/qa/rspec/commands/redhat.rb @@ -16,8 +16,8 @@ module ServiceTester stdout.match(/^logstash.noarch/) end - def package_for(version, base=ServiceTester::Base::LOCATION) - File.join(base, "logstash-#{version}.rpm") + def package_for(filename, base=ServiceTester::Base::LOCATION) + File.join(base, "#{filename}.rpm") end def install(package, host=nil) diff --git a/qa/rspec/commands/suse.rb b/qa/rspec/commands/suse.rb index 2a214a53b..028d9a5ca 100644 --- a/qa/rspec/commands/suse.rb +++ b/qa/rspec/commands/suse.rb @@ -13,8 +13,8 @@ module ServiceTester stdout.match(/^i | logstash | An extensible logging pipeline | package$/) end - def package_for(version, base=ServiceTester::Base::LOCATION) - File.join(base, "logstash-#{version}.rpm") + def package_for(filename, base=ServiceTester::Base::LOCATION) + File.join(base, "#{filename}.rpm") end def install(package, host=nil) diff --git a/rakelib/artifacts.rake b/rakelib/artifacts.rake index 581d8c798..877b9ef82 100644 --- a/rakelib/artifacts.rake +++ b/rakelib/artifacts.rake @@ -2,6 +2,9 @@ require "logstash/version" namespace "artifact" do + SNAPSHOT_BUILD = ENV["RELEASE"] != "1" + PACKAGE_SUFFIX = SNAPSHOT_BUILD ? "-SNAPSHOT" : "" + def package_files [ "LICENSE", @@ -13,6 +16,13 @@ namespace "artifact" do "lib/bootstrap/**/*", "lib/pluginmanager/**/*", "lib/systeminstall/**/*", + "logstash-core/lib/**/*", + "logstash-core/locales/**/*", + "logstash-core/*.gemspec", + "logstash-core-event-java/lib/**/*", + "logstash-core-event-java/*.gemspec", + "logstash-core-plugin-api/lib/**/*", + "logstash-core-plugin-api/*.gemspec", "patterns/**/*", "vendor/??*/**/*", # To include ruby-maven's hidden ".mvn" directory, we need to @@ -55,18 +65,75 @@ namespace "artifact" do end.flatten.uniq end - task "all" => ["prepare"] do + desc "Generate rpm, deb, tar and zip artifacts" + task "all" => ["prepare", "build"] + + desc "Build a tar.gz of default logstash plugins with all dependencies" + task "tar" => ["prepare", "generate_build_metadata"] do + puts("[artifact:tar] Building tar.gz of default plugins") + build_tar + end + + desc "Build a zip of default logstash plugins with all dependencies" + task "zip" => ["prepare", "generate_build_metadata"] do + puts("[artifact:zip] Building zip of default plugins") + build_zip + end + + desc "Build an RPM of logstash with all dependencies" + task "rpm" => ["prepare", "generate_build_metadata"] do + puts("[artifact:rpm] building rpm package") + package("centos", "5") + end + + desc "Build a DEB of logstash with all dependencies" + task "deb" => ["prepare", "generate_build_metadata"] do + puts("[artifact:deb] building deb package") + package("ubuntu", "12.04") + end + + desc "Generate logstash core gems" + task "gems" => ["prepare"] do + Rake::Task["artifact:build-logstash-core"].invoke + Rake::Task["artifact:build-logstash-core-event"].invoke + Rake::Task["artifact:build-logstash-core-plugin-api"].invoke + end + + # "all-plugins" version of tasks + desc "Generate rpm, deb, tar and zip artifacts (\"all-plugins\" version)" + task "all-all-plugins" => ["prepare-all", "build"] + + desc "Build a zip of all logstash plugins from logstash-plugins github repo" + task "zip-all-plugins" => ["prepare-all", "generate_build_metadata"] do + puts("[artifact:zip] Building zip of all plugins") + build_zip "-all-plugins" + end + + desc "Build a tar.gz of all logstash plugins from logstash-plugins github repo" + task "tar-all-plugins" => ["prepare-all", "generate_build_metadata"] do + puts("[artifact:tar] Building tar.gz of all plugins") + build_tar "-all-plugins" + end + + # Auxiliary tasks + task "build" => [:generate_build_metadata] do + Rake::Task["artifact:gems"].invoke unless SNAPSHOT_BUILD Rake::Task["artifact:deb"].invoke Rake::Task["artifact:rpm"].invoke Rake::Task["artifact:zip"].invoke Rake::Task["artifact:tar"].invoke end - task "all-all-plugins" => ["prepare-all"] do - Rake::Task["artifact:deb"].invoke - Rake::Task["artifact:rpm"].invoke - Rake::Task["artifact:zip"].invoke - Rake::Task["artifact:tar"].invoke + task "generate_build_metadata" do + return if defined?(BUILD_METADATA_FILE) + BUILD_METADATA_FILE = Tempfile.new('build.rb') + build_info = { + "build_date" => Time.now.iso8601, + "build_sha" => `git rev-parse HEAD`.chomp, + "build_snapshot" => SNAPSHOT_BUILD + } + metadata = [ "# encoding: utf-8", "BUILD_INFO = #{build_info}" ] + IO.write(BUILD_METADATA_FILE.path, metadata.join("\n")) end # We create an empty bundle config file @@ -78,8 +145,8 @@ namespace "artifact" do end # locate the "gem "logstash-core" ..." line in Gemfile, and if the :path => "..." option if specified - # build and install the local logstash-core gem otherwise just do nothing, bundler will deal with it. - task "install-logstash-core" do + # build the local logstash-core gem otherwise just do nothing, bundler will deal with it. + task "build-logstash-core" do # regex which matches a Gemfile gem definition for the logstash-core gem and captures the :path option gem_line_regex = /^\s*gem\s+["']logstash-core["'](?:\s*,\s*["'][^"^']+["'])?(?:\s*,\s*:path\s*=>\s*["']([^"^']+)["'])?/i @@ -90,15 +157,16 @@ namespace "artifact" do path = matches.first[gem_line_regex, 1] if path - Rake::Task["plugin:install-local-core-gem"].invoke("logstash-core", path) + Rake::Task["plugin:build-local-core-gem"].invoke("logstash-core", path) else - puts("[artifact:install-logstash-core] using logstash-core from Rubygems") + puts "The Gemfile should reference \"logstash-core\" gem locally through :path, but found instead: #{matches}" + exit(1) end end # # locate the "gem "logstash-core-event*" ..." line in Gemfile, and if the :path => "." option if specified - # # build and install the local logstash-core-event* gem otherwise just do nothing, bundler will deal with it. - task "install-logstash-core-event" do + # # build the local logstash-core-event* gem otherwise just do nothing, bundler will deal with it. + task "build-logstash-core-event" do # regex which matches a Gemfile gem definition for the logstash-core-event* gem and captures the gem name and :path option gem_line_regex = /^\s*gem\s+["'](logstash-core-event[^"^']*)["'](?:\s*,\s*["'][^"^']+["'])?(?:\s*,\s*:path\s*=>\s*["']([^"^']+)["'])?/i @@ -110,15 +178,16 @@ namespace "artifact" do path = matches.first[gem_line_regex, 2] if path - Rake::Task["plugin:install-local-core-gem"].invoke(name, path) + Rake::Task["plugin:build-local-core-gem"].invoke(name, path) else - puts("[artifact:install-logstash-core] using #{name} from Rubygems") + puts "The Gemfile should reference \"logstash-core-event\" gem locally through :path, but found instead: #{matches}" + exit(1) end end # locate the "gem "logstash-core-plugin-api" ..." line in Gemfile, and if the :path => "..." option if specified - # build and install the local logstash-core-plugin-api gem otherwise just do nothing, bundler will deal with it. - task "install-logstash-core-plugin-api" do + # build the local logstash-core-plugin-api gem otherwise just do nothing, bundler will deal with it. + task "build-logstash-core-plugin-api" do # regex which matches a Gemfile gem definition for the logstash-core gem and captures the :path option gem_line_regex = /^\s*gem\s+["']logstash-core-plugin-api["'](?:\s*,\s*["'][^"^']+["'])?(?:\s*,\s*:path\s*=>\s*["']([^"^']+)["'])?/i @@ -129,86 +198,87 @@ namespace "artifact" do path = matches.first[gem_line_regex, 1] if path - Rake::Task["plugin:install-local-core-gem"].invoke("logstash-core-plugin-api", path) + Rake::Task["plugin:build-local-core-gem"].invoke("logstash-core-plugin-api", path) else - puts("[artifact:install-logstash-core-plugin-api] using logstash-core from Rubygems") + puts "The Gemfile should reference \"logstash-core-plugin-api\" gem locally through :path, but found instead: #{matches}" + exit(1) end end - task "prepare" => ["bootstrap", "plugin:install-default", "install-logstash-core", "install-logstash-core-event", "install-logstash-core-plugin-api", "clean-bundle-config"] - - task "prepare-all" => ["bootstrap", "plugin:install-all", "install-logstash-core", "install-logstash-core-event", "install-logstash-core-plugin-api", "clean-bundle-config"] - - desc "Build a tar.gz of default logstash plugins with all dependencies" - task "tar" => ["prepare"] do - puts("[artifact:tar] Building tar.gz of default plugins") - build_tar + task "prepare" do + if ENV['SKIP_PREPARE'] != "1" + ["bootstrap", "plugin:install-default", "artifact:clean-bundle-config"].each {|task| Rake::Task[task].invoke } + end end - desc "Build a tar.gz of all logstash plugins from logstash-plugins github repo" - task "tar-all-plugins" => ["prepare-all"] do - puts("[artifact:tar] Building tar.gz of all plugins") - build_tar "-all-plugins" + task "prepare-all" do + if ENV['SKIP_PREPARE'] != "1" + ["bootstrap", "plugin:install-all", "artifact:clean-bundle-config"].each {|task| Rake::Task[task].invoke } + end end def build_tar(tar_suffix = nil) require "zlib" require "archive/tar/minitar" require "logstash/version" - tarpath = "build/logstash#{tar_suffix}-#{LOGSTASH_VERSION}.tar.gz" + tarpath = "build/logstash#{tar_suffix}-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}.tar.gz" puts("[artifact:tar] building #{tarpath}") gz = Zlib::GzipWriter.new(File.new(tarpath, "wb"), Zlib::BEST_COMPRESSION) tar = Archive::Tar::Minitar::Output.new(gz) files.each do |path| - stat = File.lstat(path) - path_in_tar = "logstash-#{LOGSTASH_VERSION}/#{path}" - opts = { - :size => stat.size, - :mode => stat.mode, - :mtime => stat.mtime - } - if stat.directory? - tar.tar.mkdir(path_in_tar, opts) - else - tar.tar.add_file_simple(path_in_tar, opts) do |io| - File.open(path,'rb') do |fd| - chunk = nil - size = 0 - size += io.write(chunk) while chunk = fd.read(16384) - if stat.size != size - raise "Failure to write the entire file (#{path}) to the tarball. Expected to write #{stat.size} bytes; actually write #{size}" - end - end - end - end + write_to_tar(tar, path, "logstash-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}/#{path}") end + + # add build.rb to tar + metadata_file_path_in_tar = File.join("logstash-core", "lib", "logstash", "build.rb") + path_in_tar = File.join("logstash-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}", metadata_file_path_in_tar) + write_to_tar(tar, BUILD_METADATA_FILE.path, path_in_tar) + tar.close gz.close puts "Complete: #{tarpath}" end - desc "Build a zip of default logstash plugins with all dependencies" - task "zip" => ["prepare"] do - puts("[artifact:zip] Building zip of default plugins") - build_zip - end - - desc "Build a zip of all logstash plugins from logstash-plugins github repo" - task "zip-all-plugins" => ["prepare-all"] do - puts("[artifact:zip] Building zip of all plugins") - build_zip "-all-plugins" + def write_to_tar(tar, path, path_in_tar) + stat = File.lstat(path) + opts = { + :size => stat.size, + :mode => stat.mode, + :mtime => stat.mtime + } + if stat.directory? + tar.tar.mkdir(path_in_tar, opts) + else + tar.tar.add_file_simple(path_in_tar, opts) do |io| + File.open(path,'rb') do |fd| + chunk = nil + size = 0 + size += io.write(chunk) while chunk = fd.read(16384) + if stat.size != size + raise "Failure to write the entire file (#{path}) to the tarball. Expected to write #{stat.size} bytes; actually write #{size}" + end + end + end + end end def build_zip(zip_suffix = "") require 'zip' - zippath = "build/logstash#{zip_suffix}-#{LOGSTASH_VERSION}.zip" + zippath = "build/logstash#{zip_suffix}-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}.zip" puts("[artifact:zip] building #{zippath}") File.unlink(zippath) if File.exists?(zippath) Zip::File.open(zippath, Zip::File::CREATE) do |zipfile| files.each do |path| - path_in_zip = "logstash-#{LOGSTASH_VERSION}/#{path}" + path_in_zip = "logstash-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}/#{path}" zipfile.add(path_in_zip, path) end + + # add build.rb to zip + metadata_file_path_in_zip = File.join("logstash-core", "lib", "logstash", "build.rb") + path_in_zip = File.join("logstash-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}", metadata_file_path_in_zip) + path = BUILD_METADATA_FILE.path + Zip.continue_on_exists_proc = true + zipfile.add(path_in_zip, path) end puts "Complete: #{zippath}" end @@ -221,6 +291,10 @@ namespace "artifact" do dir = FPM::Package::Dir.new + metadata_file_path = File.join("logstash-core", "lib", "logstash", "build.rb") + metadata_source_file_path = BUILD_METADATA_FILE.path + dir.input("#{metadata_source_file_path}=/usr/share/logstash/#{metadata_file_path}") + files.each do |path| next if File.directory?(path) # Omit any config dir from /usr/share/logstash for packages, since we're @@ -246,7 +320,7 @@ namespace "artifact" do # produce: logstash-5.0.0-alpha1.deb" # produce: logstash-5.0.0-alpha1.rpm - package_filename = "logstash-#{LOGSTASH_VERSION}.TYPE" + package_filename = "logstash-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}.TYPE" case platform when "redhat", "centos" @@ -350,15 +424,4 @@ namespace "artifact" do end end # def package - desc "Build an RPM of logstash with all dependencies" - task "rpm" => ["prepare"] do - puts("[artifact:rpm] building rpm package") - package("centos", "5") - end - - desc "Build a DEB of logstash with all dependencies" - task "deb" => ["prepare"] do - puts("[artifact:deb] building deb package") - package("ubuntu", "12.04") - end end diff --git a/rakelib/build.rake b/rakelib/build.rake index 2b443add8..45aec1f9b 100644 --- a/rakelib/build.rake +++ b/rakelib/build.rake @@ -4,3 +4,6 @@ end directory "build/bootstrap" => "build" do |task, args| mkdir_p task.name unless File.directory?(task.name) end +directory "build/gems" => "build" do |task, args| + mkdir_p task.name unless File.directory?(task.name) +end diff --git a/rakelib/plugin.rake b/rakelib/plugin.rake index 0bc1b7089..79be84172 100644 --- a/rakelib/plugin.rake +++ b/rakelib/plugin.rake @@ -81,7 +81,7 @@ namespace "plugin" do task.reenable # Allow this task to be run again end - task "build-local-core-gem", [:name, :path] do |task, args| + task "build-local-core-gem", [:name, :path] => ["build/gems"] do |task, args| name = args[:name] path = args[:path] @@ -89,10 +89,12 @@ namespace "plugin" do puts("[plugin:build-local-core-gem] Building #{File.join(path, name)}.gemspec") + gem_path = nil Dir.chdir(path) do spec = Gem::Specification.load("#{name}.gemspec") - Gem::Package.build(spec) + gem_path = Gem::Package.build(spec) end + FileUtils.cp(File.join(path, gem_path), "build/gems/") task.reenable # Allow this task to be run again end diff --git a/rakelib/version.rake b/rakelib/version.rake new file mode 100644 index 000000000..45655cd33 --- /dev/null +++ b/rakelib/version.rake @@ -0,0 +1,132 @@ +require 'yaml' + +def get_versions + yaml_versions = YAML.safe_load(IO.read("versions.yml")) + { + "logstash" => { + "location" => File.join("logstash-core", "lib", "logstash", "version.rb"), + "yaml_version" => yaml_versions["logstash"], + "current_version" => get_version(File.join("logstash-core", "lib", "logstash", "version.rb")), + }, + "logstash-core" => { + "location" => File.join("logstash-core", "lib", "logstash-core", "version.rb"), + "yaml_version" => yaml_versions["logstash-core"], + "current_version" => get_version(File.join("logstash-core", "lib", "logstash-core", "version.rb")), + }, + "logstash-core-event" => { + "location" => File.join("logstash-core-event", "lib", "logstash-core-event", "version.rb"), + "yaml_version" => yaml_versions["logstash-core-event"], + "current_version" => get_version(File.join("logstash-core-event", "lib", "logstash-core-event", "version.rb")), + }, + "logstash-core-event-java" => { + "location" => File.join("logstash-core-event-java", "lib", "logstash-core-event-java", "version.rb"), + "yaml_version" => yaml_versions["logstash-core-event-java"], + "current_version" => get_version(File.join("logstash-core-event-java", "lib", "logstash-core-event-java", "version.rb")), + }, + "logstash-core-plugin-api" => { + "location" => File.join("logstash-core-plugin-api", "lib", "logstash-core-plugin-api", "version.rb"), + "yaml_version" => yaml_versions["logstash-core-plugin-api"], + "current_version" => get_version(File.join("logstash-core-plugin-api", "lib", "logstash-core-plugin-api", "version.rb")), + } + } +end + +def get_version(file) + text = IO.read(file) + version = text.match(/^[A-Z_]+ = "(.+?)"/) + version[1] +end + +namespace :version do + + desc "check if the versions.yml is out of sync with .gemspecs and other references" + task :check do + out_of_sync = get_versions.select do |component, metadata| + metadata["yaml_version"] != metadata["current_version"] + end + if out_of_sync.any? + out_of_sync.each do |component, metadata| + puts "#{component} is out of sync. CURRENT: #{metadata['current_version']} | YAML: #{metadata['yaml_version']}" + end + exit(1) + end + end + + desc "push versions found in versions.yml to all component version locations" + task :sync do + versions = get_versions + # update version.rb files + versions.select do |component, metadata| + next if metadata["yaml_version"] == metadata["current_version"] + puts "Updating \"#{component}\" from \"#{metadata['current_version']}\" to \"#{metadata['yaml_version']}\"" + text = IO.read(metadata["location"]) + IO.write(metadata["location"], text.gsub(metadata["current_version"], metadata["yaml_version"])) + end + + # update dependencies + # + # logstash-core depends on logstash-core-event-java + # ./logstash-core/logstash-core.gemspec: gem.add_runtime_dependency "logstash-core-event-java", "~> 5.0.0.dev" + logstash_core_gemspec = File.join("logstash-core", "logstash-core.gemspec") + logstash_core_event_java_version = versions['logstash-core-event-java']['yaml_version'] + text = IO.read(logstash_core_gemspec) + IO.write(logstash_core_gemspec, text.sub( + / gem.add_runtime_dependency \"logstash-core-event-java\", \".+?\"/, + " gem.add_runtime_dependency \"logstash-core-event-java\", \"#{logstash_core_event_java_version}\"")) + + # logstash-core-event-java depends on logstash-code + # ./logstash-core-plugin-api/logstash-core-plugin-api.gemspec: gem.add_runtime_dependency "logstash-core", "5.0.0.dev" + logstash_core_plugin_api_gemspec = File.join("logstash-core-plugin-api", "logstash-core-plugin-api.gemspec") + logstash_core_version = versions['logstash-core']['yaml_version'] + text = IO.read(logstash_core_plugin_api_gemspec) + IO.write(logstash_core_plugin_api_gemspec, text.sub( + / gem.add_runtime_dependency \"logstash-core\", \".+?\"/, + " gem.add_runtime_dependency \"logstash-core\", \"#{logstash_core_version}\"")) + end + + desc "show version of core components" + task :show do + Rake::Task["version:sync"].invoke; Rake::Task["version:sync"].reenable + get_versions.each do |component, metadata| + puts "#{component}: #{metadata['yaml_version']}" + end + end + + desc "set version of logstash, logstash-core, logstash-core-event, logstash-core-event-java" + task :set, [:version] => [:validate] do |t, args| + hash = {} + get_versions.each do |component, metadata| + # we just assume that, usually, all components except + # "logstash-core-plugin-api" will be versioned together + # so let's skip this one and have a separate task for it + if component == "logstash-core-plugin-api" + hash[component] = metadata["yaml_version"] + else + hash[component] = args[:version] + end + end + IO.write("versions.yml", hash.to_yaml) + Rake::Task["version:sync"].invoke; Rake::Task["version:sync"].reenable + end + + desc "set version of logstash-core-plugin-api" + task :set_plugin_api, [:version] => [:validate] do |t, args| + hash = {} + get_versions.each do |component, metadata| + if component == "logstash-core-plugin-api" + hash[component] = args[:version] + else + hash[component] = metadata["yaml_version"] + end + end + IO.write("versions.yml", hash.to_yaml) + Rake::Task["version:sync"].invoke; Rake::Task["version:sync"].reenable + end + + task :validate, :version do |t, args| + unless Regexp.new('^\d+\.\d+\.\d+(?:-\w+\d+)?$').match(args[:version]) + abort("Invalid version argument: \"#{args[:version]}\". Aborting...") + end + end + +end diff --git a/versions.yml b/versions.yml new file mode 100644 index 000000000..a7e6c26df --- /dev/null +++ b/versions.yml @@ -0,0 +1,6 @@ +--- +logstash: 5.0.0 +logstash-core: 5.0.0 +logstash-core-event: 5.0.0 +logstash-core-event-java: 5.0.0 +logstash-core-plugin-api: 2.1.7