diff --git a/Gemfile b/Gemfile index f4aab97ef..0e392191e 100644 --- a/Gemfile +++ b/Gemfile @@ -2,16 +2,13 @@ # If you modify this file manually all comments and formatting will be lost. source "https://rubygems.org" -gem "logstash-core", "5.0.0.dev", :path => "./logstash-core" -gem "logstash-core-event-java", "5.0.0.dev", :path => "./logstash-core-event-java" -gem "logstash-core-plugin-api", "2.0.0", :path => "./logstash-core-plugin-api" +gem "logstash-core", :path => "./logstash-core" +gem "logstash-core-event-java", :path => "./logstash-core-event-java" +gem "logstash-core-plugin-api", :path => "./logstash-core-plugin-api" gem "file-dependencies", "0.1.6" gem "ci_reporter_rspec", "1.0.0", :group => :development gem "simplecov", :group => :development gem "coveralls", :group => :development -# Tins 1.7 requires the ruby 2.0 platform to install, -# this gem is a dependency of term-ansi-color which is a dependency of coveralls. -# 1.6 is the last supported version on jruby. gem "tins", "1.6", :group => :development gem "rspec", "~> 3.1.0", :group => :development gem "logstash-devutils", :group => :development diff --git a/Gemfile.jruby-1.9.lock b/Gemfile.jruby-1.9.lock deleted file mode 100644 index 709340a19..000000000 --- a/Gemfile.jruby-1.9.lock +++ /dev/null @@ -1,326 +0,0 @@ -GIT - remote: git://github.com/elastic/logstash-devutils.git - revision: e9af3a24824a41d0f19b025ca359e0735e820251 - branch: feature/plugin-api-2_0 - specs: - logstash-devutils (0.0.19-java) - fivemat - gem_publisher - insist (= 1.0.0) - kramdown - minitar - rake - rspec (~> 3.1.0) - rspec-wait - stud (>= 0.0.20) - -GIT - remote: git://github.com/logstash-plugins/logstash-codec-json.git - revision: 141d488f3a73706d5ae08dd41468fed285fbbdf9 - branch: feature/plugin-api-2_0 - specs: - logstash-codec-json (2.1.4) - logstash-core-plugin-api (~> 2.0) - -GIT - remote: git://github.com/logstash-plugins/logstash-codec-json_lines.git - revision: f1a3fda4d1affaa6da4b037eec78ee5f10287eb8 - branch: feature/plugin-api-2_0 - specs: - logstash-codec-json_lines (2.1.3) - logstash-codec-line (>= 2.1.0) - logstash-core-plugin-api (~> 2.0) - -GIT - remote: git://github.com/logstash-plugins/logstash-codec-line.git - revision: f912b6ca5137bef914ea0e605cb34520b0f172d4 - branch: feature/plugin-api-2_0 - specs: - logstash-codec-line (2.1.2) - logstash-core-plugin-api (~> 2.0) - -GIT - remote: git://github.com/logstash-plugins/logstash-codec-plain.git - revision: 5c5424905d5e06d7f7adc888411057c8a5681194 - branch: feature/plugin-api-2_0 - specs: - logstash-codec-plain (2.0.4) - logstash-core-plugin-api (~> 2.0) - -GIT - remote: git://github.com/logstash-plugins/logstash-filter-clone.git - revision: 4da247817809ff68a47557f022c8049536651564 - branch: feature/plugin-api-2_0 - specs: - logstash-filter-clone (2.0.6) - logstash-core-plugin-api (~> 2.0) - -GIT - remote: git://github.com/logstash-plugins/logstash-filter-grok.git - revision: 284dc6614b0cac6770cda2c18a52f8405146f1eb - branch: feature/plugin-api-2_0 - specs: - logstash-filter-grok (2.0.5) - jls-grok (~> 0.11.1) - logstash-core-plugin-api (~> 2.0) - logstash-patterns-core - -GIT - remote: git://github.com/logstash-plugins/logstash-filter-multiline.git - revision: fd1da31b7d1e0b44319c0f2865c767b36440583d - branch: feature/plugin-api-2_0 - specs: - logstash-filter-multiline (2.0.5) - jls-grok (~> 0.11.0) - logstash-core-plugin-api (~> 2.0) - logstash-patterns-core - -GIT - remote: git://github.com/logstash-plugins/logstash-filter-mutate.git - revision: f9624cc0a05354c308b2d22a5895c737eceaa08b - branch: feature/plugin-api-2_0 - specs: - logstash-filter-mutate (2.0.6) - logstash-core-plugin-api (~> 2.0) - -GIT - remote: git://github.com/logstash-plugins/logstash-input-generator.git - revision: 2557a4e7da667d6e801cac47e27ff297b6ecc79a - branch: feature/plugin-api-2_0 - specs: - logstash-input-generator (2.0.4) - logstash-codec-plain - logstash-core-plugin-api (~> 2.0) - -GIT - remote: git://github.com/logstash-plugins/logstash-input-stdin.git - revision: cca193c505f931500bd0f6be45afada2af0578ed - branch: feature/plugin-api-2_0 - specs: - logstash-input-stdin (2.0.4) - concurrent-ruby - logstash-codec-line - logstash-core-plugin-api (~> 2.0) - -GIT - remote: git://github.com/logstash-plugins/logstash-input-tcp.git - revision: b6f2705d7fc226c08dd0905561ef9373d2801a58 - branch: feature/plugin-api-2_0 - specs: - logstash-input-tcp (3.0.5) - logstash-codec-json - logstash-codec-json_lines - logstash-codec-line - logstash-codec-plain - logstash-core-plugin-api (~> 2.0) - -GIT - remote: git://github.com/logstash-plugins/logstash-output-stdout.git - revision: c150cb4f560b047372e15887e8b6bf421af22fc9 - branch: feature/plugin-api-2_0 - specs: - logstash-output-stdout (2.0.6) - logstash-codec-line - logstash-core-plugin-api (~> 2.0) - -GIT - remote: git://github.com/logstash-plugins/logstash-patterns-core.git - revision: e039ba2d3db6a01b6501fe0cab8a82453c6dc45f - branch: feature/plugin-api-2_0 - specs: - logstash-patterns-core (2.0.5) - logstash-core-plugin-api (~> 2.0) - -PATH - remote: ./logstash-core - specs: - logstash-core (5.0.0.dev-java) - cabin (~> 0.8.0) - chronic_duration (= 0.10.6) - clamp (~> 0.6.5) - concurrent-ruby (= 1.0.0) - filesize (= 0.0.4) - gems (~> 0.8.3) - i18n (= 0.6.9) - jrjackson (~> 0.3.7) - jruby-monitoring (~> 0.3.1) - jruby-openssl (= 0.9.16) - logstash-core-event-java (~> 5.0.0.dev) - minitar (~> 0.5.4) - pry (~> 0.10.1) - puma (~> 2.16, >= 2.16.0) - rubyzip (~> 1.1.7) - sinatra (~> 1.4, >= 1.4.6) - stud (~> 0.0.19) - thread_safe (~> 0.3.5) - treetop (< 1.5.0) - -PATH - remote: ./logstash-core-event-java - specs: - logstash-core-event-java (5.0.0.dev-java) - jar-dependencies - ruby-maven (~> 3.3.9) - -PATH - remote: ./logstash-core-plugin-api - specs: - logstash-core-plugin-api (2.0.0-java) - logstash-core (= 5.0.0.dev) - -GEM - remote: https://rubygems.org/ - specs: - addressable (2.3.8) - arr-pm (0.0.10) - cabin (> 0) - backports (3.6.8) - benchmark-ips (2.6.1) - builder (3.2.2) - cabin (0.8.1) - childprocess (0.5.9) - ffi (~> 1.0, >= 1.0.11) - chronic_duration (0.10.6) - numerizer (~> 0.1.1) - ci_reporter (2.0.0) - builder (>= 2.1.2) - ci_reporter_rspec (1.0.0) - ci_reporter (~> 2.0) - rspec (>= 2.14, < 4) - clamp (0.6.5) - coderay (1.1.1) - concurrent-ruby (1.0.0-java) - coveralls (0.8.13) - json (~> 1.8) - simplecov (~> 0.11.0) - term-ansicolor (~> 1.3) - thor (~> 0.19.1) - tins (~> 1.6.0) - diff-lcs (1.2.5) - docile (1.1.5) - faraday (0.9.2) - multipart-post (>= 1.2, < 3) - ffi (1.9.10-java) - file-dependencies (0.1.6) - minitar - filesize (0.0.4) - fivemat (1.3.2) - flores (0.0.6) - fpm (1.3.3) - arr-pm (~> 0.0.9) - backports (>= 2.6.2) - cabin (>= 0.6.0) - childprocess - clamp (~> 0.6) - ffi - json (>= 1.7.7) - gem_publisher (1.5.0) - gems (0.8.3) - i18n (0.6.9) - insist (1.0.0) - jar-dependencies (0.3.2) - jls-grok (0.11.2) - cabin (>= 0.6.0) - jrjackson (0.3.9-java) - jruby-monitoring (0.3.1) - jruby-openssl (0.9.16-java) - json (1.8.3-java) - kramdown (1.10.0) - method_source (0.8.2) - minitar (0.5.4) - multipart-post (2.0.0) - numerizer (0.1.1) - octokit (3.8.0) - sawyer (~> 0.6.0, >= 0.5.3) - polyglot (0.3.5) - pry (0.10.3-java) - coderay (~> 1.1.0) - method_source (~> 0.8.1) - slop (~> 3.4) - spoon (~> 0.0) - puma (2.16.0-java) - rack (1.6.4) - rack-protection (1.5.3) - rack - rack-test (0.6.3) - rack (>= 1.0) - rake (11.1.2) - rspec (3.1.0) - rspec-core (~> 3.1.0) - rspec-expectations (~> 3.1.0) - rspec-mocks (~> 3.1.0) - rspec-core (3.1.7) - rspec-support (~> 3.1.0) - rspec-expectations (3.1.2) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.1.0) - rspec-mocks (3.1.3) - rspec-support (~> 3.1.0) - rspec-support (3.1.2) - rspec-wait (0.0.8) - rspec (>= 2.11, < 3.5) - ruby-maven (3.3.10) - ruby-maven-libs (~> 3.3.1) - ruby-maven-libs (3.3.3) - rubyzip (1.1.7) - sawyer (0.6.0) - addressable (~> 2.3.5) - faraday (~> 0.8, < 0.10) - simplecov (0.11.2) - docile (~> 1.1.0) - json (~> 1.8) - simplecov-html (~> 0.10.0) - simplecov-html (0.10.0) - sinatra (1.4.7) - rack (~> 1.5) - rack-protection (~> 1.4) - tilt (>= 1.3, < 3) - slop (3.6.0) - spoon (0.0.4) - ffi - stud (0.0.22) - term-ansicolor (1.3.2) - tins (~> 1.0) - thor (0.19.1) - thread_safe (0.3.5-java) - tilt (2.0.2) - tins (1.6.0) - treetop (1.4.15) - polyglot - polyglot (>= 0.3.1) - -PLATFORMS - java - -DEPENDENCIES - benchmark-ips - ci_reporter_rspec (= 1.0.0) - coveralls - file-dependencies (= 0.1.6) - flores (~> 0.0.6) - fpm (~> 1.3.3) - gems (~> 0.8.3) - logstash-codec-json! - logstash-codec-json_lines! - logstash-codec-line! - logstash-codec-plain! - logstash-core (= 5.0.0.dev)! - logstash-core-event-java (= 5.0.0.dev)! - logstash-core-plugin-api (= 2.0.0)! - logstash-devutils! - logstash-filter-clone! - logstash-filter-grok! - logstash-filter-multiline! - logstash-filter-mutate! - logstash-input-generator! - logstash-input-stdin! - logstash-input-tcp! - logstash-output-stdout! - logstash-patterns-core! - octokit (= 3.8.0) - rack-test - rspec (~> 3.1.0) - rubyzip (~> 1.1.7) - simplecov - stud (~> 0.0.22) - tins (= 1.6) diff --git a/lib/bootstrap/bundler.rb b/lib/bootstrap/bundler.rb index 1a7bb0572..13c04b58a 100644 --- a/lib/bootstrap/bundler.rb +++ b/lib/bootstrap/bundler.rb @@ -78,7 +78,7 @@ module LogStash # @return [String, Exception] the installation captured output and any raised exception or nil if none def invoke!(options = {}) options = {:max_tries => 10, :clean => false, :install => false, :update => false, :local => false, - :all => false, :package => false, :without => [:development]}.merge(options) + :jobs => 12, :all => false, :package => false, :without => [:development]}.merge(options) options[:without] = Array(options[:without]) options[:update] = Array(options[:update]) if options[:update] diff --git a/logstash-core-event-java/lib/logstash-core-event-java/version.rb b/logstash-core-event-java/lib/logstash-core-event-java/version.rb index cfc6a1624..ceaa49b47 100644 --- a/logstash-core-event-java/lib/logstash-core-event-java/version.rb +++ b/logstash-core-event-java/lib/logstash-core-event-java/version.rb @@ -5,4 +5,4 @@ # Note to authors: this should not include dashes because 'gem' barfs if # you include a dash in the version string. -LOGSTASH_CORE_EVENT_JAVA_VERSION = "5.0.0.dev" +LOGSTASH_CORE_EVENT_JAVA_VERSION = "5.0.0" diff --git a/logstash-core-event-java/lib/logstash-core-event-java_jars.rb b/logstash-core-event-java/lib/logstash-core-event-java_jars.rb index 143d7a3e0..d45036714 100644 --- a/logstash-core-event-java/lib/logstash-core-event-java_jars.rb +++ b/logstash-core-event-java/lib/logstash-core-event-java_jars.rb @@ -1,6 +1,14 @@ # this is a generated file, to avoid over-writing it just delete this comment -require 'jar_dependencies' +begin + require 'jar_dependencies' +rescue LoadError + require 'com/fasterxml/jackson/core/jackson-core/2.7.1/jackson-core-2.7.1.jar' + require 'com/fasterxml/jackson/core/jackson-annotations/2.7.0/jackson-annotations-2.7.0.jar' + require 'com/fasterxml/jackson/core/jackson-databind/2.7.1-1/jackson-databind-2.7.1-1.jar' +end -require_jar( 'com.fasterxml.jackson.core', 'jackson-core', '2.7.1' ) -require_jar( 'com.fasterxml.jackson.core', 'jackson-annotations', '2.7.0' ) -require_jar( 'com.fasterxml.jackson.core', 'jackson-databind', '2.7.1-1' ) +if defined? Jars + require_jar( 'com.fasterxml.jackson.core', 'jackson-core', '2.7.1' ) + require_jar( 'com.fasterxml.jackson.core', 'jackson-annotations', '2.7.0' ) + require_jar( 'com.fasterxml.jackson.core', 'jackson-databind', '2.7.1-1' ) +end diff --git a/logstash-core-event/lib/logstash-core-event/version.rb b/logstash-core-event/lib/logstash-core-event/version.rb index e1e0774da..04253b622 100644 --- a/logstash-core-event/lib/logstash-core-event/version.rb +++ b/logstash-core-event/lib/logstash-core-event/version.rb @@ -5,4 +5,4 @@ # Note to authors: this should not include dashes because 'gem' barfs if # you include a dash in the version string. -LOGSTASH_CORE_EVENT_VERSION = "5.0.0.dev" +LOGSTASH_CORE_EVENT_VERSION = "5.0.0" diff --git a/logstash-core-plugin-api/lib/logstash-core-plugin-api/version.rb b/logstash-core-plugin-api/lib/logstash-core-plugin-api/version.rb index ced593939..9e4e415bd 100644 --- a/logstash-core-plugin-api/lib/logstash-core-plugin-api/version.rb +++ b/logstash-core-plugin-api/lib/logstash-core-plugin-api/version.rb @@ -1,2 +1,2 @@ # encoding: utf-8 -LOGSTASH_CORE_PLUGIN_API = "2.0.0" +LOGSTASH_CORE_PLUGIN_API = "2.1.7" diff --git a/logstash-core-plugin-api/logstash-core-plugin-api.gemspec b/logstash-core-plugin-api/logstash-core-plugin-api.gemspec index 8764ad5b2..f7787f65a 100644 --- a/logstash-core-plugin-api/logstash-core-plugin-api.gemspec +++ b/logstash-core-plugin-api/logstash-core-plugin-api.gemspec @@ -11,13 +11,13 @@ Gem::Specification.new do |gem| gem.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html" gem.license = "Apache License (2.0)" - gem.files = Dir.glob(["logstash-core-event.gemspec", "lib/**/*.rb", "spec/**/*.rb"]) + gem.files = Dir.glob(["logstash-core-plugin-api.gemspec", "lib/**/*.rb", "spec/**/*.rb"]) gem.test_files = gem.files.grep(%r{^(test|spec|features)/}) gem.name = "logstash-core-plugin-api" gem.require_paths = ["lib"] gem.version = LOGSTASH_CORE_PLUGIN_API - gem.add_runtime_dependency "logstash-core", "5.0.0.dev" + gem.add_runtime_dependency "logstash-core", "5.0.0" # Make sure we dont build this gem from a non jruby # environment. diff --git a/logstash-core/lib/logstash-core/version.rb b/logstash-core/lib/logstash-core/version.rb index 737441c5a..ad8cbbd34 100644 --- a/logstash-core/lib/logstash-core/version.rb +++ b/logstash-core/lib/logstash-core/version.rb @@ -5,4 +5,4 @@ # Note to authors: this should not include dashes because 'gem' barfs if # you include a dash in the version string. -LOGSTASH_CORE_VERSION = "5.0.0.dev" +LOGSTASH_CORE_VERSION = "5.0.0" diff --git a/logstash-core/lib/logstash/api/commands/system/basicinfo_command.rb b/logstash-core/lib/logstash/api/commands/system/basicinfo_command.rb index 7a3207399..30e74f692 100644 --- a/logstash-core/lib/logstash/api/commands/system/basicinfo_command.rb +++ b/logstash-core/lib/logstash/api/commands/system/basicinfo_command.rb @@ -1,6 +1,7 @@ # encoding: utf-8 require 'logstash/api/commands/base' require "logstash/util/duration_formatter" +require 'logstash/build' module LogStash module Api @@ -11,9 +12,7 @@ module LogStash def run { "hostname" => hostname, - "version" => { - "number" => LOGSTASH_VERSION - } + "version" => { "number" => LOGSTASH_VERSION }.merge(BUILD_INFO) } end end diff --git a/logstash-core/lib/logstash/build.rb b/logstash-core/lib/logstash/build.rb new file mode 100644 index 000000000..fb2132c22 --- /dev/null +++ b/logstash-core/lib/logstash/build.rb @@ -0,0 +1,6 @@ +# encoding: utf-8 + +# DO NOT EDIT +# this file acts as a placeholder for build information when executing +# logstash in dev mode (outside of a package build) +BUILD_INFO = {} diff --git a/logstash-core/lib/logstash/version.rb b/logstash-core/lib/logstash/version.rb index 676dde5ac..c2d7b8d14 100644 --- a/logstash-core/lib/logstash/version.rb +++ b/logstash-core/lib/logstash/version.rb @@ -11,4 +11,4 @@ # eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be # fixed. -LOGSTASH_VERSION = "5.0.0.dev" +LOGSTASH_VERSION = "5.0.0" diff --git a/logstash-core/logstash-core.gemspec b/logstash-core/logstash-core.gemspec index f63617656..938612d4e 100644 --- a/logstash-core/logstash-core.gemspec +++ b/logstash-core/logstash-core.gemspec @@ -17,7 +17,7 @@ Gem::Specification.new do |gem| gem.require_paths = ["lib"] gem.version = LOGSTASH_CORE_VERSION - gem.add_runtime_dependency "logstash-core-event-java", "~> 5.0.0.dev" + gem.add_runtime_dependency "logstash-core-event-java", "5.0.0" gem.add_runtime_dependency "cabin", "~> 0.8.0" #(Apache 2.0 license) gem.add_runtime_dependency "pry", "~> 0.10.1" #(Ruby license) diff --git a/rakelib/artifacts.rake b/rakelib/artifacts.rake index 581d8c798..877b9ef82 100644 --- a/rakelib/artifacts.rake +++ b/rakelib/artifacts.rake @@ -2,6 +2,9 @@ require "logstash/version" namespace "artifact" do + SNAPSHOT_BUILD = ENV["RELEASE"] != "1" + PACKAGE_SUFFIX = SNAPSHOT_BUILD ? "-SNAPSHOT" : "" + def package_files [ "LICENSE", @@ -13,6 +16,13 @@ namespace "artifact" do "lib/bootstrap/**/*", "lib/pluginmanager/**/*", "lib/systeminstall/**/*", + "logstash-core/lib/**/*", + "logstash-core/locales/**/*", + "logstash-core/*.gemspec", + "logstash-core-event-java/lib/**/*", + "logstash-core-event-java/*.gemspec", + "logstash-core-plugin-api/lib/**/*", + "logstash-core-plugin-api/*.gemspec", "patterns/**/*", "vendor/??*/**/*", # To include ruby-maven's hidden ".mvn" directory, we need to @@ -55,18 +65,75 @@ namespace "artifact" do end.flatten.uniq end - task "all" => ["prepare"] do + desc "Generate rpm, deb, tar and zip artifacts" + task "all" => ["prepare", "build"] + + desc "Build a tar.gz of default logstash plugins with all dependencies" + task "tar" => ["prepare", "generate_build_metadata"] do + puts("[artifact:tar] Building tar.gz of default plugins") + build_tar + end + + desc "Build a zip of default logstash plugins with all dependencies" + task "zip" => ["prepare", "generate_build_metadata"] do + puts("[artifact:zip] Building zip of default plugins") + build_zip + end + + desc "Build an RPM of logstash with all dependencies" + task "rpm" => ["prepare", "generate_build_metadata"] do + puts("[artifact:rpm] building rpm package") + package("centos", "5") + end + + desc "Build a DEB of logstash with all dependencies" + task "deb" => ["prepare", "generate_build_metadata"] do + puts("[artifact:deb] building deb package") + package("ubuntu", "12.04") + end + + desc "Generate logstash core gems" + task "gems" => ["prepare"] do + Rake::Task["artifact:build-logstash-core"].invoke + Rake::Task["artifact:build-logstash-core-event"].invoke + Rake::Task["artifact:build-logstash-core-plugin-api"].invoke + end + + # "all-plugins" version of tasks + desc "Generate rpm, deb, tar and zip artifacts (\"all-plugins\" version)" + task "all-all-plugins" => ["prepare-all", "build"] + + desc "Build a zip of all logstash plugins from logstash-plugins github repo" + task "zip-all-plugins" => ["prepare-all", "generate_build_metadata"] do + puts("[artifact:zip] Building zip of all plugins") + build_zip "-all-plugins" + end + + desc "Build a tar.gz of all logstash plugins from logstash-plugins github repo" + task "tar-all-plugins" => ["prepare-all", "generate_build_metadata"] do + puts("[artifact:tar] Building tar.gz of all plugins") + build_tar "-all-plugins" + end + + # Auxiliary tasks + task "build" => [:generate_build_metadata] do + Rake::Task["artifact:gems"].invoke unless SNAPSHOT_BUILD Rake::Task["artifact:deb"].invoke Rake::Task["artifact:rpm"].invoke Rake::Task["artifact:zip"].invoke Rake::Task["artifact:tar"].invoke end - task "all-all-plugins" => ["prepare-all"] do - Rake::Task["artifact:deb"].invoke - Rake::Task["artifact:rpm"].invoke - Rake::Task["artifact:zip"].invoke - Rake::Task["artifact:tar"].invoke + task "generate_build_metadata" do + return if defined?(BUILD_METADATA_FILE) + BUILD_METADATA_FILE = Tempfile.new('build.rb') + build_info = { + "build_date" => Time.now.iso8601, + "build_sha" => `git rev-parse HEAD`.chomp, + "build_snapshot" => SNAPSHOT_BUILD + } + metadata = [ "# encoding: utf-8", "BUILD_INFO = #{build_info}" ] + IO.write(BUILD_METADATA_FILE.path, metadata.join("\n")) end # We create an empty bundle config file @@ -78,8 +145,8 @@ namespace "artifact" do end # locate the "gem "logstash-core" ..." line in Gemfile, and if the :path => "..." option if specified - # build and install the local logstash-core gem otherwise just do nothing, bundler will deal with it. - task "install-logstash-core" do + # build the local logstash-core gem otherwise just do nothing, bundler will deal with it. + task "build-logstash-core" do # regex which matches a Gemfile gem definition for the logstash-core gem and captures the :path option gem_line_regex = /^\s*gem\s+["']logstash-core["'](?:\s*,\s*["'][^"^']+["'])?(?:\s*,\s*:path\s*=>\s*["']([^"^']+)["'])?/i @@ -90,15 +157,16 @@ namespace "artifact" do path = matches.first[gem_line_regex, 1] if path - Rake::Task["plugin:install-local-core-gem"].invoke("logstash-core", path) + Rake::Task["plugin:build-local-core-gem"].invoke("logstash-core", path) else - puts("[artifact:install-logstash-core] using logstash-core from Rubygems") + puts "The Gemfile should reference \"logstash-core\" gem locally through :path, but found instead: #{matches}" + exit(1) end end # # locate the "gem "logstash-core-event*" ..." line in Gemfile, and if the :path => "." option if specified - # # build and install the local logstash-core-event* gem otherwise just do nothing, bundler will deal with it. - task "install-logstash-core-event" do + # # build the local logstash-core-event* gem otherwise just do nothing, bundler will deal with it. + task "build-logstash-core-event" do # regex which matches a Gemfile gem definition for the logstash-core-event* gem and captures the gem name and :path option gem_line_regex = /^\s*gem\s+["'](logstash-core-event[^"^']*)["'](?:\s*,\s*["'][^"^']+["'])?(?:\s*,\s*:path\s*=>\s*["']([^"^']+)["'])?/i @@ -110,15 +178,16 @@ namespace "artifact" do path = matches.first[gem_line_regex, 2] if path - Rake::Task["plugin:install-local-core-gem"].invoke(name, path) + Rake::Task["plugin:build-local-core-gem"].invoke(name, path) else - puts("[artifact:install-logstash-core] using #{name} from Rubygems") + puts "The Gemfile should reference \"logstash-core-event\" gem locally through :path, but found instead: #{matches}" + exit(1) end end # locate the "gem "logstash-core-plugin-api" ..." line in Gemfile, and if the :path => "..." option if specified - # build and install the local logstash-core-plugin-api gem otherwise just do nothing, bundler will deal with it. - task "install-logstash-core-plugin-api" do + # build the local logstash-core-plugin-api gem otherwise just do nothing, bundler will deal with it. + task "build-logstash-core-plugin-api" do # regex which matches a Gemfile gem definition for the logstash-core gem and captures the :path option gem_line_regex = /^\s*gem\s+["']logstash-core-plugin-api["'](?:\s*,\s*["'][^"^']+["'])?(?:\s*,\s*:path\s*=>\s*["']([^"^']+)["'])?/i @@ -129,86 +198,87 @@ namespace "artifact" do path = matches.first[gem_line_regex, 1] if path - Rake::Task["plugin:install-local-core-gem"].invoke("logstash-core-plugin-api", path) + Rake::Task["plugin:build-local-core-gem"].invoke("logstash-core-plugin-api", path) else - puts("[artifact:install-logstash-core-plugin-api] using logstash-core from Rubygems") + puts "The Gemfile should reference \"logstash-core-plugin-api\" gem locally through :path, but found instead: #{matches}" + exit(1) end end - task "prepare" => ["bootstrap", "plugin:install-default", "install-logstash-core", "install-logstash-core-event", "install-logstash-core-plugin-api", "clean-bundle-config"] - - task "prepare-all" => ["bootstrap", "plugin:install-all", "install-logstash-core", "install-logstash-core-event", "install-logstash-core-plugin-api", "clean-bundle-config"] - - desc "Build a tar.gz of default logstash plugins with all dependencies" - task "tar" => ["prepare"] do - puts("[artifact:tar] Building tar.gz of default plugins") - build_tar + task "prepare" do + if ENV['SKIP_PREPARE'] != "1" + ["bootstrap", "plugin:install-default", "artifact:clean-bundle-config"].each {|task| Rake::Task[task].invoke } + end end - desc "Build a tar.gz of all logstash plugins from logstash-plugins github repo" - task "tar-all-plugins" => ["prepare-all"] do - puts("[artifact:tar] Building tar.gz of all plugins") - build_tar "-all-plugins" + task "prepare-all" do + if ENV['SKIP_PREPARE'] != "1" + ["bootstrap", "plugin:install-all", "artifact:clean-bundle-config"].each {|task| Rake::Task[task].invoke } + end end def build_tar(tar_suffix = nil) require "zlib" require "archive/tar/minitar" require "logstash/version" - tarpath = "build/logstash#{tar_suffix}-#{LOGSTASH_VERSION}.tar.gz" + tarpath = "build/logstash#{tar_suffix}-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}.tar.gz" puts("[artifact:tar] building #{tarpath}") gz = Zlib::GzipWriter.new(File.new(tarpath, "wb"), Zlib::BEST_COMPRESSION) tar = Archive::Tar::Minitar::Output.new(gz) files.each do |path| - stat = File.lstat(path) - path_in_tar = "logstash-#{LOGSTASH_VERSION}/#{path}" - opts = { - :size => stat.size, - :mode => stat.mode, - :mtime => stat.mtime - } - if stat.directory? - tar.tar.mkdir(path_in_tar, opts) - else - tar.tar.add_file_simple(path_in_tar, opts) do |io| - File.open(path,'rb') do |fd| - chunk = nil - size = 0 - size += io.write(chunk) while chunk = fd.read(16384) - if stat.size != size - raise "Failure to write the entire file (#{path}) to the tarball. Expected to write #{stat.size} bytes; actually write #{size}" - end - end - end - end + write_to_tar(tar, path, "logstash-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}/#{path}") end + + # add build.rb to tar + metadata_file_path_in_tar = File.join("logstash-core", "lib", "logstash", "build.rb") + path_in_tar = File.join("logstash-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}", metadata_file_path_in_tar) + write_to_tar(tar, BUILD_METADATA_FILE.path, path_in_tar) + tar.close gz.close puts "Complete: #{tarpath}" end - desc "Build a zip of default logstash plugins with all dependencies" - task "zip" => ["prepare"] do - puts("[artifact:zip] Building zip of default plugins") - build_zip - end - - desc "Build a zip of all logstash plugins from logstash-plugins github repo" - task "zip-all-plugins" => ["prepare-all"] do - puts("[artifact:zip] Building zip of all plugins") - build_zip "-all-plugins" + def write_to_tar(tar, path, path_in_tar) + stat = File.lstat(path) + opts = { + :size => stat.size, + :mode => stat.mode, + :mtime => stat.mtime + } + if stat.directory? + tar.tar.mkdir(path_in_tar, opts) + else + tar.tar.add_file_simple(path_in_tar, opts) do |io| + File.open(path,'rb') do |fd| + chunk = nil + size = 0 + size += io.write(chunk) while chunk = fd.read(16384) + if stat.size != size + raise "Failure to write the entire file (#{path}) to the tarball. Expected to write #{stat.size} bytes; actually write #{size}" + end + end + end + end end def build_zip(zip_suffix = "") require 'zip' - zippath = "build/logstash#{zip_suffix}-#{LOGSTASH_VERSION}.zip" + zippath = "build/logstash#{zip_suffix}-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}.zip" puts("[artifact:zip] building #{zippath}") File.unlink(zippath) if File.exists?(zippath) Zip::File.open(zippath, Zip::File::CREATE) do |zipfile| files.each do |path| - path_in_zip = "logstash-#{LOGSTASH_VERSION}/#{path}" + path_in_zip = "logstash-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}/#{path}" zipfile.add(path_in_zip, path) end + + # add build.rb to zip + metadata_file_path_in_zip = File.join("logstash-core", "lib", "logstash", "build.rb") + path_in_zip = File.join("logstash-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}", metadata_file_path_in_zip) + path = BUILD_METADATA_FILE.path + Zip.continue_on_exists_proc = true + zipfile.add(path_in_zip, path) end puts "Complete: #{zippath}" end @@ -221,6 +291,10 @@ namespace "artifact" do dir = FPM::Package::Dir.new + metadata_file_path = File.join("logstash-core", "lib", "logstash", "build.rb") + metadata_source_file_path = BUILD_METADATA_FILE.path + dir.input("#{metadata_source_file_path}=/usr/share/logstash/#{metadata_file_path}") + files.each do |path| next if File.directory?(path) # Omit any config dir from /usr/share/logstash for packages, since we're @@ -246,7 +320,7 @@ namespace "artifact" do # produce: logstash-5.0.0-alpha1.deb" # produce: logstash-5.0.0-alpha1.rpm - package_filename = "logstash-#{LOGSTASH_VERSION}.TYPE" + package_filename = "logstash-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}.TYPE" case platform when "redhat", "centos" @@ -350,15 +424,4 @@ namespace "artifact" do end end # def package - desc "Build an RPM of logstash with all dependencies" - task "rpm" => ["prepare"] do - puts("[artifact:rpm] building rpm package") - package("centos", "5") - end - - desc "Build a DEB of logstash with all dependencies" - task "deb" => ["prepare"] do - puts("[artifact:deb] building deb package") - package("ubuntu", "12.04") - end end diff --git a/rakelib/build.rake b/rakelib/build.rake index 2b443add8..45aec1f9b 100644 --- a/rakelib/build.rake +++ b/rakelib/build.rake @@ -4,3 +4,6 @@ end directory "build/bootstrap" => "build" do |task, args| mkdir_p task.name unless File.directory?(task.name) end +directory "build/gems" => "build" do |task, args| + mkdir_p task.name unless File.directory?(task.name) +end diff --git a/rakelib/plugin.rake b/rakelib/plugin.rake index 0bc1b7089..79be84172 100644 --- a/rakelib/plugin.rake +++ b/rakelib/plugin.rake @@ -81,7 +81,7 @@ namespace "plugin" do task.reenable # Allow this task to be run again end - task "build-local-core-gem", [:name, :path] do |task, args| + task "build-local-core-gem", [:name, :path] => ["build/gems"] do |task, args| name = args[:name] path = args[:path] @@ -89,10 +89,12 @@ namespace "plugin" do puts("[plugin:build-local-core-gem] Building #{File.join(path, name)}.gemspec") + gem_path = nil Dir.chdir(path) do spec = Gem::Specification.load("#{name}.gemspec") - Gem::Package.build(spec) + gem_path = Gem::Package.build(spec) end + FileUtils.cp(File.join(path, gem_path), "build/gems/") task.reenable # Allow this task to be run again end diff --git a/rakelib/version.rake b/rakelib/version.rake new file mode 100644 index 000000000..45655cd33 --- /dev/null +++ b/rakelib/version.rake @@ -0,0 +1,132 @@ +require 'yaml' + +def get_versions + yaml_versions = YAML.safe_load(IO.read("versions.yml")) + { + "logstash" => { + "location" => File.join("logstash-core", "lib", "logstash", "version.rb"), + "yaml_version" => yaml_versions["logstash"], + "current_version" => get_version(File.join("logstash-core", "lib", "logstash", "version.rb")), + }, + "logstash-core" => { + "location" => File.join("logstash-core", "lib", "logstash-core", "version.rb"), + "yaml_version" => yaml_versions["logstash-core"], + "current_version" => get_version(File.join("logstash-core", "lib", "logstash-core", "version.rb")), + }, + "logstash-core-event" => { + "location" => File.join("logstash-core-event", "lib", "logstash-core-event", "version.rb"), + "yaml_version" => yaml_versions["logstash-core-event"], + "current_version" => get_version(File.join("logstash-core-event", "lib", "logstash-core-event", "version.rb")), + }, + "logstash-core-event-java" => { + "location" => File.join("logstash-core-event-java", "lib", "logstash-core-event-java", "version.rb"), + "yaml_version" => yaml_versions["logstash-core-event-java"], + "current_version" => get_version(File.join("logstash-core-event-java", "lib", "logstash-core-event-java", "version.rb")), + }, + "logstash-core-plugin-api" => { + "location" => File.join("logstash-core-plugin-api", "lib", "logstash-core-plugin-api", "version.rb"), + "yaml_version" => yaml_versions["logstash-core-plugin-api"], + "current_version" => get_version(File.join("logstash-core-plugin-api", "lib", "logstash-core-plugin-api", "version.rb")), + } + } +end + +def get_version(file) + text = IO.read(file) + version = text.match(/^[A-Z_]+ = "(.+?)"/) + version[1] +end + +namespace :version do + + desc "check if the versions.yml is out of sync with .gemspecs and other references" + task :check do + out_of_sync = get_versions.select do |component, metadata| + metadata["yaml_version"] != metadata["current_version"] + end + if out_of_sync.any? + out_of_sync.each do |component, metadata| + puts "#{component} is out of sync. CURRENT: #{metadata['current_version']} | YAML: #{metadata['yaml_version']}" + end + exit(1) + end + end + + desc "push versions found in versions.yml to all component version locations" + task :sync do + versions = get_versions + # update version.rb files + versions.select do |component, metadata| + next if metadata["yaml_version"] == metadata["current_version"] + puts "Updating \"#{component}\" from \"#{metadata['current_version']}\" to \"#{metadata['yaml_version']}\"" + text = IO.read(metadata["location"]) + IO.write(metadata["location"], text.gsub(metadata["current_version"], metadata["yaml_version"])) + end + + # update dependencies + # + # logstash-core depends on logstash-core-event-java + # ./logstash-core/logstash-core.gemspec: gem.add_runtime_dependency "logstash-core-event-java", "~> 5.0.0.dev" + logstash_core_gemspec = File.join("logstash-core", "logstash-core.gemspec") + logstash_core_event_java_version = versions['logstash-core-event-java']['yaml_version'] + text = IO.read(logstash_core_gemspec) + IO.write(logstash_core_gemspec, text.sub( + / gem.add_runtime_dependency \"logstash-core-event-java\", \".+?\"/, + " gem.add_runtime_dependency \"logstash-core-event-java\", \"#{logstash_core_event_java_version}\"")) + + # logstash-core-event-java depends on logstash-code + # ./logstash-core-plugin-api/logstash-core-plugin-api.gemspec: gem.add_runtime_dependency "logstash-core", "5.0.0.dev" + logstash_core_plugin_api_gemspec = File.join("logstash-core-plugin-api", "logstash-core-plugin-api.gemspec") + logstash_core_version = versions['logstash-core']['yaml_version'] + text = IO.read(logstash_core_plugin_api_gemspec) + IO.write(logstash_core_plugin_api_gemspec, text.sub( + / gem.add_runtime_dependency \"logstash-core\", \".+?\"/, + " gem.add_runtime_dependency \"logstash-core\", \"#{logstash_core_version}\"")) + end + + desc "show version of core components" + task :show do + Rake::Task["version:sync"].invoke; Rake::Task["version:sync"].reenable + get_versions.each do |component, metadata| + puts "#{component}: #{metadata['yaml_version']}" + end + end + + desc "set version of logstash, logstash-core, logstash-core-event, logstash-core-event-java" + task :set, [:version] => [:validate] do |t, args| + hash = {} + get_versions.each do |component, metadata| + # we just assume that, usually, all components except + # "logstash-core-plugin-api" will be versioned together + # so let's skip this one and have a separate task for it + if component == "logstash-core-plugin-api" + hash[component] = metadata["yaml_version"] + else + hash[component] = args[:version] + end + end + IO.write("versions.yml", hash.to_yaml) + Rake::Task["version:sync"].invoke; Rake::Task["version:sync"].reenable + end + + desc "set version of logstash-core-plugin-api" + task :set_plugin_api, [:version] => [:validate] do |t, args| + hash = {} + get_versions.each do |component, metadata| + if component == "logstash-core-plugin-api" + hash[component] = args[:version] + else + hash[component] = metadata["yaml_version"] + end + end + IO.write("versions.yml", hash.to_yaml) + Rake::Task["version:sync"].invoke; Rake::Task["version:sync"].reenable + end + + task :validate, :version do |t, args| + unless Regexp.new('^\d+\.\d+\.\d+(?:-\w+\d+)?$').match(args[:version]) + abort("Invalid version argument: \"#{args[:version]}\". Aborting...") + end + end + +end diff --git a/versions.yml b/versions.yml new file mode 100644 index 000000000..a7e6c26df --- /dev/null +++ b/versions.yml @@ -0,0 +1,6 @@ +--- +logstash: 5.0.0 +logstash-core: 5.0.0 +logstash-core-event: 5.0.0 +logstash-core-event-java: 5.0.0 +logstash-core-plugin-api: 2.1.7