github-mirrors/logstash
1
0
Fork 0
mirror of https://github.com/elastic/logstash.git synced 2025-04-24 22:57:16 -04:00
Code Issues Projects Releases Packages Wiki Activity

add single command release task and other release support tooling

Browse source 
Purpose:

* manage releases through a minimum number of rake tasks
* simplify building of snapshot builds
* create staged artifacts, candidates for releases, that required no changes to become releases
* this means the snapshot release process will not involve publishing gems, therefore:
* the gem artifacts should only be published to rubygems as a final artifact, at the time of GA

Changes:

* release artifacts no longer depend on gems of core components
* all core components are used locally AS-IS to minimize code changes between snapshot, RC and GA
* `versions.yml` describes the versions of all logstash parts and package
  * `rake version:set[version]` manage the yaml file and push the changes to the gemspecs/version.rb files
  * `rake version:set_plugin_api[version]` manage the yaml file and push the changes to the gemspecs/version.rb files
  * `rake artifact:all` generates SNAPSHOT artifacts: tar.gz, zip, rpm, deb
  * `RELEASE=1 rake artifact:all` creates release candidate artifacts + 4 gems: logstash-core, logstash-core-event, logstash-core-event-java and logstash-core-plugin-api

implements #5416 and #5414

Fixes #5460
This commit is contained in:
Joao Duarte 2016-06-01 17:46:11 +01:00
parent 047c071301
commit fa0eeb579f
29 changed files with 387 additions and 144 deletions
Download patch file Download diff file Expand all files Collapse all files

6
Gemfile
View file

@ -2,9 +2,9 @@
# If you modify this file manually all comments and formatting will be lost.
source "https://rubygems.org"
gem "logstash-core", "5.0.0.alpha4.snapshot3"
gem "logstash-core-event-java", "5.0.0.alpha4.snapshot3"
gem "logstash-core-plugin-api", "2.1.9"
gem "logstash-core", :path => "./logstash-core"
gem "logstash-core-event-java", :path => "./logstash-core-event-java"
gem "logstash-core-plugin-api", :path => "./logstash-core-plugin-api"
gem "file-dependencies", "0.1.6"
gem "ci_reporter_rspec", "1.0.0", :group => :development
gem "simplecov", :group => :development

78
Gemfile.jruby-1.9.lock
View file

@ -1,3 +1,40 @@
PATH
remote: ./logstash-core
specs:
logstash-core (5.0.0.pre.alpha4-java)
cabin (~> 0.8.0)
chronic_duration (= 0.10.6)
clamp (~> 0.6.5)
concurrent-ruby (= 1.0.0)
filesize (= 0.0.4)
gems (~> 0.8.3)
i18n (= 0.6.9)
jrjackson (~> 0.3.7)
jrmonitor (~> 0.4.2)
jruby-openssl (= 0.9.16)
logstash-core-event-java (= 5.0.0.pre.alpha4)
minitar (~> 0.5.4)
pry (~> 0.10.1)
puma (~> 2.16)
rubyzip (~> 1.1.7)
sinatra (~> 1.4, >= 1.4.6)
stud (~> 0.0.19)
thread_safe (~> 0.3.5)
treetop (< 1.5.0)
PATH
remote: ./logstash-core-event-java
specs:
logstash-core-event-java (5.0.0.pre.alpha4-java)
jar-dependencies
ruby-maven (~> 3.3.9)
PATH
remote: ./logstash-core-plugin-api
specs:
logstash-core-plugin-api (2.1.9-java)
logstash-core (= 5.0.0.pre.alpha4)
GEM
remote: https://rubygems.org/
specs:
@ -47,12 +84,12 @@ GEM
domain_name (0.5.20160615)
unf (>= 0.0.5, < 1.0.0)
edn (1.1.1)
elasticsearch (1.0.17)
elasticsearch-api (= 1.0.17)
elasticsearch-transport (= 1.0.17)
elasticsearch-api (1.0.17)
elasticsearch (1.0.18)
elasticsearch-api (= 1.0.18)
elasticsearch-transport (= 1.0.18)
elasticsearch-api (1.0.18)
multi_json
elasticsearch-transport (1.0.17)
elasticsearch-transport (1.0.18)
faraday
multi_json
equalizer (0.0.10)
@ -144,31 +181,6 @@ GEM
logstash-codec-rubydebug (3.0.1)
awesome_print
logstash-core-plugin-api (~> 2.0)
logstash-core (5.0.0.alpha4.snapshot3-java)
cabin (~> 0.8.0)
chronic_duration (= 0.10.6)
clamp (~> 0.6.5)
concurrent-ruby (= 1.0.0)
filesize (= 0.0.4)
gems (~> 0.8.3)
i18n (= 0.6.9)
jrjackson (~> 0.3.7)
jrmonitor (~> 0.4.2)
jruby-openssl (= 0.9.16)
logstash-core-event-java (= 5.0.0.alpha4.snapshot3)
minitar (~> 0.5.4)
pry (~> 0.10.1)
puma (~> 2.16)
rubyzip (~> 1.1.7)
sinatra (~> 1.4, >= 1.4.6)
stud (~> 0.0.19)
thread_safe (~> 0.3.5)
treetop (< 1.5.0)
logstash-core-event-java (5.0.0.alpha4.snapshot3-java)
jar-dependencies
ruby-maven (~> 3.3.9)
logstash-core-plugin-api (2.1.9-java)
logstash-core (= 5.0.0.alpha4.snapshot3)
logstash-devutils (1.0.2-java)
fivemat
gem_publisher
@ -621,9 +633,9 @@ DEPENDENCIES
logstash-codec-netflow
logstash-codec-plain
logstash-codec-rubydebug
logstash-core (= 5.0.0.alpha4.snapshot3)
logstash-core-event-java (= 5.0.0.alpha4.snapshot3)
logstash-core-plugin-api (= 2.1.9)
logstash-core!
logstash-core-event-java!
logstash-core-plugin-api!
logstash-devutils
logstash-filter-clone
logstash-filter-csv

2
lib/bootstrap/bundler.rb
View file

@ -78,7 +78,7 @@ module LogStash
# @return [String, Exception] the installation captured output and any raised exception or nil if none
def invoke!(options = {})
options = {:max_tries => 10, :clean => false, :install => false, :update => false, :local => false,
:all => false, :package => false, :without => [:development]}.merge(options)
:jobs => 12, :all => false, :package => false, :without => [:development]}.merge(options)
options[:without] = Array(options[:without])
options[:update] = Array(options[:update]) if options[:update]

16
logstash-core-event-java/lib/logstash-core-event-java_jars.rb
View file

@ -1,6 +1,14 @@
# this is a generated file, to avoid over-writing it just delete this comment
require 'jar_dependencies'
begin
require 'jar_dependencies'
rescue LoadError
require 'com/fasterxml/jackson/core/jackson-core/2.7.1/jackson-core-2.7.1.jar'
require 'com/fasterxml/jackson/core/jackson-annotations/2.7.0/jackson-annotations-2.7.0.jar'
require 'com/fasterxml/jackson/core/jackson-databind/2.7.1-1/jackson-databind-2.7.1-1.jar'
end
require_jar( 'com.fasterxml.jackson.core', 'jackson-core', '2.7.1' )
require_jar( 'com.fasterxml.jackson.core', 'jackson-annotations', '2.7.0' )
require_jar( 'com.fasterxml.jackson.core', 'jackson-databind', '2.7.1-1' )
if defined? Jars
require_jar( 'com.fasterxml.jackson.core', 'jackson-core', '2.7.1' )
require_jar( 'com.fasterxml.jackson.core', 'jackson-annotations', '2.7.0' )
require_jar( 'com.fasterxml.jackson.core', 'jackson-databind', '2.7.1-1' )
end

2
logstash-core-event-java/logstash-core-event-java.gemspec
View file

@ -15,7 +15,7 @@ Gem::Specification.new do |gem|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
gem.name = "logstash-core-event-java"
gem.require_paths = ["lib"]
gem.version = LOGSTASH_CORE_EVENT_JAVA_VERSION.gsub(/-/, '.')
gem.version = LOGSTASH_CORE_EVENT_JAVA_VERSION
gem.platform = "java"

2
logstash-core-event/lib/logstash-core-event/version.rb
View file

@ -5,4 +5,4 @@
# Note to authors: this should not include dashes because 'gem' barfs if
# you include a dash in the version string.
LOGSTASH_CORE_EVENT_VERSION = "5.0.0-alpha1"
LOGSTASH_CORE_EVENT_VERSION = "5.0.0"

2
logstash-core-event/logstash-core-event.gemspec
View file

@ -15,7 +15,7 @@ Gem::Specification.new do |gem|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
gem.name = "logstash-core-event"
gem.require_paths = ["lib"]
gem.version = LOGSTASH_CORE_EVENT_VERSION.gsub(/-/, '.')
gem.version = LOGSTASH_CORE_EVENT_VERSION
if RUBY_PLATFORM == 'java'
gem.platform = RUBY_PLATFORM

1
logstash-core-plugin-api/lib/logstash-core-plugin-api/version.rb
View file

@ -1,3 +1,2 @@
# encoding: utf-8
# 2.1.9 tracks 5.0.0-alpha4.snapshot3
LOGSTASH_CORE_PLUGIN_API = "2.1.9"

2
logstash-core-plugin-api/logstash-core-plugin-api.gemspec
View file

@ -11,7 +11,7 @@ Gem::Specification.new do |gem|
gem.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
gem.license = "Apache License (2.0)"
gem.files = Dir.glob(["logstash-core-event.gemspec", "lib/**/*.rb", "spec/**/*.rb"])
gem.files = Dir.glob(["logstash-core-plugin-api.gemspec", "lib/**/*.rb", "spec/**/*.rb"])
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
gem.name = "logstash-core-plugin-api"
gem.require_paths = ["lib"]

4
logstash-core/lib/logstash/api/commands/system/basicinfo_command.rb
View file

@ -1,6 +1,7 @@
# encoding: utf-8
require 'logstash/api/commands/base'
require "logstash/util/duration_formatter"
require 'logstash/build'
module LogStash
module Api
@ -9,7 +10,8 @@ module LogStash
class BasicInfo < Commands::Base
def run
{} # Just return the defaults
# Just merge this stuff with the defaults
BUILD_INFO
end
end
end

6
logstash-core/lib/logstash/build.rb Normal file
View file

@ -0,0 +1,6 @@
# encoding: utf-8
# DO NOT EDIT
# this file acts as a placeholder for build information when executing
# logstash in dev mode (outside of a package build)
BUILD_INFO = {}

2
logstash-core/logstash-core.gemspec
View file

@ -15,7 +15,7 @@ Gem::Specification.new do |gem|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
gem.name = "logstash-core"
gem.require_paths = ["lib"]
gem.version = LOGSTASH_CORE_VERSION.gsub(/-/, '.')
gem.version = LOGSTASH_CORE_VERSION
gem.add_runtime_dependency "logstash-core-event-java", "5.0.0.alpha4.snapshot3"

2
qa/acceptance/spec/shared_examples/cli/logstash-plugin/install.rb
View file

@ -5,7 +5,7 @@ require "fileutils"
shared_examples "logstash install" do |logstash|
before(:each) do
logstash.install(LOGSTASH_VERSION)
logstash.install({:version => LOGSTASH_VERSION})
end
after(:each) do

2
qa/acceptance/spec/shared_examples/cli/logstash-plugin/list.rb
View file

@ -6,7 +6,7 @@ require "fileutils"
shared_examples "logstash list" do |logstash|
describe "logstash-plugin list on #{logstash.hostname}" do
before(:all) do
logstash.install(LOGSTASH_VERSION)
logstash.install({:version => LOGSTASH_VERSION})
end
after(:all) do

2
qa/acceptance/spec/shared_examples/cli/logstash-plugin/uninstall.rb
View file

@ -6,7 +6,7 @@ require "fileutils"
shared_examples "logstash uninstall" do |logstash|
describe "logstash uninstall on #{logstash.hostname}" do
before :each do
logstash.install(LOGSTASH_VERSION)
logstash.install({:version => LOGSTASH_VERSION})
end
after :each do

2
qa/acceptance/spec/shared_examples/cli/logstash-plugin/update.rb
View file

@ -5,7 +5,7 @@ require "logstash/version"
shared_examples "logstash update" do |logstash|
describe "logstash update on #{logstash.hostname}" do
before :each do
logstash.install(LOGSTASH_VERSION)
logstash.install({:version => LOGSTASH_VERSION})
end
after :each do

2
qa/acceptance/spec/shared_examples/cli/logstash/version.rb
View file

@ -5,7 +5,7 @@ require "logstash/version"
shared_examples "logstash version" do |logstash|
describe "logstash --version" do
before :all do
logstash.install(LOGSTASH_VERSION)
logstash.install({:version => LOGSTASH_VERSION})
end
after :all do

2
qa/acceptance/spec/shared_examples/installed.rb
View file

@ -5,7 +5,7 @@ require 'logstash/version'
RSpec.shared_examples "installable" do |logstash|
before(:each) do
logstash.install(LOGSTASH_VERSION)
logstash.install({:version => LOGSTASH_VERSION})
end
it "is installed on #{logstash.hostname}" do

2
qa/acceptance/spec/shared_examples/running.rb
View file

@ -5,7 +5,7 @@ require 'logstash/version'
RSpec.shared_examples "runnable" do |logstash|
before(:each) do
logstash.install(LOGSTASH_VERSION)
logstash.install({:version => LOGSTASH_VERSION})
end
it "is running on #{logstash.hostname}" do

6
qa/acceptance/spec/shared_examples/updated.rb
View file

@ -11,12 +11,14 @@ RSpec.shared_examples "updated" do |logstash|
end
before(:each) do
logstash.install(LOGSTASH_LATEST_VERSION, "./") # make sure latest version is installed
options={:version => LOGSTASH_LATEST_VERSION, :snapshot => false, :base => "./" }
logstash.install(options) # make sure latest version is installed
end
it "can be updated an run on #{logstash.hostname}" do
expect(logstash).to be_installed
# Performing the update
logstash.install(LOGSTASH_VERSION)
logstash.install({:version => LOGSTASH_VERSION})
expect(logstash).to be_installed
# starts the service to be sure it runs after the upgrade
logstash.start_service

12
qa/rspec/commands.rb
View file

@ -56,8 +56,9 @@ module ServiceTester
client.stop_service(name, host)
end
def install(version, base=ServiceTester::Base::LOCATION)
package = client.package_for(version, base)
def install(options={})
base = options.fetch(:base, ServiceTester::Base::LOCATION)
package = client.package_for(filename(options), base)
client.install(package, host)
end
@ -92,6 +93,13 @@ module ServiceTester
def to_s
"Artifact #{name}@#{host}"
end
private
def filename(options={})
snapshot = options.fetch(:snapshot, true)
"logstash-#{options[:version]}#{(snapshot ? "-SNAPSHOT" : "")}"
end
end
# Factory of commands used to select the right clients for a given type of OS and host name,

6
qa/rspec/commands/debian.rb
View file

@ -16,14 +16,14 @@ module ServiceTester
stdout.match(/^Status: install ok installed$/)
end
def package_for(version, base=ServiceTester::Base::LOCATION)
File.join(base, "logstash-#{version}.deb")
def package_for(filename, base=ServiceTester::Base::LOCATION)
File.join(base, "#{filename}.deb")
end
def install(package, host=nil)
hosts = (host.nil? ? servers : Array(host))
at(hosts, {in: :serial}) do |_|
sudo_exec!("dpkg -i #{package}")
cmd = sudo_exec!("dpkg -i #{package}")
end
end

4
qa/rspec/commands/redhat.rb
View file

@ -16,8 +16,8 @@ module ServiceTester
stdout.match(/^logstash.noarch/)
end
def package_for(version, base=ServiceTester::Base::LOCATION)
File.join(base, "logstash-#{version}.rpm")
def package_for(filename, base=ServiceTester::Base::LOCATION)
File.join(base, "#{filename}.rpm")
end
def install(package, host=nil)

4
qa/rspec/commands/suse.rb
View file

@ -13,8 +13,8 @@ module ServiceTester
stdout.match(/^i | logstash | An extensible logging pipeline | package$/)
end
def package_for(version, base=ServiceTester::Base::LOCATION)
File.join(base, "logstash-#{version}.rpm")
def package_for(filename, base=ServiceTester::Base::LOCATION)
File.join(base, "#{filename}.rpm")
end
def install(package, host=nil)

215
rakelib/artifacts.rake
View file

@ -2,6 +2,9 @@ require "logstash/version"
namespace "artifact" do
SNAPSHOT_BUILD = ENV["RELEASE"] != "1"
PACKAGE_SUFFIX = SNAPSHOT_BUILD ? "-SNAPSHOT" : ""
def package_files
[
"LICENSE",
@ -13,6 +16,13 @@ namespace "artifact" do
"lib/bootstrap/**/*",
"lib/pluginmanager/**/*",
"lib/systeminstall/**/*",
"logstash-core/lib/**/*",
"logstash-core/locales/**/*",
"logstash-core/*.gemspec",
"logstash-core-event-java/lib/**/*",
"logstash-core-event-java/*.gemspec",
"logstash-core-plugin-api/lib/**/*",
"logstash-core-plugin-api/*.gemspec",
"patterns/**/*",
"vendor/??*/**/*",
# To include ruby-maven's hidden ".mvn" directory, we need to
@ -55,18 +65,75 @@ namespace "artifact" do
end.flatten.uniq
end
task "all" => ["prepare"] do
desc "Generate rpm, deb, tar and zip artifacts"
task "all" => ["prepare", "build"]
desc "Build a tar.gz of default logstash plugins with all dependencies"
task "tar" => ["prepare", "generate_build_metadata"] do
puts("[artifact:tar] Building tar.gz of default plugins")
build_tar
end
desc "Build a zip of default logstash plugins with all dependencies"
task "zip" => ["prepare", "generate_build_metadata"] do
puts("[artifact:zip] Building zip of default plugins")
build_zip
end
desc "Build an RPM of logstash with all dependencies"
task "rpm" => ["prepare", "generate_build_metadata"] do
puts("[artifact:rpm] building rpm package")
package("centos", "5")
end
desc "Build a DEB of logstash with all dependencies"
task "deb" => ["prepare", "generate_build_metadata"] do
puts("[artifact:deb] building deb package")
package("ubuntu", "12.04")
end
desc "Generate logstash core gems"
task "gems" => ["prepare"] do
Rake::Task["artifact:build-logstash-core"].invoke
Rake::Task["artifact:build-logstash-core-event"].invoke
Rake::Task["artifact:build-logstash-core-plugin-api"].invoke
end
# "all-plugins" version of tasks
desc "Generate rpm, deb, tar and zip artifacts (\"all-plugins\" version)"
task "all-all-plugins" => ["prepare-all", "build"]
desc "Build a zip of all logstash plugins from logstash-plugins github repo"
task "zip-all-plugins" => ["prepare-all", "generate_build_metadata"] do
puts("[artifact:zip] Building zip of all plugins")
build_zip "-all-plugins"
end
desc "Build a tar.gz of all logstash plugins from logstash-plugins github repo"
task "tar-all-plugins" => ["prepare-all", "generate_build_metadata"] do
puts("[artifact:tar] Building tar.gz of all plugins")
build_tar "-all-plugins"
end
# Auxiliary tasks
task "build" => [:generate_build_metadata] do
Rake::Task["artifact:gems"].invoke unless SNAPSHOT_BUILD
Rake::Task["artifact:deb"].invoke
Rake::Task["artifact:rpm"].invoke
Rake::Task["artifact:zip"].invoke
Rake::Task["artifact:tar"].invoke
end
task "all-all-plugins" => ["prepare-all"] do
Rake::Task["artifact:deb"].invoke
Rake::Task["artifact:rpm"].invoke
Rake::Task["artifact:zip"].invoke
Rake::Task["artifact:tar"].invoke
task "generate_build_metadata" do
return if defined?(BUILD_METADATA_FILE)
BUILD_METADATA_FILE = Tempfile.new('build.rb')
build_info = {
"build_date" => Time.now.iso8601,
"build_sha" => `git rev-parse HEAD`.chomp,
"build_snapshot" => SNAPSHOT_BUILD
}
metadata = [ "# encoding: utf-8", "BUILD_INFO = #{build_info}" ]
IO.write(BUILD_METADATA_FILE.path, metadata.join("\n"))
end
# We create an empty bundle config file
@ -78,8 +145,8 @@ namespace "artifact" do
end
# locate the "gem "logstash-core" ..." line in Gemfile, and if the :path => "..." option if specified
# build and install the local logstash-core gem otherwise just do nothing, bundler will deal with it.
task "install-logstash-core" do
# build the local logstash-core gem otherwise just do nothing, bundler will deal with it.
task "build-logstash-core" do
# regex which matches a Gemfile gem definition for the logstash-core gem and captures the :path option
gem_line_regex = /^\s*gem\s+["']logstash-core["'](?:\s*,\s*["'][^"^']+["'])?(?:\s*,\s*:path\s*=>\s*["']([^"^']+)["'])?/i
@ -90,15 +157,16 @@ namespace "artifact" do
path = matches.first[gem_line_regex, 1]
if path
Rake::Task["plugin:install-local-core-gem"].invoke("logstash-core", path)
Rake::Task["plugin:build-local-core-gem"].invoke("logstash-core", path)
else
puts("[artifact:install-logstash-core] using logstash-core from Rubygems")
puts "The Gemfile should reference \"logstash-core\" gem locally through :path, but found instead: #{matches}"
exit(1)
end
end
# # locate the "gem "logstash-core-event*" ..." line in Gemfile, and if the :path => "." option if specified
# # build and install the local logstash-core-event* gem otherwise just do nothing, bundler will deal with it.
task "install-logstash-core-event" do
# # build the local logstash-core-event* gem otherwise just do nothing, bundler will deal with it.
task "build-logstash-core-event" do
# regex which matches a Gemfile gem definition for the logstash-core-event* gem and captures the gem name and :path option
gem_line_regex = /^\s*gem\s+["'](logstash-core-event[^"^']*)["'](?:\s*,\s*["'][^"^']+["'])?(?:\s*,\s*:path\s*=>\s*["']([^"^']+)["'])?/i
@ -110,15 +178,16 @@ namespace "artifact" do
path = matches.first[gem_line_regex, 2]
if path
Rake::Task["plugin:install-local-core-gem"].invoke(name, path)
Rake::Task["plugin:build-local-core-gem"].invoke(name, path)
else
puts("[artifact:install-logstash-core] using #{name} from Rubygems")
puts "The Gemfile should reference \"logstash-core-event\" gem locally through :path, but found instead: #{matches}"
exit(1)
end
end
# locate the "gem "logstash-core-plugin-api" ..." line in Gemfile, and if the :path => "..." option if specified
# build and install the local logstash-core-plugin-api gem otherwise just do nothing, bundler will deal with it.
task "install-logstash-core-plugin-api" do
# build the local logstash-core-plugin-api gem otherwise just do nothing, bundler will deal with it.
task "build-logstash-core-plugin-api" do
# regex which matches a Gemfile gem definition for the logstash-core gem and captures the :path option
gem_line_regex = /^\s*gem\s+["']logstash-core-plugin-api["'](?:\s*,\s*["'][^"^']+["'])?(?:\s*,\s*:path\s*=>\s*["']([^"^']+)["'])?/i
@ -129,86 +198,87 @@ namespace "artifact" do
path = matches.first[gem_line_regex, 1]
if path
Rake::Task["plugin:install-local-core-gem"].invoke("logstash-core-plugin-api", path)
Rake::Task["plugin:build-local-core-gem"].invoke("logstash-core-plugin-api", path)
else
puts("[artifact:install-logstash-core-plugin-api] using logstash-core from Rubygems")
puts "The Gemfile should reference \"logstash-core-plugin-api\" gem locally through :path, but found instead: #{matches}"
exit(1)
end
end
task "prepare" => ["bootstrap", "plugin:install-default", "install-logstash-core", "install-logstash-core-event", "install-logstash-core-plugin-api", "clean-bundle-config"]
task "prepare-all" => ["bootstrap", "plugin:install-all", "install-logstash-core", "install-logstash-core-event", "install-logstash-core-plugin-api", "clean-bundle-config"]
desc "Build a tar.gz of default logstash plugins with all dependencies"
task "tar" => ["prepare"] do
puts("[artifact:tar] Building tar.gz of default plugins")
build_tar
task "prepare" do
if ENV['SKIP_PREPARE'] != "1"
["bootstrap", "plugin:install-default", "artifact:clean-bundle-config"].each {|task| Rake::Task[task].invoke }
end
end
desc "Build a tar.gz of all logstash plugins from logstash-plugins github repo"
task "tar-all-plugins" => ["prepare-all"] do
puts("[artifact:tar] Building tar.gz of all plugins")
build_tar "-all-plugins"
task "prepare-all" do
if ENV['SKIP_PREPARE'] != "1"
["bootstrap", "plugin:install-all", "artifact:clean-bundle-config"].each {|task| Rake::Task[task].invoke }
end
end
def build_tar(tar_suffix = nil)
require "zlib"
require "archive/tar/minitar"
require "logstash/version"
tarpath = "build/logstash#{tar_suffix}-#{LOGSTASH_VERSION}.tar.gz"
tarpath = "build/logstash#{tar_suffix}-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}.tar.gz"
puts("[artifact:tar] building #{tarpath}")
gz = Zlib::GzipWriter.new(File.new(tarpath, "wb"), Zlib::BEST_COMPRESSION)
tar = Archive::Tar::Minitar::Output.new(gz)
files.each do |path|
stat = File.lstat(path)
path_in_tar = "logstash-#{LOGSTASH_VERSION}/#{path}"
opts = {
:size => stat.size,
:mode => stat.mode,
:mtime => stat.mtime
}
if stat.directory?
tar.tar.mkdir(path_in_tar, opts)
else
tar.tar.add_file_simple(path_in_tar, opts) do |io|
File.open(path,'rb') do |fd|
chunk = nil
size = 0
size += io.write(chunk) while chunk = fd.read(16384)
if stat.size != size
raise "Failure to write the entire file (#{path}) to the tarball. Expected to write #{stat.size} bytes; actually write #{size}"
end
end
end
end
write_to_tar(tar, path, "logstash-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}/#{path}")
end
# add build.rb to tar
metadata_file_path_in_tar = File.join("logstash-core", "lib", "logstash", "build.rb")
path_in_tar = File.join("logstash-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}", metadata_file_path_in_tar)
write_to_tar(tar, BUILD_METADATA_FILE.path, path_in_tar)
tar.close
gz.close
puts "Complete: #{tarpath}"
end
desc "Build a zip of default logstash plugins with all dependencies"
task "zip" => ["prepare"] do
puts("[artifact:zip] Building zip of default plugins")
build_zip
end
desc "Build a zip of all logstash plugins from logstash-plugins github repo"
task "zip-all-plugins" => ["prepare-all"] do
puts("[artifact:zip] Building zip of all plugins")
build_zip "-all-plugins"
def write_to_tar(tar, path, path_in_tar)
stat = File.lstat(path)
opts = {
:size => stat.size,
:mode => stat.mode,
:mtime => stat.mtime
}
if stat.directory?
tar.tar.mkdir(path_in_tar, opts)
else
tar.tar.add_file_simple(path_in_tar, opts) do |io|
File.open(path,'rb') do |fd|
chunk = nil
size = 0
size += io.write(chunk) while chunk = fd.read(16384)
if stat.size != size
raise "Failure to write the entire file (#{path}) to the tarball. Expected to write #{stat.size} bytes; actually write #{size}"
end
end
end
end
end
def build_zip(zip_suffix = "")
require 'zip'
zippath = "build/logstash#{zip_suffix}-#{LOGSTASH_VERSION}.zip"
zippath = "build/logstash#{zip_suffix}-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}.zip"
puts("[artifact:zip] building #{zippath}")
File.unlink(zippath) if File.exists?(zippath)
Zip::File.open(zippath, Zip::File::CREATE) do |zipfile|
files.each do |path|
path_in_zip = "logstash-#{LOGSTASH_VERSION}/#{path}"
path_in_zip = "logstash-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}/#{path}"
zipfile.add(path_in_zip, path)
end
# add build.rb to zip
metadata_file_path_in_zip = File.join("logstash-core", "lib", "logstash", "build.rb")
path_in_zip = File.join("logstash-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}", metadata_file_path_in_zip)
path = BUILD_METADATA_FILE.path
Zip.continue_on_exists_proc = true
zipfile.add(path_in_zip, path)
end
puts "Complete: #{zippath}"
end
@ -221,6 +291,10 @@ namespace "artifact" do
dir = FPM::Package::Dir.new
metadata_file_path = File.join("logstash-core", "lib", "logstash", "build.rb")
metadata_source_file_path = BUILD_METADATA_FILE.path
dir.input("#{metadata_source_file_path}=/usr/share/logstash/#{metadata_file_path}")
files.each do |path|
next if File.directory?(path)
# Omit any config dir from /usr/share/logstash for packages, since we're
@ -246,7 +320,7 @@ namespace "artifact" do
# produce: logstash-5.0.0-alpha1.deb"
# produce: logstash-5.0.0-alpha1.rpm
package_filename = "logstash-#{LOGSTASH_VERSION}.TYPE"
package_filename = "logstash-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}.TYPE"
case platform
when "redhat", "centos"
@ -350,15 +424,4 @@ namespace "artifact" do
end
end # def package
desc "Build an RPM of logstash with all dependencies"
task "rpm" => ["prepare"] do
puts("[artifact:rpm] building rpm package")
package("centos", "5")
end
desc "Build a DEB of logstash with all dependencies"
task "deb" => ["prepare"] do
puts("[artifact:deb] building deb package")
package("ubuntu", "12.04")
end
end

3
rakelib/build.rake
View file

@ -4,3 +4,6 @@ end
directory "build/bootstrap" => "build" do |task, args|
mkdir_p task.name unless File.directory?(task.name)
end
directory "build/gems" => "build" do |task, args|
mkdir_p task.name unless File.directory?(task.name)
end

6
rakelib/plugin.rake
View file

@ -81,7 +81,7 @@ namespace "plugin" do
task.reenable # Allow this task to be run again
end
task "build-local-core-gem", [:name, :path] do |task, args|
task "build-local-core-gem", [:name, :path] => ["build/gems"] do |task, args|
name = args[:name]
path = args[:path]
@ -89,10 +89,12 @@ namespace "plugin" do
puts("[plugin:build-local-core-gem] Building #{File.join(path, name)}.gemspec")
gem_path = nil
Dir.chdir(path) do
spec = Gem::Specification.load("#{name}.gemspec")
Gem::Package.build(spec)
gem_path = Gem::Package.build(spec)
end
FileUtils.cp(File.join(path, gem_path), "build/gems/")
task.reenable # Allow this task to be run again
end

132
rakelib/version.rake Normal file
View file

@ -0,0 +1,132 @@
require 'yaml'
def get_versions
yaml_versions = YAML.safe_load(IO.read("versions.yml"))
{
"logstash" => {
"location" => File.join("logstash-core", "lib", "logstash", "version.rb"),
"yaml_version" => yaml_versions["logstash"],
"current_version" => get_version(File.join("logstash-core", "lib", "logstash", "version.rb")),
},
"logstash-core" => {
"location" => File.join("logstash-core", "lib", "logstash-core", "version.rb"),
"yaml_version" => yaml_versions["logstash-core"],
"current_version" => get_version(File.join("logstash-core", "lib", "logstash-core", "version.rb")),
},
"logstash-core-event" => {
"location" => File.join("logstash-core-event", "lib", "logstash-core-event", "version.rb"),
"yaml_version" => yaml_versions["logstash-core-event"],
"current_version" => get_version(File.join("logstash-core-event", "lib", "logstash-core-event", "version.rb")),
},
"logstash-core-event-java" => {
"location" => File.join("logstash-core-event-java", "lib", "logstash-core-event-java", "version.rb"),
"yaml_version" => yaml_versions["logstash-core-event-java"],
"current_version" => get_version(File.join("logstash-core-event-java", "lib", "logstash-core-event-java", "version.rb")),
},
"logstash-core-plugin-api" => {
"location" => File.join("logstash-core-plugin-api", "lib", "logstash-core-plugin-api", "version.rb"),
"yaml_version" => yaml_versions["logstash-core-plugin-api"],
"current_version" => get_version(File.join("logstash-core-plugin-api", "lib", "logstash-core-plugin-api", "version.rb")),
}
}
end
def get_version(file)
text = IO.read(file)
version = text.match(/^[A-Z_]+ = "(.+?)"/)
version[1]
end
namespace :version do
desc "check if the versions.yml is out of sync with .gemspecs and other references"
task :check do
out_of_sync = get_versions.select do |component, metadata|
metadata["yaml_version"] != metadata["current_version"]
end
if out_of_sync.any?
out_of_sync.each do |component, metadata|
puts "#{component} is out of sync. CURRENT: #{metadata['current_version']} | YAML: #{metadata['yaml_version']}"
end
exit(1)
end
end
desc "push versions found in versions.yml to all component version locations"
task :sync do
versions = get_versions
# update version.rb files
versions.select do |component, metadata|
next if metadata["yaml_version"] == metadata["current_version"]
puts "Updating \"#{component}\" from \"#{metadata['current_version']}\" to \"#{metadata['yaml_version']}\""
text = IO.read(metadata["location"])
IO.write(metadata["location"], text.gsub(metadata["current_version"], metadata["yaml_version"]))
end
# update dependencies
#
# logstash-core depends on logstash-core-event-java
# ./logstash-core/logstash-core.gemspec: gem.add_runtime_dependency "logstash-core-event-java", "~> 5.0.0.dev"
logstash_core_gemspec = File.join("logstash-core", "logstash-core.gemspec")
logstash_core_event_java_version = versions['logstash-core-event-java']['yaml_version']
text = IO.read(logstash_core_gemspec)
IO.write(logstash_core_gemspec, text.sub(
/ gem.add_runtime_dependency \"logstash-core-event-java\", \".+?\"/,
" gem.add_runtime_dependency \"logstash-core-event-java\", \"#{logstash_core_event_java_version}\""))
# logstash-core-event-java depends on logstash-code
# ./logstash-core-plugin-api/logstash-core-plugin-api.gemspec: gem.add_runtime_dependency "logstash-core", "5.0.0.dev"
logstash_core_plugin_api_gemspec = File.join("logstash-core-plugin-api", "logstash-core-plugin-api.gemspec")
logstash_core_version = versions['logstash-core']['yaml_version']
text = IO.read(logstash_core_plugin_api_gemspec)
IO.write(logstash_core_plugin_api_gemspec, text.sub(
/ gem.add_runtime_dependency \"logstash-core\", \".+?\"/,
" gem.add_runtime_dependency \"logstash-core\", \"#{logstash_core_version}\""))
end
desc "show version of core components"
task :show do
Rake::Task["version:sync"].invoke; Rake::Task["version:sync"].reenable
get_versions.each do |component, metadata|
puts "#{component}: #{metadata['yaml_version']}"
end
end
desc "set version of logstash, logstash-core, logstash-core-event, logstash-core-event-java"
task :set, [:version] => [:validate] do |t, args|
hash = {}
get_versions.each do |component, metadata|
# we just assume that, usually, all components except
# "logstash-core-plugin-api" will be versioned together
# so let's skip this one and have a separate task for it
if component == "logstash-core-plugin-api"
hash[component] = metadata["yaml_version"]
else
hash[component] = args[:version]
end
end
IO.write("versions.yml", hash.to_yaml)
Rake::Task["version:sync"].invoke; Rake::Task["version:sync"].reenable
end
desc "set version of logstash-core-plugin-api"
task :set_plugin_api, [:version] => [:validate] do |t, args|
hash = {}
get_versions.each do |component, metadata|
if component == "logstash-core-plugin-api"
hash[component] = args[:version]
else
hash[component] = metadata["yaml_version"]
end
end
IO.write("versions.yml", hash.to_yaml)
Rake::Task["version:sync"].invoke; Rake::Task["version:sync"].reenable
end
task :validate, :version do |t, args|
unless Regexp.new('^\d+\.\d+\.\d+(?:-\w+\d+)?$').match(args[:version])
abort("Invalid version argument: \"#{args[:version]}\". Aborting...")
end
end
end

6
versions.yml Normal file
View file

@ -0,0 +1,6 @@
---
logstash: 5.0.0
logstash-core: 5.0.0
logstash-core-event: 5.0.0
logstash-core-event-java: 5.0.0
logstash-core-plugin-api: 2.1.7