require "tempfile" require "ftools" # fails in 1.9.2 require File.join(File.dirname(__FILE__), "VERSION") # For LOGSTASH_VERSION # Compile config grammar (ragel -> ruby) file "lib/logstash/config/grammar.rb" => ["lib/logstash/config/grammar.rl"] do sh "make -C lib/logstash/config grammar.rb" end # Taken from 'jrubyc' # Currently this code is commented out because jruby emits this: # Failure during compilation of file logstash/web/helpers/require_param.rb: # java.lang.RuntimeException: java.io.FileNotFoundException: File path # /home/jls/projects/logstash/logstash/web/helpers/require_param.rb # does not start with parent path /home/jls/projects/logstash/lib # # org/jruby/util/JavaNameMangler.java:105:in `mangleFilenameForClasspath' # org/jruby/util/JavaNameMangler.java:32:in `mangleFilenameForClasspath' #require 'jruby/jrubyc' ##args = [ "-p", "net.logstash" ] #args = ["-d", "build"] #args += Dir.glob("**/*.rb") #status = JRuby::Compiler::compile_argv(args) #if (status != 0) #puts "Compilation FAILED: #{status} error(s) encountered" #exit status #end task :clean do sh "rm -rf .bundle" #sh "rm -rf build-jar-thin" #sh "rm -rf build-jar" sh "rm -rf build" sh "rm -rf vendor" end task :compile => "lib/logstash/config/grammar.rb" do |t| target = "build/ruby" mkdir_p target if !File.directory?(target) #sh "rm -rf lib/net" Dir.chdir("lib") do rel_target = File.join("..", target) sh "jrubyc", "-t", rel_target, "logstash/runner.rb" files = Dir.glob("**/*.rb") files.each do |file| d = File.join(rel_target, File.dirname(file)) mkdir_p d if !File.directory?(d) cp file, File.join(d, File.basename(file)) end end Dir.chdir("test") do rel_target = File.join("..", target) files = Dir.glob("**/*.rb") files.each do |file| d = File.join(rel_target, File.dirname(file)) mkdir_p d if !File.directory?(d) cp file, File.join(d, File.basename(file)) end end end task :jar => [ "package:monolith:jar" ] do |t| # Nothing end VERSIONS = { :jruby => "1.6.0", # Any of CPL1.0/GPL2.0/LGPL2.1 ? Confusing, but OK. #:elasticsearch => "0.15.2", # Apache 2.0 license :elasticsearch => "0.16.0", # Apache 2.0 license :joda => "1.6.2", # Apache 2.0 license } namespace :vendor do file "vendor/jar" do |t| mkdir_p t.name if !File.directory?(t.name) end # Download jruby.jar file "vendor/jar/jruby-complete-#{VERSIONS[:jruby]}.jar" => "vendor/jar" do |t| baseurl = "http://repository.codehaus.org/org/jruby/jruby-complete" if !File.exists?(t.name) sh "wget -O #{t.name} #{baseurl}/#{VERSIONS[:jruby]}/#{File.basename(t.name)}" end end # jruby task :jruby => "vendor/jar/jruby-complete-#{VERSIONS[:jruby]}.jar" do # nothing to do, the dep does it. end # Download elasticsearch + deps (for outputs/elasticsearch) task :elasticsearch => "vendor/jar" do version = VERSIONS[:elasticsearch] tarball = "elasticsearch-#{version}.tar.gz" url = "http://github.com/downloads/elasticsearch/elasticsearch/#{tarball}" if !File.exists?(tarball) # --no-check-certificate is for github and wget not supporting wildcard # certs sanely. sh "wget -O #{tarball} --no-check-certificate #{url}" end sh "tar -zxf #{tarball} -C vendor/jar/ elasticsearch-#{version}/lib" end # elasticsearch task :joda => "vendor/jar" do version = VERSIONS[:joda] baseurl = "http://sourceforge.net/projects/joda-time/files/joda-time" tarball = "joda-time-#{version}-bin.tar.gz" url = "#{baseurl}/#{version}/#{tarball}/download" if !File.exists?(tarball) sh "wget -O #{tarball} #{url}" end sh "tar -zxf #{tarball} -C vendor/jar/ joda-time-#{version}/joda-time-#{version}.jar" end # joda task :gems => "vendor/jar" do puts "=> Installing gems to vendor/bundle/..." sh "bundle install --path #{File.join("vendor", "bundle")}" end end # vendor namespace namespace :package do task :gem do sh "gem build logstash.gemspec" end monolith_deps = [ "vendor:jruby", "vendor:gems", "vendor:elasticsearch", "compile" ] namespace :monolith do task :tar => monolith_deps do paths = %w{ bin CHANGELOG CONTRIBUTORS etc examples Gemfile Gemfile.lock INSTALL lib LICENSE patterns Rakefile README.md STYLE.md test TODO USAGE vendor/bundle vendor/jar } sh "tar -zcf logstash-monolithic-someversion.tar.gz #{paths.join(" ")}" end # package:monolith:tar task :jar => monolith_deps do builddir = "build/monolith-jar" mkdir_p builddir if !File.directory?(builddir) # Unpack all the 3rdparty jars and any jars in gems Dir.glob("vendor/{bundle,jar}/**/*.jar").each do |jar| if jar =~ /sigar.*\.jar$/ puts "=> Skipping #{jar} (sigar not needed)" next end puts "=> Unpacking #{jar} into #{builddir}/" relative_path = File.join(builddir.split(File::SEPARATOR).collect { |a| ".." }) Dir.chdir(builddir) do sh "jar xf #{relative_path}/#{jar}" end end # We compile stuff to build/... # TODO(sissel): Could probably just use 'jar uf' for this? Dir.glob("build/ruby/**/*.class").each do |file| target = File.join(builddir, file.gsub("build/ruby/", "")) mkdir_p File.dirname(target) puts "=> Copying #{file} => #{target}" File.copy(file, target) end # Purge any extra files we don't need in META-INF (like manifests and # jar signatures) ["INDEX.LIST", "MANIFEST.MF", "ECLIPSEF.RSA", "ECLIPSEF.SF"].each do |file| File.delete(File.join(builddir, "META-INF", file)) rescue nil end output = "logstash-#{LOGSTASH_VERSION}-monolithic.jar" sh "jar cfe #{output} logstash.runner -C #{builddir} ." jar_update_args = [] # Learned how to do this mostly from here: # http://blog.nicksieger.com/articles/2009/01/10/jruby-1-1-6-gems-in-a-jar # # Add bundled gems to the jar # Skip the 'cache' dir which is just the original .gem files gem_dirs = %w{bin doc gems specifications} gem_root = File.join(%w{vendor bundle jruby 1.8}) # for each dir, build args: -C vendor/bundle/jruby/1.8 bin, etc gem_jar_args = gem_dirs.collect { |d| ["-C", gem_root, d ] }.flatten jar_update_args += gem_jar_args # Add compiled our compiled ruby code jar_update_args += %w{ -C build/ruby . } # Add web stuff jar_update_args += %w{ -C lib logstash/web/public } jar_update_args += %w{ -C lib logstash/web/views } # Add test code #jar_update_args += %w{ -C test logstsah } # Add grok patterns jar_update_args << "patterns" # Update with other files and also build an index. sh "jar uf #{output} #{jar_update_args.join(" ")}" sh "jar i #{output}" end # task package:monolith:jar end # namespace monolith task :jar => [ "vendor:jruby", "vendor:gems", "compile" ] do builddir = "build/thin-jar" mkdir_p builddir if !File.directory?(builddir) # Unpack jruby relative_path = File.join(builddir.split(File::SEPARATOR).collect { |a| ".." }) Dir.glob("vendor/jar/jruby-complete-1.6.0.jar").each do |jar| puts "=> Unpacking #{jar} into #{builddir}/" Dir.chdir(builddir) do sh "jar xf #{relative_path}/#{jar}" end end ["INDEX.LIST", "MANIFEST.MF", "ECLIPSEF.RSA", "ECLIPSEF.SF"].each do |file| File.delete(File.join(builddir, "META-INF", file)) rescue nil end output = "logstash-#{LOGSTASH_VERSION}.jar" sh "jar cfe #{output} logstash.runner -C #{builddir} ." jar_update_args = [] # Learned how to do this mostly from here: # http://blog.nicksieger.com/articles/2009/01/10/jruby-1-1-6-gems-in-a-jar # # Add bundled gems to the jar # Skip the 'cache' dir which is just the original .gem files gem_dirs = %w{bin doc gems specifications} gem_root = File.join(%w{vendor bundle jruby 1.8}) # for each dir, build args: -C vendor/bundle/jruby/1.8 bin, etc gem_jar_args = gem_dirs.collect { |dir| ["-C", gem_root, dir ] }.flatten jar_update_args += gem_jar_args # Add compiled our compiled ruby code jar_update_args += %w{ -C build . } # Add web stuff jar_update_args += %w{ -C lib logstash/web/public } jar_update_args += %w{ -C lib logstash/web/views } # Add test code #jar_update_args += %w{ -C test logstsah } # Add grok patterns jar_update_args << "patterns" # Update with other files and also build an index. sh "jar uf #{output} #{jar_update_args.join(" ")}" sh "jar i #{output}" end # task package:jar end # namespace package task :test do sh "cd test; ruby logstash_test_runner.rb" end task :docs => [:docgen, :doccopy, :docindex ] do end task :require_output_env do if ENV["output"].nil? raise "No output variable set. Run like: 'rake docs output=path/to/output'" end end task :doccopy => [:require_output_env] do if ENV["output"].nil? raise "No output variable set. Run like: 'rake docs output=path/to/output'" end output = ENV["output"].gsub("VERSION", LOGSTASH_VERSION) Dir.glob("docs/**/*").each do |doc| dir = File.join(output, File.dirname(doc).gsub(/docs\/?/, "")) mkdir_p dir if !File.directory?(dir) if File.directory?(doc) mkdir_p doc else puts "Copy #{doc} => #{dir}" cp(doc, dir) end end end task :docindex => [:require_output_env] do output = ENV["output"].gsub("VERSION", LOGSTASH_VERSION) sh "ruby docs/generate_index.rb #{output} > #{output}/index.html" end task :docgen => [:require_output_env] do if ENV["output"].nil? raise "No output variable set. Run like: 'rake docgen output=path/to/output'" end output = ENV["output"].gsub("VERSION", LOGSTASH_VERSION) sh "find lib/logstash/inputs lib/logstash/filters lib/logstash/outputs -type f -not -name 'base.rb' -a -name '*.rb'| xargs ruby docs/docgen.rb -o #{output}" end task :publish do latest_gem = %x{ls -t logstash-[0-9]*.gem}.split("\n").first sh "gem push #{latest_gem}" end task :release do docs_dir = File.join(File.dirname(__FILE__), "..", "logstash.github.com", "docs", LOGSTASH_VERSION) ENV["output"] = docs_dir sh "sed -i -Re 's/1.0.[0-9]/#{LOGSTASH_VERSION}/'" sh "git tag v#{LOGSTASH_VERSION}" #Rake::Task["docs"].invoke Rake::Task["package:gem"].invoke Rake::Task["package:monolith:jar"].invoke puts "Packaging complete." puts "Run the following under ruby 1.8.7 (require bluecloth)" puts "> rake docs output=#{docs_dir}" end