Add doc generation tasks to logstash

Fixes #2256

rakelib/dependency.rake

@@ -26,4 +26,8 @@ namespace "dependency" do
     Rake::Task["gem:require"].invoke("rubyzip", ">= 0", LogStash::Environment.logstash_gem_home)
   end # task stud
 
+  task "octokit" do
+    Rake::Task["gem:require"].invoke("octokit", ">= 0", LogStash::Environment.logstash_gem_home)
+  end # task octokit
+
 end # namespace dependency

rakelib/docs.rake

@@ -0,0 +1,51 @@
+namespace "docs" do
+
+  task "generate" do
+    Rake::Task['docs:install-plugins'].invoke
+    Rake::Task['docs:generate-docs'].invoke
+    Rake::Task['docs:generate-index'].invoke
+  end
+
+  task "generate-docs" do
+    list = Dir.glob("vendor/bundle/jruby/1.9/gems/logstash-*/lib/logstash/{input,output,filter,codec}s/*.rb").join(" ")
+    cmd = "bin/logstash docgen -o asciidoc_generated #{list}"
+    system(cmd)
+  end
+
+  task "generate-index" do
+    list = [ 'inputs', 'outputs', 'filters', 'codecs' ]
+    list.each do |type|
+      cmd = "ruby docs/asciidoc_index.rb asciidoc_generated #{type}"
+      system(cmd)
+    end
+  end
+
+
+  task "install-plugins" => [ "dependency:bundler", "dependency:octokit" ] do
+    
+    # because --path creates a .bundle/config file and changes bundler path
+    # we need to remove this file so it doesn't influence following bundler calls
+    FileUtils.rm_rf(::File.join(LogStash::Environment::LOGSTASH_HOME, "tools/.bundle"))
+
+    10.times do
+      begin
+        ENV["GEM_PATH"] = LogStash::Environment.logstash_gem_home
+        ENV["BUNDLE_PATH"] = LogStash::Environment.logstash_gem_home
+        ENV["BUNDLE_GEMFILE"] = "tools/Gemfile.all"
+        Bundler.reset!
+        Bundler::CLI.start(LogStash::Environment.bundler_install_command("tools/Gemfile.all", LogStash::Environment::BUNDLE_DIR))
+        break
+      rescue => e
+        # for now catch all, looks like bundler now throws Bundler::InstallError, Errno::EBADF
+        puts(e.message)
+        puts("--> Retrying install-defaults upon exception=#{e.class}")
+        sleep(1)
+      end
+    end
+
+    # because --path creates a .bundle/config file and changes bundler path
+    # we need to remove this file so it doesn't influence following bundler calls
+    FileUtils.rm_rf(::File.join(LogStash::Environment::LOGSTASH_HOME, "tools/.bundle"))
+  end
+
+end

tools/Gemfile.all

@@ -0,0 +1,13 @@
+require 'octokit'
+skiplist = ['logstash-input-gemfire', 'logstash-output-gemfire', 'logstash-input-couchdb_changes', 'logstash-filter-metricize', 'logstash-filter-yaml']
+
+source 'https://rubygems.org'
+
+gemspec :name => "logstash", :path => ".."
+
+Octokit.auto_paginate = true
+repo_list = Octokit.organization_repositories("logstash-plugins")
+repo_list.each do |repo|
+  next if skiplist.include?(repo.name)
+  gem repo.name
+end