logstash/rubyUtils.gradle

/*
 * Licensed to Elasticsearch B.V. under one or more contributor
 * license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright
 * ownership. Elasticsearch B.V. licenses this file to you under
 * the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *	http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */


buildscript {
    repositories {
        mavenCentral()
    }
    dependencies {
        classpath "org.yaml:snakeyaml:${snakeYamlVersion}"
        classpath "de.undercouch:gradle-download-task:4.0.4"
        classpath "org.jruby:jruby-core:9.4.9.0"
    }
}

import de.undercouch.gradle.tasks.download.Download
import de.undercouch.gradle.tasks.download.Verify
import org.yaml.snakeyaml.Yaml


import org.jruby.Ruby
import org.jruby.embed.PathType
import org.jruby.embed.ScriptingContainer

import java.lang.annotation.Annotation
import java.nio.file.Files
import java.nio.file.Paths


ext {
    bundle = this.&bundle
    bundleWithEnv = this.&bundleWithEnv
    bundleQAGems = this.&bundleQAGems
    gem = this.&gem
    buildGem = this.&buildGem
    rake = this.&rake
    setupJruby = this.&setupJruby
    generateRubySupportFilesForPlugin = this.&generateRubySupportFilesForPlugin
    validatePluginJar = this.&validatePluginJar
    versionMap = new HashMap()
    pluginInfo = new PluginInfo()
}


/**
 * Executes a bundler bin script with given parameters.
 * @param projectDir Gradle projectDir
 * @param buildDir Gradle buildDir
 * @param pwd Current worker directory to execute in
 * @param bundleBin Bundler Bin Script
 * @param args CLI Args to Use with Bundler
 */
void bundle(File projectDir, File buildDir, String pwd, String bundleBin, Iterable<String> args) {
    bundleWithEnv(projectDir, buildDir, pwd, bundleBin, args, Collections.emptyMap())
}

/**
 * Executes a bundler bin script with given parameters.
 * @param projectDir Gradle projectDir
 * @param buildDir Gradle buildDir
 * @param pwd Current worker directory to execute in
 * @param bundleBin Bundler Bin Script
 * @param args CLI Args to Use with Bundler
 * @param env Environment Variables to Set
 */
void bundleWithEnv(File projectDir, File buildDir, String pwd, String bundleBin, Iterable<String> args, Map<String, String> env) {
    executeJruby projectDir, buildDir, { ScriptingContainer jruby ->
        jruby.environment.putAll(env)
        jruby.currentDirectory = pwd
        jruby.argv = args.toList().toArray()
        jruby.runScriptlet(PathType.ABSOLUTE, bundleBin)
    }
}

void bundleQAGems(File projectDir, String qaBuildPath) {
    def jruby = new ScriptingContainer()
    jruby.setLoadPaths(["${projectDir}/vendor/jruby/lib/ruby/stdlib".toString()])
    try {
        jruby.currentDirectory = qaBuildPath
        jruby.runScriptlet("""
                require "bundler"
                require "bundler/cli"
                Bundler::CLI.start(['install', '--path', "${qaBuildPath}/vendor", '--gemfile', "${projectDir}/qa/integration/Gemfile"])
                """)
    } finally {
        jruby.terminate()
        Ruby.clearGlobalRuntime()
    }
}

/**
 * Installs a Gem with the given version to the given path.
 * @param projectDir Gradle projectDir
 * @param buildDir Gradle buildDir
 * @param gem Gem Name
 * @param version Version to Install
 * @param path Path to Install to
 */
void gem(File projectDir, File buildDir, String gem, String version, String path) {
    executeJruby projectDir, buildDir, { ScriptingContainer jruby ->
        jruby.currentDirectory = projectDir
        jruby.runScriptlet("""
                require 'rubygems/commands/install_command'
                cmd = Gem::Commands::InstallCommand.new
                cmd.handle_options ['--no-document', '${gem}', '-v', '${version}', '-i', '${path}']
                begin
                    cmd.execute
                rescue Gem::SystemExitException => e
                    raise e unless e.exit_code == 0
                end
                """
        )
    }
}

void buildGem(File projectDir, File buildDir, String gemspec) {
    executeJruby projectDir, buildDir, { ScriptingContainer jruby ->
        jruby.currentDirectory = projectDir
        jruby.runScriptlet("""
                require 'rubygems/commands/build_command'
                cmd = Gem::Commands::BuildCommand.new
                cmd.handle_options ['${gemspec}']
                begin
                    cmd.execute
                rescue Gem::SystemExitException => e
                    raise e unless e.exit_code == 0
                end
                """
        )
    }
}

/**
 * Executes RSpec for a given plugin.
 * @param projectDir Gradle projectDir
 * @param buildDir Gradle buildDir
 * @param plugin Plugin to run specs for
 * @param args Optional arguments to pass to the rake task
 */
void rake(File projectDir, File buildDir, String task, String... args) {
    executeJruby projectDir, buildDir, { ScriptingContainer jruby ->
        jruby.currentDirectory = projectDir
        jruby.runScriptlet("require 'rake'; require 'time'")
        def rakeArgs = args ? "'${args.join("','")}'" : ""
        jruby.runScriptlet("""
                begin
                    rake = Rake.application
                    rake.init
                    rake.load_rakefile
                    rake['${task}'].invoke(${rakeArgs})
                rescue => e
                    puts "Rake task error: #{e.class}: #{e.message}"
                    puts "Backtrace: #{e.backtrace.join("\\n")}"
                    raise e
                end
                """
        )
    }
}

void setupJruby(File projectDir, File buildDir) {
    executeJruby projectDir, buildDir, { ScriptingContainer jruby ->
        jruby.currentDirectory = projectDir
        jruby.runScriptlet("require '${projectDir}/lib/bootstrap/environment'")
        jruby.runScriptlet("LogStash::Bundler.invoke!")
        jruby.runScriptlet("LogStash::Bundler.genericize_platform")
    }
}

/**
 * Executes Closure using a fresh JRuby environment, safely tearing it down afterwards.
 * @param projectDir Gradle projectDir
 * @param buildDir Gradle buildDir
 * @param block Closure to run
 */
Object executeJruby(File projectDir, File buildDir, Closure<?> /* Object*/ block) {
    def jruby = new ScriptingContainer()
    def env = jruby.environment
    def gemDir = "${projectDir}/vendor/bundle/jruby/3.1.0".toString()
    jruby.setLoadPaths(["${projectDir}/vendor/jruby/lib/ruby/stdlib".toString()])
    env.put "USE_RUBY", "1"
    env.put "GEM_HOME", gemDir
    env.put "GEM_SPEC_CACHE", "${buildDir}/cache".toString()
    env.put "GEM_PATH", gemDir
    // Pass through ORG_GRADLE_PROJECT_fedrampHighMode if it exists in the project properties
    // See https://docs.gradle.org/current/userguide/build_environment.html#setting_a_project_property
    // For more information about setting properties via env vars prefixed with ORG_GRADLE_PROJECT
    if (project.hasProperty('fedrampHighMode') && project.property('fedrampHighMode').toBoolean()) {
        env.put "ORG_GRADLE_PROJECT_fedrampHighMode", "true"
    }
    try {
        block(jruby)
    } finally {
        jruby.terminate()
        Ruby.clearGlobalRuntime()
    }
}

//===============================================================================
// Ruby variables
//===============================================================================

def versionsPath = project.hasProperty("LOGSTASH_CORE_PATH") ? LOGSTASH_CORE_PATH + "/../versions.yml" : "${projectDir}/versions.yml"
versionMap = (Map) (new Yaml()).load(new File("${versionsPath}").text)

String jRubyURL
String jRubyVersion
String jRubySha1
Boolean doChecksum

if (versionMap["jruby-runtime-override"]) {
    jRubyVersion = versionMap["jruby-runtime-override"]["version"]
    jRubyURL = versionMap["jruby-runtime-override"]["url"]
    doChecksum = false
} else {
    jRubyVersion = versionMap["jruby"]["version"]
    jRubySha1 = versionMap["jruby"]["sha1"]
    jRubyURL = "https://repo1.maven.org/maven2/org/jruby/jruby-dist/${jRubyVersion}/jruby-dist-${jRubyVersion}-bin.tar.gz"
    doChecksum = true
}
def jrubyTarPath = "${projectDir}/vendor/_/jruby-dist-${jRubyVersion}-bin.tar.gz"

def customJRubyDir = project.hasProperty("custom.jruby.path") ? project.property("custom.jruby.path") : ""
def customJRubyVersion = customJRubyDir == "" ? "" : Files.readAllLines(Paths.get(customJRubyDir, "VERSION")).get(0).trim()
def customJRubyTar = customJRubyDir == "" ? "" : (customJRubyDir + "/maven/jruby-dist/target/jruby-dist-${customJRubyVersion}-bin.tar.gz")

tasks.register("downloadJRuby", Download) {
    description "Download JRuby artifact from this specific URL: ${jRubyURL}"
    src jRubyURL
    onlyIfNewer true
    inputs.file(versionsPath)
    outputs.file(jrubyTarPath)
    dest new File("${projectDir}/vendor/_", "jruby-dist-${jRubyVersion}-bin.tar.gz")
}

downloadJRuby.onlyIf { customJRubyDir == "" }

tasks.register("verifyFile", Verify) {
    dependsOn downloadJRuby
    description "Verify the SHA1 of the download JRuby artifact"
    inputs.file(jrubyTarPath)
    outputs.file(jrubyTarPath)
    src new File(jrubyTarPath)
    algorithm 'SHA-1'
    checksum jRubySha1
}

verifyFile.onlyIf { customJRubyDir == "" }
verifyFile.onlyIf { doChecksum }

tasks.register("buildCustomJRuby", Exec) {
    description "Build tar.gz and .jar artifacts from JRuby source directory"
    workingDir (customJRubyDir == "" ? "./" : customJRubyDir)
    commandLine './mvnw', 'clean', 'install', '-Pdist', '-Pcomplete'
    standardOutput = new ByteArrayOutputStream()
    errorOutput = new ByteArrayOutputStream()
    ext.output = {
        standardOutput.toString() + errorOutput.toString()
    }
}

buildCustomJRuby.onlyIf { customJRubyDir != "" }

tasks.register("installCustomJRuby", Copy) {
    dependsOn buildCustomJRuby
    description "Install custom built JRuby in the vendor directory"
    inputs.file(customJRubyTar)
    outputs.dir("${projectDir}/vendor/jruby")
    from tarTree(customJRubyTar == "" ? jrubyTarPath : customJRubyTar)
    eachFile { f ->
        f.path = f.path.replaceFirst("^jruby-${customJRubyVersion}", '')
    }
    includeEmptyDirs = false
    into "${projectDir}/vendor/jruby"
}

installCustomJRuby.onlyIf { customJRubyDir != "" }

tasks.register("downloadAndInstallJRuby", Copy) {
    dependsOn=[verifyFile, installCustomJRuby]
    description "Install JRuby in the vendor directory"
    inputs.file(jrubyTarPath)
    outputs.dir("${projectDir}/vendor/jruby")
    from tarTree(downloadJRuby.dest)
    eachFile { f ->
        f.path = f.path.replaceFirst("^jruby-${jRubyVersion}", '')
    }
    exclude "**/did_you_mean-*/evaluation/**" // licensing issue https://github.com/jruby/jruby/issues/6471
    exclude "vendor/bundle/jruby/**/gems/ruby-maven-libs-3.3.9/**/*"
    exclude "**/lib/jni/**/**"

    includeEmptyDirs = false
    into "${projectDir}/vendor/jruby"
}

downloadAndInstallJRuby.onlyIf { customJRubyDir == "" }

//===============================================================================
// Ruby auto-gen utilities for Java plugins
//===============================================================================

class PluginInfo {
    public String[] licenses
    public String longDescription
    public String[] authors
    public String[] email
    public String homepage
    public String pluginType
    public String pluginClass
    public String pluginName

    String pluginFullName() {
        return "logstash-" + pluginType + "-" + pluginName
    }
}

void generateRubySupportFilesForPlugin(String projectDescription, String projectGroup, String version) {
    File gemFile = file("Gemfile")
    gemFile.write("# AUTOGENERATED BY THE GRADLE SCRIPT. EDITS WILL BE OVERWRITTEN.\n")
    gemFile.append("source 'https://rubygems.org'\n")
    gemFile.append("\n")
    gemFile.append("gemspec\n")
    gemFile.append("\n")
    gemFile.append("logstash_path = ENV[\"LOGSTASH_PATH\"] || \"../../logstash\"\n")
    gemFile.append("use_logstash_source = ENV[\"LOGSTASH_SOURCE\"] && ENV[\"LOGSTASH_SOURCE\"].to_s == \"1\"\n")
    gemFile.append("\n")
    gemFile.append("if Dir.exist?(logstash_path) && use_logstash_source\n")
    gemFile.append("  gem 'logstash-core', :path => \"#{logstash_path}/logstash-core\"\n")
    gemFile.append("  gem 'logstash-core-plugin-api', :path => \"#{logstash_path}/logstash-core-plugin-api\"\n")
    gemFile.append("end\n")

    File gemspecFile = file(pluginInfo.pluginFullName() + ".gemspec")
    gemspecFile.write("# AUTOGENERATED BY THE GRADLE SCRIPT. EDITS WILL BE OVERWRITTEN.\n")
    gemspecFile.append("Gem::Specification.new do |s|\n")
    gemspecFile.append("  s.name            = '" + pluginInfo.pluginFullName() + "'\n")
    gemspecFile.append("  s.version         = ::File.read('VERSION').split('\\n').first\n")
    gemspecFile.append("  s.licenses        = ['" + String.join("', '", pluginInfo.licenses) + "']\n")
    gemspecFile.append("  s.summary         = '" + projectDescription + "'\n")
    gemspecFile.append("  s.description     = '" + pluginInfo.longDescription + "'\n")
    gemspecFile.append("  s.authors         = ['" + String.join("', '", pluginInfo.authors) + "']\n")
    gemspecFile.append("  s.email           = ['" + String.join("', '", pluginInfo.email) + "']\n")
    gemspecFile.append("  s.homepage        = '" + pluginInfo.homepage + "'\n")
    gemspecFile.append("  s.platform        = 'java'\n")
    gemspecFile.append("  s.require_paths   = ['lib', 'vendor/jar-dependencies']\n")
    gemspecFile.append("\n")
    gemspecFile.append("  s.files = Dir[\"lib/**/*\",\"*.gemspec\",\"*.md\",\"CONTRIBUTORS\",\"Gemfile\",\"LICENSE\",\"NOTICE.TXT\", \"vendor/jar-dependencies/**/*.jar\", \"vendor/jar-dependencies/**/*.rb\", \"VERSION\", \"docs/**/*\"]\n")
    gemspecFile.append("\n")
    gemspecFile.append("  # Special flag to let us know this is actually a logstash plugin\n")
    gemspecFile.append("  s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => '" + pluginInfo.pluginType + "', 'java_plugin' => 'true'}\n")
    gemspecFile.append("\n")
    gemspecFile.append("  # Gem dependencies\n")
    gemspecFile.append("  s.add_runtime_dependency \"logstash-core-plugin-api\", \">= 1.60\", \"<= 2.99\"\n")
    gemspecFile.append("  s.add_runtime_dependency 'jar-dependencies'\n")
    gemspecFile.append("  s.add_development_dependency 'logstash-devutils'\n")
    gemspecFile.append("end\n")

    String moduleName = pluginInfo.pluginType.substring(0, 1).toUpperCase() + pluginInfo.pluginType.substring(1) + "s"
    File pluginRb = file("lib/logstash/" + pluginInfo.pluginType + "s/" + pluginInfo.pluginName + ".rb")
    Files.createDirectories(pluginRb.toPath().getParent())
    pluginRb.write("# AUTOGENERATED BY THE GRADLE SCRIPT. EDITS WILL BE OVERWRITTEN.\n")
    pluginRb.append("# encoding: utf-8\n")
    pluginRb.append("require \"logstash/" + pluginInfo.pluginType + "s/base\"\n")
    pluginRb.append("require \"logstash/namespace\"\n")
    pluginRb.append("require \"" + pluginInfo.pluginFullName() + "_jars\"\n")
    pluginRb.append("require \"java\"\n")
    pluginRb.append("\n")
    pluginRb.append("class LogStash::" + moduleName + "::" + pluginInfo.pluginClass + " < LogStash::" + moduleName + "::Base\n")
    pluginRb.append("  config_name \"" + pluginInfo.pluginName + "\"\n")
    pluginRb.append("\n")
    pluginRb.append("  def self.javaClass() Java::" + projectGroup + "." + pluginInfo.pluginClass + ".java_class; end\n")
    pluginRb.append("end\n")

    File pluginJarsRb = file("lib/" + pluginInfo.pluginFullName() + "_jars.rb")
    pluginJarsRb.write("# AUTOGENERATED BY THE GRADLE SCRIPT. EDITS WILL BE OVERWRITTEN.\n")
    pluginJarsRb.append("# encoding: utf-8\n")
    pluginJarsRb.append("\n")
    pluginJarsRb.append("require 'jar_dependencies'\n")
    pluginJarsRb.append("require_jar('" + projectGroup + "', '" + pluginInfo.pluginFullName() + "', '" + version +"')\n")
}

void validatePluginJar(File pluginJar, String group) {
    List<String> validationErrors = new ArrayList<>()

    if (group.equals('org.logstash') || group.startsWith('org.logstash.') || group.equals('co.elastic.logstash') || group.startsWith('co.elastic.logstash.')) {
        validationErrors.add("The plugin should not be placed in the 'org.logstash' or 'co.elastic.logstash' packages")
        throw new GradleScriptException("Plugin validation errors:" + System.lineSeparator() +
                String.join(System.lineSeparator(), validationErrors), null)
    }

    URLClassLoader cl = URLClassLoader.newInstance([pluginJar.toURI().toURL()] as URL[])
    String pluginClassName = group + "." + pluginInfo.pluginClass

    Class<?> pluginClass = null
    try {
        pluginClass = cl.loadClass(pluginClassName)
    } catch (ClassNotFoundException ex) {
        validationErrors.add(String.format("Unable to locate plugin class defined in build.gradle as '%s' in jar '%s'", pluginClassName, pluginJar))
        throw new GradleScriptException("Plugin validation errors:" + System.lineSeparator() +
                String.join(System.lineSeparator(), validationErrors), null)
    }

    if (pluginClass != null) {

        Annotation[] logstashPlugin = pluginClass.getAnnotations().findAll({ x -> x.annotationType().toString().equals("interface co.elastic.logstash.api.LogstashPlugin") })
        if (logstashPlugin.length != 1) {
            validationErrors.add("There must be a single @LogstashPlugin annotation on the plugin class")
        } else {
            String pluginAnnotation = logstashPlugin[0].name()

            if (pluginAnnotation != pluginInfo.pluginName) {
                validationErrors.add("The 'name' property on the @LogstashPlugin (which is '" + pluginAnnotation + "') must match the 'pluginName' property which is defined as '" + pluginInfo.pluginName + "' in the build.gradle file")
            }

            if (pluginAnnotation.replace("_", "").toLowerCase() != pluginInfo.pluginClass.toLowerCase()) {
                validationErrors.add("The 'name' property on the @LogstashPlugin (which is '" + pluginAnnotation + "') must match the plugin class name '" + pluginInfo.pluginClass + "' excluding casing and underscores")
            }
        }
    }

    if (validationErrors.size() > 0) {
        throw new GradleScriptException("Plugin validation errors:" + System.lineSeparator() +
                String.join(System.lineSeparator(), validationErrors), null)
    }
}