diff --git a/apache_stdout.conf b/apache_stdout.conf new file mode 100644 index 000000000..8e9968248 --- /dev/null +++ b/apache_stdout.conf @@ -0,0 +1,88 @@ +input { + file { + id => "logfileRead" + start_position => beginning + ignore_older => 0 + path => "/Users/andrewvc/projects/ls_apache_materials/apache_access_logs" + } + + stdin {id => logStdin} +} + +filter { + grok { + id => "apacheCommonLog" + match => { + "message" => '%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "%{WORD:verb} %{DATA:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:response:int} (?:-|%{NUMBER:bytes:int}) %{QS:referrer} %{QS:agent}' + } + } + + geoip { + id => "clientGeo" + source => clientip + target => geoip + } + + useragent { + id => "clientUA" + source => agent + target => useragent + } + + + date { + id => "clientDate" + match => [ "timestamp", "dd/MMM/YYYY:HH:mm:ss Z" ] + locale => en + } + + if [geoip][country_code2] == "US" { + mutate { + id => "addUsRegion" + add_field => { "aws-region" => "us-east-1" } + } + } else if [geoip][country_code2] == "CA" { + if [referrer] =~ /google/ { + sleep { + id => "pointlessSleep" + time => 0.001 + } + } + } else { + mutate { + id => addOtherRegion + add_field => { "aws-region" => "eu-central-1" } + } + } + + + if [request] =~ /(?i)\.(png|jpg|gif)$/ { + grok { + id => grokImage + match => { + request => "%{(?i)\.(png|jpg|gif)$:extension}" + } + add_tag => ["image"] + } + + mutate { + id => addCanadianRegion + add_field => { "aws-region" => "ca-central-1" } + } + } else if [request] =~ /articles/ { + mutate { + id => tagArticle + add_tag => ["article"] + } + } +} + +output { + elasticsearch { + id => "mainEs" + index => "%{@type}-" + } + if [geoip][country_code2] != "US" { + stdout { id => "linuxStdout" codec => json_lines } + } +} diff --git a/logstash-core/.lock b/logstash-core/.lock new file mode 100644 index 000000000..e69de29bb diff --git a/logstash-core/lib/logstash/agent.rb b/logstash-core/lib/logstash/agent.rb index b367dba5e..b23af958b 100644 --- a/logstash-core/lib/logstash/agent.rb +++ b/logstash-core/lib/logstash/agent.rb @@ -403,6 +403,7 @@ class LogStash::Agent if !t.alive? return false elsif pipeline.running? + dispatcher.fire(:pipeline_started, pipeline) return true else sleep 0.01 @@ -416,6 +417,7 @@ class LogStash::Agent @logger.warn("stopping pipeline", :id => id) pipeline.shutdown { LogStash::ShutdownWatcher.start(pipeline) } @pipelines[id].thread.join + dispatcher.fire(:pipeline_stopped, pipeline) end def start_pipelines diff --git a/logstash-core/lib/logstash/compiler.rb b/logstash-core/lib/logstash/compiler.rb new file mode 100644 index 000000000..56017eed7 --- /dev/null +++ b/logstash-core/lib/logstash/compiler.rb @@ -0,0 +1,38 @@ +require 'logstash/util/loggable' +require 'logstash/compiler/lscl/lscl_grammar' + +java_import org.logstash.config.ir.Pipeline +java_import org.logstash.config.ir.graph.Graph; +java_import org.logstash.config.ir.graph.PluginVertex; + +module LogStash; class Compiler + include ::LogStash::Util::Loggable + + def self.compile_pipeline(config_str, source_file=nil) + graph_sections = self.compile_graph(config_str, source_file) + pipeline = org.logstash.config.ir.Pipeline.new( + graph_sections[:input], + graph_sections[:filter], + graph_sections[:output] + ) + end + + def self.compile_ast(config_str, source_file=nil) + grammar = LogStashCompilerLSCLGrammarParser.new + config = grammar.parse(config_str) + + if config.nil? + raise ConfigurationError, grammar.failure_reason + end + + config + end + + def self.compile_imperative(config_str, source_file=nil) + compile_ast(config_str, source_file).compile(source_file) + end + + def self.compile_graph(config_str, source_file=nil) + Hash[compile_imperative(config_str, source_file).map {|section,icompiled| [section, icompiled.toGraph]}] + end +end; end diff --git a/logstash-core/lib/logstash/compiler/lscl.rb b/logstash-core/lib/logstash/compiler/lscl.rb new file mode 100644 index 000000000..f0e4814e9 --- /dev/null +++ b/logstash-core/lib/logstash/compiler/lscl.rb @@ -0,0 +1,566 @@ +# encoding: utf-8 +require 'logstash/errors' +require "treetop" +require "logstash/compiler/treetop_monkeypatches" +java_import org.logstash.config.ir.DSL +java_import org.logstash.config.ir.SourceMetadata + +module LogStashCompilerLSCLGrammar; module LogStash; module Compiler; module LSCL; module AST + # Helpers for parsing LSCL files + module Helpers + def source_meta + line, column = line_and_column + org.logstash.config.ir.SourceMetadata.new(source_file, line, column, self.text_value) + end + + def source_file=(value) + set_meta(:source_file, value) + end + + def source_file + get_meta(:source_file) + end + + def compose(*statements) + compose_for(section_type.to_sym).call(source_meta, *statements) + end + + def compose_for(section_sym) + if section_sym == :filter + jdsl.method(:iComposeSequence) + else + jdsl.method(:iComposeParallel) + end + end + + def line_and_column + start = self.interval.first + [self.input.line_of(start), self.input.column_of(start)] + end + + def empty_source_meta() + org.logstash.config.ir.SourceMetadata.new() + end + + def jdsl + org.logstash.config.ir.DSL + end + + def self.jdsl + org.logstash.config.ir.DSL + end + + AND_METHOD = jdsl.method(:eAnd) + OR_METHOD = jdsl.method(:eOr) + end + + class Node < Treetop::Runtime::SyntaxNode + include Helpers + + def section_type + if recursive_select_parent(Plugin).any? + return "codec" + else + section = recursive_select_parent(PluginSection) + return section.first.plugin_type.text_value + end + end + end + + class Config < Node + include Helpers + + def compile(source_file=nil) + # There is no way to move vars across nodes in treetop :( + self.source_file = source_file + + sections = recursive_select(PluginSection) + + section_map = { + :input => [], + :filter => [], + :output => [] + } + + sections.each do |section| + section_name = section.plugin_type.text_value.to_sym + section_expr = section.expr + raise "Unknown section name #{section_name}!" if ![:input, :output, :filter].include?(section_name) + ::Array[section_expr].each do |se| + section_map[section_name].concat se + end + end + + section_map.keys.each do |key| + section_map[key] = compose_for(key).call(empty_source_meta, *section_map[key]) + end + + section_map + end + end + + class Comment < Node; end + class Whitespace < Node; end + + class PluginSection < Node + def expr + recursive_select(Branch, Plugin).map(&:expr) + end + end + + class Plugins < Node; end + class Plugin < Node + def expr + jdsl.iPlugin(source_meta, plugin_type_enum, self.plugin_name, self.expr_attributes) + end + + def plugin_type_enum + case section_type + when "input" + Java::OrgLogstashConfigIr::PluginDefinition::Type::INPUT + when "codec" + Java::OrgLogstashConfigIr::PluginDefinition::Type::CODEC + when "filter" + Java::OrgLogstashConfigIr::PluginDefinition::Type::FILTER + when "output" + Java::OrgLogstashConfigIr::PluginDefinition::Type::OUTPUT + end + end + + def plugin_name + return name.text_value + end + + def expr_attributes + # Turn attributes into a hash map + self.attributes.recursive_select(Attribute).map(&:expr).map {|k,v| + if v.java_kind_of?(Java::OrgLogstashConfigIrExpression::ValueExpression) + [k, v.get] + else + [k,v] + end + }.reduce({}) do |hash,kv| + k,v = kv + hash[k] = v + hash + end + + end + end + + class Name < Node + def expr + return text_value + end + end + + class Attribute < Node + def expr + [name.text_value, value.expr] + end + end + + class RValue < Node; end + class Value < RValue; end + + class Bareword < Value + def expr + jdsl.eValue(source_meta, text_value) + end + end + + class String < Value + def expr + jdsl.eValue(source_meta, text_value[1...-1]) + end + end + + class RegExp < Value + def expr + # Strip the slashes off + jdsl.eRegex(text_value[1..-2]) + end + end + + class Number < Value + def expr + jdsl.eValue(source_meta, text_value.include?(".") ? + text_value.to_f : + text_value.to_i) + end + end + + class Array < Value + def expr + jdsl.eValue(source_meta, recursive_select(Value).map(&:expr).map(&:get)) + end + end + + class Hash < Value + def validate! + duplicate_values = find_duplicate_keys + + if duplicate_values.size > 0 + raise ConfigurationError.new( + I18n.t("logstash.runner.configuration.invalid_plugin_settings_duplicate_keys", + :keys => duplicate_values.join(', '), + :line => input.line_of(interval.first), + :column => input.column_of(interval.first), + :byte => interval.first + 1, + :after => input[0..interval.first] + ) + ) + end + end + + def find_duplicate_keys + values = recursive_select(HashEntry).collect { |hash_entry| hash_entry.name.text_value } + values.find_all { |v| values.count(v) > 1 }.uniq + end + + def expr + validate! + ::Hash[recursive_select(HashEntry).map(&:expr)] + end + end + + class HashEntries < Node; end + + class HashEntry < Node + def expr + return [name.expr.get, value.expr.get()] + end + end + + class Branch < Node + def expr + # Build this stuff as s-expressions for convenience at first + # This will turn if/elsif/else blocks into nested if/else trees + + exprs = [] + else_stack = [] # For turning if / elsif / else into nested ifs + + self.recursive_select(Plugin, If, Elsif, Else).each do |node| + if node.is_a?(If) + exprs << :if + exprs << expr_cond(node) + exprs << expr_body(node) + elsif node.is_a?(Elsif) + condition = expr_cond(node) + body = expr_body(node) + else_stack << [:if, condition, body] + elsif node.is_a?(Else) + body = expr_body(node) + if else_stack.size >= 1 + else_stack.last << body + else + exprs << body + end + end + end + + else_stack.reverse.each_cons(2) do |cons| + later,earlier = cons + earlier << later + end + exprs << else_stack.first + + # Then convert to the imperative java IR + javaify_sexpr(exprs) + end + + def javaify_sexpr(sexpr) + return nil if sexpr.nil? + + head, tail = sexpr.first + tail = sexpr[1..-1] + + if head == :if + condition, t_branch, f_branch = tail + + java_t_branch = t_branch && javaify_sexpr(t_branch) + java_f_branch = f_branch && javaify_sexpr(f_branch) + + if java_t_branch || java_f_branch + jdsl.iIf(condition, java_t_branch || jdsl.noop, java_f_branch || jdsl.noop) + else + jdsl.noop() + end + elsif head == :compose + tail && tail.size > 0 ? compose(*tail) : jdsl.noop + else + raise "Unknown expression #{sexpr}!" + end + end + + def expr_cond(node) + node.elements.find {|e| e.is_a?(Condition)}.expr + end + + def expr_body(node) + [:compose, *node.recursive_select(Plugin, Branch).map(&:expr)] + end + end + + # Branch covers all these + class BranchEntry < Node; end + class If < BranchEntry; end + class Elsif < BranchEntry; end + class Else < BranchEntry; end + + class Condition < Node + include Helpers + + def expr + first_element = elements.first + rest_elements = elements.size > 1 ? elements[1].recursive_select(BooleanOperator, Expression, SelectorElement) : [] + + all_elements = [first_element, *rest_elements] + + if all_elements.size == 1 + elem = all_elements.first + if elem.is_a?(Selector) + eventValue = elem.recursive_select(SelectorElement).first.expr + jdsl.eTruthy(source_meta, eventValue) + elsif elem.is_a?(RegexpExpression) + elem.expr + else + join_conditions(all_elements) + end + else + join_conditions(all_elements) + end + end + + def precedence(op) + # Believe this is right for logstash? + case op + when AND_METHOD + 2 + when OR_METHOD + 1 + else + raise ArgumentError, "Unexpected operator #{op}" + end + end + + # Converts an sexpr of :and or :or to the java imperative IR + def jconvert(sexpr) + raise "jconvert cannot handle nils!" if sexpr.nil? + + if sexpr.java_kind_of?(Java::OrgLogstashConfigIrExpression::Expression) + return sexpr + end + + op, left, right = sexpr + + left_c = jconvert(left) + right_c = jconvert(right) + + case op + when :and + return jdsl.eAnd(left, right); + when :or + return jdsl.eOr(left, right); + else + raise "Unknown op #{jop}" + end + end + + def join_conditions(all_elements) + # Use Dijkstra's shunting yard algorithm + out = [] + operators = [] + + all_elements.each do |e| + e_exp = e.expr + + if e.is_a?(BooleanOperator) + if operators.last && precedence(operators.last) > precedence(e_exp) + out << operators.pop + end + operators << e_exp + else + out << e_exp + end + end + operators.reverse.each {|o| out << o} + + stack = [] + expr = [] + out.each do |e| + if e.is_a?(Symbol) + rval, lval = stack.pop, stack.pop + stack << jconvert([e, lval, rval]) + elsif e.nil? + raise "Nil expr encountered! This should not happen!" + else + stack << e + end + end + + stack_to_expr(stack) + end + + def stack_to_expr(stack) + raise "Got an empty stack! This should not happen!" if stack.empty? + stack = stack.reverse # We need to work the stack in reverse order + + working_stack = [] + while elem = stack.pop + if elem.is_a?(::Method) + right, left = working_stack.pop, working_stack.pop + working_stack << elem.call(left, right) + else + working_stack << elem + end + end + + raise "Invariant violated! Stack size > 1" if working_stack.size > 1 + + working_stack.first + end + end + + module Expression + def expr + # If we have a more specific type (like a Negative expression) use that + if defined?(super) + return super + end + + exprs = self.recursive_select(Condition, Selector).map(&:expr) + + raise "Exprs should only have one part!" if exprs.size != 1 + exprs.first + end + end + + module NegativeExpression + include Helpers + + def expr + exprs = self.recursive_select(Condition, Selector).map(&:expr) + raise "Negative exprs should only have one part!" if exprs.size != 1 + jdsl.eNot(source_meta, exprs.first) + end + end + + module ComparisonExpression + include Helpers + + def expr + lval, comparison_method, rval = self.recursive_select(Selector, Expression, ComparisonOperator, Number, String).map(&:expr) + comparison_method.call(source_meta, lval, rval) + end + end + + module InExpression + include Helpers + + def expr + item, list = recursive_select(RValue) + jdsl.eIn(source_meta, item.expr, list.expr) + end + end + + module NotInExpression + include Helpers + + def expr + item, list = recursive_select(RValue) + jdsl.eNot(source_meta, jdsl.eIn(item.expr, list.expr)) + end + end + + # Not implemented because no one uses this + class MethodCall < Node; end + + class RegexpExpression < Node + def expr + selector, operator_method, regexp = recursive_select( + Selector, + LogStash::Compiler::LSCL::AST::RegExpOperator, + LogStash::Compiler::LSCL::AST::RegExp, + LogStash::Compiler::LSCL::AST::String # Strings work as rvalues! :p + ).map(&:expr) + + # Handle string rvalues, they just get turned into regexps + # Maybe we really shouldn't handle these anymore... + if regexp.class == org.logstash.config.ir.expression.ValueExpression + regexp = jdsl.eRegex(regexp.get) + end + + raise "Expected a selector in #{text_value}!" unless selector + raise "Expected a regexp in #{text_value}!" unless regexp + + operator_method.call(source_meta, selector, regexp); + end + end + + module BranchOrPlugin; end + + module ComparisonOperator + include Helpers + + def expr + case self.text_value + when "==" + jdsl.method(:eEq) + when "!=" + jdsl.method(:eNeq) + when ">" + jdsl.method(:eGt) + when "<" + jdsl.method(:eLt) + when ">=" + jdsl.method(:eGte) + when "<=" + jdsl.method(:eLte) + else + raise "Unknown operator #{self.text_value}" + end + end + end + + module RegExpOperator + include Helpers + + def expr + if self.text_value == '!~' + jdsl.method(:eRegexNeq) + elsif self.text_value == '=~' + jdsl.method(:eRegexEq) + else + raise "Unknown regex operator #{self.text_value}" + end + end + end + + module BooleanOperator + include Helpers + + def expr + case self.text_value + when "and" + AND_METHOD + when "or" + OR_METHOD + else + raise "Unknown operator #{self.text_value}" + end + end + end + + class Selector < RValue + def expr + jdsl.eEventValue(source_meta, text_value) + end + end + + class SelectorElement < Node; + def expr + jdsl.eEventValue(source_meta, text_value) + end + end +end; end; end; end; end; diff --git a/logstash-core/lib/logstash/compiler/lscl/lscl_grammar.rb b/logstash-core/lib/logstash/compiler/lscl/lscl_grammar.rb new file mode 100644 index 000000000..e391b93d6 --- /dev/null +++ b/logstash-core/lib/logstash/compiler/lscl/lscl_grammar.rb @@ -0,0 +1,3503 @@ +# Autogenerated from a Treetop grammar. Edits may be lost. + + +require "treetop" +require "logstash/compiler/lscl.rb" + +module LogStashCompilerLSCLGrammar + include Treetop::Runtime + + def root + @root ||= :config + end + + module Config0 + def _ + elements[0] + end + + def plugin_section + elements[1] + end + end + + module Config1 + def _1 + elements[0] + end + + def plugin_section + elements[1] + end + + def _2 + elements[2] + end + + def _3 + elements[4] + end + end + + def _nt_config + start_index = index + if node_cache[:config].has_key?(index) + cached = node_cache[:config][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0, s0 = index, [] + r1 = _nt__ + s0 << r1 + if r1 + r2 = _nt_plugin_section + s0 << r2 + if r2 + r3 = _nt__ + s0 << r3 + if r3 + s4, i4 = [], index + loop do + i5, s5 = index, [] + r6 = _nt__ + s5 << r6 + if r6 + r7 = _nt_plugin_section + s5 << r7 + end + if s5.last + r5 = instantiate_node(SyntaxNode,input, i5...index, s5) + r5.extend(Config0) + else + @index = i5 + r5 = nil + end + if r5 + s4 << r5 + else + break + end + end + r4 = instantiate_node(SyntaxNode,input, i4...index, s4) + s0 << r4 + if r4 + r8 = _nt__ + s0 << r8 + end + end + end + end + if s0.last + r0 = instantiate_node(LogStash::Compiler::LSCL::AST::Config,input, i0...index, s0) + r0.extend(Config1) + else + @index = i0 + r0 = nil + end + + node_cache[:config][start_index] = r0 + + r0 + end + + module Comment0 + end + + def _nt_comment + start_index = index + if node_cache[:comment].has_key?(index) + cached = node_cache[:comment][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + s0, i0 = [], index + loop do + i1, s1 = index, [] + r3 = _nt_whitespace + if r3 + r2 = r3 + else + r2 = instantiate_node(SyntaxNode,input, index...index) + end + s1 << r2 + if r2 + if has_terminal?("#", false, index) + r4 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure("#") + r4 = nil + end + s1 << r4 + if r4 + s5, i5 = [], index + loop do + if has_terminal?('\G[^\\r\\n]', true, index) + r6 = true + @index += 1 + else + r6 = nil + end + if r6 + s5 << r6 + else + break + end + end + r5 = instantiate_node(SyntaxNode,input, i5...index, s5) + s1 << r5 + if r5 + if has_terminal?("\r", false, index) + r8 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure("\r") + r8 = nil + end + if r8 + r7 = r8 + else + r7 = instantiate_node(SyntaxNode,input, index...index) + end + s1 << r7 + if r7 + if has_terminal?("\n", false, index) + r9 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure("\n") + r9 = nil + end + s1 << r9 + end + end + end + end + if s1.last + r1 = instantiate_node(SyntaxNode,input, i1...index, s1) + r1.extend(Comment0) + else + @index = i1 + r1 = nil + end + if r1 + s0 << r1 + else + break + end + end + if s0.empty? + @index = i0 + r0 = nil + else + r0 = instantiate_node(LogStash::Compiler::LSCL::AST::Comment,input, i0...index, s0) + end + + node_cache[:comment][start_index] = r0 + + r0 + end + + def _nt__ + start_index = index + if node_cache[:_].has_key?(index) + cached = node_cache[:_][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + s0, i0 = [], index + loop do + i1 = index + r2 = _nt_comment + if r2 + r1 = r2 + else + r3 = _nt_whitespace + if r3 + r1 = r3 + else + @index = i1 + r1 = nil + end + end + if r1 + s0 << r1 + else + break + end + end + r0 = instantiate_node(LogStash::Compiler::LSCL::AST::Whitespace,input, i0...index, s0) + + node_cache[:_][start_index] = r0 + + r0 + end + + def _nt_whitespace + start_index = index + if node_cache[:whitespace].has_key?(index) + cached = node_cache[:whitespace][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + s0, i0 = [], index + loop do + if has_terminal?('\G[ \\t\\r\\n]', true, index) + r1 = true + @index += 1 + else + r1 = nil + end + if r1 + s0 << r1 + else + break + end + end + if s0.empty? + @index = i0 + r0 = nil + else + r0 = instantiate_node(LogStash::Compiler::LSCL::AST::Whitespace,input, i0...index, s0) + end + + node_cache[:whitespace][start_index] = r0 + + r0 + end + + module PluginSection0 + def branch_or_plugin + elements[0] + end + + def _ + elements[1] + end + end + + module PluginSection1 + def plugin_type + elements[0] + end + + def _1 + elements[1] + end + + def _2 + elements[3] + end + + end + + def _nt_plugin_section + start_index = index + if node_cache[:plugin_section].has_key?(index) + cached = node_cache[:plugin_section][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0, s0 = index, [] + r1 = _nt_plugin_type + s0 << r1 + if r1 + r2 = _nt__ + s0 << r2 + if r2 + if has_terminal?("{", false, index) + r3 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure("{") + r3 = nil + end + s0 << r3 + if r3 + r4 = _nt__ + s0 << r4 + if r4 + s5, i5 = [], index + loop do + i6, s6 = index, [] + r7 = _nt_branch_or_plugin + s6 << r7 + if r7 + r8 = _nt__ + s6 << r8 + end + if s6.last + r6 = instantiate_node(SyntaxNode,input, i6...index, s6) + r6.extend(PluginSection0) + else + @index = i6 + r6 = nil + end + if r6 + s5 << r6 + else + break + end + end + r5 = instantiate_node(SyntaxNode,input, i5...index, s5) + s0 << r5 + if r5 + if has_terminal?("}", false, index) + r9 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure("}") + r9 = nil + end + s0 << r9 + end + end + end + end + end + if s0.last + r0 = instantiate_node(LogStash::Compiler::LSCL::AST::PluginSection,input, i0...index, s0) + r0.extend(PluginSection1) + else + @index = i0 + r0 = nil + end + + node_cache[:plugin_section][start_index] = r0 + + r0 + end + + def _nt_branch_or_plugin + start_index = index + if node_cache[:branch_or_plugin].has_key?(index) + cached = node_cache[:branch_or_plugin][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0 = index + r1 = _nt_branch + if r1 + r0 = r1 + else + r2 = _nt_plugin + if r2 + r0 = r2 + else + @index = i0 + r0 = nil + end + end + + node_cache[:branch_or_plugin][start_index] = r0 + + r0 + end + + def _nt_plugin_type + start_index = index + if node_cache[:plugin_type].has_key?(index) + cached = node_cache[:plugin_type][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0 = index + if has_terminal?("input", false, index) + r1 = instantiate_node(SyntaxNode,input, index...(index + 5)) + @index += 5 + else + terminal_parse_failure("input") + r1 = nil + end + if r1 + r0 = r1 + else + if has_terminal?("filter", false, index) + r2 = instantiate_node(SyntaxNode,input, index...(index + 6)) + @index += 6 + else + terminal_parse_failure("filter") + r2 = nil + end + if r2 + r0 = r2 + else + if has_terminal?("output", false, index) + r3 = instantiate_node(SyntaxNode,input, index...(index + 6)) + @index += 6 + else + terminal_parse_failure("output") + r3 = nil + end + if r3 + r0 = r3 + else + @index = i0 + r0 = nil + end + end + end + + node_cache[:plugin_type][start_index] = r0 + + r0 + end + + module Plugins0 + def _ + elements[0] + end + + def plugin + elements[1] + end + end + + module Plugins1 + def plugin + elements[0] + end + + end + + def _nt_plugins + start_index = index + if node_cache[:plugins].has_key?(index) + cached = node_cache[:plugins][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i1, s1 = index, [] + r2 = _nt_plugin + s1 << r2 + if r2 + s3, i3 = [], index + loop do + i4, s4 = index, [] + r5 = _nt__ + s4 << r5 + if r5 + r6 = _nt_plugin + s4 << r6 + end + if s4.last + r4 = instantiate_node(SyntaxNode,input, i4...index, s4) + r4.extend(Plugins0) + else + @index = i4 + r4 = nil + end + if r4 + s3 << r4 + else + break + end + end + r3 = instantiate_node(SyntaxNode,input, i3...index, s3) + s1 << r3 + end + if s1.last + r1 = instantiate_node(SyntaxNode,input, i1...index, s1) + r1.extend(Plugins1) + else + @index = i1 + r1 = nil + end + if r1 + r0 = r1 + else + r0 = instantiate_node(SyntaxNode,input, index...index) + end + + node_cache[:plugins][start_index] = r0 + + r0 + end + + module Plugin0 + def whitespace + elements[0] + end + + def _ + elements[1] + end + + def attribute + elements[2] + end + end + + module Plugin1 + def attribute + elements[0] + end + + end + + module Plugin2 + def name + elements[0] + end + + def _1 + elements[1] + end + + def _2 + elements[3] + end + + def attributes + elements[4] + end + + def _3 + elements[5] + end + + end + + def _nt_plugin + start_index = index + if node_cache[:plugin].has_key?(index) + cached = node_cache[:plugin][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0, s0 = index, [] + r1 = _nt_name + s0 << r1 + if r1 + r2 = _nt__ + s0 << r2 + if r2 + if has_terminal?("{", false, index) + r3 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure("{") + r3 = nil + end + s0 << r3 + if r3 + r4 = _nt__ + s0 << r4 + if r4 + i6, s6 = index, [] + r7 = _nt_attribute + s6 << r7 + if r7 + s8, i8 = [], index + loop do + i9, s9 = index, [] + r10 = _nt_whitespace + s9 << r10 + if r10 + r11 = _nt__ + s9 << r11 + if r11 + r12 = _nt_attribute + s9 << r12 + end + end + if s9.last + r9 = instantiate_node(SyntaxNode,input, i9...index, s9) + r9.extend(Plugin0) + else + @index = i9 + r9 = nil + end + if r9 + s8 << r9 + else + break + end + end + r8 = instantiate_node(SyntaxNode,input, i8...index, s8) + s6 << r8 + end + if s6.last + r6 = instantiate_node(SyntaxNode,input, i6...index, s6) + r6.extend(Plugin1) + else + @index = i6 + r6 = nil + end + if r6 + r5 = r6 + else + r5 = instantiate_node(SyntaxNode,input, index...index) + end + s0 << r5 + if r5 + r13 = _nt__ + s0 << r13 + if r13 + if has_terminal?("}", false, index) + r14 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure("}") + r14 = nil + end + s0 << r14 + end + end + end + end + end + end + if s0.last + r0 = instantiate_node(LogStash::Compiler::LSCL::AST::Plugin,input, i0...index, s0) + r0.extend(Plugin2) + else + @index = i0 + r0 = nil + end + + node_cache[:plugin][start_index] = r0 + + r0 + end + + def _nt_name + start_index = index + if node_cache[:name].has_key?(index) + cached = node_cache[:name][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0 = index + s1, i1 = [], index + loop do + if has_terminal?('\G[A-Za-z0-9_-]', true, index) + r2 = true + @index += 1 + else + r2 = nil + end + if r2 + s1 << r2 + else + break + end + end + if s1.empty? + @index = i1 + r1 = nil + else + r1 = instantiate_node(LogStash::Compiler::LSCL::AST::Name,input, i1...index, s1) + end + if r1 + r0 = r1 + else + r3 = _nt_string + if r3 + r0 = r3 + else + @index = i0 + r0 = nil + end + end + + node_cache[:name][start_index] = r0 + + r0 + end + + module Attribute0 + def name + elements[0] + end + + def _1 + elements[1] + end + + def _2 + elements[3] + end + + def value + elements[4] + end + end + + def _nt_attribute + start_index = index + if node_cache[:attribute].has_key?(index) + cached = node_cache[:attribute][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0, s0 = index, [] + r1 = _nt_name + s0 << r1 + if r1 + r2 = _nt__ + s0 << r2 + if r2 + if has_terminal?("=>", false, index) + r3 = instantiate_node(SyntaxNode,input, index...(index + 2)) + @index += 2 + else + terminal_parse_failure("=>") + r3 = nil + end + s0 << r3 + if r3 + r4 = _nt__ + s0 << r4 + if r4 + r5 = _nt_value + s0 << r5 + end + end + end + end + if s0.last + r0 = instantiate_node(LogStash::Compiler::LSCL::AST::Attribute,input, i0...index, s0) + r0.extend(Attribute0) + else + @index = i0 + r0 = nil + end + + node_cache[:attribute][start_index] = r0 + + r0 + end + + def _nt_value + start_index = index + if node_cache[:value].has_key?(index) + cached = node_cache[:value][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0 = index + r1 = _nt_plugin + if r1 + r0 = r1 + else + r2 = _nt_bareword + if r2 + r0 = r2 + else + r3 = _nt_string + if r3 + r0 = r3 + else + r4 = _nt_number + if r4 + r0 = r4 + else + r5 = _nt_array + if r5 + r0 = r5 + else + r6 = _nt_hash + if r6 + r0 = r6 + else + @index = i0 + r0 = nil + end + end + end + end + end + end + + node_cache[:value][start_index] = r0 + + r0 + end + + def _nt_array_value + start_index = index + if node_cache[:array_value].has_key?(index) + cached = node_cache[:array_value][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0 = index + r1 = _nt_bareword + if r1 + r0 = r1 + else + r2 = _nt_string + if r2 + r0 = r2 + else + r3 = _nt_number + if r3 + r0 = r3 + else + r4 = _nt_array + if r4 + r0 = r4 + else + r5 = _nt_hash + if r5 + r0 = r5 + else + @index = i0 + r0 = nil + end + end + end + end + end + + node_cache[:array_value][start_index] = r0 + + r0 + end + + module Bareword0 + end + + def _nt_bareword + start_index = index + if node_cache[:bareword].has_key?(index) + cached = node_cache[:bareword][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0, s0 = index, [] + if has_terminal?('\G[A-Za-z_]', true, index) + r1 = true + @index += 1 + else + r1 = nil + end + s0 << r1 + if r1 + s2, i2 = [], index + loop do + if has_terminal?('\G[A-Za-z0-9_]', true, index) + r3 = true + @index += 1 + else + r3 = nil + end + if r3 + s2 << r3 + else + break + end + end + if s2.empty? + @index = i2 + r2 = nil + else + r2 = instantiate_node(SyntaxNode,input, i2...index, s2) + end + s0 << r2 + end + if s0.last + r0 = instantiate_node(LogStash::Compiler::LSCL::AST::Bareword,input, i0...index, s0) + r0.extend(Bareword0) + else + @index = i0 + r0 = nil + end + + node_cache[:bareword][start_index] = r0 + + r0 + end + + module DoubleQuotedString0 + end + + module DoubleQuotedString1 + end + + def _nt_double_quoted_string + start_index = index + if node_cache[:double_quoted_string].has_key?(index) + cached = node_cache[:double_quoted_string][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0, s0 = index, [] + if has_terminal?('"', false, index) + r1 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure('"') + r1 = nil + end + s0 << r1 + if r1 + s2, i2 = [], index + loop do + i3 = index + if has_terminal?('\"', false, index) + r4 = instantiate_node(SyntaxNode,input, index...(index + 2)) + @index += 2 + else + terminal_parse_failure('\"') + r4 = nil + end + if r4 + r3 = r4 + else + i5, s5 = index, [] + i6 = index + if has_terminal?('"', false, index) + r7 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure('"') + r7 = nil + end + if r7 + r6 = nil + else + @index = i6 + r6 = instantiate_node(SyntaxNode,input, index...index) + end + s5 << r6 + if r6 + if index < input_length + r8 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure("any character") + r8 = nil + end + s5 << r8 + end + if s5.last + r5 = instantiate_node(SyntaxNode,input, i5...index, s5) + r5.extend(DoubleQuotedString0) + else + @index = i5 + r5 = nil + end + if r5 + r3 = r5 + else + @index = i3 + r3 = nil + end + end + if r3 + s2 << r3 + else + break + end + end + r2 = instantiate_node(SyntaxNode,input, i2...index, s2) + s0 << r2 + if r2 + if has_terminal?('"', false, index) + r9 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure('"') + r9 = nil + end + s0 << r9 + end + end + if s0.last + r0 = instantiate_node(LogStash::Compiler::LSCL::AST::String,input, i0...index, s0) + r0.extend(DoubleQuotedString1) + else + @index = i0 + r0 = nil + end + + node_cache[:double_quoted_string][start_index] = r0 + + r0 + end + + module SingleQuotedString0 + end + + module SingleQuotedString1 + end + + def _nt_single_quoted_string + start_index = index + if node_cache[:single_quoted_string].has_key?(index) + cached = node_cache[:single_quoted_string][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0, s0 = index, [] + if has_terminal?("'", false, index) + r1 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure("'") + r1 = nil + end + s0 << r1 + if r1 + s2, i2 = [], index + loop do + i3 = index + if has_terminal?("\\'", false, index) + r4 = instantiate_node(SyntaxNode,input, index...(index + 2)) + @index += 2 + else + terminal_parse_failure("\\'") + r4 = nil + end + if r4 + r3 = r4 + else + i5, s5 = index, [] + i6 = index + if has_terminal?("'", false, index) + r7 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure("'") + r7 = nil + end + if r7 + r6 = nil + else + @index = i6 + r6 = instantiate_node(SyntaxNode,input, index...index) + end + s5 << r6 + if r6 + if index < input_length + r8 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure("any character") + r8 = nil + end + s5 << r8 + end + if s5.last + r5 = instantiate_node(SyntaxNode,input, i5...index, s5) + r5.extend(SingleQuotedString0) + else + @index = i5 + r5 = nil + end + if r5 + r3 = r5 + else + @index = i3 + r3 = nil + end + end + if r3 + s2 << r3 + else + break + end + end + r2 = instantiate_node(SyntaxNode,input, i2...index, s2) + s0 << r2 + if r2 + if has_terminal?("'", false, index) + r9 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure("'") + r9 = nil + end + s0 << r9 + end + end + if s0.last + r0 = instantiate_node(LogStash::Compiler::LSCL::AST::String,input, i0...index, s0) + r0.extend(SingleQuotedString1) + else + @index = i0 + r0 = nil + end + + node_cache[:single_quoted_string][start_index] = r0 + + r0 + end + + def _nt_string + start_index = index + if node_cache[:string].has_key?(index) + cached = node_cache[:string][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0 = index + r1 = _nt_double_quoted_string + if r1 + r0 = r1 + else + r2 = _nt_single_quoted_string + if r2 + r0 = r2 + else + @index = i0 + r0 = nil + end + end + + node_cache[:string][start_index] = r0 + + r0 + end + + module Regexp0 + end + + module Regexp1 + end + + def _nt_regexp + start_index = index + if node_cache[:regexp].has_key?(index) + cached = node_cache[:regexp][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0, s0 = index, [] + if has_terminal?('/', false, index) + r1 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure('/') + r1 = nil + end + s0 << r1 + if r1 + s2, i2 = [], index + loop do + i3 = index + if has_terminal?('\/', false, index) + r4 = instantiate_node(SyntaxNode,input, index...(index + 2)) + @index += 2 + else + terminal_parse_failure('\/') + r4 = nil + end + if r4 + r3 = r4 + else + i5, s5 = index, [] + i6 = index + if has_terminal?('/', false, index) + r7 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure('/') + r7 = nil + end + if r7 + r6 = nil + else + @index = i6 + r6 = instantiate_node(SyntaxNode,input, index...index) + end + s5 << r6 + if r6 + if index < input_length + r8 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure("any character") + r8 = nil + end + s5 << r8 + end + if s5.last + r5 = instantiate_node(SyntaxNode,input, i5...index, s5) + r5.extend(Regexp0) + else + @index = i5 + r5 = nil + end + if r5 + r3 = r5 + else + @index = i3 + r3 = nil + end + end + if r3 + s2 << r3 + else + break + end + end + r2 = instantiate_node(SyntaxNode,input, i2...index, s2) + s0 << r2 + if r2 + if has_terminal?('/', false, index) + r9 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure('/') + r9 = nil + end + s0 << r9 + end + end + if s0.last + r0 = instantiate_node(LogStash::Compiler::LSCL::AST::RegExp,input, i0...index, s0) + r0.extend(Regexp1) + else + @index = i0 + r0 = nil + end + + node_cache[:regexp][start_index] = r0 + + r0 + end + + module Number0 + end + + module Number1 + end + + def _nt_number + start_index = index + if node_cache[:number].has_key?(index) + cached = node_cache[:number][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0, s0 = index, [] + if has_terminal?("-", false, index) + r2 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure("-") + r2 = nil + end + if r2 + r1 = r2 + else + r1 = instantiate_node(SyntaxNode,input, index...index) + end + s0 << r1 + if r1 + s3, i3 = [], index + loop do + if has_terminal?('\G[0-9]', true, index) + r4 = true + @index += 1 + else + r4 = nil + end + if r4 + s3 << r4 + else + break + end + end + if s3.empty? + @index = i3 + r3 = nil + else + r3 = instantiate_node(SyntaxNode,input, i3...index, s3) + end + s0 << r3 + if r3 + i6, s6 = index, [] + if has_terminal?(".", false, index) + r7 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure(".") + r7 = nil + end + s6 << r7 + if r7 + s8, i8 = [], index + loop do + if has_terminal?('\G[0-9]', true, index) + r9 = true + @index += 1 + else + r9 = nil + end + if r9 + s8 << r9 + else + break + end + end + r8 = instantiate_node(SyntaxNode,input, i8...index, s8) + s6 << r8 + end + if s6.last + r6 = instantiate_node(SyntaxNode,input, i6...index, s6) + r6.extend(Number0) + else + @index = i6 + r6 = nil + end + if r6 + r5 = r6 + else + r5 = instantiate_node(SyntaxNode,input, index...index) + end + s0 << r5 + end + end + if s0.last + r0 = instantiate_node(LogStash::Compiler::LSCL::AST::Number,input, i0...index, s0) + r0.extend(Number1) + else + @index = i0 + r0 = nil + end + + node_cache[:number][start_index] = r0 + + r0 + end + + module Array0 + def _1 + elements[0] + end + + def _2 + elements[2] + end + + def value + elements[3] + end + end + + module Array1 + def value + elements[0] + end + + end + + module Array2 + def _1 + elements[1] + end + + def _2 + elements[3] + end + + end + + def _nt_array + start_index = index + if node_cache[:array].has_key?(index) + cached = node_cache[:array][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0, s0 = index, [] + if has_terminal?("[", false, index) + r1 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure("[") + r1 = nil + end + s0 << r1 + if r1 + r2 = _nt__ + s0 << r2 + if r2 + i4, s4 = index, [] + r5 = _nt_value + s4 << r5 + if r5 + s6, i6 = [], index + loop do + i7, s7 = index, [] + r8 = _nt__ + s7 << r8 + if r8 + if has_terminal?(",", false, index) + r9 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure(",") + r9 = nil + end + s7 << r9 + if r9 + r10 = _nt__ + s7 << r10 + if r10 + r11 = _nt_value + s7 << r11 + end + end + end + if s7.last + r7 = instantiate_node(SyntaxNode,input, i7...index, s7) + r7.extend(Array0) + else + @index = i7 + r7 = nil + end + if r7 + s6 << r7 + else + break + end + end + r6 = instantiate_node(SyntaxNode,input, i6...index, s6) + s4 << r6 + end + if s4.last + r4 = instantiate_node(SyntaxNode,input, i4...index, s4) + r4.extend(Array1) + else + @index = i4 + r4 = nil + end + if r4 + r3 = r4 + else + r3 = instantiate_node(SyntaxNode,input, index...index) + end + s0 << r3 + if r3 + r12 = _nt__ + s0 << r12 + if r12 + if has_terminal?("]", false, index) + r13 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure("]") + r13 = nil + end + s0 << r13 + end + end + end + end + if s0.last + r0 = instantiate_node(LogStash::Compiler::LSCL::AST::Array,input, i0...index, s0) + r0.extend(Array2) + else + @index = i0 + r0 = nil + end + + node_cache[:array][start_index] = r0 + + r0 + end + + module Hash0 + def _1 + elements[1] + end + + def _2 + elements[3] + end + + end + + def _nt_hash + start_index = index + if node_cache[:hash].has_key?(index) + cached = node_cache[:hash][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0, s0 = index, [] + if has_terminal?("{", false, index) + r1 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure("{") + r1 = nil + end + s0 << r1 + if r1 + r2 = _nt__ + s0 << r2 + if r2 + r4 = _nt_hashentries + if r4 + r3 = r4 + else + r3 = instantiate_node(SyntaxNode,input, index...index) + end + s0 << r3 + if r3 + r5 = _nt__ + s0 << r5 + if r5 + if has_terminal?("}", false, index) + r6 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure("}") + r6 = nil + end + s0 << r6 + end + end + end + end + if s0.last + r0 = instantiate_node(LogStash::Compiler::LSCL::AST::Hash,input, i0...index, s0) + r0.extend(Hash0) + else + @index = i0 + r0 = nil + end + + node_cache[:hash][start_index] = r0 + + r0 + end + + module Hashentries0 + def whitespace + elements[0] + end + + def hashentry + elements[1] + end + end + + module Hashentries1 + def hashentry + elements[0] + end + + end + + def _nt_hashentries + start_index = index + if node_cache[:hashentries].has_key?(index) + cached = node_cache[:hashentries][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0, s0 = index, [] + r1 = _nt_hashentry + s0 << r1 + if r1 + s2, i2 = [], index + loop do + i3, s3 = index, [] + r4 = _nt_whitespace + s3 << r4 + if r4 + r5 = _nt_hashentry + s3 << r5 + end + if s3.last + r3 = instantiate_node(SyntaxNode,input, i3...index, s3) + r3.extend(Hashentries0) + else + @index = i3 + r3 = nil + end + if r3 + s2 << r3 + else + break + end + end + r2 = instantiate_node(SyntaxNode,input, i2...index, s2) + s0 << r2 + end + if s0.last + r0 = instantiate_node(LogStash::Compiler::LSCL::AST::HashEntries,input, i0...index, s0) + r0.extend(Hashentries1) + else + @index = i0 + r0 = nil + end + + node_cache[:hashentries][start_index] = r0 + + r0 + end + + module Hashentry0 + def name + elements[0] + end + + def _1 + elements[1] + end + + def _2 + elements[3] + end + + def value + elements[4] + end + end + + def _nt_hashentry + start_index = index + if node_cache[:hashentry].has_key?(index) + cached = node_cache[:hashentry][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0, s0 = index, [] + i1 = index + r2 = _nt_number + if r2 + r1 = r2 + else + r3 = _nt_bareword + if r3 + r1 = r3 + else + r4 = _nt_string + if r4 + r1 = r4 + else + @index = i1 + r1 = nil + end + end + end + s0 << r1 + if r1 + r5 = _nt__ + s0 << r5 + if r5 + if has_terminal?("=>", false, index) + r6 = instantiate_node(SyntaxNode,input, index...(index + 2)) + @index += 2 + else + terminal_parse_failure("=>") + r6 = nil + end + s0 << r6 + if r6 + r7 = _nt__ + s0 << r7 + if r7 + r8 = _nt_value + s0 << r8 + end + end + end + end + if s0.last + r0 = instantiate_node(LogStash::Compiler::LSCL::AST::HashEntry,input, i0...index, s0) + r0.extend(Hashentry0) + else + @index = i0 + r0 = nil + end + + node_cache[:hashentry][start_index] = r0 + + r0 + end + + module Branch0 + def _ + elements[0] + end + + def else_if + elements[1] + end + end + + module Branch1 + def _ + elements[0] + end + + def else + elements[1] + end + end + + module Branch2 + def if + elements[0] + end + + end + + def _nt_branch + start_index = index + if node_cache[:branch].has_key?(index) + cached = node_cache[:branch][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0, s0 = index, [] + r1 = _nt_if + s0 << r1 + if r1 + s2, i2 = [], index + loop do + i3, s3 = index, [] + r4 = _nt__ + s3 << r4 + if r4 + r5 = _nt_else_if + s3 << r5 + end + if s3.last + r3 = instantiate_node(SyntaxNode,input, i3...index, s3) + r3.extend(Branch0) + else + @index = i3 + r3 = nil + end + if r3 + s2 << r3 + else + break + end + end + r2 = instantiate_node(SyntaxNode,input, i2...index, s2) + s0 << r2 + if r2 + i7, s7 = index, [] + r8 = _nt__ + s7 << r8 + if r8 + r9 = _nt_else + s7 << r9 + end + if s7.last + r7 = instantiate_node(SyntaxNode,input, i7...index, s7) + r7.extend(Branch1) + else + @index = i7 + r7 = nil + end + if r7 + r6 = r7 + else + r6 = instantiate_node(SyntaxNode,input, index...index) + end + s0 << r6 + end + end + if s0.last + r0 = instantiate_node(LogStash::Compiler::LSCL::AST::Branch,input, i0...index, s0) + r0.extend(Branch2) + else + @index = i0 + r0 = nil + end + + node_cache[:branch][start_index] = r0 + + r0 + end + + module If0 + def branch_or_plugin + elements[0] + end + + def _ + elements[1] + end + end + + module If1 + def _1 + elements[1] + end + + def condition + elements[2] + end + + def _2 + elements[3] + end + + def _3 + elements[5] + end + + end + + def _nt_if + start_index = index + if node_cache[:if].has_key?(index) + cached = node_cache[:if][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0, s0 = index, [] + if has_terminal?("if", false, index) + r1 = instantiate_node(SyntaxNode,input, index...(index + 2)) + @index += 2 + else + terminal_parse_failure("if") + r1 = nil + end + s0 << r1 + if r1 + r2 = _nt__ + s0 << r2 + if r2 + r3 = _nt_condition + s0 << r3 + if r3 + r4 = _nt__ + s0 << r4 + if r4 + if has_terminal?("{", false, index) + r5 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure("{") + r5 = nil + end + s0 << r5 + if r5 + r6 = _nt__ + s0 << r6 + if r6 + s7, i7 = [], index + loop do + i8, s8 = index, [] + r9 = _nt_branch_or_plugin + s8 << r9 + if r9 + r10 = _nt__ + s8 << r10 + end + if s8.last + r8 = instantiate_node(SyntaxNode,input, i8...index, s8) + r8.extend(If0) + else + @index = i8 + r8 = nil + end + if r8 + s7 << r8 + else + break + end + end + r7 = instantiate_node(SyntaxNode,input, i7...index, s7) + s0 << r7 + if r7 + if has_terminal?("}", false, index) + r11 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure("}") + r11 = nil + end + s0 << r11 + end + end + end + end + end + end + end + if s0.last + r0 = instantiate_node(LogStash::Compiler::LSCL::AST::If,input, i0...index, s0) + r0.extend(If1) + else + @index = i0 + r0 = nil + end + + node_cache[:if][start_index] = r0 + + r0 + end + + module ElseIf0 + def branch_or_plugin + elements[0] + end + + def _ + elements[1] + end + end + + module ElseIf1 + def _1 + elements[1] + end + + def _2 + elements[3] + end + + def condition + elements[4] + end + + def _3 + elements[5] + end + + def _4 + elements[7] + end + + end + + def _nt_else_if + start_index = index + if node_cache[:else_if].has_key?(index) + cached = node_cache[:else_if][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0, s0 = index, [] + if has_terminal?("else", false, index) + r1 = instantiate_node(SyntaxNode,input, index...(index + 4)) + @index += 4 + else + terminal_parse_failure("else") + r1 = nil + end + s0 << r1 + if r1 + r2 = _nt__ + s0 << r2 + if r2 + if has_terminal?("if", false, index) + r3 = instantiate_node(SyntaxNode,input, index...(index + 2)) + @index += 2 + else + terminal_parse_failure("if") + r3 = nil + end + s0 << r3 + if r3 + r4 = _nt__ + s0 << r4 + if r4 + r5 = _nt_condition + s0 << r5 + if r5 + r6 = _nt__ + s0 << r6 + if r6 + if has_terminal?("{", false, index) + r7 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure("{") + r7 = nil + end + s0 << r7 + if r7 + r8 = _nt__ + s0 << r8 + if r8 + s9, i9 = [], index + loop do + i10, s10 = index, [] + r11 = _nt_branch_or_plugin + s10 << r11 + if r11 + r12 = _nt__ + s10 << r12 + end + if s10.last + r10 = instantiate_node(SyntaxNode,input, i10...index, s10) + r10.extend(ElseIf0) + else + @index = i10 + r10 = nil + end + if r10 + s9 << r10 + else + break + end + end + r9 = instantiate_node(SyntaxNode,input, i9...index, s9) + s0 << r9 + if r9 + if has_terminal?("}", false, index) + r13 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure("}") + r13 = nil + end + s0 << r13 + end + end + end + end + end + end + end + end + end + if s0.last + r0 = instantiate_node(LogStash::Compiler::LSCL::AST::Elsif,input, i0...index, s0) + r0.extend(ElseIf1) + else + @index = i0 + r0 = nil + end + + node_cache[:else_if][start_index] = r0 + + r0 + end + + module Else0 + def branch_or_plugin + elements[0] + end + + def _ + elements[1] + end + end + + module Else1 + def _1 + elements[1] + end + + def _2 + elements[3] + end + + end + + def _nt_else + start_index = index + if node_cache[:else].has_key?(index) + cached = node_cache[:else][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0, s0 = index, [] + if has_terminal?("else", false, index) + r1 = instantiate_node(SyntaxNode,input, index...(index + 4)) + @index += 4 + else + terminal_parse_failure("else") + r1 = nil + end + s0 << r1 + if r1 + r2 = _nt__ + s0 << r2 + if r2 + if has_terminal?("{", false, index) + r3 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure("{") + r3 = nil + end + s0 << r3 + if r3 + r4 = _nt__ + s0 << r4 + if r4 + s5, i5 = [], index + loop do + i6, s6 = index, [] + r7 = _nt_branch_or_plugin + s6 << r7 + if r7 + r8 = _nt__ + s6 << r8 + end + if s6.last + r6 = instantiate_node(SyntaxNode,input, i6...index, s6) + r6.extend(Else0) + else + @index = i6 + r6 = nil + end + if r6 + s5 << r6 + else + break + end + end + r5 = instantiate_node(SyntaxNode,input, i5...index, s5) + s0 << r5 + if r5 + if has_terminal?("}", false, index) + r9 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure("}") + r9 = nil + end + s0 << r9 + end + end + end + end + end + if s0.last + r0 = instantiate_node(LogStash::Compiler::LSCL::AST::Else,input, i0...index, s0) + r0.extend(Else1) + else + @index = i0 + r0 = nil + end + + node_cache[:else][start_index] = r0 + + r0 + end + + module Condition0 + def _1 + elements[0] + end + + def boolean_operator + elements[1] + end + + def _2 + elements[2] + end + + def expression + elements[3] + end + end + + module Condition1 + def expression + elements[0] + end + + end + + def _nt_condition + start_index = index + if node_cache[:condition].has_key?(index) + cached = node_cache[:condition][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0, s0 = index, [] + r1 = _nt_expression + s0 << r1 + if r1 + s2, i2 = [], index + loop do + i3, s3 = index, [] + r4 = _nt__ + s3 << r4 + if r4 + r5 = _nt_boolean_operator + s3 << r5 + if r5 + r6 = _nt__ + s3 << r6 + if r6 + r7 = _nt_expression + s3 << r7 + end + end + end + if s3.last + r3 = instantiate_node(SyntaxNode,input, i3...index, s3) + r3.extend(Condition0) + else + @index = i3 + r3 = nil + end + if r3 + s2 << r3 + else + break + end + end + r2 = instantiate_node(SyntaxNode,input, i2...index, s2) + s0 << r2 + end + if s0.last + r0 = instantiate_node(LogStash::Compiler::LSCL::AST::Condition,input, i0...index, s0) + r0.extend(Condition1) + else + @index = i0 + r0 = nil + end + + node_cache[:condition][start_index] = r0 + + r0 + end + + module Expression0 + def _1 + elements[1] + end + + def condition + elements[2] + end + + def _2 + elements[3] + end + + end + + def _nt_expression + start_index = index + if node_cache[:expression].has_key?(index) + cached = node_cache[:expression][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0 = index + i1, s1 = index, [] + if has_terminal?("(", false, index) + r2 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure("(") + r2 = nil + end + s1 << r2 + if r2 + r3 = _nt__ + s1 << r3 + if r3 + r4 = _nt_condition + s1 << r4 + if r4 + r5 = _nt__ + s1 << r5 + if r5 + if has_terminal?(")", false, index) + r6 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure(")") + r6 = nil + end + s1 << r6 + end + end + end + end + if s1.last + r1 = instantiate_node(SyntaxNode,input, i1...index, s1) + r1.extend(Expression0) + else + @index = i1 + r1 = nil + end + if r1 + r0 = r1 + r0.extend(LogStash::Compiler::LSCL::AST::Expression) + else + r7 = _nt_negative_expression + if r7 + r0 = r7 + r0.extend(LogStash::Compiler::LSCL::AST::Expression) + else + r8 = _nt_in_expression + if r8 + r0 = r8 + r0.extend(LogStash::Compiler::LSCL::AST::Expression) + else + r9 = _nt_not_in_expression + if r9 + r0 = r9 + r0.extend(LogStash::Compiler::LSCL::AST::Expression) + else + r10 = _nt_compare_expression + if r10 + r0 = r10 + r0.extend(LogStash::Compiler::LSCL::AST::Expression) + else + r11 = _nt_regexp_expression + if r11 + r0 = r11 + r0.extend(LogStash::Compiler::LSCL::AST::Expression) + else + r12 = _nt_rvalue + if r12 + r0 = r12 + r0.extend(LogStash::Compiler::LSCL::AST::Expression) + else + @index = i0 + r0 = nil + end + end + end + end + end + end + end + + node_cache[:expression][start_index] = r0 + + r0 + end + + module NegativeExpression0 + def _1 + elements[1] + end + + def _2 + elements[3] + end + + def condition + elements[4] + end + + def _3 + elements[5] + end + + end + + module NegativeExpression1 + def _ + elements[1] + end + + def selector + elements[2] + end + end + + def _nt_negative_expression + start_index = index + if node_cache[:negative_expression].has_key?(index) + cached = node_cache[:negative_expression][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0 = index + i1, s1 = index, [] + if has_terminal?("!", false, index) + r2 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure("!") + r2 = nil + end + s1 << r2 + if r2 + r3 = _nt__ + s1 << r3 + if r3 + if has_terminal?("(", false, index) + r4 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure("(") + r4 = nil + end + s1 << r4 + if r4 + r5 = _nt__ + s1 << r5 + if r5 + r6 = _nt_condition + s1 << r6 + if r6 + r7 = _nt__ + s1 << r7 + if r7 + if has_terminal?(")", false, index) + r8 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure(")") + r8 = nil + end + s1 << r8 + end + end + end + end + end + end + if s1.last + r1 = instantiate_node(SyntaxNode,input, i1...index, s1) + r1.extend(NegativeExpression0) + else + @index = i1 + r1 = nil + end + if r1 + r0 = r1 + r0.extend(LogStash::Compiler::LSCL::AST::NegativeExpression) + else + i9, s9 = index, [] + if has_terminal?("!", false, index) + r10 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure("!") + r10 = nil + end + s9 << r10 + if r10 + r11 = _nt__ + s9 << r11 + if r11 + r12 = _nt_selector + s9 << r12 + end + end + if s9.last + r9 = instantiate_node(SyntaxNode,input, i9...index, s9) + r9.extend(NegativeExpression1) + else + @index = i9 + r9 = nil + end + if r9 + r0 = r9 + r0.extend(LogStash::Compiler::LSCL::AST::NegativeExpression) + else + @index = i0 + r0 = nil + end + end + + node_cache[:negative_expression][start_index] = r0 + + r0 + end + + module InExpression0 + def rvalue1 + elements[0] + end + + def _1 + elements[1] + end + + def in_operator + elements[2] + end + + def _2 + elements[3] + end + + def rvalue2 + elements[4] + end + end + + def _nt_in_expression + start_index = index + if node_cache[:in_expression].has_key?(index) + cached = node_cache[:in_expression][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0, s0 = index, [] + r1 = _nt_rvalue + s0 << r1 + if r1 + r2 = _nt__ + s0 << r2 + if r2 + r3 = _nt_in_operator + s0 << r3 + if r3 + r4 = _nt__ + s0 << r4 + if r4 + r5 = _nt_rvalue + s0 << r5 + end + end + end + end + if s0.last + r0 = instantiate_node(LogStash::Compiler::LSCL::AST::InExpression,input, i0...index, s0) + r0.extend(InExpression0) + else + @index = i0 + r0 = nil + end + + node_cache[:in_expression][start_index] = r0 + + r0 + end + + module NotInExpression0 + def rvalue1 + elements[0] + end + + def _1 + elements[1] + end + + def not_in_operator + elements[2] + end + + def _2 + elements[3] + end + + def rvalue2 + elements[4] + end + end + + def _nt_not_in_expression + start_index = index + if node_cache[:not_in_expression].has_key?(index) + cached = node_cache[:not_in_expression][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0, s0 = index, [] + r1 = _nt_rvalue + s0 << r1 + if r1 + r2 = _nt__ + s0 << r2 + if r2 + r3 = _nt_not_in_operator + s0 << r3 + if r3 + r4 = _nt__ + s0 << r4 + if r4 + r5 = _nt_rvalue + s0 << r5 + end + end + end + end + if s0.last + r0 = instantiate_node(LogStash::Compiler::LSCL::AST::NotInExpression,input, i0...index, s0) + r0.extend(NotInExpression0) + else + @index = i0 + r0 = nil + end + + node_cache[:not_in_expression][start_index] = r0 + + r0 + end + + def _nt_in_operator + start_index = index + if node_cache[:in_operator].has_key?(index) + cached = node_cache[:in_operator][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + if has_terminal?("in", false, index) + r0 = instantiate_node(SyntaxNode,input, index...(index + 2)) + @index += 2 + else + terminal_parse_failure("in") + r0 = nil + end + + node_cache[:in_operator][start_index] = r0 + + r0 + end + + module NotInOperator0 + def _ + elements[1] + end + + end + + def _nt_not_in_operator + start_index = index + if node_cache[:not_in_operator].has_key?(index) + cached = node_cache[:not_in_operator][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0, s0 = index, [] + if has_terminal?("not ", false, index) + r1 = instantiate_node(SyntaxNode,input, index...(index + 4)) + @index += 4 + else + terminal_parse_failure("not ") + r1 = nil + end + s0 << r1 + if r1 + r2 = _nt__ + s0 << r2 + if r2 + if has_terminal?("in", false, index) + r3 = instantiate_node(SyntaxNode,input, index...(index + 2)) + @index += 2 + else + terminal_parse_failure("in") + r3 = nil + end + s0 << r3 + end + end + if s0.last + r0 = instantiate_node(SyntaxNode,input, i0...index, s0) + r0.extend(NotInOperator0) + else + @index = i0 + r0 = nil + end + + node_cache[:not_in_operator][start_index] = r0 + + r0 + end + + def _nt_rvalue + start_index = index + if node_cache[:rvalue].has_key?(index) + cached = node_cache[:rvalue][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0 = index + r1 = _nt_string + if r1 + r0 = r1 + else + r2 = _nt_number + if r2 + r0 = r2 + else + r3 = _nt_selector + if r3 + r0 = r3 + else + r4 = _nt_array + if r4 + r0 = r4 + else + r5 = _nt_method_call + if r5 + r0 = r5 + else + r6 = _nt_regexp + if r6 + r0 = r6 + else + @index = i0 + r0 = nil + end + end + end + end + end + end + + node_cache[:rvalue][start_index] = r0 + + r0 + end + + module MethodCall0 + def _1 + elements[0] + end + + def _2 + elements[2] + end + + def rvalue + elements[3] + end + end + + module MethodCall1 + def rvalue + elements[0] + end + + end + + module MethodCall2 + def method + elements[0] + end + + def _1 + elements[1] + end + + def _2 + elements[3] + end + + def _3 + elements[5] + end + + end + + def _nt_method_call + start_index = index + if node_cache[:method_call].has_key?(index) + cached = node_cache[:method_call][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0, s0 = index, [] + r1 = _nt_method + s0 << r1 + if r1 + r2 = _nt__ + s0 << r2 + if r2 + if has_terminal?("(", false, index) + r3 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure("(") + r3 = nil + end + s0 << r3 + if r3 + r4 = _nt__ + s0 << r4 + if r4 + i6, s6 = index, [] + r7 = _nt_rvalue + s6 << r7 + if r7 + s8, i8 = [], index + loop do + i9, s9 = index, [] + r10 = _nt__ + s9 << r10 + if r10 + if has_terminal?(",", false, index) + r11 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure(",") + r11 = nil + end + s9 << r11 + if r11 + r12 = _nt__ + s9 << r12 + if r12 + r13 = _nt_rvalue + s9 << r13 + end + end + end + if s9.last + r9 = instantiate_node(SyntaxNode,input, i9...index, s9) + r9.extend(MethodCall0) + else + @index = i9 + r9 = nil + end + if r9 + s8 << r9 + else + break + end + end + r8 = instantiate_node(SyntaxNode,input, i8...index, s8) + s6 << r8 + end + if s6.last + r6 = instantiate_node(SyntaxNode,input, i6...index, s6) + r6.extend(MethodCall1) + else + @index = i6 + r6 = nil + end + if r6 + r5 = r6 + else + r5 = instantiate_node(SyntaxNode,input, index...index) + end + s0 << r5 + if r5 + r14 = _nt__ + s0 << r14 + if r14 + if has_terminal?(")", false, index) + r15 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure(")") + r15 = nil + end + s0 << r15 + end + end + end + end + end + end + if s0.last + r0 = instantiate_node(LogStash::Compiler::LSCL::AST::MethodCall,input, i0...index, s0) + r0.extend(MethodCall2) + else + @index = i0 + r0 = nil + end + + node_cache[:method_call][start_index] = r0 + + r0 + end + + def _nt_method + start_index = index + if node_cache[:method].has_key?(index) + cached = node_cache[:method][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + r0 = _nt_bareword + + node_cache[:method][start_index] = r0 + + r0 + end + + module CompareExpression0 + def rvalue1 + elements[0] + end + + def _1 + elements[1] + end + + def compare_operator + elements[2] + end + + def _2 + elements[3] + end + + def rvalue2 + elements[4] + end + end + + def _nt_compare_expression + start_index = index + if node_cache[:compare_expression].has_key?(index) + cached = node_cache[:compare_expression][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0, s0 = index, [] + r1 = _nt_rvalue + s0 << r1 + if r1 + r2 = _nt__ + s0 << r2 + if r2 + r3 = _nt_compare_operator + s0 << r3 + if r3 + r4 = _nt__ + s0 << r4 + if r4 + r5 = _nt_rvalue + s0 << r5 + end + end + end + end + if s0.last + r0 = instantiate_node(LogStash::Compiler::LSCL::AST::ComparisonExpression,input, i0...index, s0) + r0.extend(CompareExpression0) + else + @index = i0 + r0 = nil + end + + node_cache[:compare_expression][start_index] = r0 + + r0 + end + + def _nt_compare_operator + start_index = index + if node_cache[:compare_operator].has_key?(index) + cached = node_cache[:compare_operator][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0 = index + if has_terminal?("==", false, index) + r1 = instantiate_node(SyntaxNode,input, index...(index + 2)) + @index += 2 + else + terminal_parse_failure("==") + r1 = nil + end + if r1 + r0 = r1 + r0.extend(LogStash::Compiler::LSCL::AST::ComparisonOperator) + else + if has_terminal?("!=", false, index) + r2 = instantiate_node(SyntaxNode,input, index...(index + 2)) + @index += 2 + else + terminal_parse_failure("!=") + r2 = nil + end + if r2 + r0 = r2 + r0.extend(LogStash::Compiler::LSCL::AST::ComparisonOperator) + else + if has_terminal?("<=", false, index) + r3 = instantiate_node(SyntaxNode,input, index...(index + 2)) + @index += 2 + else + terminal_parse_failure("<=") + r3 = nil + end + if r3 + r0 = r3 + r0.extend(LogStash::Compiler::LSCL::AST::ComparisonOperator) + else + if has_terminal?(">=", false, index) + r4 = instantiate_node(SyntaxNode,input, index...(index + 2)) + @index += 2 + else + terminal_parse_failure(">=") + r4 = nil + end + if r4 + r0 = r4 + r0.extend(LogStash::Compiler::LSCL::AST::ComparisonOperator) + else + if has_terminal?("<", false, index) + r5 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure("<") + r5 = nil + end + if r5 + r0 = r5 + r0.extend(LogStash::Compiler::LSCL::AST::ComparisonOperator) + else + if has_terminal?(">", false, index) + r6 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure(">") + r6 = nil + end + if r6 + r0 = r6 + r0.extend(LogStash::Compiler::LSCL::AST::ComparisonOperator) + else + @index = i0 + r0 = nil + end + end + end + end + end + end + + node_cache[:compare_operator][start_index] = r0 + + r0 + end + + module RegexpExpression0 + def rvalue + elements[0] + end + + def _1 + elements[1] + end + + def regexp_operator + elements[2] + end + + def _2 + elements[3] + end + + end + + def _nt_regexp_expression + start_index = index + if node_cache[:regexp_expression].has_key?(index) + cached = node_cache[:regexp_expression][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0, s0 = index, [] + r1 = _nt_rvalue + s0 << r1 + if r1 + r2 = _nt__ + s0 << r2 + if r2 + r3 = _nt_regexp_operator + s0 << r3 + if r3 + r4 = _nt__ + s0 << r4 + if r4 + i5 = index + r6 = _nt_string + if r6 + r5 = r6 + else + r7 = _nt_regexp + if r7 + r5 = r7 + else + @index = i5 + r5 = nil + end + end + s0 << r5 + end + end + end + end + if s0.last + r0 = instantiate_node(LogStash::Compiler::LSCL::AST::RegexpExpression,input, i0...index, s0) + r0.extend(RegexpExpression0) + else + @index = i0 + r0 = nil + end + + node_cache[:regexp_expression][start_index] = r0 + + r0 + end + + def _nt_regexp_operator + start_index = index + if node_cache[:regexp_operator].has_key?(index) + cached = node_cache[:regexp_operator][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0 = index + if has_terminal?("=~", false, index) + r1 = instantiate_node(SyntaxNode,input, index...(index + 2)) + @index += 2 + else + terminal_parse_failure("=~") + r1 = nil + end + if r1 + r0 = r1 + r0.extend(LogStash::Compiler::LSCL::AST::RegExpOperator) + else + if has_terminal?("!~", false, index) + r2 = instantiate_node(SyntaxNode,input, index...(index + 2)) + @index += 2 + else + terminal_parse_failure("!~") + r2 = nil + end + if r2 + r0 = r2 + r0.extend(LogStash::Compiler::LSCL::AST::RegExpOperator) + else + @index = i0 + r0 = nil + end + end + + node_cache[:regexp_operator][start_index] = r0 + + r0 + end + + def _nt_boolean_operator + start_index = index + if node_cache[:boolean_operator].has_key?(index) + cached = node_cache[:boolean_operator][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0 = index + if has_terminal?("and", false, index) + r1 = instantiate_node(SyntaxNode,input, index...(index + 3)) + @index += 3 + else + terminal_parse_failure("and") + r1 = nil + end + if r1 + r0 = r1 + r0.extend(LogStash::Compiler::LSCL::AST::BooleanOperator) + else + if has_terminal?("or", false, index) + r2 = instantiate_node(SyntaxNode,input, index...(index + 2)) + @index += 2 + else + terminal_parse_failure("or") + r2 = nil + end + if r2 + r0 = r2 + r0.extend(LogStash::Compiler::LSCL::AST::BooleanOperator) + else + if has_terminal?("xor", false, index) + r3 = instantiate_node(SyntaxNode,input, index...(index + 3)) + @index += 3 + else + terminal_parse_failure("xor") + r3 = nil + end + if r3 + r0 = r3 + r0.extend(LogStash::Compiler::LSCL::AST::BooleanOperator) + else + if has_terminal?("nand", false, index) + r4 = instantiate_node(SyntaxNode,input, index...(index + 4)) + @index += 4 + else + terminal_parse_failure("nand") + r4 = nil + end + if r4 + r0 = r4 + r0.extend(LogStash::Compiler::LSCL::AST::BooleanOperator) + else + @index = i0 + r0 = nil + end + end + end + end + + node_cache[:boolean_operator][start_index] = r0 + + r0 + end + + def _nt_selector + start_index = index + if node_cache[:selector].has_key?(index) + cached = node_cache[:selector][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + s0, i0 = [], index + loop do + r1 = _nt_selector_element + if r1 + s0 << r1 + else + break + end + end + if s0.empty? + @index = i0 + r0 = nil + else + r0 = instantiate_node(LogStash::Compiler::LSCL::AST::Selector,input, i0...index, s0) + end + + node_cache[:selector][start_index] = r0 + + r0 + end + + module SelectorElement0 + end + + def _nt_selector_element + start_index = index + if node_cache[:selector_element].has_key?(index) + cached = node_cache[:selector_element][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0, s0 = index, [] + if has_terminal?("[", false, index) + r1 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure("[") + r1 = nil + end + s0 << r1 + if r1 + s2, i2 = [], index + loop do + if has_terminal?('\G[^\\],]', true, index) + r3 = true + @index += 1 + else + r3 = nil + end + if r3 + s2 << r3 + else + break + end + end + if s2.empty? + @index = i2 + r2 = nil + else + r2 = instantiate_node(SyntaxNode,input, i2...index, s2) + end + s0 << r2 + if r2 + if has_terminal?("]", false, index) + r4 = instantiate_node(SyntaxNode,input, index...(index + 1)) + @index += 1 + else + terminal_parse_failure("]") + r4 = nil + end + s0 << r4 + end + end + if s0.last + r0 = instantiate_node(LogStash::Compiler::LSCL::AST::SelectorElement,input, i0...index, s0) + r0.extend(SelectorElement0) + else + @index = i0 + r0 = nil + end + + node_cache[:selector_element][start_index] = r0 + + r0 + end + +end + +class LogStashCompilerLSCLGrammarParser < Treetop::Runtime::CompiledParser + include LogStashCompilerLSCLGrammar +end + diff --git a/logstash-core/lib/logstash/compiler/lscl/lscl_grammar.treetop b/logstash-core/lib/logstash/compiler/lscl/lscl_grammar.treetop new file mode 100644 index 000000000..095c8e912 --- /dev/null +++ b/logstash-core/lib/logstash/compiler/lscl/lscl_grammar.treetop @@ -0,0 +1,241 @@ +require "treetop" +require "logstash/compiler/lscl.rb" + +grammar LogStashCompilerLSCLGrammar + rule config + _ plugin_section _ (_ plugin_section)* _ + end + + rule comment + (whitespace? "#" [^\r\n]* "\r"? "\n")+ + end + + rule _ + (comment / whitespace)* + end + + rule whitespace + [ \t\r\n]+ + end + + rule plugin_section + plugin_type _ "{" + _ (branch_or_plugin _)* + "}" + + end + + rule branch_or_plugin + branch / plugin + end + + rule plugin_type + ("input" / "filter" / "output") + end + + rule plugins + (plugin (_ plugin)*)? + + end + + rule plugin + name _ "{" + _ + attributes:( attribute (whitespace _ attribute)*)? + _ + "}" + + end + + rule name + ( + ([A-Za-z0-9_-]+ ) + / string + ) + end + + rule attribute + name _ "=>" _ value + + end + + rule value + plugin / bareword / string / number / array / hash + end + + rule array_value + bareword / string / number / array / hash + end + + rule bareword + [A-Za-z_] [A-Za-z0-9_]+ + + end + + rule double_quoted_string + ( '"' ( '\"' / !'"' . )* '"' ) + end + + rule single_quoted_string + ( "'" ( "\\'" / !"'" . )* "'" ) + end + + rule string + double_quoted_string / single_quoted_string + end + + rule regexp + ( '/' ( '\/' / !'/' . )* '/' ) + end + + rule number + "-"? [0-9]+ ("." [0-9]*)? + + end + + rule array + "[" + _ + ( + value (_ "," _ value)* + )? + _ + "]" + + end + + rule hash + "{" + _ + hashentries? + _ + "}" + + end + + rule hashentries + hashentry (whitespace hashentry)* + + end + + rule hashentry + name:(number / bareword / string) _ "=>" _ value + + end + + # Conditions + rule branch + if (_ else_if)* (_ else)? + + end + + rule if + "if" _ condition _ "{" _ (branch_or_plugin _)* "}" + + end + + rule else_if + "else" _ "if" _ condition _ "{" _ ( branch_or_plugin _)* "}" + + end + + rule else + "else" _ "{" _ (branch_or_plugin _)* "}" + + end + + rule condition + expression (_ boolean_operator _ expression)* + + end + + rule expression + ( + ("(" _ condition _ ")") + / negative_expression + / in_expression + / not_in_expression + / compare_expression + / regexp_expression + / rvalue + ) + end + + rule negative_expression + ( + ("!" _ "(" _ condition _ ")") + / ("!" _ selector) + ) + end + + rule in_expression + rvalue _ in_operator _ rvalue + + end + + rule not_in_expression + rvalue _ not_in_operator _ rvalue + + end + + rule in_operator + "in" + end + + rule not_in_operator + "not " _ "in" + end + + rule rvalue + string / number / selector / array / method_call / regexp + end + + rule method_call + method _ "(" _ + ( + rvalue ( _ "," _ rvalue )* + )? + _ ")" + + end + + rule method + bareword + end + + rule compare_expression + rvalue _ compare_operator _ rvalue + + end + + rule compare_operator + ("==" / "!=" / "<=" / ">=" / "<" / ">") + + end + + rule regexp_expression + rvalue _ regexp_operator _ (string / regexp) + + end + + rule regexp_operator + ("=~" / "!~") + end + + + rule boolean_operator + ("and" / "or" / "xor" / "nand") + + end + + rule selector + selector_element+ + + end + + rule selector_element + "[" [^\],]+ "]" + + end + +end diff --git a/logstash-core/lib/logstash/compiler/treetop_monkeypatches.rb b/logstash-core/lib/logstash/compiler/treetop_monkeypatches.rb new file mode 100644 index 000000000..753b15aa4 --- /dev/null +++ b/logstash-core/lib/logstash/compiler/treetop_monkeypatches.rb @@ -0,0 +1,92 @@ +class Treetop::Runtime::SyntaxNode + def get_meta(key) + @ast_metadata ||= {} + return @ast_metadata[key] if @ast_metadata[key] + return self.parent.get_meta(key) if self.parent + nil + end + + def set_meta(key, value) + @ast_metadata ||= {} + @ast_metadata[key] = value + end + def compile + return "" if elements.nil? + return elements.collect(&:compile).reject(&:empty?).join("") + end + + # Traverse the syntax tree recursively. + # The order should respect the order of the configuration file as it is read + # and written by humans (and the order in which it is parsed). + def recurse(e, depth=0, &block) + r = block.call(e, depth) + e.elements.each { |e| recurse(e, depth + 1, &block) } if r && e.elements + nil + end + + def recursive_inject(results=[], &block) + if !elements.nil? + elements.each do |element| + if block.call(element) + results << element + else + element.recursive_inject(results, &block) + end + end + end + return results + end + + # When Treetop parses the configuration file + # it will generate a tree, the generated tree will contain + # a few `Empty` nodes to represent the actual space/tab or newline in the file. + # Some of theses node will point to our concrete class. + # To fetch a specific types of object we need to follow each branch + # and ignore the empty nodes. + def recursive_select(*klasses) + return recursive_inject { |e| klasses.any? {|k| e.is_a?(k)} } + end + + def recursive_inject_parent(results=[], &block) + if !parent.nil? + if block.call(parent) + results << parent + else + parent.recursive_inject_parent(results, &block) + end + end + return results + end + + def recursive_select_parent(results=[], klass) + return recursive_inject_parent(results) { |e| e.is_a?(klass) } + end + + # Monkeypatch Treetop::Runtime::SyntaxNode's inspect method to skip + # any Whitespace or SyntaxNodes with no children. + def _inspect(indent="") + em = extension_modules + interesting_methods = methods-[em.last ? em.last.methods : nil]-self.class.instance_methods + im = interesting_methods.size > 0 ? " (#{interesting_methods.join(",")})" : "" + tv = text_value + tv = "...#{tv[-20..-1]}" if tv.size > 20 + + indent + + self.class.to_s.sub(/.*:/,'') + + em.map{|m| "+"+m.to_s.sub(/.*:/,'')}*"" + + " offset=#{interval.first}" + + ", #{tv.inspect}" + + im + + (elements && elements.size > 0 ? + ":" + + (elements.select { |e| !e.is_a?(LogStash::Config::AST::Whitespace) && e.elements && e.elements.size > 0 }||[]).map{|e| + begin + "\n"+e.inspect(indent+" ") + rescue # Defend against inspect not taking a parameter + "\n"+indent+" "+e.inspect + end + }.join("") : + "" + ) + end +end diff --git a/logstash-core/lib/logstash/config/config_ast.rb b/logstash-core/lib/logstash/config/config_ast.rb index 16f47fc16..a079eec57 100644 --- a/logstash-core/lib/logstash/config/config_ast.rb +++ b/logstash-core/lib/logstash/config/config_ast.rb @@ -2,61 +2,7 @@ require 'logstash/errors' require "treetop" -class Treetop::Runtime::SyntaxNode - - def compile - return "" if elements.nil? - return elements.collect(&:compile).reject(&:empty?).join("") - end - - # Traverse the syntax tree recursively. - # The order should respect the order of the configuration file as it is read - # and written by humans (and the order in which it is parsed). - def recurse(e, depth=0, &block) - r = block.call(e, depth) - e.elements.each { |e| recurse(e, depth + 1, &block) } if r && e.elements - nil - end - - def recursive_inject(results=[], &block) - if !elements.nil? - elements.each do |element| - if block.call(element) - results << element - else - element.recursive_inject(results, &block) - end - end - end - return results - end - - # When Treetop parses the configuration file - # it will generate a tree, the generated tree will contain - # a few `Empty` nodes to represent the actual space/tab or newline in the file. - # Some of theses node will point to our concrete class. - # To fetch a specific types of object we need to follow each branch - # and ignore the empty nodes. - def recursive_select(klass) - return recursive_inject { |e| e.is_a?(klass) } - end - - def recursive_inject_parent(results=[], &block) - if !parent.nil? - if block.call(parent) - results << parent - else - parent.recursive_inject_parent(results, &block) - end - end - return results - end - - def recursive_select_parent(results=[], klass) - return recursive_inject_parent(results) { |e| e.is_a?(klass) } - end -end - +require "logstash/compiler/treetop_monkeypatches" module LogStash; module Config; module AST diff --git a/logstash-core/lib/logstash/instrument/metric_store.rb b/logstash-core/lib/logstash/instrument/metric_store.rb index 119297c53..9b9a7f770 100644 --- a/logstash-core/lib/logstash/instrument/metric_store.rb +++ b/logstash-core/lib/logstash/instrument/metric_store.rb @@ -222,7 +222,7 @@ module LogStash module Instrument key_candidates = extract_filter_keys(key_paths.shift) key_candidates.each do |key_candidate| - raise MetricNotFound, "For path: #{key_candidate}" if map[key_candidate].nil? + raise MetricNotFound, "For path: #{key_candidate}. Map keys: #{map.keys}" if map[key_candidate].nil? if key_paths.empty? # End of the user requested path if map[key_candidate].is_a?(Concurrent::Map) diff --git a/logstash-core/lib/logstash/pipeline.rb b/logstash-core/lib/logstash/pipeline.rb index b73c0e5d3..ac940a86a 100644 --- a/logstash-core/lib/logstash/pipeline.rb +++ b/logstash-core/lib/logstash/pipeline.rb @@ -21,16 +21,20 @@ require "logstash/instrument/wrapped_write_client" require "logstash/output_delegator" require "logstash/filter_delegator" require "logstash/queue_factory" +require 'logstash/compiler' module LogStash; class BasePipeline include LogStash::Util::Loggable - attr_reader :config_str, :config_hash, :inputs, :filters, :outputs, :pipeline_id - + attr_reader :config_str, :config_hash, :inputs, :filters, :outputs, :pipeline_id, :lir + def initialize(config_str, settings = SETTINGS) @logger = self.logger @config_str = config_str @config_hash = Digest::SHA1.hexdigest(@config_str) + + @lir = compile_lir + # Every time #plugin is invoked this is incremented to give each plugin # a unique id when auto-generating plugin ids @plugin_counter ||= 0 @@ -63,6 +67,10 @@ module LogStash; class BasePipeline raise e end end + + def compile_lir + LogStash::Compiler.compile_pipeline(self.config_str) + end def plugin(plugin_type, name, *args) @plugin_counter += 1 @@ -166,6 +174,8 @@ module LogStash; class Pipeline < BasePipeline @running = Concurrent::AtomicBoolean.new(false) @flushing = Concurrent::AtomicReference.new(false) end # def initialize + + def ready? @ready.value @@ -539,7 +549,6 @@ module LogStash; class Pipeline < BasePipeline end end - # Calculate the uptime in milliseconds # # @return [Fixnum] Uptime in milliseconds, 0 if the pipeline is not started diff --git a/logstash-core/spec/logstash/compiler/compiler_spec.rb b/logstash-core/spec/logstash/compiler/compiler_spec.rb new file mode 100644 index 000000000..df7b749ac --- /dev/null +++ b/logstash-core/spec/logstash/compiler/compiler_spec.rb @@ -0,0 +1,583 @@ +require "spec_helper" +require "logstash/compiler" +java_import Java::OrgLogstashConfigIr::DSL + +describe LogStash::Compiler do + def j + Java::OrgLogstashConfigIr::DSL + end + + # Static import of these useful enums + INPUT = Java::OrgLogstashConfigIr::PluginDefinition::Type::INPUT + FILTER = Java::OrgLogstashConfigIr::PluginDefinition::Type::FILTER + OUTPUT = Java::OrgLogstashConfigIr::PluginDefinition::Type::OUTPUT + CODEC = Java::OrgLogstashConfigIr::PluginDefinition::Type::OUTPUT + + describe "compiling to Pipeline" do + subject(:source_file) { "fake_sourcefile" } + subject(:compiled) { described_class.compile_pipeline(source, source_file) } + + describe "complex configs" do + shared_examples_for "compilable LSCL files" do |path| + describe "parsing #{path}" do + let(:source) { File.read(path) } + + it "should compile" do + expect(compiled).to be_java_kind_of(Java::OrgLogstashConfigIr::Pipeline) + end + + it "should have a hash" do + expect(compiled.uniqueHash) + end + end + end + + Dir.glob(File.join(SUPPORT_DIR, "lscl_configs", "*.conf")).each do |path| + it_should_behave_like "compilable LSCL files", path + end + end + end + + describe "compiling imperative" do + let(:source_file) { "fake_sourcefile" } + subject(:compiled) { described_class.compile_imperative(source, source_file) } + + describe "an empty file" do + let(:source) { "input {} output {}" } + + it "should have an empty input block" do + expect(compiled[:input]).to ir_eql(j.noop) + end + + it "should have an empty filter block" do + expect(compiled[:filter]).to ir_eql(j.noop) + end + + it "should have an empty output block" do + expect(compiled[:output]).to ir_eql(j.noop) + end + end + + describe "SourceMetadata" do + let(:source) { "input { generator {} } output { }" } + + it "should attach correct source text for components" do + expect(compiled[:input].get_meta.getSourceText).to eql("generator {}") + end + end + + context "plugins" do + subject(:c_plugin) { compiled[:input] } + let(:source) { "input { #{plugin_source} } " } + + describe "a simple plugin" do + let(:plugin_source) { "generator {}" } + + it "should contain the plugin" do + expect(c_plugin).to ir_eql(j.iPlugin(INPUT, "generator")) + end + end + + describe "a plugin with mixed parameter types" do + let(:plugin_source) { "generator { aarg => [1] hasharg => {foo => bar} iarg => 123 farg => 123.123 sarg => 'hello'}" } + let(:expected_plugin_args) do + { + "aarg" => [1], + "hasharg" => {"foo" => "bar"}, + "iarg" => 123, + "farg" => 123.123, + "sarg" => 'hello' + } + end + + it "should contain the plugin" do + expect(c_plugin).to ir_eql(j.iPlugin(INPUT, "generator", expected_plugin_args)) + end + end + end + + context "inputs" do + subject(:input) { compiled[:input] } + + describe "a single input" do + let(:source) { "input { generator {} }" } + + it "should contain the single input" do + expect(input).to ir_eql(j.iPlugin(INPUT, "generator")) + end + end + + describe "two inputs" do + let(:source) { "input { generator { count => 1 } generator { count => 2 } } output { }" } + + it "should contain both inputs" do + expect(input).to ir_eql(j.iComposeParallel( + j.iPlugin(INPUT, "generator", {"count" => 1}), + j.iPlugin(INPUT, "generator", {"count" => 2}) + )) + end + end + end + + shared_examples_for "complex grammar" do |section| + let(:section_name_enum) { + case section + when :input + INPUT + when :filter + FILTER + when :output + OUTPUT + else + raise "Unknown section" + end + } + + let(:section) { section } + let (:compiled_section) { compiled[section] } + + def splugin(*args) + j.iPlugin(section_name_enum, *args) + end + + def compose(*statements) + if section == :filter + j.iComposeSequence(*statements) + else + j.iComposeParallel(*statements) + end + end + + describe "multiple section declarations" do + let(:source) do + <<-EOS + #{section} { + aplugin { count => 1 } + } + + #{section} { + aplugin { count => 2 } + } + EOS + end + + + it "should contain both section declarations, in order" do + expect(compiled_section).to ir_eql(compose( + splugin("aplugin", {"count" => 1}), + splugin("aplugin", {"count" => 2}) + )) + end + end + + describe "two plugins" do + let(:source) do + # We care about line/column for this test, hence the indentation + <<-EOS + #{section} { + aplugin { count => 1 } + aplugin { count => 2 } + } + EOS + end + + it "should contain both" do + expect(compiled_section).to ir_eql(compose( + splugin("aplugin", {"count" => 1}), + splugin("aplugin", {"count" => 2}) + )) + end + + it "should attach source_metadata with correct info to the statements" do + meta = compiled_section.statements.first.meta + expect(meta.getSourceText).to eql("aplugin { count => 1 }") + expect(meta.getSourceLine).to eql(2) + expect(meta.getSourceColumn).to eql(13) + expect(meta.getSourceFile).to eql(source_file) + expect(compiled_section.statements.first.meta) + expect(compiled_section) + end + end + + describe "if conditions" do + describe "conditional expressions" do + let(:source) { "#{section} { if (#{expression}) { aplugin {} } }" } + let(:c_expression) { compiled_section.getBooleanExpression } + + describe "logical expressions" do + describe "simple and" do + let(:expression) { "2 > 1 and 1 < 2" } + + it "should compile correctly" do + expect(c_expression).to ir_eql( + j.eAnd( + j.eGt(j.eValue(2), j.eValue(1)), + j.eLt(j.eValue(1), j.eValue(2)) + )) + end + end + + describe "'in' array" do + let(:expression) { "'foo' in ['foo', 'bar']" } + + it "should compile correctly" do + expect(c_expression).to ir_eql( + j.eIn( + j.eValue('foo'), + j.eValue(['foo', 'bar']) + )) + end + end + + describe "'not in' array" do + let(:expression) { "'foo' not in ['foo', 'bar']" } + + it "should compile correctly" do + expect(c_expression).to ir_eql( + j.eNot( + j.eIn( + j.eValue('foo'), + j.eValue(['foo', 'bar']) + ))) + end + end + + describe "'not'" do + let(:expression) { "!(1 > 2)" } + + it "should compile correctly" do + expect(c_expression).to ir_eql(j.eNot(j.eGt(j.eValue(1), j.eValue(2)))) + end + end + + describe "and or precedence" do + let(:expression) { "2 > 1 and 1 < 2 or 3 < 2" } + + it "should compile correctly" do + expect(c_expression).to ir_eql( + j.eOr( + j.eAnd( + j.eGt(j.eValue(2), j.eValue(1)), + j.eLt(j.eValue(1), j.eValue(2)) + ), + j.eLt(j.eValue(3), j.eValue(2)) + ) + ) + end + + describe "multiple or" do + let(:expression) { "2 > 1 or 1 < 2 or 3 < 2" } + + it "should compile correctly" do + expect(c_expression).to ir_eql( + j.eOr( + j.eGt(j.eValue(2), j.eValue(1)), + j.eOr( + j.eLt(j.eValue(1), j.eValue(2)), + j.eLt(j.eValue(3), j.eValue(2)) + ) + ) + ) + end + end + + describe "a complex expression" do + let(:expression) { "1 > 2 and 3 > 4 or 6 > 7 and 8 > 9" } + false and false or true and true + + it "should compile correctly" do + expect(c_expression).to ir_eql( + j.eOr( + j.eAnd( + j.eGt(j.eValue(1), j.eValue(2)), + j.eGt(j.eValue(3), j.eValue(4)) + ), + j.eAnd( + j.eGt(j.eValue(6), j.eValue(7)), + j.eGt(j.eValue(8), j.eValue(9)) + ) + ) + ) + end + end + + describe "a complex nested expression" do + let(:expression) { "1 > 2 and (1 > 2 and 3 > 4 or 6 > 7 and 8 > 9) or 6 > 7 and 8 > 9" } + false and false or true and true + + it "should compile correctly" do + expect(c_expression).to ir_eql( + j.eOr( + j.eAnd( + j.eGt(j.eValue(1), j.eValue(2)), + j.eOr( + j.eAnd( + j.eGt(j.eValue(1), j.eValue(2)), + j.eGt(j.eValue(3), j.eValue(4)) + ), + j.eAnd( + j.eGt(j.eValue(6), j.eValue(7)), + j.eGt(j.eValue(8), j.eValue(9)) + ) + ) + ), + j.eAnd( + j.eGt(j.eValue(6), j.eValue(7)), + j.eGt(j.eValue(8), j.eValue(9)) + ) + ) + ) + end + end + end + end + + describe "comparisons" do + describe "field not null" do + let(:expression) { "[foo]"} + + it "should compile correctly" do + expect(c_expression).to ir_eql(j.eTruthy(j.eEventValue("[foo]"))) + end + end + + describe "'=='" do + let(:expression) { "[foo] == 5"} + + it "should compile correctly" do + expect(c_expression).to ir_eql(j.eEq(j.eEventValue("[foo]"), j.eValue(5.to_java))) + end + end + + describe "'!='" do + let(:expression) { "[foo] != 5"} + + it "should compile correctly" do + expect(c_expression).to ir_eql(j.eNeq(j.eEventValue("[foo]"), j.eValue(5.to_java))) + end + end + + describe "'>'" do + let(:expression) { "[foo] > 5"} + + it "should compile correctly" do + expect(c_expression).to ir_eql(j.eGt(j.eEventValue("[foo]"), j.eValue(5.to_java))) + end + end + + describe "'<'" do + let(:expression) { "[foo] < 5"} + + it "should compile correctly" do + expect(c_expression).to ir_eql(j.eLt(j.eEventValue("[foo]"), j.eValue(5.to_java))) + end + end + + describe "'>='" do + let(:expression) { "[foo] >= 5"} + + it "should compile correctly" do + expect(c_expression).to ir_eql(j.eGte(j.eEventValue("[foo]"), j.eValue(5.to_java))) + end + end + + describe "'<='" do + let(:expression) { "[foo] <= 5"} + + it "should compile correctly" do + expect(c_expression).to ir_eql(j.eLte(j.eEventValue("[foo]"), j.eValue(5.to_java))) + end + end + + describe "'=~'" do + let(:expression) { "[foo] =~ /^abc$/"} + + it "should compile correctly" do + expect(c_expression).to ir_eql(j.eRegexEq(j.eEventValue("[foo]"), j.eRegex('^abc$'))) + end + + # Believe it or not, "\.\." is a valid regexp! + describe "when given a quoted regexp" do + let(:expression) { '[foo] =~ "\\.\\."' } + + it "should compile correctly" do + expect(c_expression).to ir_eql(j.eRegexEq(j.eEventValue("[foo]"), j.eRegex('\\.\\.'))) + end + end + end + + describe "'!~'" do + let(:expression) { "[foo] !~ /^abc$/"} + + it "should compile correctly" do + expect(c_expression).to ir_eql(j.eRegexNeq(j.eEventValue("[foo]"), j.eRegex('^abc$'))) + end + end + end + end + + describe "only true branch" do + let (:source) { "#{section} { if [foo] == [bar] { grok {} } }" } + + it "should compile correctly" do + expect(compiled_section).to ir_eql(j.iIf( + j.eEq(j.eEventValue("[foo]"), j.eEventValue("[bar]")), + splugin("grok") + ) + ) + end + end + + describe "only false branch" do + let (:source) { "#{section} { if [foo] == [bar] { } else { fplugin {} } }" } + + it "should compile correctly" do + expect(compiled_section).to ir_eql(j.iIf( + j.eEq(j.eEventValue("[foo]"), j.eEventValue("[bar]")), + j.noop, + splugin("fplugin"), + ) + ) + end + end + + describe "empty if statement" do + let (:source) { "#{section} { if [foo] == [bar] { } }" } + + it "should compile correctly" do + expect(compiled_section).to ir_eql(j.iIf( + j.eEq(j.eEventValue("[foo]"), j.eEventValue("[bar]")), + j.noop, + j.noop + ) + ) + end + end + + describe "if else" do + let (:source) { "#{section} { if [foo] == [bar] { tplugin {} } else { fplugin {} } }" } + + it "should compile correctly" do + expect(compiled_section).to ir_eql(j.iIf( + j.eEq(j.eEventValue("[foo]"), j.eEventValue("[bar]")), + splugin("tplugin"), + splugin("fplugin") + ) + ) + end + end + + describe "if elsif else" do + let (:source) { "#{section} { if [foo] == [bar] { tplugin {} } else if [bar] == [baz] { eifplugin {} } else { fplugin {} } }" } + + it "should compile correctly" do + expect(compiled_section).to ir_eql(j.iIf( + j.eEq(j.eEventValue("[foo]"), j.eEventValue("[bar]")), + splugin("tplugin"), + j.iIf( + j.eEq(j.eEventValue("[bar]"), j.eEventValue("[baz]")), + splugin("eifplugin"), + splugin("fplugin") + ) + ) + ) + end + end + + describe "if elsif elsif else" do + let (:source) do + <<-EOS + #{section} { + if [foo] == [bar] { tplugin {} } + else if [bar] == [baz] { eifplugin {} } + else if [baz] == [bot] { eeifplugin {} } + else { fplugin {} } + } + EOS + end + + it "should compile correctly" do + expect(compiled_section).to ir_eql(j.iIf( + j.eEq(j.eEventValue("[foo]"), j.eEventValue("[bar]")), + splugin("tplugin"), + j.iIf( + j.eEq(j.eEventValue("[bar]"), j.eEventValue("[baz]")), + splugin("eifplugin"), + j.iIf( + j.eEq(j.eEventValue("[baz]"), j.eEventValue("[bot]")), + splugin("eeifplugin"), + splugin("fplugin") + ) + ) + ) + ) + end + + describe "nested ifs" do + let (:source) do + <<-EOS + #{section} { + if [foo] == [bar] { + if [bar] == [baz] { aplugin {} } + } else { + if [bar] == [baz] { bplugin {} } + else if [baz] == [bot] { cplugin {} } + else { dplugin {} } + } + } + EOS + end + + it "should compile correctly" do + expect(compiled_section).to ir_eql(j.iIf( + j.eEq(j.eEventValue("[foo]"), j.eEventValue("[bar]")), + j.iIf(j.eEq(j.eEventValue("[bar]"), j.eEventValue("[baz]")), + splugin("aplugin"), + j.noop + ), + j.iIf( + j.eEq(j.eEventValue("[bar]"), j.eEventValue("[baz]")), + splugin("bplugin"), + j.iIf( + j.eEq(j.eEventValue("[baz]"), j.eEventValue("[bot]")), + splugin("cplugin"), + splugin("dplugin") + ) + ) + ) + ) + end + end + end + end + end + + context "filters" do + subject(:filter) { compiled[:filter] } + + describe "a single filter" do + let(:source) { "input { } filter { grok {} } output { }" } + + it "should contain the single input" do + expect(filter).to ir_eql(j.iPlugin(FILTER, "grok")) + end + end + + it_should_behave_like "complex grammar", :filter + end + + context "outputs" do + subject(:output) { compiled[:output] } + + describe "a single output" do + let(:source) { "input { } output { stdout {} }" } + + it "should contain the single input" do + expect(output).to ir_eql(j.iPlugin(OUTPUT, "stdout")) + end + end + + it_should_behave_like "complex grammar", :output + end + end +end diff --git a/logstash-core/spec/support/helpers.rb b/logstash-core/spec/support/helpers.rb index 6d55ab803..98ac9e0b8 100644 --- a/logstash-core/spec/support/helpers.rb +++ b/logstash-core/spec/support/helpers.rb @@ -14,3 +14,17 @@ def clear_data_dir FileUtils.rm_rf(File.join(data_path, f)) end end + +RSpec::Matchers.define :ir_eql do |expected| + match do |actual| + next unless expected.java_kind_of?(org.logstash.config.ir.SourceComponent) && actual.java_kind_of?(org.logstash.config.ir.SourceComponent) + + expected.sourceComponentEquals(actual) + end + + failure_message do |actual| + "actual value \n#{actual.to_s}\nis not .sourceComponentEquals to the expected value: \n#{expected.to_s}\n" + end +end + +SUPPORT_DIR = Pathname.new(::File.join(::File.dirname(__FILE__), "support")) diff --git a/logstash-core/src/main/java/org/logstash/common/Util.java b/logstash-core/src/main/java/org/logstash/common/Util.java new file mode 100644 index 000000000..be0b73af0 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/common/Util.java @@ -0,0 +1,38 @@ +package org.logstash.common; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +/** + * Created by andrewvc on 12/23/16. + */ +public class Util { + // Modified from http://stackoverflow.com/a/11009612/11105 + + public static MessageDigest defaultMessageDigest() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } + + public static String digest(String base) { + MessageDigest digest = defaultMessageDigest(); + byte[] hash = digest.digest(base.getBytes(StandardCharsets.UTF_8)); + return bytesToHexString(hash); + } + + public static String bytesToHexString(byte[] bytes) { + StringBuilder hexString = new StringBuilder(); + + for (byte aHash : bytes) { + String hex = Integer.toHexString(0xff & aHash); + if (hex.length() == 1) hexString.append('0'); + hexString.append(hex); + } + + return hexString.toString(); + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/BaseSourceComponent.java b/logstash-core/src/main/java/org/logstash/config/ir/BaseSourceComponent.java new file mode 100644 index 000000000..f686a0e48 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/BaseSourceComponent.java @@ -0,0 +1,26 @@ +package org.logstash.config.ir; + +/** + * Created by andrewvc on 9/6/16. + * + * This class is useful to inherit from for things that need to be source components + * since it handles storage of the meta property for you and reduces a lot of boilerplate. + * + */ +public abstract class BaseSourceComponent implements SourceComponent { + private final SourceMetadata meta; + + public BaseSourceComponent(SourceMetadata meta) { + this.meta = meta; + } + + public SourceMetadata getMeta() { + return meta; + } + + public abstract boolean sourceComponentEquals(SourceComponent sourceComponent); + + public String toString(int indent) { + return "toString(int indent) should be implemented for " + this.getClass().getName(); + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/DSL.java b/logstash-core/src/main/java/org/logstash/config/ir/DSL.java new file mode 100644 index 000000000..b6170e74e --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/DSL.java @@ -0,0 +1,296 @@ +package org.logstash.config.ir; + +import org.logstash.config.ir.expression.*; +import org.logstash.config.ir.expression.binary.*; +import org.logstash.config.ir.expression.unary.Not; +import org.logstash.config.ir.expression.unary.Truthy; +import org.logstash.config.ir.graph.Graph; +import org.logstash.config.ir.graph.IfVertex; +import org.logstash.config.ir.graph.PluginVertex; +import org.logstash.config.ir.imperative.*; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +/** + * Created by andrewvc on 9/15/16. + */ +public class DSL { + public static EventValueExpression eEventValue(SourceMetadata meta, String fieldName) { + return new EventValueExpression(meta, fieldName); + } + + public static EventValueExpression eEventValue(String fieldName) { + return eEventValue(new SourceMetadata(), fieldName); + } + + public static ValueExpression eValue(SourceMetadata meta, Object value) throws InvalidIRException { + return new ValueExpression(meta, value); + } + + public static ValueExpression eValue(Object value) throws InvalidIRException { + return eValue(new SourceMetadata(), value); + } + + public static ValueExpression eRegex(SourceMetadata meta, String pattern) throws InvalidIRException { + return new RegexValueExpression(meta, pattern); + } + + public static ValueExpression eRegex(String pattern) throws InvalidIRException { + return eRegex(new SourceMetadata(), pattern); + } + + public static ValueExpression eValue(long value) { + try { + return eValue(new SourceMetadata(), value); + } catch (InvalidIRException e) { + e.printStackTrace(); // Can't happen with an int + return null; + } + } + + public static ValueExpression eValue(double value) { + try { + return eValue(new SourceMetadata(), value); + } catch (InvalidIRException e) { + e.printStackTrace(); // Can't happen with an int + return null; + } + } + + public static Gt eGt(SourceMetadata meta, Expression left, Expression right) throws InvalidIRException { + return new Gt(meta, left, right); + } + + public static Gt eGt(Expression left, Expression right) throws InvalidIRException { + return new Gt(null, left, right); + } + + public static Gte eGte(SourceMetadata meta, Expression left, Expression right) throws InvalidIRException { + return new Gte(meta, left, right); + } + + public static Gte eGte(Expression left, Expression right) throws InvalidIRException { + return new Gte(null, left, right); + } + + public static Lt eLt(SourceMetadata meta, Expression left, Expression right) throws InvalidIRException { + return new Lt(meta, left, right); + } + + public static Lt eLt(Expression left, Expression right) throws InvalidIRException { + return new Lt(null, left, right); + } + + public static Lte eLte(SourceMetadata meta, Expression left, Expression right) throws InvalidIRException { + return new Lte(meta, left, right); + } + public static Lte eLte(Expression left, Expression right) throws InvalidIRException { + return new Lte(null, left, right); + } + + public static Eq eEq(SourceMetadata meta, Expression left, Expression right) throws InvalidIRException { + return new Eq(meta, left, right); + } + + public static Eq eEq(Expression left, Expression right) throws InvalidIRException { + return new Eq(null, left, right); + } + + public static And eAnd(SourceMetadata meta, Expression left, Expression right) throws InvalidIRException { + return new And(meta, left, right); + } + + public static And eAnd(Expression left, Expression right) throws InvalidIRException { + return new And(null, left, right); + } + + public static Or eOr(SourceMetadata meta, Expression left, Expression right) throws InvalidIRException { + return new Or(meta, left, right); + } + + public static Or eOr(Expression left, Expression right) throws InvalidIRException { + return new Or(null, left, right); + } + + public static RegexEq eRegexEq(SourceMetadata meta, Expression left, ValueExpression right) throws InvalidIRException { + return new RegexEq(meta, left, right); + } + + public static RegexEq eRegexEq(Expression left, ValueExpression right) throws InvalidIRException { + return new RegexEq(null, left, right); + } + + public static Expression eRegexNeq(SourceMetadata meta, Expression left, ValueExpression right) throws InvalidIRException { + return eNot(eRegexEq(meta, left, right)); + } + + public static Expression eRegexNeq(Expression left, ValueExpression right) throws InvalidIRException { + return eNot(eRegexEq(left, right)); + } + + public static Neq eNeq(SourceMetadata meta, Expression left, Expression right) throws InvalidIRException { + return new Neq(meta, left, right); + } + public static Neq eNeq(Expression left, Expression right) throws InvalidIRException { + return new Neq(null, left, right); + } + + public static In eIn(SourceMetadata meta, Expression left, Expression right) throws InvalidIRException { + return new In(meta, left, right); + } + + public static In eIn(Expression left, Expression right) throws InvalidIRException { + return new In(null, left, right); + } + + public static Not eNot(SourceMetadata meta, Expression expr) throws InvalidIRException { + return new Not(meta, expr); + } + + public static Not eNot(Expression expr) throws InvalidIRException { + return new Not(null, expr); + } + + public static BooleanExpression eTruthy(SourceMetadata meta, Expression expr) throws InvalidIRException { + if (expr instanceof BooleanExpression) { + return (BooleanExpression) expr; + } + return new Truthy(meta, expr); + } + public static BooleanExpression eTruthy(Expression expr) throws InvalidIRException { + return eTruthy(null, expr); + } + + public static Statement iCompose(ComposedStatement.IFactory factory, SourceMetadata meta, Statement... statements) throws InvalidIRException { + if (statements.length == 0 ) { + return new NoopStatement(meta); + } else if (statements.length == 1 ) { + return statements[0]; + } else { + return factory.make(meta, Arrays.asList(statements)); + } + } + + public static Statement iComposeSequence(SourceMetadata meta, Statement... statements) throws InvalidIRException { + return iCompose(ComposedSequenceStatement::new, meta, statements); + } + + public static Statement iComposeSequence(Statement... statements) throws InvalidIRException { + return iComposeSequence(null, statements); + } + + public static Statement iComposeParallel(SourceMetadata meta, Statement... statements) throws InvalidIRException { + return iCompose(ComposedParallelStatement::new, meta, statements); + } + + public static Statement iComposeParallel(Statement... statements) throws InvalidIRException { + return iComposeParallel(null, statements); + } + + public static NoopStatement noop(SourceMetadata meta) { + return new NoopStatement(meta); + } + + public static NoopStatement noop() { + return new NoopStatement(new SourceMetadata()); + } + + public static PluginStatement iPlugin(SourceMetadata meta, PluginDefinition.Type pluginType, String pluginName, Map pluginArguments) { + return new PluginStatement(meta, new PluginDefinition(pluginType, pluginName, pluginArguments)); + } + + public static PluginStatement iPlugin(PluginDefinition.Type type, String pluginName, Map pluginArguments) { + return iPlugin(new SourceMetadata(), type, pluginName, pluginArguments); + } + + public static PluginStatement iPlugin(PluginDefinition.Type type, String pluginName, MapBuilder argBuilder) { + return iPlugin(type, pluginName, argBuilder.build()); + } + + public static PluginStatement iPlugin(PluginDefinition.Type type, String pluginName, String id) { + return iPlugin(type, pluginName, argumentBuilder().put("id", id).build()); + } + + public static PluginStatement iPlugin(PluginDefinition.Type type, String pluginName) { + return iPlugin(type, pluginName, pargs()); + } + + public static IfStatement iIf(SourceMetadata meta, + Expression condition, + Statement ifTrue, + Statement ifFalse) throws InvalidIRException { + BooleanExpression booleanExpression = eTruthy(meta, condition); + return new IfStatement(meta, booleanExpression, ifTrue, ifFalse); + } + + public static IfStatement iIf(Expression condition, + Statement ifTrue, + Statement ifFalse) throws InvalidIRException { + return iIf(new SourceMetadata(), condition, ifTrue, ifFalse); + } + + public static IfStatement iIf(Expression condition, + Statement ifTrue) throws InvalidIRException { + return iIf(new SourceMetadata(), condition, ifTrue, noop()); + } + + public static class MapBuilder { + private final HashMap map; + + public MapBuilder() { + this.map = new HashMap<>(); + } + + public MapBuilder put(K k, V v) { + map.put(k, v); + return this; + } + + public Map build() { + return map; + } + } + + static MapBuilder mapBuilder() { + return new MapBuilder<>(); + } + + public static MapBuilder argumentBuilder() { + return mapBuilder(); + } + + public static MapBuilder pargs() { + return argumentBuilder(); + } + + public static Graph graph() { + return new Graph(); + } + + public static PluginVertex gPlugin(SourceMetadata sourceMetadata, PluginDefinition.Type pluginType, String pluginName, Map pluginArgs) { + return new PluginVertex(sourceMetadata, new PluginDefinition(pluginType, pluginName, pluginArgs)); + } + + public static PluginVertex gPlugin(PluginDefinition.Type type, String pluginName, Map pluginArgs) { + return gPlugin(new SourceMetadata(), type, pluginName, pluginArgs); + } + + public static PluginVertex gPlugin(PluginDefinition.Type type, String pluginName, String id) { + return gPlugin(type, pluginName, argumentBuilder().put("id", id).build()); + } + + public static PluginVertex gPlugin(PluginDefinition.Type type, String pluginName) { + return gPlugin(new SourceMetadata(), type, pluginName, new HashMap<>()); + } + + + public static IfVertex gIf(SourceMetadata meta, BooleanExpression expression) { + return new IfVertex(meta, expression); + } + + public static IfVertex gIf(BooleanExpression expression) { + return new IfVertex(new SourceMetadata(), expression); + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/Hashable.java b/logstash-core/src/main/java/org/logstash/config/ir/Hashable.java new file mode 100644 index 000000000..5b4445385 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/Hashable.java @@ -0,0 +1,14 @@ +package org.logstash.config.ir; + +import org.logstash.common.Util; + +/** + * Created by andrewvc on 12/23/16. + */ +public interface Hashable { + String hashSource(); + + default String uniqueHash() { + return Util.digest(this.hashSource()); + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/InvalidIRException.java b/logstash-core/src/main/java/org/logstash/config/ir/InvalidIRException.java new file mode 100644 index 000000000..5d0ef41fb --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/InvalidIRException.java @@ -0,0 +1,16 @@ +package org.logstash.config.ir; + +import org.logstash.config.ir.graph.algorithms.TopologicalSort; + +/** + * Created by andrewvc on 9/6/16. + */ +public class InvalidIRException extends Exception { + public InvalidIRException(String s) { + super(s); + } + + public InvalidIRException(String s, Exception e) { + super(s,e); + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/Pipeline.java b/logstash-core/src/main/java/org/logstash/config/ir/Pipeline.java new file mode 100644 index 000000000..7c9fef7ef --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/Pipeline.java @@ -0,0 +1,102 @@ +package org.logstash.config.ir; + +import org.logstash.config.ir.graph.Graph; +import org.logstash.config.ir.graph.PluginVertex; +import org.logstash.config.ir.graph.QueueVertex; +import org.logstash.config.ir.graph.Vertex; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Created by andrewvc on 9/20/16. + */ +public class Pipeline implements Hashable { + public Graph getGraph() { + return graph; + } + + public QueueVertex getQueue() { + return queue; + } + + //public QueueVertex getFilterOut() { + // return filterOut; + //} + + private final Graph graph; + private final QueueVertex queue; + //private final QueueVertex filterOut; + + public Pipeline(Graph inputSection, Graph filterSection, Graph outputSection) throws InvalidIRException { + // Validate all incoming graphs, we can't turn an invalid graph into a Pipeline! + inputSection.validate(); + filterSection.validate(); + outputSection.validate(); + + Graph tempGraph = inputSection.copy(); // The input section are our roots, so we can import that wholesale + + // Connect all the input vertices out to the queue + queue = new QueueVertex(); + tempGraph = tempGraph.chain(queue); + + // Now we connect the queue to the root of the filter section + tempGraph = tempGraph.chain(filterSection); + + // Finally, connect the filter out node to all the outputs + this.graph = tempGraph.chain(outputSection); + } + + public List getPostQueue() throws InvalidIRException { + return graph.getSortedVerticesAfter(queue); + } + + public List getInputPluginVertices() { + return getPluginVertices(PluginDefinition.Type.INPUT); + } + + public List getFilterPluginVertices() { + return getPluginVertices(PluginDefinition.Type.FILTER); + } + + public List getOutputPluginVertices() { + return getPluginVertices(PluginDefinition.Type.OUTPUT); + } + + @Override + public String toString() { + String summary = String.format("[Pipeline] Inputs: %d Filters: %d Outputs %d", + getInputPluginVertices().size(), + getFilterPluginVertices().size(), + getOutputPluginVertices().size()); + return summary + "\n" + graph.toString(); + } + + + // Return plugin vertices by type + public Stream pluginVertices(PluginDefinition.Type type) { + return pluginVertices() + .filter(v -> v.getPluginDefinition().getType().equals(type)); + } + + // Return plugin vertices by type + public List getPluginVertices(PluginDefinition.Type type) { + return pluginVertices(type).collect(Collectors.toList()); + } + + public List getPluginVertices() { + return pluginVertices().collect(Collectors.toList()); + } + + public Stream pluginVertices() { + return graph.vertices() + .filter(v -> v instanceof PluginVertex) + .map(v -> (PluginVertex) v); + } + + @Override + public String hashSource() { + return this.graph.uniqueHash(); + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/PluginDefinition.java b/logstash-core/src/main/java/org/logstash/config/ir/PluginDefinition.java new file mode 100644 index 000000000..e78ebc477 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/PluginDefinition.java @@ -0,0 +1,102 @@ +package org.logstash.config.ir; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.util.HashSet; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * Created by andrewvc on 9/20/16. + */ +public class PluginDefinition implements SourceComponent, Hashable { + private static ObjectMapper om = new ObjectMapper(); + + @Override + public String hashSource() { + try { + String serializedArgs = om.writeValueAsString(this.getArguments()); + return this.getClass().getCanonicalName() + "|" + + this.getType().toString() + "|" + + this.getName() + "|" + + serializedArgs; + } catch (JsonProcessingException e) { + throw new IllegalArgumentException("Could not serialize plugin args as JSON", e); + } + } + + public enum Type { + INPUT, + FILTER, + OUTPUT, + CODEC + } + + private final Type type; + private final String name; + private final Map arguments; + + public Type getType() { + return type; + } + + public String getName() { + return name; + } + + public Map getArguments() { + return arguments; + } + + public PluginDefinition(Type type, String name, Map arguments) { + this.type = type; + this.name = name; + this.arguments = arguments; + } + + public String toString() { + return type.toString().toLowerCase() + "-" + name + arguments; + } + + public int hashCode() { + return Objects.hash(type, name, arguments); + } + + @Override + public boolean equals(Object o) { + if (o == null) return false; + if (o instanceof PluginDefinition) { + PluginDefinition oPlugin = (PluginDefinition) o; + return type.equals(oPlugin.type) && name.equals(oPlugin.name) && arguments.equals(oPlugin.arguments); + } + return false; + } + + @Override + public boolean sourceComponentEquals(SourceComponent o) { + if (o == null) return false; + if (o instanceof PluginDefinition) { + PluginDefinition oPluginDefinition = (PluginDefinition) o; + + Set allArgs = new HashSet<>(); + allArgs.addAll(getArguments().keySet()); + allArgs.addAll(oPluginDefinition.getArguments().keySet()); + + // Compare all arguments except the unique id + boolean argsMatch = allArgs.stream(). + filter(k -> !k.equals("id")). + allMatch(k -> Objects.equals(getArguments().get(k), oPluginDefinition.getArguments().get(k))); + + + return argsMatch && type.equals(oPluginDefinition.type) && name.equals(oPluginDefinition.name); + } + return false; + } + + @Override + public SourceMetadata getMeta() { + return null; + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/SourceComponent.java b/logstash-core/src/main/java/org/logstash/config/ir/SourceComponent.java new file mode 100644 index 000000000..f68afc060 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/SourceComponent.java @@ -0,0 +1,9 @@ +package org.logstash.config.ir; + +/** + * Created by andrewvc on 9/16/16. + */ +public interface SourceComponent { + boolean sourceComponentEquals(SourceComponent sourceComponent); + SourceMetadata getMeta(); +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/SourceMetadata.java b/logstash-core/src/main/java/org/logstash/config/ir/SourceMetadata.java new file mode 100644 index 000000000..8a9a3205a --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/SourceMetadata.java @@ -0,0 +1,52 @@ +package org.logstash.config.ir; + +import java.util.Objects; + +/** + * Created by andrewvc on 9/6/16. + */ +public class SourceMetadata { + private final String sourceFile; + + public String getSourceFile() { + return sourceFile; + } + + public Integer getSourceLine() { + return sourceLine; + } + + public Integer getSourceColumn() { + return sourceColumn; + } + + public String getSourceText() { + return sourceText; + } + + private final Integer sourceLine; + private final Integer sourceColumn; + private final String sourceText; + + public SourceMetadata(String sourceFile, Integer sourceLine, Integer sourceChar, String sourceText) { + this.sourceFile = sourceFile; + this.sourceLine = sourceLine; + this.sourceColumn = sourceChar; + this.sourceText = sourceText; + } + + public SourceMetadata() { + this.sourceFile = null; + this.sourceLine = null; + this.sourceColumn = null; + this.sourceText = null; + } + + public int hashCode() { + return Objects.hash(this.sourceFile, this.sourceLine, this.sourceColumn, this.sourceText); + } + + public String toString() { + return sourceFile + ":" + sourceLine + ":" + sourceColumn + ":```\n" + sourceText + "\n```"; + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/expression/BinaryBooleanExpression.java b/logstash-core/src/main/java/org/logstash/config/ir/expression/BinaryBooleanExpression.java new file mode 100644 index 000000000..8c99275b2 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/expression/BinaryBooleanExpression.java @@ -0,0 +1,53 @@ +package org.logstash.config.ir.expression; + +import org.logstash.config.ir.SourceComponent; +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.SourceMetadata; + +/** + * Created by andrewvc on 9/6/16. + */ +public abstract class BinaryBooleanExpression extends BooleanExpression { + @Override + public boolean sourceComponentEquals(SourceComponent sourceComponent) { + if (sourceComponent == null) return false; + if (this == sourceComponent) return true; + if (this.getClass().equals(sourceComponent.getClass())) { + BinaryBooleanExpression other = (BinaryBooleanExpression) sourceComponent; + return (this.getLeft().sourceComponentEquals(other.getLeft()) && + this.getRight().sourceComponentEquals(other.getRight())); + } + return false; + } + + private final Expression left; + private final Expression right; + + public Expression getRight() { + return right; + } + + public Expression getLeft() { + return left; + } + + public BinaryBooleanExpression(SourceMetadata meta, + Expression left, + Expression right) throws InvalidIRException { + super(meta); + this.left = left; + this.right = right; + } + + public abstract String rubyOperator(); + + @Override + public String toRubyString() { + return "(" + getLeft().toRubyString() + rubyOperator() + getRight().toRubyString() + ")"; + } + + @Override + public String hashSource() { + return this.getClass().getCanonicalName() + "[" + getLeft().hashSource() + "|" + getRight().hashSource() + "]"; + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/expression/BooleanExpression.java b/logstash-core/src/main/java/org/logstash/config/ir/expression/BooleanExpression.java new file mode 100644 index 000000000..066f175b9 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/expression/BooleanExpression.java @@ -0,0 +1,12 @@ +package org.logstash.config.ir.expression; + +import org.logstash.config.ir.SourceMetadata; + +/** + * Created by andrewvc on 9/14/16. + */ +public abstract class BooleanExpression extends Expression { + public BooleanExpression(SourceMetadata meta) { + super(meta); + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/expression/EventValueExpression.java b/logstash-core/src/main/java/org/logstash/config/ir/expression/EventValueExpression.java new file mode 100644 index 000000000..e49a2b848 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/expression/EventValueExpression.java @@ -0,0 +1,46 @@ +package org.logstash.config.ir.expression; + +import org.logstash.config.ir.SourceComponent; +import org.logstash.config.ir.SourceMetadata; + +/** + * Created by andrewvc on 9/13/16. + */ +public class EventValueExpression extends Expression { + private final String fieldName; + + public EventValueExpression(SourceMetadata meta, String fieldName) { + super(meta); + this.fieldName = fieldName; + } + + public String getFieldName() { + return fieldName; + } + + @Override + public boolean sourceComponentEquals(SourceComponent sourceComponent) { + if (sourceComponent == null) return false; + if (this == sourceComponent) return true; + if (sourceComponent instanceof EventValueExpression) { + EventValueExpression other = (EventValueExpression) sourceComponent; + return (this.getFieldName().equals(other.getFieldName())); + } + return false; + } + + @Override + public String toString() { + return "event.get('" + fieldName + "')"; + } + + @Override + public String toRubyString() { + return "event.getField('" + fieldName + "')"; + } + + @Override + public String hashSource() { + return this.getClass().getCanonicalName() + "|" + fieldName; + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/expression/Expression.java b/logstash-core/src/main/java/org/logstash/config/ir/expression/Expression.java new file mode 100644 index 000000000..1d5e9a447 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/expression/Expression.java @@ -0,0 +1,48 @@ +package org.logstash.config.ir.expression; + +import org.jruby.RubyInstanceConfig; +import org.jruby.embed.AttributeName; +import org.jruby.embed.ScriptingContainer; +import org.logstash.config.ir.Hashable; +import org.logstash.config.ir.BaseSourceComponent; +import org.logstash.config.ir.SourceMetadata; + +/* + * [foo] == "foostr" eAnd [bar] > 10 + * eAnd(eEq(eventValueExpr("foo"), value("foostr")), eEq(eEventValue("bar"), value(10))) + * + * if [foo] + * notnull(eEventValue("foo")) + * Created by andrewvc on 9/6/16. + */ +public abstract class Expression extends BaseSourceComponent implements Hashable { + private Object compiled; + private ScriptingContainer container; + + public Expression(SourceMetadata meta) { + super(meta); + } + + public boolean eval() { + return true; + } + + public void compile() { + container = new ScriptingContainer(); + container.setCompileMode(RubyInstanceConfig.CompileMode.JIT); + container.setAttribute(AttributeName.SHARING_VARIABLES, false); + container.runScriptlet("def start(event)\n" + this.toString() + "\nend"); + } + + @Override + public String toString(int indent) { + return toString(); + } + + @Override + public String toString() { + return toRubyString(); + } + + public abstract String toRubyString(); +} \ No newline at end of file diff --git a/logstash-core/src/main/java/org/logstash/config/ir/expression/RegexValueExpression.java b/logstash-core/src/main/java/org/logstash/config/ir/expression/RegexValueExpression.java new file mode 100644 index 000000000..4cd85aa4e --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/expression/RegexValueExpression.java @@ -0,0 +1,53 @@ +package org.logstash.config.ir.expression; + +import org.joni.Option; +import org.joni.Regex; +import org.logstash.config.ir.SourceComponent; +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.SourceMetadata; + +/** + * Created by andrewvc on 9/15/16. + */ +public class RegexValueExpression extends ValueExpression { + private final Regex regex; + + public RegexValueExpression(SourceMetadata meta, Object value) throws InvalidIRException { + super(meta, value); + + if (!(value instanceof String)) { + throw new InvalidIRException("Regex value expressions can only take strings!"); + } + + byte[] patternBytes = getSource().getBytes(); + this.regex = new Regex(patternBytes, 0, patternBytes.length, Option.NONE); + } + + @Override + public Object get() { + return this.regex; + } + + public String getSource() { + return (String) value; + } + + @Override + public String toString() { + return this.value.toString(); + } + + @Override + public boolean sourceComponentEquals(SourceComponent other) { + if (other == null) return false; + if (other instanceof RegexValueExpression) { + return (((RegexValueExpression) other).getSource().equals(getSource())); + } + return false; + } + + @Override + public String toRubyString() { + return (String) value; + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/expression/UnaryBooleanExpression.java b/logstash-core/src/main/java/org/logstash/config/ir/expression/UnaryBooleanExpression.java new file mode 100644 index 000000000..51e175caa --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/expression/UnaryBooleanExpression.java @@ -0,0 +1,27 @@ +package org.logstash.config.ir.expression; + +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.SourceMetadata; + +/** + * Created by andrewvc on 9/13/16. + */ +public abstract class UnaryBooleanExpression extends BooleanExpression { + private final Expression expression; + + public Expression getExpression() { + return expression; + } + + public UnaryBooleanExpression(SourceMetadata meta, + Expression expression) throws InvalidIRException { + super(meta); + if (expression == null) throw new InvalidIRException("Unary expressions cannot operate on null!"); + this.expression = expression; + } + + @Override + public String hashSource() { + return this.getClass().getCanonicalName() + "[" + this.expression.hashSource() + "]"; + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/expression/ValueExpression.java b/logstash-core/src/main/java/org/logstash/config/ir/expression/ValueExpression.java new file mode 100644 index 000000000..e03ea73a4 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/expression/ValueExpression.java @@ -0,0 +1,73 @@ +package org.logstash.config.ir.expression; + +import org.logstash.config.ir.SourceComponent; +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.SourceMetadata; + +import java.math.BigDecimal; +import java.util.List; + +/** + * Created by andrewvc on 9/13/16. + */ +public class ValueExpression extends Expression { + protected final Object value; + + public ValueExpression(SourceMetadata meta, Object value) throws InvalidIRException { + super(meta); + + if (!(value == null || + value instanceof Short || + value instanceof Long || + value instanceof Integer || + value instanceof Float || + value instanceof Double || + value instanceof BigDecimal || + value instanceof String || + value instanceof List || + value instanceof java.time.Instant + )) { + // This *should* be caught by the treetop grammar, but we need this case just in case there's a bug + // somewhere + throw new InvalidIRException("Invalid eValue " + value + " with class " + value.getClass().getName()); + } + + this.value = value; + } + + public Object get() { + return value; + } + + @Override + public boolean sourceComponentEquals(SourceComponent sourceComponent) { + if (sourceComponent == null) return false; + if (this == sourceComponent) return true; + if (sourceComponent instanceof ValueExpression) { + ValueExpression other = (ValueExpression) sourceComponent; + if (this.get() == null) { + return (other.get() == null); + } else { + return (this.get().equals(other.get())); + } + } + return false; + } + + @Override + public String toRubyString() { + if (value == null) { + return "null"; + } + if (value instanceof String) { + return "'" + get() + "'"; + } + + return get().toString(); + } + + @Override + public String hashSource() { + return this.getClass().getCanonicalName() + "|" + value; + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/expression/binary/And.java b/logstash-core/src/main/java/org/logstash/config/ir/expression/binary/And.java new file mode 100644 index 000000000..434cea5c1 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/expression/binary/And.java @@ -0,0 +1,20 @@ +package org.logstash.config.ir.expression.binary; + +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.SourceMetadata; +import org.logstash.config.ir.expression.BinaryBooleanExpression; +import org.logstash.config.ir.expression.Expression; + +/** + * Created by andrewvc on 9/21/16. + */ +public class And extends BinaryBooleanExpression { + public And(SourceMetadata meta, Expression left, Expression right) throws InvalidIRException { + super(meta, left, right); + } + + @Override + public String rubyOperator() { + return "&&"; + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/expression/binary/Eq.java b/logstash-core/src/main/java/org/logstash/config/ir/expression/binary/Eq.java new file mode 100644 index 000000000..b2772a18d --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/expression/binary/Eq.java @@ -0,0 +1,20 @@ +package org.logstash.config.ir.expression.binary; + +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.SourceMetadata; +import org.logstash.config.ir.expression.BinaryBooleanExpression; +import org.logstash.config.ir.expression.Expression; + +/** + * Created by andrewvc on 9/21/16. + */ +public class Eq extends BinaryBooleanExpression { + public Eq(SourceMetadata meta, Expression left, Expression right) throws InvalidIRException { + super(meta, left, right); + } + + @Override + public String rubyOperator() { + return "=="; + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/expression/binary/Gt.java b/logstash-core/src/main/java/org/logstash/config/ir/expression/binary/Gt.java new file mode 100644 index 000000000..e611a7717 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/expression/binary/Gt.java @@ -0,0 +1,20 @@ +package org.logstash.config.ir.expression.binary; + +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.SourceMetadata; +import org.logstash.config.ir.expression.BinaryBooleanExpression; +import org.logstash.config.ir.expression.Expression; + +/** + * Created by andrewvc on 9/21/16. + */ +public class Gt extends BinaryBooleanExpression { + public Gt(SourceMetadata meta, Expression left, Expression right) throws InvalidIRException { + super(meta, left, right); + } + + @Override + public String rubyOperator() { + return ">"; + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/expression/binary/Gte.java b/logstash-core/src/main/java/org/logstash/config/ir/expression/binary/Gte.java new file mode 100644 index 000000000..3e9840c6e --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/expression/binary/Gte.java @@ -0,0 +1,20 @@ +package org.logstash.config.ir.expression.binary; + +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.SourceMetadata; +import org.logstash.config.ir.expression.BinaryBooleanExpression; +import org.logstash.config.ir.expression.Expression; + +/** + * Created by andrewvc on 9/21/16. + */ +public class Gte extends BinaryBooleanExpression { + public Gte(SourceMetadata meta, Expression left, Expression right) throws InvalidIRException { + super(meta, left, right); + } + + @Override + public String rubyOperator() { + return ">="; + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/expression/binary/In.java b/logstash-core/src/main/java/org/logstash/config/ir/expression/binary/In.java new file mode 100644 index 000000000..0a4302c46 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/expression/binary/In.java @@ -0,0 +1,20 @@ +package org.logstash.config.ir.expression.binary; + +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.SourceMetadata; +import org.logstash.config.ir.expression.BinaryBooleanExpression; +import org.logstash.config.ir.expression.Expression; + +/** + * Created by andrewvc on 9/21/16. + */ +public class In extends BinaryBooleanExpression { + public In(SourceMetadata meta, Expression left, Expression right) throws InvalidIRException { + super(meta, left, right); + } + + @Override + public String rubyOperator() { + return ".include?"; + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/expression/binary/Lt.java b/logstash-core/src/main/java/org/logstash/config/ir/expression/binary/Lt.java new file mode 100644 index 000000000..ea453b9a4 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/expression/binary/Lt.java @@ -0,0 +1,20 @@ +package org.logstash.config.ir.expression.binary; + +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.SourceMetadata; +import org.logstash.config.ir.expression.BinaryBooleanExpression; +import org.logstash.config.ir.expression.Expression; + +/** + * Created by andrewvc on 9/21/16. + */ +public class Lt extends BinaryBooleanExpression { + public Lt(SourceMetadata meta, Expression left, Expression right) throws InvalidIRException { + super(meta, left, right); + } + + @Override + public String rubyOperator() { + return "<"; + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/expression/binary/Lte.java b/logstash-core/src/main/java/org/logstash/config/ir/expression/binary/Lte.java new file mode 100644 index 000000000..0a165023d --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/expression/binary/Lte.java @@ -0,0 +1,20 @@ +package org.logstash.config.ir.expression.binary; + +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.SourceMetadata; +import org.logstash.config.ir.expression.BinaryBooleanExpression; +import org.logstash.config.ir.expression.Expression; + +/** + * Created by andrewvc on 9/21/16. + */ +public class Lte extends BinaryBooleanExpression { + public Lte(SourceMetadata meta, Expression left, Expression right) throws InvalidIRException { + super(meta, left, right); + } + + @Override + public String rubyOperator() { + return "<="; + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/expression/binary/Neq.java b/logstash-core/src/main/java/org/logstash/config/ir/expression/binary/Neq.java new file mode 100644 index 000000000..5358a1239 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/expression/binary/Neq.java @@ -0,0 +1,20 @@ +package org.logstash.config.ir.expression.binary; + +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.SourceMetadata; +import org.logstash.config.ir.expression.BinaryBooleanExpression; +import org.logstash.config.ir.expression.Expression; + +/** + * Created by andrewvc on 9/21/16. + */ +public class Neq extends BinaryBooleanExpression { + public Neq(SourceMetadata meta, Expression left, Expression right) throws InvalidIRException { + super(meta, left, right); + } + + @Override + public String rubyOperator() { + return "!="; + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/expression/binary/Or.java b/logstash-core/src/main/java/org/logstash/config/ir/expression/binary/Or.java new file mode 100644 index 000000000..290200e2b --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/expression/binary/Or.java @@ -0,0 +1,20 @@ +package org.logstash.config.ir.expression.binary; + +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.SourceMetadata; +import org.logstash.config.ir.expression.BinaryBooleanExpression; +import org.logstash.config.ir.expression.Expression; + +/** + * Created by andrewvc on 9/21/16. + */ +public class Or extends BinaryBooleanExpression { + public Or(SourceMetadata meta, Expression left, Expression right) throws InvalidIRException { + super(meta, left, right); + } + + @Override + public String rubyOperator() { + return "||"; + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/expression/binary/RegexEq.java b/logstash-core/src/main/java/org/logstash/config/ir/expression/binary/RegexEq.java new file mode 100644 index 000000000..44ec686d1 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/expression/binary/RegexEq.java @@ -0,0 +1,25 @@ +package org.logstash.config.ir.expression.binary; + +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.SourceMetadata; +import org.logstash.config.ir.expression.BinaryBooleanExpression; +import org.logstash.config.ir.expression.Expression; +import org.logstash.config.ir.expression.RegexValueExpression; + +/** + * Created by andrewvc on 9/21/16. + */ +public class RegexEq extends BinaryBooleanExpression { + public RegexEq(SourceMetadata meta, Expression left, Expression right) throws InvalidIRException { + super(meta, left, right); + + if (!(right instanceof RegexValueExpression)) { + throw new InvalidIRException("You must use a regexp operator with a regexp rval!" + right); + } + } + + @Override + public String rubyOperator() { + return "=~"; + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/expression/unary/Not.java b/logstash-core/src/main/java/org/logstash/config/ir/expression/unary/Not.java new file mode 100644 index 000000000..3e3d7fc70 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/expression/unary/Not.java @@ -0,0 +1,28 @@ +package org.logstash.config.ir.expression.unary; + +import org.logstash.config.ir.SourceComponent; +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.SourceMetadata; +import org.logstash.config.ir.expression.Expression; +import org.logstash.config.ir.expression.UnaryBooleanExpression; + +/** + * Created by andrewvc on 9/21/16. + */ +public class Not extends UnaryBooleanExpression { + public Not(SourceMetadata meta, Expression expression) throws InvalidIRException { + super(meta, expression); + } + + @Override + public String toRubyString() { + return "!(" + getExpression().toRubyString() + ")"; + } + + @Override + public boolean sourceComponentEquals(SourceComponent sourceComponent) { + return sourceComponent != null && + (sourceComponent instanceof Not && + ((Not) sourceComponent).getExpression().sourceComponentEquals(getExpression())); + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/expression/unary/Truthy.java b/logstash-core/src/main/java/org/logstash/config/ir/expression/unary/Truthy.java new file mode 100644 index 000000000..a8138b0e0 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/expression/unary/Truthy.java @@ -0,0 +1,28 @@ +package org.logstash.config.ir.expression.unary; + +import org.logstash.config.ir.SourceComponent; +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.SourceMetadata; +import org.logstash.config.ir.expression.Expression; +import org.logstash.config.ir.expression.UnaryBooleanExpression; + +/** + * Created by andrewvc on 9/21/16. + */ +public class Truthy extends UnaryBooleanExpression { + public Truthy(SourceMetadata meta, Expression expression) throws InvalidIRException { + super(meta, expression); + } + + @Override + public String toRubyString() { + return "(" + this.getExpression() + ")"; + } + + @Override + public boolean sourceComponentEquals(SourceComponent sourceComponent) { + return sourceComponent != null && + sourceComponent instanceof Truthy && + ((Truthy) sourceComponent).getExpression().sourceComponentEquals(this.getExpression()); + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/graph/BooleanEdge.java b/logstash-core/src/main/java/org/logstash/config/ir/graph/BooleanEdge.java new file mode 100644 index 000000000..645933115 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/graph/BooleanEdge.java @@ -0,0 +1,85 @@ +package org.logstash.config.ir.graph; + +import org.logstash.common.Util; +import org.logstash.config.ir.SourceComponent; +import org.logstash.config.ir.InvalidIRException; + +/** + * Created by andrewvc on 9/15/16. + */ +public class BooleanEdge extends Edge { + public static class BooleanEdgeFactory extends EdgeFactory { + public Boolean getEdgeType() { + return edgeType; + } + + private final Boolean edgeType; + + public BooleanEdgeFactory(Boolean edgeType) { + this.edgeType = edgeType; + } + + public BooleanEdge make(Vertex in, Vertex out) throws InvalidIRException { + return new BooleanEdge(edgeType, in, out); + } + + public boolean equals(Object other) { + if (other == null) return false; + if (other instanceof BooleanEdgeFactory) { + return ((BooleanEdgeFactory) other).getEdgeType().equals(edgeType); + } + return false; + } + + public String toString() { + return "BooleanEdge.BooleanEdgeFactory[" + edgeType + "]"; + } + } + public static BooleanEdge.BooleanEdgeFactory trueFactory = new BooleanEdge.BooleanEdgeFactory(true); + public static BooleanEdge.BooleanEdgeFactory falseFactory = new BooleanEdge.BooleanEdgeFactory(false); + + private final Boolean edgeType; + + public Boolean getEdgeType() { + return edgeType; + } + + public BooleanEdge(Boolean edgeType, Vertex outVertex, Vertex inVertex) throws InvalidIRException { + super(outVertex, inVertex); + this.edgeType = edgeType; + } + + @Override + public String individualHashSource() { + return this.getClass().getCanonicalName() + "|" + this.getEdgeType() + "|"; + } + + @Override + public String getId() { + return Util.digest(this.getFrom().getId() + "[" + this.getEdgeType() + "]->" + this.getTo().getId()); + } + + public String toString() { + return getFrom() + " -|" + this.edgeType + "|-> " + getTo(); + } + + @Override + public boolean sourceComponentEquals(SourceComponent sourceComponent) { + if (sourceComponent == null) return false; + if (sourceComponent == this) return true; + if (sourceComponent instanceof BooleanEdge) { + BooleanEdge otherE = (BooleanEdge) sourceComponent; + + return this.getFrom().sourceComponentEquals(otherE.getFrom()) && + this.getTo().sourceComponentEquals(otherE.getTo()) && + this.getEdgeType().equals(otherE.getEdgeType()); + } + return false; + } + + @Override + public BooleanEdge copy(Vertex from, Vertex to) throws InvalidIRException { + return new BooleanEdge(this.edgeType, from, to); + } + +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/graph/Edge.java b/logstash-core/src/main/java/org/logstash/config/ir/graph/Edge.java new file mode 100644 index 000000000..b9a3d8dab --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/graph/Edge.java @@ -0,0 +1,99 @@ +package org.logstash.config.ir.graph; + +import org.logstash.config.ir.SourceComponent; +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.SourceMetadata; + +import java.util.stream.Stream; + +/** + * Created by andrewvc on 9/15/16. + */ +public abstract class Edge implements SourceComponent { + private Graph graph; + + public void setGraph(Graph graph) { + if (this.graph == graph) { + return; + } else if (this.graph == null) { + this.graph = graph; + } else { + throw new IllegalArgumentException("Attempted to set graph for edge that already has one!" + this); + } + } + + public abstract Edge copy(Vertex from, Vertex to) throws InvalidIRException; + + public static abstract class EdgeFactory { + public abstract Edge make(Vertex from, Vertex to) throws InvalidIRException; + } + + private final Vertex from; + private final Vertex to; + + public Stream ancestors() { + // Without all the distinct calls this can be slow + return Stream.concat(this.from.incomingEdges(), this.from.incomingEdges().flatMap(Edge::ancestors).distinct()).distinct(); + } + + public Stream descendants() { + // Without all the distinct calls this can be slow + return Stream.concat(this.to.outgoingEdges(), this.to.outgoingEdges().flatMap(Edge::ancestors).distinct()).distinct(); + } + + public Stream lineage() { + return Stream.concat(Stream.concat(ancestors(), Stream.of(this)), descendants()); + } + + public Edge(Vertex from, Vertex to) throws InvalidIRException { + this.from = from; + this.to = to; + + if (this.from == this.to) { + throw new InvalidIRException("Cannot create a cyclic vertex! " + to); + } + + if (!this.from.acceptsOutgoingEdge(this)) { + throw new Vertex.InvalidEdgeTypeException(String.format("Invalid outgoing edge %s for edge %s", this.from, this)); + } + + if (!this.to.acceptsIncomingEdge(this)) { + throw new Vertex.InvalidEdgeTypeException(String.format("Invalid incoming edge %s for edge %s", this.from, this)); + } + } + + public Vertex getTo() { + return to; + } + + public Vertex getFrom() { + return from; + } + + public String toString() { + return getFrom() + " -> " + getTo(); + } + + @Override + public boolean sourceComponentEquals(SourceComponent sourceComponent) { + if (sourceComponent == null) return false; + if (sourceComponent == this) return true; + if (sourceComponent.getClass() == this.getClass()) { + Edge otherE = (Edge) sourceComponent; + + return this.getFrom().sourceComponentEquals(otherE.getFrom()) && + this.getTo().sourceComponentEquals(otherE.getTo()); + } + return false; + } + + public abstract String individualHashSource(); + + + public abstract String getId(); + + @Override + public SourceMetadata getMeta() { + return null; + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/graph/Graph.java b/logstash-core/src/main/java/org/logstash/config/ir/graph/Graph.java new file mode 100644 index 000000000..709ea2c22 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/graph/Graph.java @@ -0,0 +1,434 @@ +package org.logstash.config.ir.graph; + +import org.logstash.common.Util; +import org.logstash.config.ir.Hashable; +import org.logstash.config.ir.SourceComponent; +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.SourceMetadata; +import org.logstash.config.ir.graph.algorithms.BreadthFirst; +import org.logstash.config.ir.graph.algorithms.GraphDiff; +import org.logstash.config.ir.graph.algorithms.TopologicalSort; + +import java.security.MessageDigest; +import java.util.*; +import java.util.function.BiFunction; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Created by andrewvc on 9/15/16. + */ +public class Graph implements SourceComponent, Hashable { + public final Set vertices = new HashSet<>(); + private final Set edges = new HashSet<>(); + private Map vertexRanks = new HashMap<>(); + private final Map> outgoingEdgeLookup = new HashMap<>(); + private final Map> incomingEdgeLookup = new HashMap<>(); + private List sortedVertices; + + // Builds a graph that has the specified vertices and edges + // Note that this does *not* validate the result + public Graph(Collection vertices, Collection edges) throws InvalidIRException { + for (Vertex vertex : vertices) { this.addVertex(vertex, false); } + for (Edge edge : edges) { this.addEdge(edge, false); } + this.refresh(); + } + + public Graph() {} + + public static Graph empty() { + return new Graph(); + } + + public void addVertex(Vertex v) throws InvalidIRException { + addVertex(v, true); + } + + private void addVertex(Vertex v, boolean doRefresh) throws InvalidIRException { + // If this belongs to another graph use a copy + if (v.getGraph() != null && v.getGraph() != this) { + throw new InvalidIRException("Attempted to add vertex already belonging to a graph!"); + } + + v.setGraph(this); + + this.vertices.add(v); + + if (doRefresh) this.refresh(); + } + + // Takes an arbitrary vertex from any graph and brings it into this one. + // It may have to copy it. The actual vertex that gets used is returned + public Vertex importVertex(Vertex v) throws InvalidIRException { + if (v.getGraph() != this) { + if (v.getGraph() == null) { + this.addVertex(v); + return v; + } else { + Vertex copy = v.copy(); + this.addVertex(copy); + return copy; + } + } else { + return v; + } + } + + public Vertex getVertexById(String id) { + return this.vertices().filter(v -> v.getId().equals(id)).findAny().get(); + } + + // Use chainVertices instead + private Graph addEdge(Edge e) throws InvalidIRException { + return addEdge(e, true); + } + + private Graph addEdge(Edge e, boolean doRefresh) throws InvalidIRException { + if (!(this.getVertices().contains(e.getFrom()) && this.getVertices().contains(e.getTo()))) { + throw new InvalidIRException("Attempted to add edge referencing vertices not in this graph!"); + } + + this.edges.add(e); + + BiFunction, Set> lookupComputeFunction = (vertex, edgeSet) -> { + if (edgeSet == null) edgeSet = new HashSet<>(); + edgeSet.add(e); + return edgeSet; + }; + this.outgoingEdgeLookup.compute(e.getFrom(), lookupComputeFunction); + this.incomingEdgeLookup.compute(e.getTo(), lookupComputeFunction); + + e.setGraph(this); + if (doRefresh) this.refresh(); + return this; + } + + protected Collection getOutgoingEdges(Vertex v) { + return this.outgoingEdgeLookup.getOrDefault(v, Collections.emptySet()); + } + + protected Collection getIncomingEdges(Vertex v) { + return this.incomingEdgeLookup.getOrDefault(v, Collections.emptySet()); + } + + // Returns a copy of this graph + public Graph copy() throws InvalidIRException { + return Graph.combine(this).graph; + } + + // Returns a new graph that is the union of all provided graphs. + // If a single graph is passed in this will return a copy of it + public static GraphCombinationResult combine(Graph... graphs) throws InvalidIRException { + Map oldToNewVertices = new HashMap<>(); + Map oldToNewEdges = new HashMap<>(); + + for (Graph graph : graphs) { + graph.vertices().forEach(v -> oldToNewVertices.put(v, v.copy())); + + for (Edge e : graph.getEdges()) { + Edge copy = e.copy(oldToNewVertices.get(e.getFrom()), oldToNewVertices.get(e.getTo())); + oldToNewEdges.put(e, copy); + } + } + + Graph newGraph = new Graph(oldToNewVertices.values(), oldToNewEdges.values()); + return new GraphCombinationResult(newGraph, oldToNewVertices, oldToNewEdges); + } + + public static final class GraphCombinationResult { + public final Graph graph; + public final Map oldToNewVertices; + public final Map oldToNewEdges; + + GraphCombinationResult(Graph graph, Map oldToNewVertices, Map oldToNewEdges) { + this.graph = graph; + this.oldToNewVertices = oldToNewVertices; + this.oldToNewEdges = oldToNewEdges; + } + } + + /* + Return a copy of this graph with the other graph's nodes to this one by connection this graph's leaves to + the other graph's root + */ + public Graph chain(Graph otherGraph) throws InvalidIRException { + if (otherGraph.getVertices().size() == 0) return this.copy(); + if (this.isEmpty()) return otherGraph.copy(); + + GraphCombinationResult combineResult = Graph.combine(this, otherGraph); + + // Build these lists here since we do mutate the graph in place later + // This isn't strictly necessary, but makes things less confusing + Collection fromLeaves = allLeaves().map(combineResult.oldToNewVertices::get).collect(Collectors.toSet()); + Collection toRoots = otherGraph.roots().map(combineResult.oldToNewVertices::get).collect(Collectors.toSet()); + + return combineResult.graph.chain(fromLeaves, toRoots); + } + + public Graph chain(Vertex... otherVertex) throws InvalidIRException { + chain(this.getAllLeaves(), Arrays.asList(otherVertex)); + return this; + } + + // This does *not* return a copy for performance reasons + private Graph chain(Collection fromLeaves, Collection toVertices) throws InvalidIRException { + for (Vertex leaf : fromLeaves) { + for (Edge.EdgeFactory unusedEf : leaf.getUnusedOutgoingEdgeFactories()) { + for (Vertex toVertex : toVertices) { + this.chainVertices(unusedEf, leaf, toVertex); + } + } + } + + return this; + } + + public Collection chainVerticesById(String... vertexIds) throws InvalidIRException { + return chainVerticesById(PlainEdge.factory, vertexIds); + } + + public Collection chainVerticesById(Edge.EdgeFactory edgeFactory, String... vertexIds) throws InvalidIRException { + Vertex[] argVertices = new Vertex[vertexIds.length]; + for (int i = 0; i < vertexIds.length; i ++) { + String id = vertexIds[i]; + Vertex v = getVertexById(id); + if (v==null) throw new InvalidIRException("Could not chain vertex, id not found in graph: !" + id + "\n" + this); + argVertices[i] = v; + } + return chainVertices(edgeFactory, argVertices); + } + + // Will not validate the graph after running! + // You must invoke validate the graph yourself + // after invoking + public Collection chainVerticesUnsafe(Edge.EdgeFactory edgeFactory, Vertex... argVertices) throws InvalidIRException { + List importedVertices = new ArrayList<>(argVertices.length); + for (Vertex va : argVertices) { + importedVertices.add(this.importVertex(va)); + } + + List newEdges = new ArrayList<>(); + for (int i = 0; i < importedVertices.size()-1; i++) { + Vertex from = importedVertices.get(i); + Vertex to = importedVertices.get(i+1); + + this.addVertex(from, false); + this.addVertex(to, false); + + Edge edge = edgeFactory.make(from, to); + newEdges.add(edge); + this.addEdge(edge, false); + } + + refresh(); + + return newEdges; + } + + public Collection chainVertices(Edge.EdgeFactory edgeFactory, Vertex... argVertices) throws InvalidIRException { + Collection edges = chainVerticesUnsafe(edgeFactory, argVertices); + validate(); + return edges; + } + + public Edge chainVertices(Vertex a, Vertex b) throws InvalidIRException { + return chainVertices(PlainEdge.factory, a, b).stream().findFirst().get(); + } + + public Collection chainVertices(Vertex... vertices) throws InvalidIRException { + return chainVertices(PlainEdge.factory, vertices); + } + + public Collection chainVertices(boolean bool, Vertex... vertices) throws InvalidIRException { + Edge.EdgeFactory factory = bool ? BooleanEdge.trueFactory : BooleanEdge.falseFactory; + return chainVertices(factory, vertices); + } + + // Many of the operations we perform involve modifying one graph by adding vertices/edges + // from another. This method ensures that all the vertices/edges we know about having been pulled into + // this graph. Methods in this class that add or remove externally provided vertices/edges + // should call this method to ensure that the rest of the graph these items depend on are pulled + // in. + public void refresh() throws InvalidIRException { + this.calculateRanks(); + this.calculateTopologicalSort(); + } + + private void calculateTopologicalSort() throws InvalidIRException { + try { + this.sortedVertices = TopologicalSort.sortVertices(this); + } catch (TopologicalSort.UnexpectedGraphCycleError unexpectedGraphCycleError) { + throw new InvalidIRException("Graph is not a dag!", unexpectedGraphCycleError); + } + } + + private void calculateRanks() { + vertexRanks = BreadthFirst.breadthFirst(this.getRoots()).vertexDistances; + } + + public Integer rank(Vertex vertex) { + Integer rank = vertexRanks.get(vertex); + // This should never happen + if (rank == null) throw new RuntimeException("Attempted to get rank from vertex where it is not yet calculated: " + this); + return rank; + } + + public Map> verticesByHash() { + return this.vertices().collect(Collectors.groupingBy(Vertex::uniqueHash)); + } + + public void validate() throws InvalidIRException { + if (this.isEmpty()) return; + + if (this.getVertices().stream().noneMatch(Vertex::isLeaf)) { + throw new InvalidIRException("Graph has no leaf vertices!\n" + this.toString()); + } + + List> duplicates = verticesByHash().values().stream().filter((group) -> group.size() > 1).collect(Collectors.toList()); + if (!duplicates.isEmpty()) { + Stream errorMessageGroups = duplicates.stream(). + map((group) -> group.stream().map(Object::toString).collect(Collectors.joining("==="))); + + String joinedErrorMessageGroups = errorMessageGroups.collect(Collectors.joining("\n---\n")); + + throw new InvalidIRException("Some nodes on the graph are fully redundant!\n" + this + "|" + joinedErrorMessageGroups); + } + } + + + public Stream roots() { + return vertices.stream().filter(Vertex::isRoot); + } + + public Collection getRoots() { + return roots().collect(Collectors.toList()); + } + + // Vertices which are partially leaves in that they support multiple + // outgoing edge types but only have one or fewer attached + public Stream allLeaves() { + return vertices.stream().filter(Vertex::isPartialLeaf); + } + + // Get all leaves whether partial or not + public Collection getAllLeaves() { + return allLeaves().collect(Collectors.toList()); + } + + public Stream leaves() { + return vertices.stream().filter(Vertex::isLeaf); + } + + public Collection getLeaves() { + return leaves().collect(Collectors.toList()); + } + + public Set getVertices() { + return vertices; + } + + public Set getEdges() { + return edges; + } + + public String toString() { + Stream edgesToFormat; + try { + edgesToFormat = sortedEdges(); + } catch (InvalidIRException e) { + // Even if this isn't a valid graph we still need to print it + edgesToFormat = edges.stream(); + } + + String edgelessVerticesStr; + if (this.isolatedVertices().count() > 0) { + edgelessVerticesStr = "\n== Vertices Without Edges ==\n" + + this.isolatedVertices().map(Vertex::toString).collect(Collectors.joining("\n")); + } else { + edgelessVerticesStr = ""; + } + + return "**GRAPH**\n" + + "Vertices: " + this.vertices.size()+ " Edges: " + this.edges().count() + "\n" + + "----------------------" + + edgesToFormat.map(Edge::toString).collect(Collectors.joining("\n")) + + edgelessVerticesStr + + "\n**GRAPH**"; + } + + public Stream isolatedVertices() { + return this.getVertices().stream().filter(v -> v.getOutgoingEdges().isEmpty() && v.getIncomingEdges().isEmpty()); + } + + public List getSortedVertices() { + return this.sortedVertices; + } + + public Stream sortedEdges() throws InvalidIRException { + return getSortedVertices().stream(). + flatMap(Vertex::outgoingEdges); + } + + public List getSortedVerticesBefore(Vertex end) throws InvalidIRException { + return getSortedVerticesBetween(null, end); + } + + public List getSortedVerticesAfter(Vertex start) throws InvalidIRException { + return getSortedVerticesBetween(start, null); + } + + public List getSortedVerticesBetween(Vertex start, Vertex end) throws InvalidIRException { + List sortedVertices = getSortedVertices(); + + int startIndex = start == null ? 0 : sortedVertices.indexOf(start); + int endIndex = end == null ? sortedVertices.size() : sortedVertices.indexOf(end); + + return sortedVertices.subList(startIndex+1, endIndex); + } + + @Override + public boolean sourceComponentEquals(SourceComponent sourceComponent) { + if (sourceComponent == this) return true; + if (sourceComponent instanceof Graph) { + Graph otherGraph = (Graph) sourceComponent; + GraphDiff.DiffResult diff = GraphDiff.diff(this, otherGraph); + return diff.isIdentical(); + + } + return false; + } + + // returns true if this graph has a .sourceComponentEquals equivalent edge + public boolean hasEquivalentEdge(Edge otherE) { + return this.getEdges().stream().anyMatch(e -> e.sourceComponentEquals(otherE)); + } + + public boolean hasEquivalentVertex(Vertex otherV) { + return this.getVertices().stream().anyMatch(v -> v.sourceComponentEquals(otherV)); + } + + @Override + public SourceMetadata getMeta() { + return null; + } + + public boolean isEmpty() { + return (this.getVertices().size() == 0); + } + + public Stream vertices() { + return this.vertices.stream(); + } + + public Stream edges() { + return this.edges.stream(); + } + + @Override + public String hashSource() { + MessageDigest lineageDigest = Util.defaultMessageDigest(); + List sources = this.vertices.stream().parallel().map(Vertex::uniqueHash).sorted().map(String::getBytes).collect(Collectors.toList()); + sources.forEach(lineageDigest::update); + return Util.bytesToHexString(lineageDigest.digest()); + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/graph/IfVertex.java b/logstash-core/src/main/java/org/logstash/config/ir/graph/IfVertex.java new file mode 100644 index 000000000..6df0b993f --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/graph/IfVertex.java @@ -0,0 +1,103 @@ +package org.logstash.config.ir.graph; + +import org.logstash.config.ir.SourceComponent; +import org.logstash.config.ir.SourceMetadata; +import org.logstash.config.ir.expression.BooleanExpression; + +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Created by andrewvc on 9/15/16. + */ +public class IfVertex extends Vertex { + public BooleanExpression getBooleanExpression() { + return booleanExpression; + } + + private final BooleanExpression booleanExpression; + + public IfVertex(SourceMetadata meta, BooleanExpression booleanExpression) { + super(meta); + this.booleanExpression = booleanExpression; + } + + public String toString() { + return "[if " + booleanExpression.toString(0) + "]"; + } + + @Override + public boolean sourceComponentEquals(SourceComponent other) { + if (other == null) return false; + if (other == this) return true; + if (other instanceof IfVertex) { + IfVertex otherV = (IfVertex) other; + // We don't check the id because we're comparing functional equality, not + // identity + return otherV.booleanExpression.sourceComponentEquals(this.booleanExpression); + } + return false; + } + + // An IfVertex has no real metadata in and of itself, but its expression does! + @Override + public SourceMetadata getMeta() { + return null; + } + + public boolean hasEdgeType(boolean type) { + for (Edge e : getOutgoingEdges()) { + BooleanEdge bEdge = (BooleanEdge) e; // There should only be boolean edges here! + if (bEdge.getEdgeType() == type) return true; + } + return false; + } + + public Collection getUnusedOutgoingEdgeFactories() { + List l = new LinkedList<>(); + if (!hasEdgeType(true)) l.add(BooleanEdge.trueFactory); + if (!hasEdgeType(false)) l.add(BooleanEdge.falseFactory); + return l; + } + + public boolean acceptsOutgoingEdge(Edge e) { + return (e instanceof BooleanEdge); + } + + @Override + public String getId() { + return this.uniqueHash(); + } + + public Collection getOutgoingBooleanEdges() { + // Wish there was a way to do this as a java a cast without an operation + return getOutgoingEdges().stream().map(e -> (BooleanEdge) e).collect(Collectors.toList()); + } + + public Collection getOutgoingBooleanEdgesByType(Boolean edgeType) { + return getOutgoingBooleanEdges().stream().filter(e -> e.getEdgeType().equals(edgeType)).collect(Collectors.toList()); + } + + // The easiest readable version of this for a human. + // If the original source is available we use that, otherwise we serialize the expression + public String humanReadableExpression() { + String sourceText = this.booleanExpression.getMeta() != null ? this.booleanExpression.getMeta().getSourceText() : null; + if (sourceText != null) { + return sourceText; + } else { + return this.getBooleanExpression().toRubyString(); + } + } + + @Override + public IfVertex copy() { + return new IfVertex(getMeta(),getBooleanExpression()); + } + + @Override + public String individualHashSource() { + return this.getClass().getCanonicalName() + "{" + this.booleanExpression.hashSource() + "}"; + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/graph/PlainEdge.java b/logstash-core/src/main/java/org/logstash/config/ir/graph/PlainEdge.java new file mode 100644 index 000000000..64ee58430 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/graph/PlainEdge.java @@ -0,0 +1,37 @@ +package org.logstash.config.ir.graph; + +import org.logstash.common.Util; +import org.logstash.config.ir.InvalidIRException; + +/** + * Created by andrewvc on 9/19/16. + */ +public class PlainEdge extends Edge { + public static class PlainEdgeFactory extends Edge.EdgeFactory { + @Override + public PlainEdge make(Vertex from, Vertex to) throws InvalidIRException { + return new PlainEdge(from, to); + } + } + + public static PlainEdgeFactory factory = new PlainEdgeFactory(); + + @Override + public String individualHashSource() { + return this.getClass().getCanonicalName(); + } + + @Override + public String getId() { + return Util.digest(this.getFrom().getId() + "->" + this.getTo().getId()); + } + + public PlainEdge(Vertex from, Vertex to) throws InvalidIRException { + super(from, to); + } + + @Override + public PlainEdge copy(Vertex from, Vertex to) throws InvalidIRException { + return new PlainEdge(from, to); + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/graph/PluginVertex.java b/logstash-core/src/main/java/org/logstash/config/ir/graph/PluginVertex.java new file mode 100644 index 000000000..93a7b4b15 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/graph/PluginVertex.java @@ -0,0 +1,85 @@ +package org.logstash.config.ir.graph; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.logstash.common.Util; +import org.logstash.config.ir.SourceComponent; +import org.logstash.config.ir.PluginDefinition; +import org.logstash.config.ir.SourceMetadata; + +/** + * Created by andrewvc on 9/15/16. + */ +public class PluginVertex extends Vertex { + private final SourceMetadata meta; + private final String id; + private final PluginDefinition pluginDefinition; + + public String getId() { + if (id != null) return id; + if (this.getGraph() == null) { + throw new RuntimeException("Attempted to get ID from PluginVertex before attaching it to a graph!"); + } + return this.uniqueHash(); + } + + public PluginDefinition getPluginDefinition() { + return pluginDefinition; + } + @Override + public SourceMetadata getMeta() { + return meta; + } + + + public PluginVertex(SourceMetadata meta, PluginDefinition pluginDefinition) { + super(meta); + this.meta = meta; + + this.pluginDefinition = pluginDefinition; + + Object argId = this.pluginDefinition.getArguments().get("id"); + this.id = argId != null ? argId.toString() : null; + } + + public String toString() { + return "P[" + pluginDefinition + "|" + this.getMeta() + "]"; + } + + @Override + public String individualHashSource() { + ObjectMapper objectMapper = new ObjectMapper(); + try { + return Util.digest(this.getClass().getCanonicalName() + "|" + + (this.id != null ? this.id : "NOID") + "|" + + this.pluginDefinition.getName() + "|" + + this.pluginDefinition.getType().toString() + "|" + + objectMapper.writeValueAsString(this.pluginDefinition.getArguments())); + } catch (JsonProcessingException e) { + // This is basically impossible given the constrained values in the plugin definition + throw new RuntimeException(e); + } + } + + public String individualHash() { + return Util.digest(individualHashSource()); + } + + @Override + public PluginVertex copy() { + return new PluginVertex(meta, getPluginDefinition()); + } + + @Override + public boolean sourceComponentEquals(SourceComponent other) { + if (other == null) return false; + if (other == this) return true; + if (other instanceof PluginVertex) { + PluginVertex otherV = (PluginVertex) other; + // We don't test ID equality because we're testing + // Semantics, and ids have nothing to do with that + return otherV.getPluginDefinition().sourceComponentEquals(this.getPluginDefinition()); + } + return false; + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/graph/QueueVertex.java b/logstash-core/src/main/java/org/logstash/config/ir/graph/QueueVertex.java new file mode 100644 index 000000000..4f675a191 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/graph/QueueVertex.java @@ -0,0 +1,44 @@ +package org.logstash.config.ir.graph; + +import org.logstash.config.ir.SourceComponent; +import org.logstash.config.ir.SourceMetadata; + +/** + * Created by andrewvc on 9/15/16. + */ +public class QueueVertex extends Vertex { + public QueueVertex() { + super(null); + } + + @Override + public String getId() { + return "__QUEUE__"; + } + + @Override + public String individualHashSource() { + return this.getClass().getCanonicalName(); + } + + public String toString() { + return this.getId(); + } + + @Override + public QueueVertex copy() { + return new QueueVertex(); + } + + @Override + public boolean sourceComponentEquals(SourceComponent other) { + if (other == null) return false; + return other instanceof QueueVertex; + } + + // Special vertices really have no metadata + @Override + public SourceMetadata getMeta() { + return null; + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/graph/Vertex.java b/logstash-core/src/main/java/org/logstash/config/ir/graph/Vertex.java new file mode 100644 index 000000000..2948cfb2b --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/graph/Vertex.java @@ -0,0 +1,201 @@ +package org.logstash.config.ir.graph; + +import org.logstash.common.Util; +import org.logstash.config.ir.Hashable; +import org.logstash.config.ir.SourceComponent; +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.SourceMetadata; +import org.logstash.config.ir.graph.algorithms.DepthFirst; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.Collection; +import java.util.Collections; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Created by andrewvc on 9/15/16. + */ +public abstract class Vertex implements SourceComponent, Hashable { + private final SourceMetadata sourceMetadata; + private Graph graph = this.getGraph(); + + public Vertex() { + this.sourceMetadata = null; + } + + public Vertex(SourceMetadata sourceMetadata) { + this.sourceMetadata = sourceMetadata; + } + + public abstract Vertex copy(); + + public static class InvalidEdgeTypeException extends InvalidIRException { + public InvalidEdgeTypeException(String s) { + super(s); + } + } + + public Graph getGraph() { + return this.graph; + } + + public void setGraph(Graph graph) { + if (this.graph == graph) { + return; + } else if (this.graph == null) { + this.graph = graph; + } else { + throw new IllegalArgumentException("Cannot set graph property on Vertex that is already assigned to an existing graph!"); + } + } + + public boolean isRoot() { + return getIncomingEdges().isEmpty(); + } + + public boolean isLeaf() { + return getOutgoingEdges().isEmpty(); + } + + public boolean hasIncomingEdges() { + return !getIncomingEdges().isEmpty(); + } + + public boolean hasOutgoingEdges() { + return !getOutgoingEdges().isEmpty(); + } + + public Collection getIncomingEdges() { + return incomingEdges().collect(Collectors.toSet()); + } + + public Collection getOutgoingEdges() { + return outgoingEdges().collect(Collectors.toSet()); + } + + public Collection getOutgoingVertices() { + return outgoingVertices().collect(Collectors.toList()); + } + + public Stream outgoingVertices() { + return outgoingEdges().map(Edge::getTo); + } + + public Collection getIncomingVertices() { + return incomingVertices().collect(Collectors.toList()); + } + + public Stream incomingVertices() { + return incomingEdges().map(Edge::getFrom); + } + + public Stream incomingEdges() { + return this.getGraph().getIncomingEdges(this).stream(); + } + + public Stream outgoingEdges() { + return this.getGraph().getOutgoingEdges(this).stream(); + } + + public Stream ancestors() { + return DepthFirst.reverseDepthFirst(this).filter(v -> v != this); + } + + public Stream roots() { + return ancestors().filter(Vertex::isRoot); + } + + public Stream descendants() { + return DepthFirst.depthFirst(this).filter(v -> v != this); + } + + public Stream lineage() { + return Stream.concat(Stream.concat(ancestors(), Stream.of(this)), descendants()); + } + + // Rank is the shortest distance to a root for this vertex + public int rank() { + return this.graph.rank(this); + } + + @Override + public String uniqueHash() { + // Sort the lineage to ensure consistency. We prepend each item with a lexicographically sortable + // encoding of its rank (using hex notation) so that the sort order is identical to the traversal order. + // This is a required since there may be individually identical components in different locations in the graph. + // It is, however, illegal to have functionally identical vertices, that is to say two vertices with the same + // contents that have the same lineage. + + MessageDigest lineageDigest = Util.defaultMessageDigest(); + + lineageDigest.update(hashPrefix().getBytes()); + + // The lineage can be quite long and we want to avoid the quadratic complexity of string concatenation + // Thus, in this case there's no real way to get the hash source, we just hash as we go. + lineage(). + map(Vertex::contextualHashSource). + sorted(). + forEachOrdered(v -> { + byte[] bytes = v.getBytes(StandardCharsets.UTF_8); + lineageDigest.update(bytes); + }); + + String digest = Util.bytesToHexString(lineageDigest.digest()); + + return digest; + } + + @Override + public String hashSource() { + // In this case the source can be quite large, so we never actually use this function. + return this.uniqueHash(); + } + + public String hashPrefix() { + return String.format("Vertex[%08x]=", this.rank()) + this.individualHashSource() + "|"; + } + + public String contextualHashSource() { + // This string must be lexicographically sortable hence the ID at the front. It also must have the individualHashSource + // repeated at the front for the case of a graph with two nodes at the same rank, same contents, but different lineages + StringBuilder result = new StringBuilder(); + result.append(hashPrefix()); + result.append(individualHashSource()); + + result.append("I:"); + this.incomingEdges().map(Edge::individualHashSource).sorted().forEachOrdered(result::append); + result.append("O:"); + this.outgoingEdges().map(Edge::individualHashSource).sorted().forEachOrdered(result::append); + + return result.toString(); + } + + public abstract String individualHashSource(); + + // Can be overriden in subclasses to define multiple + // expected Edge classes this Vertex can take. + // If any EdgeFactory instances are returned this Vertex is considered + // a partial leaf. + public Collection getUnusedOutgoingEdgeFactories() { + if (!this.hasOutgoingEdges()) { + return Collections.singletonList(new PlainEdge.PlainEdgeFactory()); + } + return Collections.emptyList(); + } + + public boolean isPartialLeaf() { + return getUnusedOutgoingEdgeFactories().size() > 0; + } + + public boolean acceptsIncomingEdge(Edge e) { + return true; + } + + public boolean acceptsOutgoingEdge(Edge e) { + return true; + } + + public abstract String getId(); +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/graph/algorithms/BreadthFirst.java b/logstash-core/src/main/java/org/logstash/config/ir/graph/algorithms/BreadthFirst.java new file mode 100644 index 000000000..26b2f1b1a --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/graph/algorithms/BreadthFirst.java @@ -0,0 +1,64 @@ +package org.logstash.config.ir.graph.algorithms; + +import org.logstash.config.ir.graph.Vertex; + +import java.util.*; +import java.util.function.Consumer; +import java.util.stream.Stream; + +/** + * Created by andrewvc on 1/5/17. + * + */ +public class BreadthFirst { + public static BfsResult breadthFirst(Collection roots) { + return breadthFirst(roots, false, null); + } + + /* This isn't as pretty as the DFS search with its streaminess, but for our current uses we only really + * care about using this to get the calculated vertexDistances, so that's fine. */ + public static BfsResult breadthFirst(Collection roots, + boolean reverse, + Consumer> consumer + ) { + Map vertexDistances = new HashMap<>(); + Map vertexParents = new HashMap<>(); + + Deque queue = new ArrayDeque<>(roots); + roots.forEach(v -> vertexDistances.put(v, 0)); + + while (!queue.isEmpty()) { + Vertex currentVertex = queue.removeFirst(); + Integer currentDistance = vertexDistances.get(currentVertex); + + if (consumer != null) { + consumer.accept(new AbstractMap.SimpleImmutableEntry<>(currentVertex, currentDistance)); + } + + Stream nextVertices = reverse ? currentVertex.incomingVertices() : currentVertex.outgoingVertices(); + nextVertices.forEach(nextVertex -> { + if (vertexDistances.get(nextVertex) == null) { + vertexDistances.put(nextVertex, currentDistance+1); + vertexParents.put(nextVertex, currentVertex); + queue.push(nextVertex); + } + }); + } + + return new BfsResult(vertexDistances, vertexParents); +} + + public static final class BfsResult { + public final Map vertexDistances; + private final Map vertexParents; + + BfsResult(Map vertexDistances, Map vertexParents) { + this.vertexDistances = vertexDistances; + this.vertexParents = vertexParents; + } + + public Collection getVertices() { + return vertexDistances.keySet(); + } + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/graph/algorithms/DepthFirst.java b/logstash-core/src/main/java/org/logstash/config/ir/graph/algorithms/DepthFirst.java new file mode 100644 index 000000000..cc0ef0fc6 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/graph/algorithms/DepthFirst.java @@ -0,0 +1,71 @@ +package org.logstash.config.ir.graph.algorithms; + +import org.logstash.config.ir.graph.Graph; +import org.logstash.config.ir.graph.Vertex; + +import java.util.*; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +/** + * Created by andrewvc on 1/5/17. + */ +public class DepthFirst { + public static Stream depthFirst(Graph g) { + return depthFirst(g.getRoots()); + } + + public static Stream reverseDepthFirst(Graph g) { + return reverseDepthFirst(g.getLeaves()); + } + + public static Stream depthFirst(Vertex v) { + return depthFirst(Collections.singleton(v)); + } + + public static Stream reverseDepthFirst(Vertex v) { + return depthFirst(Collections.singleton(v)); + } + + public static Stream depthFirst(Collection v) { + return streamify(new Traversal(v, false)); + } + + public static Stream reverseDepthFirst(Collection v) { + return streamify(new Traversal(v, true)); + } + + private static Stream streamify(Traversal t) { + return StreamSupport.stream(Spliterators.spliteratorUnknownSize(t, Spliterator.DISTINCT),false); + } + + public static class Traversal implements Iterator { + private final Set visited = new HashSet<>(); + private final Deque pending; + private final boolean reverse; + + Traversal(Collection initialVertices, boolean reverse) { + this.reverse = reverse; + this.pending = new ArrayDeque<>(initialVertices); + } + + @Override + public boolean hasNext() { + return !pending.isEmpty(); + } + + @Override + public Vertex next() { + Vertex current = pending.removeFirst(); + this.visited.add(current); + + Stream next = reverse ? current.incomingVertices() : current.outgoingVertices(); + next.forEach(v -> { + if (!visited.contains(v)) { + this.pending.add(v); + } + }); + return current; + } + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/graph/algorithms/GraphDiff.java b/logstash-core/src/main/java/org/logstash/config/ir/graph/algorithms/GraphDiff.java new file mode 100644 index 000000000..05517fb36 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/graph/algorithms/GraphDiff.java @@ -0,0 +1,100 @@ +package org.logstash.config.ir.graph.algorithms; + +import org.logstash.config.ir.graph.Edge; +import org.logstash.config.ir.graph.Graph; +import org.logstash.config.ir.graph.Vertex; + +import java.util.Collection; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Created by andrewvc on 1/5/17. + */ +public class GraphDiff { + public static DiffResult diff(Graph left, Graph right) { + List removedEdges = left.edges().filter(e -> !right.hasEquivalentEdge(e)).collect(Collectors.toList()); + List removedVertices = left.vertices().filter(v -> !right.hasEquivalentVertex(v)).collect(Collectors.toList()); + List addedEdges = right.edges().filter(e -> !left.hasEquivalentEdge(e)).collect(Collectors.toList()); + List addedVertices = right.vertices().filter(v -> !left.hasEquivalentVertex(v)).collect(Collectors.toList()); + return new DiffResult(removedEdges, addedEdges, removedVertices, addedVertices); + } + + public static class DiffResult { + private final List removedVertices; + private final List addedVertices; + private final Collection removedEdges; + private final Collection addedEdges; + + public Collection getRemovedEdges() { + return removedEdges; + } + + public Collection getAddedEdges() { + return addedEdges; + } + + public Collection getRemovedVertices() { + return removedVertices; + } + + public Collection getAddedVertices() { + return addedVertices; + } + + public DiffResult(Collection removedEdges, Collection addedEdges, List removedVertices, List addedVertices) { + this.removedEdges = removedEdges; + this.addedEdges = addedEdges; + this.removedVertices = removedVertices; + this.addedVertices = addedVertices; + } + + public String summary() { + String template = "(-%d,+%d Edges | -%d,+%d Vertices)"; + return String.format(template, removedEdges.size(), addedEdges.size(), removedVertices.size(), addedVertices.size()); + } + + public boolean hasSameEdges() { + return addedEdges.isEmpty() && removedEdges.isEmpty(); + } + + public boolean hasSameVertices() { + return addedVertices.isEmpty() && removedVertices.isEmpty(); + } + + public boolean isIdentical() { + return hasSameEdges() && hasSameVertices(); + } + + public String toString() { + if (isIdentical()) return "Identical Graphs"; + + StringBuilder output = new StringBuilder(); + output.append(this.summary()); + + if (!hasSameEdges()) { + output.append("\n"); + output.append(detailedDiffFor("Edges", removedEdges, addedEdges)); + } + if (!hasSameVertices()) { + output.append("\n"); + output.append(detailedDiffFor("Vertices", removedVertices, addedVertices)); + } + output.append("\n"); + + return output.toString(); + } + + private String detailedDiffFor(String name, Collection removed, Collection added) { + return (name + " GraphDiff: " + "\n") + + "--------------------------\n" + + Stream.concat(removed.stream().map(c -> "-" + c.toString()), + added.stream().map(c -> "+" + c.toString())). + map(Object::toString). + collect(Collectors.joining("\n")) + + "\n--------------------------"; + } + } +} \ No newline at end of file diff --git a/logstash-core/src/main/java/org/logstash/config/ir/graph/algorithms/ShortestPath.java b/logstash-core/src/main/java/org/logstash/config/ir/graph/algorithms/ShortestPath.java new file mode 100644 index 000000000..11437fd52 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/graph/algorithms/ShortestPath.java @@ -0,0 +1,106 @@ +package org.logstash.config.ir.graph.algorithms; + +import org.logstash.config.ir.graph.Vertex; + +import java.security.cert.CollectionCertStoreParameters; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Created by andrewvc on 1/5/17. + * This class isn't currently used anywhere, I wrote it for some code that is now removed, however, I'm sure it will be + * useful shortly, so we should hold onto it for a while. + */ +public class ShortestPath { + static class InvalidShortestPathArguments extends Exception { + private final Collection invalidVertices; + + public InvalidShortestPathArguments(Collection invalidVertices) { + super(); + this.invalidVertices = invalidVertices; + + } + + @Override + public String getMessage() { + String verticesMessage = invalidVertices.stream().map(Vertex::toString).collect(Collectors.joining(", ")); + return "Attempted to determine path for vertex that is not in the search space!" + verticesMessage; + } + } + + + public static List shortestPath(Vertex from, Vertex to) throws InvalidShortestPathArguments { + return shortestPath(from, Collections.singleton(to)).get(to); + } + + public static Map> shortestPath(Vertex from, Collection to) throws InvalidShortestPathArguments { + return shortestPath(from, to, false); + } + + // Finds the shortest paths to the specified vertices traversing edges backward using Dijkstra's algorithm. + // The items in `to` must be ancestors of this Vertex! + public static Map> shortestReversePath(Vertex from, Collection to) throws InvalidShortestPathArguments { + return shortestPath(from, to, true); + } + + // Finds the shortest paths to the specified vertices using Dijkstra's algorithm. + // The items in `to` must be ancestors of this Vertex! + public static Map> shortestPath(Vertex from, Collection to, boolean reverseSearch) throws InvalidShortestPathArguments { + Map vertexDistances = new HashMap<>(); + Map vertexPathPrevious = new HashMap<>(); + + List pending = new ArrayList<>(); + Stream searchSpace = reverseSearch ? from.ancestors() : from.descendants(); + searchSpace.forEach((vertex) -> { + pending.add(vertex); + // Max value is an unknown distance + // using this is more convenient and concise than null in later code + vertexDistances.put(vertex, Integer.MAX_VALUE); + }); + + pending.add(from); + vertexDistances.put(from, 0); + + Collection invalidVertices = to.stream().filter(v -> !pending.contains(v)).collect(Collectors.toList()); + if (!invalidVertices.isEmpty()) { + throw new InvalidShortestPathArguments(invalidVertices); + } + + while (!pending.isEmpty()) { + Vertex current = pending.stream().min(Comparator.comparing(vertexDistances::get)).get(); + int currentDistance = vertexDistances.get(current); + pending.remove(current); + + Stream toProcess = reverseSearch ? current.incomingVertices() : current.outgoingVertices(); + + toProcess.forEach((v) -> { + Integer curDistance = vertexDistances.get(v); + int altDistance = currentDistance + 1; // Fixed cost per edge of 1 + if (altDistance < curDistance) { + vertexDistances.put(v, altDistance); + vertexPathPrevious.put(v, current); + } + }); + } + + Map> result = new HashMap<>(to.size()); + + for (Vertex toVertex : to) { + int toVertexDistance = vertexDistances.get(toVertex); + + List path = new ArrayList<>(toVertexDistance+1); + Vertex pathCurrentVertex = toVertex; + while (pathCurrentVertex != from) { + path.add(pathCurrentVertex); + pathCurrentVertex = vertexPathPrevious.get(pathCurrentVertex); + } + path.add(from); + Collections.reverse(path); + result.put(toVertex, path); + } + + return result; + } + +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/graph/algorithms/TopologicalSort.java b/logstash-core/src/main/java/org/logstash/config/ir/graph/algorithms/TopologicalSort.java new file mode 100644 index 000000000..a3fed6425 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/graph/algorithms/TopologicalSort.java @@ -0,0 +1,51 @@ +package org.logstash.config.ir.graph.algorithms; + +import org.logstash.config.ir.graph.Edge; +import org.logstash.config.ir.graph.Graph; +import org.logstash.config.ir.graph.Vertex; + +import java.util.*; + +/** + * Created by andrewvc on 1/7/17. + */ +public class TopologicalSort { + public static class UnexpectedGraphCycleError extends Exception { + UnexpectedGraphCycleError(Graph g) { + super("Graph has cycles, is not a DAG! " + g); + } + } + + // Uses Kahn's algorithm to do a topological sort and detect cycles + public static List sortVertices(Graph g) throws UnexpectedGraphCycleError { + if (g.getEdges().size() == 0) return new ArrayList<>(g.getVertices()); + + List sorted = new ArrayList<>(g.getVertices().size()); + + Deque pending = new LinkedList<>(); + pending.addAll(g.getRoots()); + + Set traversedEdges = new HashSet<>(); + + while (!pending.isEmpty()) { + Vertex currentVertex = pending.removeFirst(); + sorted.add(currentVertex); + + currentVertex.getOutgoingEdges().forEach(edge -> { + traversedEdges.add(edge); + Vertex toVertex = edge.getTo(); + if (toVertex.getIncomingEdges().stream().allMatch(traversedEdges::contains)) { + pending.add(toVertex); + } + }); + } + + // Check for cycles + if (g.edges().noneMatch(traversedEdges::contains)) { + throw new UnexpectedGraphCycleError(g); + } + + return sorted; + } + +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/imperative/ComposedParallelStatement.java b/logstash-core/src/main/java/org/logstash/config/ir/imperative/ComposedParallelStatement.java new file mode 100644 index 000000000..d054a414d --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/imperative/ComposedParallelStatement.java @@ -0,0 +1,32 @@ +package org.logstash.config.ir.imperative; + +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.SourceMetadata; +import org.logstash.config.ir.graph.Graph; + +import java.util.List; + +/** + * Created by andrewvc on 9/22/16. + */ +public class ComposedParallelStatement extends ComposedStatement { + public ComposedParallelStatement(SourceMetadata meta, List statements) throws InvalidIRException { + super(meta, statements); + } + + @Override + protected String composeTypeString() { + return "composed-parallel"; + } + + @Override + public Graph toGraph() throws InvalidIRException { + Graph g = Graph.empty(); + + for (Statement s : getStatements()) { + g = Graph.combine(g, s.toGraph()).graph; + } + + return g; + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/imperative/ComposedSequenceStatement.java b/logstash-core/src/main/java/org/logstash/config/ir/imperative/ComposedSequenceStatement.java new file mode 100644 index 000000000..823da140b --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/imperative/ComposedSequenceStatement.java @@ -0,0 +1,33 @@ +package org.logstash.config.ir.imperative; + +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.SourceMetadata; +import org.logstash.config.ir.graph.Graph; + +import java.util.List; + +/** + * Created by andrewvc on 9/22/16. + */ +public class ComposedSequenceStatement extends ComposedStatement { + public ComposedSequenceStatement(SourceMetadata meta, List statements) throws InvalidIRException { + super(meta, statements); + } + + @Override + protected String composeTypeString() { + return "do-sequence"; + } + + @Override + public Graph toGraph() throws InvalidIRException { + Graph g = Graph.empty(); + + for (Statement statement : getStatements()) { + Graph sg = statement.toGraph(); + g = g.chain(sg); + } + + return g; + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/imperative/ComposedStatement.java b/logstash-core/src/main/java/org/logstash/config/ir/imperative/ComposedStatement.java new file mode 100644 index 000000000..c70fc4825 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/imperative/ComposedStatement.java @@ -0,0 +1,65 @@ +package org.logstash.config.ir.imperative; + +import org.logstash.config.ir.SourceComponent; +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.SourceMetadata; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * Created by andrewvc on 9/6/16. + */ +public abstract class ComposedStatement extends Statement { + public interface IFactory { + ComposedStatement make(SourceMetadata meta, List statements) throws InvalidIRException; + } + + private final List statements; + + public ComposedStatement(SourceMetadata meta, List statements) throws InvalidIRException { + super(meta); + if (statements == null || statements.stream().anyMatch(s -> s == null)) { + throw new InvalidIRException("Nulls eNot allowed for list eOr in statement list"); + } + this.statements = statements; + } + + public List getStatements() { + return this.statements; + } + + public int size() { + return getStatements().size(); + } + + @Override + public boolean sourceComponentEquals(SourceComponent sourceComponent) { + if (sourceComponent == null) return false; + if (this == sourceComponent) return true; + if (sourceComponent.getClass().equals(this.getClass())) { + ComposedStatement other = (ComposedStatement) sourceComponent; + if (this.size() != other.size()) { + return false; + } + for (int i = 0; i < size(); i++) { + Statement s = this.getStatements().get(i); + Statement os = other.getStatements().get(i); + if (!(s.sourceComponentEquals(os))) return false; + } + return true; + } + return false; + } + + @Override + public String toString(int indent) { + return "(" + this.composeTypeString() + "\n" + + getStatements().stream(). + map(s -> s.toString(indent+2)). + collect(Collectors.joining("\n")) + + "\n"; + } + + protected abstract String composeTypeString(); +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/imperative/IfStatement.java b/logstash-core/src/main/java/org/logstash/config/ir/imperative/IfStatement.java new file mode 100644 index 000000000..aea135f41 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/imperative/IfStatement.java @@ -0,0 +1,105 @@ +package org.logstash.config.ir.imperative; + +import org.logstash.config.ir.SourceComponent; +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.SourceMetadata; +import org.logstash.config.ir.expression.BooleanExpression; +import org.logstash.config.ir.graph.BooleanEdge; +import org.logstash.config.ir.graph.Graph; +import org.logstash.config.ir.graph.IfVertex; +import org.logstash.config.ir.graph.Vertex; + +import java.util.Collection; +import java.util.stream.Collectors; + +/** + * Created by andrewvc on 9/6/16. + * if 5 { + * + * } + */ + +public class IfStatement extends Statement { + private final BooleanExpression booleanExpression; + private final Statement trueStatement; + private final Statement falseStatement; + + public BooleanExpression getBooleanExpression() { + return booleanExpression; + } + + public Statement getTrueStatement() { + return trueStatement; + } + + public Statement getFalseStatement() { + return falseStatement; + } + + public IfStatement(SourceMetadata meta, + BooleanExpression booleanExpression, + Statement trueStatement, + Statement falseStatement + ) throws InvalidIRException { + super(meta); + + if (booleanExpression == null) throw new InvalidIRException("Boolean expr must eNot be null!"); + if (trueStatement == null) throw new InvalidIRException("If Statement needs true statement!"); + if (falseStatement == null) throw new InvalidIRException("If Statement needs false statement!"); + + this.booleanExpression = booleanExpression; + this.trueStatement = trueStatement; + this.falseStatement = falseStatement; + } + + @Override + public boolean sourceComponentEquals(SourceComponent sourceComponent) { + if (sourceComponent == null) return false; + if (sourceComponent == this) return true; + if (sourceComponent instanceof IfStatement) { + IfStatement other = (IfStatement) sourceComponent; + + + return (this.booleanExpression.sourceComponentEquals(other.getBooleanExpression()) && + this.trueStatement.sourceComponentEquals(other.trueStatement) && + this.falseStatement.sourceComponentEquals(other.falseStatement)); + } + return false; + } + + @Override + public String toString(int indent) { + return indentPadding(indent) + + "(if " + booleanExpression.toString(0) + + "\n" + + this.trueStatement + + "\n" + + this.falseStatement + + ")"; + } + + + @Override + public Graph toGraph() throws InvalidIRException { + Graph trueGraph = getTrueStatement().toGraph(); + Graph falseGraph = getFalseStatement().toGraph(); + + Graph.GraphCombinationResult combination = Graph.combine(trueGraph, falseGraph); + Graph newGraph = combination.graph; + Collection trueRoots = trueGraph.roots().map(combination.oldToNewVertices::get).collect(Collectors.toList()); + Collection falseRoots = falseGraph.roots().map(combination.oldToNewVertices::get).collect(Collectors.toList()); + + IfVertex ifVertex = new IfVertex(this.getMeta(), this.booleanExpression); + newGraph.addVertex(ifVertex); + + for (Vertex v : trueRoots) { + newGraph.chainVerticesUnsafe(BooleanEdge.trueFactory, ifVertex, v); + } + + for (Vertex v : falseRoots) { + newGraph.chainVerticesUnsafe(BooleanEdge.falseFactory, ifVertex, v); + } + + return newGraph; + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/imperative/NoopStatement.java b/logstash-core/src/main/java/org/logstash/config/ir/imperative/NoopStatement.java new file mode 100644 index 000000000..11bda0457 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/imperative/NoopStatement.java @@ -0,0 +1,33 @@ +package org.logstash.config.ir.imperative; + +import org.logstash.config.ir.SourceComponent; +import org.logstash.config.ir.SourceMetadata; +import org.logstash.config.ir.graph.Graph; + +/** + * Created by andrewvc on 9/15/16. + */ +public class NoopStatement extends Statement { + + public NoopStatement(SourceMetadata meta) { + super(meta); + } + + @Override + public boolean sourceComponentEquals(SourceComponent sourceComponent) { + if (sourceComponent == null) return false; + if (sourceComponent instanceof NoopStatement) return true; + return false; + } + + @Override + public String toString(int indent) { + return indentPadding(indent) + "(Noop)"; + } + + @Override + public Graph toGraph() { + return Graph.empty(); + } + +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/imperative/PluginStatement.java b/logstash-core/src/main/java/org/logstash/config/ir/imperative/PluginStatement.java new file mode 100644 index 000000000..5dff4d6cf --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/imperative/PluginStatement.java @@ -0,0 +1,45 @@ +package org.logstash.config.ir.imperative; + +import org.logstash.config.ir.SourceComponent; +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.PluginDefinition; +import org.logstash.config.ir.SourceMetadata; +import org.logstash.config.ir.graph.Graph; +import org.logstash.config.ir.graph.PluginVertex; +import org.logstash.config.ir.graph.Vertex; + +/** + * Created by andrewvc on 9/6/16. + */ +public class PluginStatement extends Statement { + private final PluginDefinition pluginDefinition; + + public PluginStatement(SourceMetadata meta, PluginDefinition pluginDefinition) { + super(meta); + this.pluginDefinition = pluginDefinition; + } + + @Override + public boolean sourceComponentEquals(SourceComponent sourceComponent) { + if (sourceComponent == null) return false; + if (sourceComponent == this) return true; + if (sourceComponent instanceof PluginStatement) { + PluginStatement other = (PluginStatement) sourceComponent; + return this.pluginDefinition.equals(other.pluginDefinition); + } + return false; + } + + @Override + public String toString(int indent) { + return indentPadding(indent) + this.pluginDefinition; + } + + @Override + public Graph toGraph() throws InvalidIRException { + Vertex pluginVertex = new PluginVertex(getMeta(), pluginDefinition); + Graph g = Graph.empty(); + g.addVertex(pluginVertex); + return g; + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/imperative/Statement.java b/logstash-core/src/main/java/org/logstash/config/ir/imperative/Statement.java new file mode 100644 index 000000000..f242f08b0 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/imperative/Statement.java @@ -0,0 +1,27 @@ +package org.logstash.config.ir.imperative; + +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.BaseSourceComponent; +import org.logstash.config.ir.SourceMetadata; +import org.logstash.config.ir.graph.Graph; + +/** + * Created by andrewvc on 9/6/16. + */ +public abstract class Statement extends BaseSourceComponent { + public Statement(SourceMetadata meta) { + super(meta); + } + + public abstract Graph toGraph() throws InvalidIRException; + + public String toString() { + return toString(2); + } + + public abstract String toString(int indent); + + public String indentPadding(int length) { + return String.format("%" + length + "s", ""); + } +} diff --git a/logstash-core/src/test/java/org/logstash/config/ir/IRHelpers.java b/logstash-core/src/test/java/org/logstash/config/ir/IRHelpers.java new file mode 100644 index 000000000..8a888fc99 --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/config/ir/IRHelpers.java @@ -0,0 +1,139 @@ +package org.logstash.config.ir; + +import org.hamcrest.MatcherAssert; +import org.logstash.config.ir.expression.BooleanExpression; +import org.logstash.config.ir.expression.ValueExpression; +import org.logstash.config.ir.expression.unary.Truthy; +import org.logstash.config.ir.graph.Edge; +import org.logstash.config.ir.graph.Graph; +import org.logstash.config.ir.graph.Vertex; +import org.logstash.config.ir.graph.algorithms.GraphDiff; + +import java.util.HashMap; +import java.util.Objects; +import java.util.UUID; + +import static org.logstash.config.ir.DSL.*; +import static org.logstash.config.ir.PluginDefinition.Type.*; + +/** + * Created by andrewvc on 9/19/16. + */ +public class IRHelpers { + public static void assertSyntaxEquals(SourceComponent left, SourceComponent right) { + String message = String.format("Expected '%s' to equal '%s'", left, right); + MatcherAssert.assertThat(message, left.sourceComponentEquals(right)); + } + + public static void assertSyntaxEquals(Graph left, Graph right) { + String message = String.format("Expected \n'%s'\n to equal \n'%s'\n%s", left, right, GraphDiff.diff(left, right)); + MatcherAssert.assertThat(message, left.sourceComponentEquals(right)); + } + + public static Vertex createTestVertex() { + return createTestVertex(UUID.randomUUID().toString()); + } + + public static Vertex createTestVertex(String id) { + return new TestVertex(id); + } + + static class TestVertex extends Vertex { + private String id; + + public TestVertex(String id) { + this.id = id; + } + + @Override + public Vertex copy() { + return new TestVertex(id); + } + + @Override + public String individualHashSource() { + return "TVertex" + "|" + id; + } + + @Override + public String getId() { + return this.id; + } + + public String toString() { + return "TestVertex-" + id; + } + + @Override + public boolean sourceComponentEquals(SourceComponent other) { + if (other == null) return false; + if (other instanceof TestVertex) { + return Objects.equals(getId(), ((TestVertex) other).getId()); + } + return false; + } + + @Override + public SourceMetadata getMeta() { + return null; + } + } + + public static Edge createTestEdge() throws InvalidIRException { + Vertex v1 = createTestVertex(); + Vertex v2 = createTestVertex(); + return new TestEdge(v1,v2); + + } + + public static Edge createTestEdge(Vertex from, Vertex to) throws InvalidIRException { + return new TestEdge(from, to); + } + + public static final class TestEdge extends Edge { + TestEdge(Vertex from, Vertex to) throws InvalidIRException { + super(from, to); + } + + @Override + public Edge copy(Vertex from, Vertex to) throws InvalidIRException { + return new TestEdge(from, to); + } + + @Override + public String individualHashSource() { + return "TEdge"; + } + + @Override + public String getId() { + return individualHashSource(); + } + } + + public static BooleanExpression createTestExpression() throws InvalidIRException { + return new Truthy(null, new ValueExpression(null, 1)); + } + + public static SourceMetadata testMetadata() { + return new SourceMetadata("/fake/file", 1, 2, ""); + } + + public static PluginDefinition testPluginDefinition() { + return new PluginDefinition(PluginDefinition.Type.FILTER, "testDefinition", new HashMap()); + } + + public static Pipeline samplePipeline() throws InvalidIRException { + Graph inputSection = iComposeParallel(iPlugin(INPUT, "generator"), iPlugin(INPUT, "stdin")).toGraph(); + Graph filterSection = iIf(eEq(eEventValue("[foo]"), eEventValue("[bar]")), + iPlugin(FILTER, "grok"), + iPlugin(FILTER, "kv")).toGraph(); + Graph outputSection = iIf(eGt(eEventValue("[baz]"), eValue(1000)), + iComposeParallel( + iPlugin(OUTPUT, "s3"), + iPlugin(OUTPUT, "elasticsearch")), + iPlugin(OUTPUT, "stdout")).toGraph(); + + return new Pipeline(inputSection, filterSection, outputSection); + } +} diff --git a/logstash-core/src/test/java/org/logstash/config/ir/PipelineTest.java b/logstash-core/src/test/java/org/logstash/config/ir/PipelineTest.java new file mode 100644 index 000000000..2ed331703 --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/config/ir/PipelineTest.java @@ -0,0 +1,32 @@ +package org.logstash.config.ir; + +import org.junit.Test; +import org.logstash.config.ir.graph.Graph; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.logstash.config.ir.DSL.*; +import static org.logstash.config.ir.PluginDefinition.Type.*; + +/** + * Created by andrewvc on 9/20/16. + */ +public class PipelineTest { + @Test + public void testPipelineCreation() throws InvalidIRException { + Graph inputSection = iComposeParallel(iPlugin(INPUT, "generator"), iPlugin(INPUT, "stdin")).toGraph(); + Graph filterSection = iIf(eEq(eEventValue("[foo]"), eEventValue("[bar]")), + iPlugin(FILTER, "grok"), + iPlugin(FILTER, "kv")).toGraph(); + Graph outputSection = iIf(eGt(eEventValue("[baz]"), eValue(1000)), + iComposeParallel( + iPlugin(OUTPUT, "s3"), + iPlugin(OUTPUT, "elasticsearch")), + iPlugin(OUTPUT, "stdout")).toGraph(); + + Pipeline pipeline = new Pipeline(inputSection, filterSection, outputSection); + assertEquals(2, pipeline.getInputPluginVertices().size()); + assertEquals(2, pipeline.getFilterPluginVertices().size()); + assertEquals(3, pipeline.getOutputPluginVertices().size()); + } +} diff --git a/logstash-core/src/test/java/org/logstash/config/ir/graph/BooleanEdgeTest.java b/logstash-core/src/test/java/org/logstash/config/ir/graph/BooleanEdgeTest.java new file mode 100644 index 000000000..cb503ca90 --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/config/ir/graph/BooleanEdgeTest.java @@ -0,0 +1,35 @@ +package org.logstash.config.ir.graph; + +import org.junit.experimental.theories.DataPoint; +import org.junit.experimental.theories.Theories; +import org.junit.experimental.theories.Theory; +import org.junit.runner.RunWith; +import org.logstash.config.ir.InvalidIRException; + +import static org.hamcrest.CoreMatchers.*; +import static org.junit.Assert.assertThat; +import static org.logstash.config.ir.IRHelpers.*; + +/** + * Created by andrewvc on 11/21/16. + */ +@RunWith(Theories.class) +public class BooleanEdgeTest { + @DataPoint + public static Boolean TRUE = true; + @DataPoint + public static Boolean FALSE = false; + + @Theory + public void testBasicBooleanEdgeProperties(Boolean edgeType) throws InvalidIRException { + BooleanEdge be = new BooleanEdge(edgeType, createTestVertex(), createTestVertex()); + assertThat(be.getEdgeType(), is(edgeType)); + } + + @Theory + public void testFactoryCreation(Boolean edgeType) throws InvalidIRException { + BooleanEdge.BooleanEdgeFactory factory = new BooleanEdge.BooleanEdgeFactory(edgeType); + BooleanEdge be = factory.make(createTestVertex(), createTestVertex()); + assertThat(be.getEdgeType(), is(edgeType)); + } +} diff --git a/logstash-core/src/test/java/org/logstash/config/ir/graph/EdgeTest.java b/logstash-core/src/test/java/org/logstash/config/ir/graph/EdgeTest.java new file mode 100644 index 000000000..d9d32d894 --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/config/ir/graph/EdgeTest.java @@ -0,0 +1,21 @@ +package org.logstash.config.ir.graph; + +import org.junit.Test; +import org.logstash.config.ir.IRHelpers; +import org.logstash.config.ir.InvalidIRException; + +import static org.junit.Assert.*; +import static org.hamcrest.CoreMatchers.*; + + +/** + * Created by andrewvc on 11/21/16. + */ +public class EdgeTest { + @Test + public void testBasicEdge() throws InvalidIRException { + Edge e = IRHelpers.createTestEdge(); + assertThat("From is edge", e.getFrom(), notNullValue()); + assertThat("To is edge", e.getTo(), notNullValue()); + } +} diff --git a/logstash-core/src/test/java/org/logstash/config/ir/graph/GraphTest.java b/logstash-core/src/test/java/org/logstash/config/ir/graph/GraphTest.java new file mode 100644 index 000000000..ae877065d --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/config/ir/graph/GraphTest.java @@ -0,0 +1,175 @@ +package org.logstash.config.ir.graph; + +import org.junit.Test; +import org.logstash.config.ir.DSL; +import org.logstash.config.ir.IRHelpers; +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.PluginDefinition; +import org.logstash.config.ir.imperative.IfStatement; + +import java.util.Collection; +import java.util.Collections; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; +import static org.logstash.config.ir.IRHelpers.createTestExpression; +import static org.logstash.config.ir.IRHelpers.createTestVertex; + +/** + * Created by andrewvc on 11/18/16. + */ +public class GraphTest { + @Test + public void testGraphBasics() throws InvalidIRException { + Graph g = Graph.empty(); + Vertex v1 = IRHelpers.createTestVertex(); + Vertex v2 = IRHelpers.createTestVertex(); + g.chainVertices(v1, v2); + Edge e = v1.outgoingEdges().findFirst().get(); + assertEquals("Connects vertex edges correctly", v1.getOutgoingEdges(), v2.getIncomingEdges()); + assertEquals("Has one edge", g.getEdges(), Collections.singleton(e)); + assertTrue("Has v1", g.getVertices().contains(v1)); + assertTrue("Has v2", g.getVertices().contains(v2)); + } + + // Expect an Invalid IR Exception from the cycle + @Test(expected = org.logstash.config.ir.InvalidIRException.class) + public void testGraphCycleDetection() throws InvalidIRException { + Graph g = Graph.empty(); + Vertex v1 = IRHelpers.createTestVertex(); + Vertex v2 = IRHelpers.createTestVertex(); + g.chainVertices(v1, v2); + g.chainVertices(v2, v1); + } + + @Test + public void chaining() throws InvalidIRException { + Graph fromGraph = Graph.empty(); + fromGraph.chainVertices(createTestVertex("fromV1"), createTestVertex("fromV2")); + + Graph toGraph = Graph.empty(); + toGraph.chainVertices(createTestVertex("toV1"), createTestVertex("toV2")); + + Graph result = fromGraph.chain(toGraph); + assertEquals(3, result.getEdges().size()); + assertVerticesConnected(result, "fromV2", "toV1"); + } + + @Test + public void chainingIntoMultipleRoots() throws InvalidIRException { + Graph fromGraph = Graph.empty(); + fromGraph.chainVertices(createTestVertex("fromV1"), createTestVertex("fromV2")); + + Graph toGraph = Graph.empty(); + toGraph.chainVertices(createTestVertex("toV1"), createTestVertex("toV2")); + toGraph.addVertex(createTestVertex("toV3")); + + Graph result = fromGraph.chain(toGraph); + assertEquals(4, result.getEdges().size()); + assertVerticesConnected(result, "fromV2", "toV1"); + assertVerticesConnected(result, "fromV2", "toV3"); + } + + @Test + public void SimpleConsistencyTest() throws InvalidIRException { + Graph g1 = Graph.empty(); + g1.addVertex(createTestVertex("a")); + Graph g2 = Graph.empty(); + g2.addVertex(createTestVertex("a")); + + assertEquals(g1.uniqueHash(), g2.uniqueHash()); + } + + @Test + public void complexConsistencyTest() throws InvalidIRException { + Graph g1 = IRHelpers.samplePipeline().getGraph(); + Graph g2 = IRHelpers.samplePipeline().getGraph(); + + assertEquals(g1.uniqueHash(), g2.uniqueHash()); + } + + @Test + public void testThreading() throws InvalidIRException { + Graph graph = Graph.empty(); + Vertex v1 = IRHelpers.createTestVertex(); + Vertex v2 = IRHelpers.createTestVertex(); + graph.chainVertices(v1, v2); + assertVerticesConnected(v1, v2); + Edge v1Edge = v1.outgoingEdges().findFirst().get(); + Edge v2Edge = v2.incomingEdges().findFirst().get(); + assertThat(v1Edge, is(v2Edge)); + assertThat(v1Edge, instanceOf(PlainEdge.class)); + } + + @Test + public void testThreadingMulti() throws InvalidIRException { + Graph graph = Graph.empty(); + Vertex v1 = IRHelpers.createTestVertex(); + Vertex v2 = IRHelpers.createTestVertex(); + Vertex v3 = IRHelpers.createTestVertex(); + Collection multiEdges = graph.chainVertices(v1, v2, v3); + + assertThat(v1.getOutgoingVertices(), is(Collections.singletonList(v2))); + assertThat(v2.getIncomingVertices(), is(Collections.singletonList(v1))); + assertThat(v2.getOutgoingVertices(), is(Collections.singletonList(v3))); + assertThat(v3.getIncomingVertices(), is(Collections.singletonList(v2))); + } + + @Test + public void testThreadingTyped() throws InvalidIRException { + Graph graph = Graph.empty(); + Vertex if1 = new IfVertex(null, createTestExpression()); + Vertex condT = IRHelpers.createTestVertex(); + Edge tEdge = graph.chainVertices(BooleanEdge.trueFactory, if1, condT).stream().findFirst().get(); + assertThat(tEdge, instanceOf(BooleanEdge.class)); + BooleanEdge tBooleanEdge = (BooleanEdge) tEdge; + assertThat(tBooleanEdge.getEdgeType(), is(true)); + } + + @Test + public void copyTest() throws InvalidIRException { + Graph left = Graph.empty(); + left.addVertex(createTestVertex("t1")); + Graph right = left.copy(); + + Vertex lv = left.getVertexById("t1"); + Vertex rv = right.getVertexById("t1"); + assertTrue(lv.sourceComponentEquals(rv)); + assertTrue(rv.sourceComponentEquals(lv)); + } + + @Test + public void uniqueHashingOfSimilarLeaves() throws InvalidIRException { + // the initial implementation didn't handle this well, so we'll leave it here as a tricky test + + IfStatement imperative = DSL.iIf( + DSL.eTruthy(DSL.eValue("1")), + DSL.iPlugin(PluginDefinition.Type.FILTER, "drop"), + DSL.iIf( + DSL.eTruthy(DSL.eValue("2")), + DSL.iPlugin(PluginDefinition.Type.FILTER, "drop"), + DSL.iIf( + DSL.eTruthy(DSL.eValue("3")), + DSL.iPlugin(PluginDefinition.Type.FILTER, "drop") + ) + ) + ); + + Graph g = imperative.toGraph(); + g.validate(); + } + + private void assertVerticesConnected(Graph graph, String fromId, String toId) { + Vertex from = graph.getVertexById(fromId); + assertNotNull(from); + Vertex to = graph.getVertexById(toId); + assertNotNull(to); + assertVerticesConnected(from, to); + } + + public void assertVerticesConnected(Vertex from, Vertex to) { + assertTrue("No connection: " + from + " -> " + to, from.getOutgoingVertices().contains(to)); + assertTrue("No reverse connection: " + from + " -> " + to, to.getIncomingVertices().contains(from)); + } +} diff --git a/logstash-core/src/test/java/org/logstash/config/ir/graph/IfVertexTest.java b/logstash-core/src/test/java/org/logstash/config/ir/graph/IfVertexTest.java new file mode 100644 index 000000000..d7ad532b9 --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/config/ir/graph/IfVertexTest.java @@ -0,0 +1,66 @@ +package org.logstash.config.ir.graph; + +import org.junit.Test; +import org.logstash.config.ir.InvalidIRException; + +import static org.hamcrest.CoreMatchers.*; +import static org.junit.Assert.assertThat; +import static org.logstash.config.ir.IRHelpers.*; + +/** + * Created by andrewvc on 11/22/16. + */ +public class IfVertexTest { + @Test + public void testIfVertexCreation() throws InvalidIRException { + testIfVertex(); + } + + @Test(expected = Vertex.InvalidEdgeTypeException.class) + public void testDoesNotAcceptNonBooleanEdges() throws InvalidIRException { + Graph graph = Graph.empty(); + IfVertex ifV = testIfVertex(); + Vertex otherV = createTestVertex(); + graph.chainVertices(PlainEdge.factory, ifV, otherV); + } + + @Test + public void testEdgeTypeHandling() throws InvalidIRException { + Graph graph = Graph.empty(); + IfVertex ifV = testIfVertex(); + graph.addVertex(ifV); + Vertex trueV = createTestVertex(); + graph.addVertex(trueV); + + assertThat(ifV.hasEdgeType(true), is(false)); + assertThat(ifV.hasEdgeType(false), is(false)); + assertThat(ifV.getUnusedOutgoingEdgeFactories().size(), is(2)); + + graph.chainVertices(BooleanEdge.trueFactory, ifV, trueV); + + assertThat(ifV.hasEdgeType(true), is(true)); + assertThat(ifV.hasEdgeType(false), is(false)); + assertThat(ifV.getUnusedOutgoingEdgeFactories().size(), is(1)); + assertThat( + ifV.getUnusedOutgoingEdgeFactories().stream().findFirst().get(), + is(BooleanEdge.falseFactory) + ); + + Vertex falseV = createTestVertex(); + graph.chainVertices(BooleanEdge.falseFactory, ifV, falseV); + + assertThat(ifV.hasEdgeType(false), is(true)); + assertThat(ifV.getUnusedOutgoingEdgeFactories().isEmpty(), is(true)); + + + BooleanEdge trueEdge = ifV.getOutgoingBooleanEdgesByType(true).stream().findAny().get(); + BooleanEdge falseEdge = ifV.getOutgoingBooleanEdgesByType(false).stream().findAny().get(); + assertThat(trueEdge.getEdgeType(), is(true)); + assertThat(falseEdge.getEdgeType(), is(false)); + } + + public IfVertex testIfVertex() throws InvalidIRException { + return new IfVertex(testMetadata(), createTestExpression()); + } + +} diff --git a/logstash-core/src/test/java/org/logstash/config/ir/graph/PlainEdgeTest.java b/logstash-core/src/test/java/org/logstash/config/ir/graph/PlainEdgeTest.java new file mode 100644 index 000000000..bbae087ef --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/config/ir/graph/PlainEdgeTest.java @@ -0,0 +1,21 @@ +package org.logstash.config.ir.graph; + +import org.junit.Test; +import org.logstash.config.ir.IRHelpers; +import org.logstash.config.ir.InvalidIRException; + +/** + * Created by andrewvc on 11/22/16. + */ +public class PlainEdgeTest { + @Test + public void creationDoesNotRaiseException() throws InvalidIRException { + new PlainEdge(IRHelpers.createTestVertex(), IRHelpers.createTestVertex()); + } + + @Test + public void testFactoryCreationDoesNotRaiseException() throws InvalidIRException { + PlainEdge.PlainEdgeFactory factory = new PlainEdge.PlainEdgeFactory(); + factory.make(IRHelpers.createTestVertex(), IRHelpers.createTestVertex()); + } +} diff --git a/logstash-core/src/test/java/org/logstash/config/ir/graph/PluginVertexTest.java b/logstash-core/src/test/java/org/logstash/config/ir/graph/PluginVertexTest.java new file mode 100644 index 000000000..b11b35a8c --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/config/ir/graph/PluginVertexTest.java @@ -0,0 +1,38 @@ +package org.logstash.config.ir.graph; + +import org.junit.Test; +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.PluginDefinition; + +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.CoreMatchers.*; +import static org.junit.Assert.assertThat; +import static org.logstash.config.ir.IRHelpers.*; + +/** + * Created by andrewvc on 11/22/16. + */ +public class PluginVertexTest { + @Test + public void testConstructionIdHandlingWhenNoExplicitId() throws InvalidIRException { + PluginDefinition pluginDefinition = testPluginDefinition(); + PluginVertex pluginVertex = new PluginVertex(testMetadata(), pluginDefinition); + Graph graph = Graph.empty(); + graph.addVertex(pluginVertex); + assertThat(pluginVertex.getId(), notNullValue()); + } + + @Test + public void testConstructionIdHandlingWhenExplicitId() { + String customId = "mycustomid"; + Map pluginArguments = new HashMap<>(); + pluginArguments.put("id", customId); + PluginDefinition pluginDefinition = new PluginDefinition(PluginDefinition.Type.FILTER, "myPlugin", pluginArguments); + PluginVertex pluginVertex = new PluginVertex(testMetadata(), pluginDefinition); + + assertThat(pluginVertex.getId(), is(customId)); + assertThat(pluginVertex.getPluginDefinition().getArguments().get("id"), is(customId)); + } +} diff --git a/logstash-core/src/test/java/org/logstash/config/ir/graph/QueueVertexTest.java b/logstash-core/src/test/java/org/logstash/config/ir/graph/QueueVertexTest.java new file mode 100644 index 000000000..855791c38 --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/config/ir/graph/QueueVertexTest.java @@ -0,0 +1,13 @@ +package org.logstash.config.ir.graph; + +import org.junit.Test; + +/** + * Created by andrewvc on 11/22/16. + */ +public class QueueVertexTest { + @Test + public void testConstruction() { + new QueueVertex(); + } +} diff --git a/logstash-core/src/test/java/org/logstash/config/ir/graph/VertexTest.java b/logstash-core/src/test/java/org/logstash/config/ir/graph/VertexTest.java new file mode 100644 index 000000000..8a16223ef --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/config/ir/graph/VertexTest.java @@ -0,0 +1,65 @@ +package org.logstash.config.ir.graph; + +import org.junit.Test; +import org.logstash.config.ir.IRHelpers; +import org.logstash.config.ir.InvalidIRException; + +import static org.junit.Assert.*; + +/** + * Created by andrewvc on 11/21/16. + */ +public class VertexTest { + @Test + public void TestVertexBasics() throws InvalidIRException { + Graph graph = Graph.empty(); + Vertex v1 = IRHelpers.createTestVertex(); + Vertex v2 = IRHelpers.createTestVertex(); + graph.chainVertices(v1, v2); + + assertTrue("v1 has v2 as an outgoing vertex", v1.outgoingVertices().anyMatch(v2::equals)); + assertTrue("v2 has v1 as an incoming vertex", v2.incomingVertices().anyMatch(v1::equals)); + } + + @Test + public void testIsLeafAndRoot() throws InvalidIRException { + Graph graph = Graph.empty(); + + Vertex v = IRHelpers.createTestVertex(); + graph.addVertex(v); + + // Nodes should be leaves and roots if they are isolated + assertTrue(v.isLeaf()); + assertTrue(v.isRoot()); + + Vertex otherV = IRHelpers.createTestVertex(); + graph.chainVertices(v, otherV); + + assertFalse(v.isLeaf()); + assertTrue(v.isRoot()); + assertTrue(otherV.isLeaf()); + assertFalse(otherV.isRoot()); + } + + @Test + public void testPartialLeafOnUnconnectedVertex() throws InvalidIRException { + Graph g = Graph.empty(); + Vertex v = IRHelpers.createTestVertex(); + g.addVertex(v); + assertEquals(v.getUnusedOutgoingEdgeFactories().size(), 1); + assertTrue(v.isPartialLeaf()); + } + + @Test + public void testPartialLeafOnConnectedVertex() throws InvalidIRException { + Vertex v = IRHelpers.createTestVertex(); + Vertex otherV = IRHelpers.createTestVertex(); + Graph graph = Graph.empty(); + graph.chainVertices(v, otherV); + + assertEquals(v.getUnusedOutgoingEdgeFactories().size(), 0); + assertFalse(v.isPartialLeaf()); + } + + +} diff --git a/logstash-core/src/test/java/org/logstash/config/ir/graph/algorithms/BreadthFirstTest.java b/logstash-core/src/test/java/org/logstash/config/ir/graph/algorithms/BreadthFirstTest.java new file mode 100644 index 000000000..96c0dc8ee --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/config/ir/graph/algorithms/BreadthFirstTest.java @@ -0,0 +1,32 @@ +package org.logstash.config.ir.graph.algorithms; + +import org.junit.Test; +import org.logstash.config.ir.IRHelpers; +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.graph.Graph; + +import java.util.concurrent.atomic.AtomicInteger; + +import static junit.framework.TestCase.assertEquals; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * Created by andrewvc on 1/5/17. + */ +public class BreadthFirstTest { + @Test + public void testBFSBasic() throws InvalidIRException { + Graph g = Graph.empty(); + g.chainVertices(IRHelpers.createTestVertex(), IRHelpers.createTestVertex(), IRHelpers.createTestVertex()); + + // We don't *really* need threadsafety for the count, + // but since we're using a lambda we need something that's final + final AtomicInteger visitCount = new AtomicInteger(); + BreadthFirst.BfsResult res = BreadthFirst.breadthFirst(g.getRoots(), false, (v -> visitCount.incrementAndGet())); + + assertEquals("It should visit each node once", visitCount.get(), 3); + assertThat(res.getVertices(), is(g.getVertices())); + } + +} diff --git a/logstash-core/src/test/java/org/logstash/config/ir/graph/algorithms/DepthFirstTest.java b/logstash-core/src/test/java/org/logstash/config/ir/graph/algorithms/DepthFirstTest.java new file mode 100644 index 000000000..65c27bd49 --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/config/ir/graph/algorithms/DepthFirstTest.java @@ -0,0 +1,28 @@ +package org.logstash.config.ir.graph.algorithms; + +import org.junit.Test; +import org.logstash.config.ir.IRHelpers; +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.graph.Graph; +import org.logstash.config.ir.graph.Vertex; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static junit.framework.TestCase.assertEquals; + +/** + * Created by andrewvc on 1/5/17. + */ +public class DepthFirstTest { + @Test + public void testDFSBasic() throws InvalidIRException { + Graph g = Graph.empty(); + g.chainVertices(IRHelpers.createTestVertex(), IRHelpers.createTestVertex(), IRHelpers.createTestVertex()); + final AtomicInteger visitCount = new AtomicInteger(); + final List visited = new ArrayList<>(); + DepthFirst.depthFirst(g).forEach(v -> visitCount.incrementAndGet()); + assertEquals("It should visit each node once", visitCount.get(), 3); + } +} diff --git a/logstash-core/src/test/java/org/logstash/config/ir/graph/algorithms/GraphDiffTest.java b/logstash-core/src/test/java/org/logstash/config/ir/graph/algorithms/GraphDiffTest.java new file mode 100644 index 000000000..4cfda61da --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/config/ir/graph/algorithms/GraphDiffTest.java @@ -0,0 +1,66 @@ +package org.logstash.config.ir.graph.algorithms; + +import org.junit.Test; +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.graph.Edge; +import org.logstash.config.ir.graph.Graph; +import org.logstash.config.ir.graph.Vertex; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.logstash.config.ir.IRHelpers.createTestVertex; + +/** + * Created by andrewvc on 1/5/17. + */ +public class GraphDiffTest { + @Test + public void testIdenticalGraphs() throws InvalidIRException { + Graph left = simpleGraph(); + Graph right = simpleGraph(); + GraphDiff.DiffResult result = GraphDiff.diff(left, right); + + assertTrue(result.isIdentical()); + + assertTrue(result.getAddedEdges().isEmpty()); + assertTrue(result.getRemovedEdges().isEmpty()); + assertTrue(result.getAddedVertices().isEmpty()); + assertTrue(result.getRemovedVertices().isEmpty()); + } + + @Test + public void testDifferentSimpleGraphs() throws InvalidIRException { + Graph left = simpleGraph(); + + Graph right = left.copy(); + Vertex new1 = createTestVertex("new1"); + right.addVertex(new1); + right.chainVerticesById("t3", "new1"); + + GraphDiff.DiffResult result = GraphDiff.diff(left, right); + + assertFalse(result.isIdentical()); + + assertThat(result.getAddedVertices().size(), is(1)); + assertThat(result.getAddedVertices().stream().findAny().get().getId(), is("new1")); + + assertThat(result.getAddedEdges().size(), is(1)); + Edge expectedEdge = new1.incomingEdges().findAny().get(); + assertTrue(result.getAddedEdges().stream().findAny().get().sourceComponentEquals(expectedEdge)); + + assertTrue(result.getRemovedVertices().isEmpty()); + assertTrue(result.getRemovedEdges().isEmpty()); + } + + public Graph simpleGraph() throws InvalidIRException { + Graph graph = Graph.empty(); + graph.addVertex(createTestVertex("t1")); + graph.addVertex(createTestVertex("t2")); + graph.addVertex(createTestVertex("t3")); + graph.chainVerticesById("t1", "t2", "t3"); + graph.chainVerticesById("t1", "t3"); + return graph; + } +} diff --git a/logstash-core/src/test/java/org/logstash/config/ir/graph/algorithms/ShortestPathTest.java b/logstash-core/src/test/java/org/logstash/config/ir/graph/algorithms/ShortestPathTest.java new file mode 100644 index 000000000..f35111fe3 --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/config/ir/graph/algorithms/ShortestPathTest.java @@ -0,0 +1,38 @@ +package org.logstash.config.ir.graph.algorithms; + +import org.junit.Test; +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.graph.Graph; +import org.logstash.config.ir.graph.Vertex; + +import java.util.Arrays; +import java.util.List; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.logstash.config.ir.IRHelpers.createTestVertex; + +/** + * Created by andrewvc on 1/5/17. + */ +public class ShortestPathTest { + @Test + public void testShortestPathBasic() throws InvalidIRException, ShortestPath.InvalidShortestPathArguments { + Graph g = Graph.empty(); + Vertex v1 = createTestVertex("v1"); + g.addVertex(v1); + Vertex v2 = createTestVertex("v2"); + g.addVertex(v2); + Vertex v3 = createTestVertex("v3"); + g.addVertex(v3); + Vertex v4 = createTestVertex("v4"); + g.addVertex(v4); + + g.chainVertices(v1, v2, v3, v4); + g.chainVertices(v2, v4); + + List path = ShortestPath.shortestPath(v1, v4); + List expected = Arrays.asList(v1,v2,v4); + assertThat(path, is(expected)); + } +} diff --git a/logstash-core/src/test/java/org/logstash/config/ir/graph/algorithms/TopologicalSortTest.java b/logstash-core/src/test/java/org/logstash/config/ir/graph/algorithms/TopologicalSortTest.java new file mode 100644 index 000000000..fc91eb21a --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/config/ir/graph/algorithms/TopologicalSortTest.java @@ -0,0 +1,46 @@ +package org.logstash.config.ir.graph.algorithms; + +import org.junit.Test; +import org.logstash.config.ir.IRHelpers; +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.graph.Graph; +import org.logstash.config.ir.graph.Vertex; + +import java.util.Arrays; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.AnyOf.anyOf; +import static org.hamcrest.core.Is.is; + +/** + * Created by andrewvc on 1/7/17. + */ +public class TopologicalSortTest { + @Test(expected = InvalidIRException.class) + public void testGraphCycleDetection() throws InvalidIRException { + Graph g = Graph.empty(); + Vertex v1 = IRHelpers.createTestVertex(); + Vertex v2 = IRHelpers.createTestVertex(); + Vertex v3 = IRHelpers.createTestVertex(); + g.chainVertices(v1, v2); + g.chainVertices(v2, v3); + g.chainVertices(v2, v1); + } + + @Test + public void testSortOrder() throws InvalidIRException, TopologicalSort.UnexpectedGraphCycleError { + Graph g = Graph.empty(); + Vertex v1 = IRHelpers.createTestVertex(); + Vertex v2 = IRHelpers.createTestVertex(); + Vertex v3 = IRHelpers.createTestVertex(); + Vertex v4 = IRHelpers.createTestVertex(); + g.chainVertices(v3, v1, v2); + g.chainVertices(v4, v1, v2); + assertThat(TopologicalSort.sortVertices(g), + anyOf( + is(Arrays.asList(v3,v4,v1,v2)), + is(Arrays.asList(v4,v3,v1,v2)) + )); + } + +} diff --git a/logstash-core/src/test/java/org/logstash/config/ir/imperative/DSLTest.java b/logstash-core/src/test/java/org/logstash/config/ir/imperative/DSLTest.java new file mode 100644 index 000000000..d6dc03b3e --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/config/ir/imperative/DSLTest.java @@ -0,0 +1,58 @@ +package org.logstash.config.ir.imperative; + +import org.junit.Test; +import org.logstash.config.ir.BaseSourceComponent; +import org.logstash.config.ir.InvalidIRException; + +import static org.logstash.config.ir.DSL.*; +import static org.logstash.config.ir.IRHelpers.assertSyntaxEquals; +import static org.logstash.config.ir.PluginDefinition.Type.*; + +/** + * Created by andrewvc on 9/13/16. + */ +public class DSLTest { + @Test + public void testDSLOnePluginEquality() { + assertSyntaxEquals(iPlugin(FILTER, "foo"), iPlugin(FILTER, "foo")); + } + + @Test + public void testComposedPluginEquality() throws InvalidIRException { + assertSyntaxEquals(composedPlugins(), composedPlugins()); + } + + @Test + public void testDSLComplexEquality() throws InvalidIRException { + assertSyntaxEquals(complexExpression(), complexExpression()); + } + + @Test + public void testComposeSingle() throws InvalidIRException { + assertSyntaxEquals(iPlugin(FILTER, "grok"), iComposeSequence(iPlugin(FILTER, "grok"))); + } + + @Test + public void testComposeMulti() throws InvalidIRException { + Statement composed = iComposeSequence(iPlugin(FILTER, "grok"), iPlugin(FILTER, "foo")); + assertSyntaxEquals(iComposeSequence(iPlugin(FILTER, "grok"), iPlugin(FILTER, "foo")), composed); +} + + + public BaseSourceComponent composedPlugins() throws InvalidIRException { + return iComposeSequence(iPlugin(FILTER, "json"), iPlugin(FILTER, "stuff")); + } + + public BaseSourceComponent complexExpression() throws InvalidIRException { + return iComposeSequence( + iPlugin(FILTER, "grok"), + iPlugin(FILTER, "kv"), + iIf(eAnd(eTruthy(eValue(5l)), eTruthy(eValue(null))), + iPlugin(FILTER, "grok"), + iComposeSequence(iPlugin(FILTER, "json"), iPlugin(FILTER, "stuff")) + ) + ); + } + + +} diff --git a/logstash-core/src/test/java/org/logstash/config/ir/imperative/ImperativeToGraphtest.java b/logstash-core/src/test/java/org/logstash/config/ir/imperative/ImperativeToGraphtest.java new file mode 100644 index 000000000..36db351c2 --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/config/ir/imperative/ImperativeToGraphtest.java @@ -0,0 +1,166 @@ +package org.logstash.config.ir.imperative; + +import org.junit.Test; +import org.logstash.config.ir.InvalidIRException; +import org.logstash.config.ir.graph.Graph; +import org.logstash.config.ir.graph.IfVertex; +import org.logstash.config.ir.graph.PluginVertex; + +import static org.logstash.config.ir.DSL.*; +import static org.logstash.config.ir.IRHelpers.assertSyntaxEquals; +import static org.logstash.config.ir.PluginDefinition.Type.*; + +/** + * Created by andrewvc on 9/15/16. + */ +public class ImperativeToGraphtest { + + @Test + public void convertSimpleExpression() throws InvalidIRException { + Graph imperative = iComposeSequence(iPlugin(FILTER, "json"), iPlugin(FILTER, "stuff")).toGraph(); + imperative.validate(); // Verify this is a valid graph + + Graph regular = Graph.empty(); + regular.chainVertices(gPlugin(FILTER, "json"), gPlugin(FILTER, "stuff")); + + assertSyntaxEquals(imperative, regular); + } + + @Test + public void testIdsDontAffectSourceComponentEquality() throws InvalidIRException { + Graph imperative = iComposeSequence(iPlugin(FILTER, "json", "oneid"), iPlugin(FILTER, "stuff", "anotherid")).toGraph(); + imperative.validate(); // Verify this is a valid graph + + Graph regular = Graph.empty(); + regular.chainVertices(gPlugin(FILTER, "json", "someotherid"), gPlugin(FILTER, "stuff", "graphid")); + + assertSyntaxEquals(imperative, regular); + } + + @Test + public void convertComplexExpression() throws InvalidIRException { + Graph imperative = iComposeSequence( + iPlugin(FILTER, "p1"), + iPlugin(FILTER, "p2"), + iIf(eAnd(eTruthy(eValue(5l)), eTruthy(eValue(null))), + iPlugin(FILTER, "p3"), + iComposeSequence(iPlugin(FILTER, "p4"), iPlugin(FILTER, "p5")) + ) + ).toGraph(); + imperative.validate(); // Verify this is a valid graph + + PluginVertex p1 = gPlugin(FILTER, "p1"); + PluginVertex p2 = gPlugin(FILTER, "p2"); + PluginVertex p3 = gPlugin(FILTER, "p3"); + PluginVertex p4 = gPlugin(FILTER, "p4"); + PluginVertex p5 = gPlugin(FILTER, "p5"); + IfVertex testIf = gIf(eAnd(eTruthy(eValue(5l)), eTruthy(eValue(null)))); + + Graph expected = Graph.empty(); + expected.chainVertices(p1,p2,testIf); + expected.chainVertices(true, testIf, p3); + expected.chainVertices(false, testIf, p4); + expected.chainVertices(p4, p5); + + assertSyntaxEquals(expected, imperative); + } + + // This test has an imperative grammar with nested ifs and dangling + // partial leaves. This makes sure they all wire-up right + @Test + public void deepDanglingPartialLeaves() throws InvalidIRException { + Graph imperative = iComposeSequence( + iPlugin(FILTER, "p0"), + iIf(eTruthy(eValue(1)), + iPlugin(FILTER, "p1"), + iIf(eTruthy(eValue(3)), + iPlugin(FILTER, "p5")) + ), + iIf(eTruthy(eValue(2)), + iPlugin(FILTER, "p3"), + iPlugin(FILTER, "p4") + ), + iPlugin(FILTER, "pLast") + + ).toGraph(); + imperative.validate(); // Verify this is a valid graph + + IfVertex if1 = gIf(eTruthy(eValue(1))); + IfVertex if2 = gIf(eTruthy(eValue(2))); + IfVertex if3 = gIf(eTruthy(eValue(3))); + PluginVertex p0 = gPlugin(FILTER, "p0"); + PluginVertex p1 = gPlugin(FILTER, "p1"); + PluginVertex p2 = gPlugin(FILTER, "p2"); + PluginVertex p3 = gPlugin(FILTER, "p3"); + PluginVertex p4 = gPlugin(FILTER, "p4"); + PluginVertex p5 = gPlugin(FILTER, "p5"); + PluginVertex pLast = gPlugin(FILTER, "pLast"); + + Graph expected = Graph.empty(); + expected.chainVertices(p0, if1); + expected.chainVertices(true, if1, p1); + expected.chainVertices(false, if1, if3); + expected.chainVertices(true, if3, p5); + expected.chainVertices(false, if3, if2); + expected.chainVertices(p5, if2); + expected.chainVertices(p1, if2); + expected.chainVertices(true, if2, p3); + expected.chainVertices(false, if2, p4); + expected.chainVertices(p3, pLast); + expected.chainVertices(p4,pLast); + + assertSyntaxEquals(imperative, expected); + } + + // This is a good test for what the filter block will do, where there + // will be a composed set of ifs potentially, all of which must terminate at a + // single node + @Test + public void convertComplexExpressionWithTerminal() throws InvalidIRException { + Graph imperative = iComposeSequence( + iPlugin(FILTER, "p1"), + iIf(eTruthy(eValue(1)), + iComposeSequence( + iIf(eTruthy(eValue(2)), noop(), iPlugin(FILTER, "p2")), + iIf(eTruthy(eValue(3)), iPlugin(FILTER, "p3"), noop()) + ), + iComposeSequence( + iIf(eTruthy(eValue(4)), iPlugin(FILTER, "p4")), + iPlugin(FILTER, "p5") + ) + ), + iPlugin(FILTER, "terminal") + ).toGraph(); + imperative.validate(); // Verify this is a valid graph + + PluginVertex p1 = gPlugin(FILTER,"p1"); + PluginVertex p2 = gPlugin(FILTER, "p2"); + PluginVertex p3 = gPlugin(FILTER, "p3"); + PluginVertex p4 = gPlugin(FILTER, "p4"); + PluginVertex p5 = gPlugin(FILTER, "p5"); + PluginVertex terminal = gPlugin(FILTER, "terminal"); + + IfVertex if1 = gIf(eTruthy(eValue(1))); + IfVertex if2 = gIf(eTruthy(eValue(2))); + IfVertex if3 = gIf(eTruthy(eValue(3))); + IfVertex if4 = gIf(eTruthy(eValue(4))); + + Graph expected = Graph.empty(); + expected.chainVertices(p1, if1); + expected.chainVertices(true, if1, if2); + expected.chainVertices(false, if1, if4); + expected.chainVertices(true, if2, if3); + expected.chainVertices(false, if2, p2); + expected.chainVertices(p2, if3); + expected.chainVertices(true, if3, p3); + expected.chainVertices(false, if3, terminal); + expected.chainVertices(p3, terminal); + expected.chainVertices(true, if4, p4); + expected.chainVertices(false, if4, p5); + expected.chainVertices(p4, p5); + expected.chainVertices(p5, terminal); + + assertSyntaxEquals(imperative, expected); + + } +} diff --git a/rakelib/compile.rake b/rakelib/compile.rake index fd113a476..86496c0c6 100644 --- a/rakelib/compile.rake +++ b/rakelib/compile.rake @@ -10,10 +10,17 @@ namespace "compile" do desc "Compile the config grammar" task "grammar" => "logstash-core/lib/logstash/config/grammar.rb" + + def safe_system(*args) + if !system(*args) + status = $? + raise "Got exit status #{status.exitstatus} attempting to execute #{args.inspect}!" + end + end task "logstash-core-java" do puts("Building logstash-core using gradle") - system("./gradlew", "jar") + safe_system("./gradlew", "jar") end desc "Build everything" diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 44919e332..e1e3ac421 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -10,8 +10,6 @@ require "flores/rspec" require "flores/random" require "pathname" -SUPPORT_DIR = Pathname.new(::File.join(::File.dirname(__FILE__), "support")) - class JSONIOThingy < IO def initialize; end def flush; end diff --git a/spec/support/lscl_configs/apache_logstash.conf b/spec/support/lscl_configs/apache_logstash.conf new file mode 100644 index 000000000..ef8030d4c --- /dev/null +++ b/spec/support/lscl_configs/apache_logstash.conf @@ -0,0 +1,39 @@ +input { + stdin { } +} + + +filter { + grok { + match => { + "message" => '%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "%{WORD:verb} %{DATA:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:response:int} (?:-|%{NUMBER:bytes:int}) %{QS:referrer} %{QS:agent}' + } + } + + date { + match => [ "timestamp", "dd/MMM/YYYY:HH:mm:ss Z" ] + locale => en + } + + geoip { + source => "clientip" + } + + useragent { + source => "agent" + target => "useragent" + } +} + +output { + stdout { + codec => dots {} + } + + elasticsearch { + index => "apache_elastic_example" + template => "./apache_template.json" + template_name => "apache_elastic_example" + template_overwrite => true + } +} diff --git a/spec/support/lscl_configs/nginx_json_logstash.conf b/spec/support/lscl_configs/nginx_json_logstash.conf new file mode 100644 index 000000000..5fc0dada1 --- /dev/null +++ b/spec/support/lscl_configs/nginx_json_logstash.conf @@ -0,0 +1,42 @@ +input { + stdin { + codec => json + } +} + +filter { + + date { + match => ["time", "dd/MMM/YYYY:HH:mm:ss Z" ] + locale => en + } + + geoip { + source => "remote_ip" + target => "geoip" + } + + useragent { + source => "agent" + target => "user_agent" + } + + grok { + match => [ "request" , "%{WORD:request_action} %{DATA:request1} HTTP/%{NUMBER:http_version}" ] + } +} + +output { + stdout { + codec => dots {} + } + + elasticsearch { + index => "nginx_json_elastic_stack_example" + document_type => "logs" + template => "./nginx_json_template.json" + template_name => "nginx_json_elastic_stack_example" + template_overwrite => true + } + +} diff --git a/spec/support/lscl_configs/nginx_logstash.conf b/spec/support/lscl_configs/nginx_logstash.conf new file mode 100644 index 000000000..4c43456d6 --- /dev/null +++ b/spec/support/lscl_configs/nginx_logstash.conf @@ -0,0 +1,39 @@ +input { + stdin { } +} + +filter { + grok { + match => { + "message" => '%{IPORHOST:remote_ip} - %{DATA:user_name} \[%{HTTPDATE:time}\] "%{WORD:request_action} %{DATA:request} HTTP/%{NUMBER:http_version}" %{NUMBER:response} %{NUMBER:bytes} "%{DATA:referrer}" "%{DATA:agent}"' + } + } + + date { + match => [ "time", "dd/MMM/YYYY:HH:mm:ss Z" ] + locale => en + } + + geoip { + source => "remote_ip" + target => "geoip" + } + + useragent { + source => "agent" + target => "user_agent" + } +} + +output { +stdout { + codec => dots {} + } + elasticsearch { + index => "nginx_elastic_stack_example" + document_type => "logs" + template => "./nginx_template.json" + template_name => "nginx_elastic_stack_example" + template_overwrite => true + } +} diff --git a/spec/support/lscl_configs/nyc_collision_logstash.conf b/spec/support/lscl_configs/nyc_collision_logstash.conf new file mode 100644 index 000000000..5df5950a2 --- /dev/null +++ b/spec/support/lscl_configs/nyc_collision_logstash.conf @@ -0,0 +1,156 @@ +input { + stdin { } +} + +filter { + csv { + columns => ["date","time","borough","zip_code","latitude","longitude","location","on_street_name","cross_street_name","off_street_name","number_of_persons_injured","number_of_persons_killed","number_of_pedestrians_injured","number_of_pedestrians_killed","number_of_cyclist_injured","number_of_cyclist_killed","number_of_motorist_injured","number_of_motorist_killed","contributing_factor_vehicle_1","contributing_factor_vehicle_2","contributing_factor_vehicle_3","contributing_factor_vehicle_4","contributing_factor_vehicle_5","unique_key","vehicle_type_code_1","vehicle_type_code_2","vehicle_type_code_3","vehicle_type_code_4","vehicle_type_code_5"] + } + +# Drop the first (header) row in the file + + if ([date] == "DATE") { + drop { } + } + +# Combine latitude and longitude into single coords field + if [latitude] and [longitude] { + mutate { + add_field => { + "coords" => "%{longitude}" + "tmplat" => "%{latitude}" + } + } + mutate { + merge => ["coords", "tmplat"] + } + mutate { + # Convert our new array of strings back to float + convert => [ "coords", "float" ] + # Delete our temporary latitude field + remove_field => [ "tmplat" ] + } + } + + if [on_street_name] and [cross_street_name] { + ruby { + # create new intersection field that combines cross street & on street names + code => "event.set('intersection',[event.get('on_street_name'), event.get('cross_street_name')].sort.join('--'))" + } + } + + # Merge date and time into datetime + mutate { + add_field => { + "datetime" => "%{date} %{time}" + "contributing_factor_vehicle" => "%{contributing_factor_vehicle_1}" + "vehicle_type" => "%{vehicle_type_code_1}" + } + + + + # convert to integer type + convert => ["number_of_persons_injured","integer","number_of_persons_killed","integer","number_of_pedestrians_injured","integer","number_of_pedestrians_killed","integer","number_of_cyclist_injured","integer","number_of_cyclist_killed","integer","number_of_motorist_injured","integer","number_of_motorist_killed","integer"] + strip => ["on_street_name", "cross_street_name"] + } + + if ![number_of_persons_killed] + { + mutate { + add_field => {"number_of_persons_killed" => "0"} + } + } + + if ![number_of_persons_injured] + { + mutate { + add_field => {"number_of_persons_injured" => "0"} + } + } + + + ruby { + # Get total number of persons impacted + code => "event.set('number_persons_impacted',event.get('number_of_persons_killed') + event.get('number_of_persons_injured'))" + } + + +# Combine contributing_factor_vehicle_X (X=1,2,3,4,5) fields into a single field + if [contributing_factor_vehicle_2] and "Unspecified" != [contributing_factor_vehicle_2] and [contributing_factor_vehicle_2] not in [contributing_factor_vehicle] { + mutate { + merge => ["contributing_factor_vehicle", "contributing_factor_vehicle_2"] + } + } + + if [contributing_factor_vehicle_3] and "Unspecified" != [contributing_factor_vehicle_3] and [contributing_factor_vehicle_3] not in [contributing_factor_vehicle] { + mutate { + merge => ["contributing_factor_vehicle", "contributing_factor_vehicle_3"] + } + } + + if [contributing_factor_vehicle_4] and "Unspecified" != [contributing_factor_vehicle_4] and [contributing_factor_vehicle_4] not in [contributing_factor_vehicle] { + mutate { + merge => ["contributing_factor_vehicle", "contributing_factor_vehicle_4"] + } + } + + if [contributing_factor_vehicle_5] and "Unspecified" != [contributing_factor_vehicle_5] and [contributing_factor_vehicle_5] not in [contributing_factor_vehicle] { + mutate { + merge => ["contributing_factor_vehicle", "contributing_factor_vehicle_5"] + } + } + + # Combine vehicle_type_code_X (X=1,2,3,4,5) fields into a single field + if [vehicle_type_code_2] and "Unspecified" != [vehicle_type_code_2] and [vehicle_type_code_2] not in [vehicle_type] { + mutate { + merge => ["vehicle_type", "vehicle_type_code_2"] + } + } + if [vehicle_type_code_3] and "Unspecified" != [vehicle_type_code_3] and [vehicle_type_code_3] not in [vehicle_type] { + mutate { + merge => ["vehicle_type", "vehicle_type_code_3"] + } + } + if [vehicle_type_code_4] and "Unspecified" != [vehicle_type_code_4] and [vehicle_type_code_4] not in [vehicle_type] { + mutate { + merge => ["vehicle_type", "vehicle_type_code_4"] + } + } + if [vehicle_type_code_5] and "Unspecified" != [vehicle_type_code_5] and [vehicle_type_code_5] not in [vehicle_type] { + mutate { + merge => ["vehicle_type", "vehicle_type_code_5"] + } + } + + # Map @timestamp (event timestamp) to datetime + date { + match => [ "datetime", "MM/dd/YY HH:mm", "MM/dd/YY H:mm"] + timezone => "EST" + } + + # Grab hour of day from time + grok { + match => { "time" => "%{DATA:hour_of_day}:%{GREEDYDATA}" } + } + mutate { + convert => ["hour_of_day", "integer"] + } + + # Remove extra fields + mutate { + remove_field => ["datetime", "contributing_factor_vehicle_1", "contributing_factor_vehicle_2", "contributing_factor_vehicle_3", "contributing_factor_vehicle_4", "contributing_factor_vehicle_5","vehicle_type_code_1", "vehicle_type_code_2", "vehicle_type_code_3", "vehicle_type_code_4", "vehicle_type_code_5"] + } +} + +output { + + #stdout {codec => rubydebug} + stdout { codec => dots } + + elasticsearch { + index => "nyc_visionzero" + template => "./nyc_collision_template.json" + template_name => "nyc_visionzero" + template_overwrite => true + } +} diff --git a/spec/support/lscl_configs/twitter_logstash.conf b/spec/support/lscl_configs/twitter_logstash.conf new file mode 100644 index 000000000..0a16e4341 --- /dev/null +++ b/spec/support/lscl_configs/twitter_logstash.conf @@ -0,0 +1,26 @@ +input { + twitter { + consumer_key => "INSERT YOUR CONSUMER KEY" + consumer_secret => "INSERT YOUR CONSUMER SECRET" + oauth_token => "INSERT YOUR ACCESS TOKEN" + oauth_token_secret => "INSERT YOUR ACCESS TOKEN SECRET" + keywords => [ "thor", "spiderman", "wolverine", "ironman", "hulk"] + full_tweet => true + } +} + +filter { } + +output { + stdout { + codec => dots + } + elasticsearch { + hosts => "localhost:9200" + index => "twitter_elastic_example" + document_type => "tweets" + template => "./twitter_template.json" + template_name => "twitter_elastic_example" + template_overwrite => true + } +} diff --git a/spec/support/lscl_configs/usfec_logstash.conf b/spec/support/lscl_configs/usfec_logstash.conf new file mode 100644 index 000000000..47cc18ec3 --- /dev/null +++ b/spec/support/lscl_configs/usfec_logstash.conf @@ -0,0 +1,30 @@ +input { + stdin { + codec => json_lines {} + } +} + +filter { + if !([transactionDate] == "") + { + date { + match => [ "transactionDate", "MMddyyyy", "MM/dd/yyyy"] + } + } + mutate { + convert => ["transactionAmount", "float"] + replace => { "type" => "%{recordType}" } + } +} + +output { + #stdout { codec => rubydebug } + stdout { codec => dots } + elasticsearch { + hosts => "localhost:9200" + index => "usfec_%{recordType}" + template => "usfec_template.json" + template_name => "usfec" + template_overwrite => true + } +}