github-mirrors/logstash
1
0
Fork 0
mirror of https://github.com/elastic/logstash.git synced 2025-04-24 14:47:19 -04:00
Code Issues Projects Releases Packages Wiki Activity

Use file hashing where possible in LIR

Browse source 
This commit further improves hashing performance, using SourceWithMetadata
objects where present to determine Vertex IDs and the hash of the source
file. If those objects are not present LIR will revert to graph hashing,
which is slower, but always works. This is still useful for things like
testing, so it makes sense to leave that code in.

In the future it might be nice to extract the hashing code using the
strategy pattern to clean things up.

This commit also improves the Hashable interface, moving hashSource into
its own interface, letting classes that do their own hashing simply
implement uniqueHash instead of requiring them to also implement
hashSource.

A requirement of these changes was also that SourceWithMetadata no
longer accept null or empty arguments. This patch now enforces that
requirement. This also is nicer from an API standpoint, as you now know
that if a SourceWithMetadata object exists, it will actually have all
its fields in use.

Fixes #7408
This commit is contained in:
Andrew Cholakian 2017-06-12 12:07:54 -05:00
parent dc52add283
commit f5c6c5a4b1
27 changed files with 245 additions and 102 deletions
Download patch file Download diff file Expand all files Collapse all files

9
logstash-core/lib/logstash/compiler/lscl.rb
View file

@ -46,10 +46,6 @@ module LogStashCompilerLSCLGrammar; module LogStash; module Compiler; module LSC
[self.input.line_of(start), self.input.column_of(start)]
end
def empty_source_meta()
org.logstash.common.SourceWithMetadata.new(base_protocol, base_id, nil)
end
def jdsl
org.logstash.config.ir.DSL
end
@ -99,11 +95,12 @@ module LogStashCompilerLSCLGrammar; module LogStash; module Compiler; module LSC
end
end
compiled_section_map = {}
section_map.keys.each do |key|
section_map[key] = compose_for(key).call(empty_source_meta, *section_map[key])
compiled_section_map[key] = compose_for(key).call(*section_map[key])
end
section_map
compiled_section_map
end
end

10
logstash-core/lib/logstash/config/source/local.rb
View file

@ -19,7 +19,7 @@ module LogStash module Config module Source
class Local < Base
class ConfigStringLoader
def self.read(config_string)
[org.logstash.common.SourceWithMetadata.new("string", "config_string", config_string)]
[org.logstash.common.SourceWithMetadata.new("string", "config_string", 0, 0, config_string)]
end
end
@ -54,7 +54,7 @@ module LogStash module Config module Source
config_string = ::File.read(file)
if valid_encoding?(config_string)
part = org.logstash.common.SourceWithMetadata.new("file", file, config_string)
part = org.logstash.common.SourceWithMetadata.new("file", file, 0, 0, config_string)
config_parts << part
else
encoding_issue_files << file
@ -121,7 +121,7 @@ module LogStash module Config module Source
# since we have fetching config we wont follow any redirection.
case response.code.to_i
when 200
[org.logstash.common.SourceWithMetadata.new(uri.scheme, uri.to_s, response.body)]
[org.logstash.common.SourceWithMetadata.new(uri.scheme, uri.to_s, 0, 0, response.body)]
when 302
raise LogStash::ConfigLoadingError, I18n.t("logstash.runner.configuration.fetch-failed", :path => uri.to_s, :message => "We don't follow redirection for remote configuration")
when 404
@ -177,12 +177,12 @@ module LogStash module Config module Source
# this is for backward compatibility reason
def add_missing_default_inputs_or_outputs(config_parts)
if !config_parts.any? { |part| INPUT_BLOCK_RE.match(part.text) }
config_parts << org.logstash.common.SourceWithMetadata.new(self.class.name, "default input", LogStash::Config::Defaults.input)
config_parts << org.logstash.common.SourceWithMetadata.new(self.class.name, "default input", 0, 0, LogStash::Config::Defaults.input)
end
# include a default stdout output if no outputs given
if !config_parts.any? { |part| OUTPUT_BLOCK_RE.match(part.text) }
config_parts << org.logstash.common.SourceWithMetadata.new(self.class.name, "default output", LogStash::Config::Defaults.output)
config_parts << org.logstash.common.SourceWithMetadata.new(self.class.name, "default output", 0, 0, LogStash::Config::Defaults.output)
end
end

2
logstash-core/lib/logstash/config/source/modules.rb
View file

@ -14,7 +14,7 @@ module LogStash module Config module Source
pipelines = LogStash::Config::ModulesCommon.pipeline_configs(@settings)
pipelines.map do |hash|
PipelineConfig.new(self, hash["pipeline_id"].to_sym,
org.logstash.common.SourceWithMetadata.new("module", hash["alt_name"], hash["config_string"]),
org.logstash.common.SourceWithMetadata.new("module", hash["alt_name"], 0, 0, hash["config_string"]),
hash["settings"])
end
end

2
logstash-core/lib/logstash/pipeline.rb
View file

@ -90,7 +90,7 @@ module LogStash; class BasePipeline
end
def compile_lir
source_with_metadata = SourceWithMetadata.new("str", "pipeline", self.config_str)
source_with_metadata = SourceWithMetadata.new("str", "pipeline", 0, 0, self.config_str)
LogStash::Compiler.compile_sources(source_with_metadata)
end

8
logstash-core/spec/logstash/compiler/compiler_spec.rb
View file

@ -28,8 +28,8 @@ describe LogStash::Compiler do
describe "compiling to Pipeline" do
subject(:source_id) { "fake_sourcefile" }
let(:source_with_metadata) { org.logstash.common.SourceWithMetadata.new(source_protocol, source_id, source) }
subject(:compiled) { described_class.compile_pipeline(source_with_metadata) }
let(:source_with_metadata) { org.logstash.common.SourceWithMetadata.new(source_protocol, source_id, 0, 0, source) }
subject(:compiled) { puts "PCOMP"; described_class.compile_pipeline(source_with_metadata) }
describe "compiling multiple sources" do
let(:sources) do
@ -40,7 +40,7 @@ describe LogStash::Compiler do
end
let(:sources_with_metadata) do
sources.map.with_index do |source, idx|
org.logstash.common.SourceWithMetadata.new("#{source_protocol}_#{idx}", "#{source_id}_#{idx}", source)
org.logstash.common.SourceWithMetadata.new("#{source_protocol}_#{idx}", "#{source_id}_#{idx}", 0, 0, source)
end
end
@ -92,7 +92,7 @@ describe LogStash::Compiler do
describe "compiling imperative" do
let(:source_id) { "fake_sourcefile" }
let(:source_with_metadata) { org.logstash.common.SourceWithMetadata.new(source_protocol, source_id, source) }
let(:source_with_metadata) { org.logstash.common.SourceWithMetadata.new(source_protocol, source_id, 0, 0, source) }
subject(:compiled) { described_class.compile_imperative(source_with_metadata) }
describe "an empty file" do

14
logstash-core/spec/logstash/config/pipeline_config_spec.rb
View file

@ -7,13 +7,13 @@ describe LogStash::Config::PipelineConfig do
let(:pipeline_id) { :main }
let(:ordered_config_parts) do
[
org.logstash.common.SourceWithMetadata.new("file", "/tmp/1", "input { generator1 }"),
org.logstash.common.SourceWithMetadata.new("file", "/tmp/2", "input { generator2 }"),
org.logstash.common.SourceWithMetadata.new("file", "/tmp/3", "input { generator3 }"),
org.logstash.common.SourceWithMetadata.new("file", "/tmp/4", "input { generator4 }"),
org.logstash.common.SourceWithMetadata.new("file", "/tmp/5", "input { generator5 }"),
org.logstash.common.SourceWithMetadata.new("file", "/tmp/6", "input { generator6 }"),
org.logstash.common.SourceWithMetadata.new("string", "config_string", "input { generator1 }"),
org.logstash.common.SourceWithMetadata.new("file", "/tmp/1", 0, 0, "input { generator1 }"),
org.logstash.common.SourceWithMetadata.new("file", "/tmp/2", 0, 0, "input { generator2 }"),
org.logstash.common.SourceWithMetadata.new("file", "/tmp/3", 0, 0, "input { generator3 }"),
org.logstash.common.SourceWithMetadata.new("file", "/tmp/4", 0, 0, "input { generator4 }"),
org.logstash.common.SourceWithMetadata.new("file", "/tmp/5", 0, 0, "input { generator5 }"),
org.logstash.common.SourceWithMetadata.new("file", "/tmp/6", 0, 0, "input { generator6 }"),
org.logstash.common.SourceWithMetadata.new("string", "config_string", 0, 0, "input { generator1 }"),
]
end

1
logstash-core/spec/logstash/config/source/local_spec.rb
View file

@ -367,6 +367,7 @@ describe LogStash::Config::Source::Local do
file = Stud::Temporary.file
path = file.path
file.write(config_string)
file.close # we need to flush the write
path
end
let(:settings) { mock_settings( "path.config" => config_path) }

6
logstash-core/spec/logstash/config/source/multi_local_spec.rb
View file

@ -29,7 +29,7 @@ describe LogStash::Config::Source::MultiLocal do
end
context "when `config.path` are set`" do
let(:config_file) { temporary_file("") }
let(:config_file) { temporary_file("input {} output {}") }
let(:settings) do
mock_settings("path.config" => config_file)
@ -85,8 +85,8 @@ describe LogStash::Config::Source::MultiLocal do
describe "#pipeline_configs" do
let(:retrieved_pipelines) do
[
{ "pipeline.id" => "main", "config.string" => "" },
{ "pipeline.id" => "backup", "config.string" => "" }
{ "pipeline.id" => "main", "config.string" => "input {} output {}" },
{ "pipeline.id" => "backup", "config.string" => "input {} output {}" }
]
end
before(:each) do

2
logstash-core/spec/logstash/config/source_loader_spec.rb
View file

@ -4,7 +4,7 @@ require "logstash/config/source/base"
require_relative "../../support/helpers"
def temporary_pipeline_config(id, source, reader = "random_reader")
config_part = org.logstash.common.SourceWithMetadata.new("local", "...", "input {} output {}")
config_part = org.logstash.common.SourceWithMetadata.new("local", "...", 0, 0, "input {} output {}")
LogStash::Config::PipelineConfig.new(source, id, [config_part], LogStash::SETTINGS)
end

2
logstash-core/spec/support/helpers.rb
View file

@ -63,7 +63,7 @@ def mock_pipeline_config(pipeline_id, config_string = nil, settings = {})
settings = mock_settings(settings)
end
config_part = org.logstash.common.SourceWithMetadata.new("config_string", "config_string", config_string)
config_part = org.logstash.common.SourceWithMetadata.new("config_string", "config_string", 0, 0, config_string)
LogStash::Config::PipelineConfig.new(LogStash::Config::Source::Local, pipeline_id, config_part, settings)
end

12
logstash-core/src/main/java/org/logstash/common/IncompleteSourceWithMetadataException.java Normal file
View file

@ -0,0 +1,12 @@
package org.logstash.common;
import org.logstash.config.ir.InvalidIRException;
/**
* Created by andrewvc on 6/12/17.
*/
public class IncompleteSourceWithMetadataException extends InvalidIRException {
public IncompleteSourceWithMetadataException(String message) {
super(message);
}
}

55
logstash-core/src/main/java/org/logstash/common/SourceWithMetadata.java
View file

@ -1,11 +1,19 @@
package org.logstash.common;
import org.logstash.config.ir.HashableWithSource;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Objects;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
* Created by andrewvc on 9/6/16.
*/
public class SourceWithMetadata {
public class SourceWithMetadata implements HashableWithSource {
// Either 'file' or something else
private final String protocol;
// A Unique identifier for the source within the given protocol
@ -35,32 +43,49 @@ public class SourceWithMetadata {
return text;
}
public SourceWithMetadata(String protocol, String id, Integer line, Integer column, String text) {
private static final Pattern emptyString = Pattern.compile("^\\s*$");
public SourceWithMetadata(String protocol, String id, Integer line, Integer column, String text) throws IncompleteSourceWithMetadataException {
this.protocol = protocol;
this.id = id;
this.line = line;
this.column = column;
this.text = text;
List<Object> badAttributes = this.attributes().stream().filter(a -> {
if (a == null) return true;
if (a instanceof String) {
return emptyString.matcher((String) a).matches();
}
return false;
}).collect(Collectors.toList());
if (!badAttributes.isEmpty()){
String message = "Missing attributes in SourceWithMetadata: (" + badAttributes + ") "
+ this.toString();
throw new IncompleteSourceWithMetadataException(message);
}
}
// Convenience method for dealing with files
public SourceWithMetadata(String path, Integer line, Integer column, String text) {
this("file", path, line, column, text);
}
public SourceWithMetadata(String protocol, String id, String text) {
this(protocol, id, 1, 1, text);
}
public SourceWithMetadata() {
this(null, null, null, null, null);
public SourceWithMetadata(String protocol, String id, String text) throws IncompleteSourceWithMetadataException {
this(protocol, id, 0, 0, text);
}
public int hashCode() {
return Objects.hash(this.id, this.line, this.column, this.text);
return Objects.hash(attributes().toArray());
}
public String toString() {
return "[protocol]" + id + ":" + line + ":" + column + ":```\n" + text + "\n```";
return "[" + protocol + "]" + id + ":" + line + ":" + column + ":```\n" + text + "\n```";
}
@Override
public String hashSource() {
return attributes().stream().map(Object::toString).collect(Collectors.joining("|"));
}
// Fields used in the hashSource and hashCode methods to ensure uniqueness
private Collection<Object> attributes() {
return Arrays.asList(this.getId(), this.getProtocol(), this.getLine(), this.getColumn(), this.getText());
}
}

24
logstash-core/src/main/java/org/logstash/config/ir/DSL.java
View file

@ -23,7 +23,7 @@ public class DSL {
}
public static EventValueExpression eEventValue(String fieldName) {
return eEventValue(new SourceWithMetadata(), fieldName);
return eEventValue(null, fieldName);
}
public static ValueExpression eValue(SourceWithMetadata meta, Object value) throws InvalidIRException {
@ -31,7 +31,7 @@ public class DSL {
}
public static ValueExpression eValue(Object value) throws InvalidIRException {
return eValue(new SourceWithMetadata(), value);
return eValue(null, value);
}
public static ValueExpression eRegex(SourceWithMetadata meta, String pattern) throws InvalidIRException {
@ -39,12 +39,12 @@ public class DSL {
}
public static ValueExpression eRegex(String pattern) throws InvalidIRException {
return eRegex(new SourceWithMetadata(), pattern);
return eRegex(null, pattern);
}
public static ValueExpression eValue(long value) {
try {
return eValue(new SourceWithMetadata(), value);
return eValue(null, value);
} catch (InvalidIRException e) {
e.printStackTrace(); // Can't happen with an int
return null;
@ -53,7 +53,7 @@ public class DSL {
public static ValueExpression eValue(double value) {
try {
return eValue(new SourceWithMetadata(), value);
return eValue(null, value);
} catch (InvalidIRException e) {
e.printStackTrace(); // Can't happen with an int
return null;
@ -195,7 +195,7 @@ public class DSL {
}
public static NoopStatement noop() {
return new NoopStatement(new SourceWithMetadata());
return new NoopStatement(null);
}
public static PluginStatement iPlugin(SourceWithMetadata meta, PluginDefinition.Type pluginType, String pluginName, Map<String, Object> pluginArguments) {
@ -203,7 +203,7 @@ public class DSL {
}
public static PluginStatement iPlugin(PluginDefinition.Type type, String pluginName, Map<String, Object> pluginArguments) {
return iPlugin(new SourceWithMetadata(), type, pluginName, pluginArguments);
return iPlugin(null, type, pluginName, pluginArguments);
}
public static PluginStatement iPlugin(PluginDefinition.Type type, String pluginName, MapBuilder<String, Object> argBuilder) {
@ -229,12 +229,12 @@ public class DSL {
public static IfStatement iIf(Expression condition,
Statement ifTrue,
Statement ifFalse) throws InvalidIRException {
return iIf(new SourceWithMetadata(), condition, ifTrue, ifFalse);
return iIf(null, condition, ifTrue, ifFalse);
}
public static IfStatement iIf(Expression condition,
Statement ifTrue) throws InvalidIRException {
return iIf(new SourceWithMetadata(), condition, ifTrue, noop());
return iIf(null, condition, ifTrue, noop());
}
public static class MapBuilder<K,V> {
@ -275,7 +275,7 @@ public class DSL {
}
public static PluginVertex gPlugin(PluginDefinition.Type type, String pluginName, Map<String, Object> pluginArgs) {
return gPlugin(new SourceWithMetadata(), type, pluginName, pluginArgs);
return gPlugin(null, type, pluginName, pluginArgs);
}
public static PluginVertex gPlugin(PluginDefinition.Type type, String pluginName, String id) {
@ -283,7 +283,7 @@ public class DSL {
}
public static PluginVertex gPlugin(PluginDefinition.Type type, String pluginName) {
return gPlugin(new SourceWithMetadata(), type, pluginName, new HashMap<>());
return gPlugin(null, type, pluginName, new HashMap<>());
}
@ -292,6 +292,6 @@ public class DSL {
}
public static IfVertex gIf(BooleanExpression expression) {
return new IfVertex(new SourceWithMetadata(), expression);
return new IfVertex(null, expression);
}
}

6
logstash-core/src/main/java/org/logstash/config/ir/Hashable.java
View file

@ -6,9 +6,5 @@ import org.logstash.common.Util;
* Created by andrewvc on 12/23/16.
*/
public interface Hashable {
String hashSource();
default String uniqueHash() {
return Util.digest(this.hashSource());
}
String uniqueHash();
}

15
logstash-core/src/main/java/org/logstash/config/ir/HashableWithSource.java Normal file
View file

@ -0,0 +1,15 @@
package org.logstash.config.ir;
import org.logstash.common.Util;
import org.logstash.config.ir.Hashable;
/**
* Created by andrewvc on 6/12/17.
*/
public interface HashableWithSource extends Hashable {
@Override
default String uniqueHash() {
return Util.digest(hashSource());
}
String hashSource();
}

10
logstash-core/src/main/java/org/logstash/config/ir/PipelineIR.java
View file

@ -13,6 +13,8 @@ import java.util.stream.Stream;
* Created by andrewvc on 9/20/16.
*/
public class PipelineIR implements Hashable {
private String uniqueHash;
public Graph getGraph() {
return graph;
}
@ -47,6 +49,10 @@ public class PipelineIR implements Hashable {
this.graph = tempGraph.chain(outputSection);
this.graph.validate();
if (this.getOriginalSource() != null && this.getOriginalSource().matches("^\\S+$")) {
uniqueHash = this.graph.uniqueHash();
}
}
public String getOriginalSource() {
@ -101,7 +107,7 @@ public class PipelineIR implements Hashable {
}
@Override
public String hashSource() {
return this.graph.uniqueHash();
public String uniqueHash() {
return this.uniqueHash;
}
}

2
logstash-core/src/main/java/org/logstash/config/ir/PluginDefinition.java
View file

@ -12,7 +12,7 @@ import java.util.Set;
/**
* Created by andrewvc on 9/20/16.
*/
public class PluginDefinition implements SourceComponent, Hashable {
public class PluginDefinition implements SourceComponent, HashableWithSource {
private static ObjectMapper om = new ObjectMapper();
@Override

6
logstash-core/src/main/java/org/logstash/config/ir/expression/BinaryBooleanExpression.java
View file

@ -1,5 +1,6 @@
package org.logstash.config.ir.expression;
import org.logstash.common.Util;
import org.logstash.config.ir.SourceComponent;
import org.logstash.config.ir.InvalidIRException;
import org.logstash.common.SourceWithMetadata;
@ -46,8 +47,7 @@ public abstract class BinaryBooleanExpression extends BooleanExpression {
return "(" + getLeft().toRubyString() + rubyOperator() + getRight().toRubyString() + ")";
}
@Override
public String hashSource() {
return this.getClass().getCanonicalName() + "[" + getLeft().hashSource() + "|" + getRight().hashSource() + "]";
public String uniqueHash() {
return Util.digest(this.getClass().getCanonicalName() + "[" + getLeft().hashSource() + "|" + getRight().hashSource() + "]");
}
}

7
logstash-core/src/main/java/org/logstash/config/ir/expression/Expression.java
View file

@ -6,6 +6,7 @@ import org.jruby.embed.ScriptingContainer;
import org.logstash.config.ir.Hashable;
import org.logstash.config.ir.BaseSourceComponent;
import org.logstash.common.SourceWithMetadata;
import org.logstash.config.ir.HashableWithSource;
/*
* [foo] == "foostr" eAnd [bar] > 10
@ -15,7 +16,7 @@ import org.logstash.common.SourceWithMetadata;
* notnull(eEventValue("foo"))
* Created by andrewvc on 9/6/16.
*/
public abstract class Expression extends BaseSourceComponent implements Hashable {
public abstract class Expression extends BaseSourceComponent implements HashableWithSource {
private ScriptingContainer container;
public Expression(SourceWithMetadata meta) {
@ -40,4 +41,8 @@ public abstract class Expression extends BaseSourceComponent implements Hashable
}
public abstract String toRubyString();
public String hashSource() {
return toRubyString();
}
}

7
logstash-core/src/main/java/org/logstash/config/ir/graph/Graph.java
View file

@ -268,10 +268,6 @@ public class Graph implements SourceComponent, Hashable {
return rank;
}
public Map<String, List<Vertex>> verticesByHash() {
return this.vertices().parallel().collect(Collectors.groupingBy(Vertex::uniqueHash));
}
public void validate() throws InvalidIRException {
if (this.isEmpty()) return;
@ -426,8 +422,7 @@ public class Graph implements SourceComponent, Hashable {
return this.edges.stream();
}
@Override
public String hashSource() {
public String uniqueHash() {
MessageDigest lineageDigest = Util.defaultMessageDigest();
// We only need to calculate the hashes of the leaves since those hashes are sensitive to changes

7
logstash-core/src/main/java/org/logstash/config/ir/graph/IfVertex.java
View file

@ -13,6 +13,8 @@ import java.util.stream.Collectors;
* Created by andrewvc on 9/15/16.
*/
public class IfVertex extends Vertex {
private volatile String generatedId;
public BooleanExpression getBooleanExpression() {
return booleanExpression;
}
@ -66,11 +68,6 @@ public class IfVertex extends Vertex {
return (e instanceof BooleanEdge);
}
@Override
public String getId() {
return this.uniqueHash();
}
public Collection<BooleanEdge> getOutgoingBooleanEdges() {
// Wish there was a way to do this as a java a cast without an operation
return getOutgoingEdges().stream().map(e -> (BooleanEdge) e).collect(Collectors.toList());

28
logstash-core/src/main/java/org/logstash/config/ir/graph/PluginVertex.java
View file

@ -7,21 +7,15 @@ import org.logstash.config.ir.SourceComponent;
import org.logstash.config.ir.PluginDefinition;
import org.logstash.common.SourceWithMetadata;
import java.util.UUID;
/**
* Created by andrewvc on 9/15/16.
*/
public class PluginVertex extends Vertex {
private final SourceWithMetadata meta;
private final String id;
private final PluginDefinition pluginDefinition;
public String getId() {
if (id != null) return id;
if (this.getGraph() == null) {
throw new RuntimeException("Attempted to get ID from PluginVertex before attaching it to a graph!");
}
return this.uniqueHash();
}
private volatile String generatedId;
public PluginDefinition getPluginDefinition() {
return pluginDefinition;
@ -31,15 +25,11 @@ public class PluginVertex extends Vertex {
return meta;
}
public PluginVertex(SourceWithMetadata meta, PluginDefinition pluginDefinition) {
super(meta);
// We know that if the ID value exists it will be as a string
super(meta, (String) pluginDefinition.getArguments().get("id"));
this.meta = meta;
this.pluginDefinition = pluginDefinition;
Object argId = this.pluginDefinition.getArguments().get("id");
this.id = argId != null ? argId.toString() : null;
}
public String toString() {
@ -51,7 +41,7 @@ public class PluginVertex extends Vertex {
ObjectMapper objectMapper = new ObjectMapper();
try {
return Util.digest(this.getClass().getCanonicalName() + "|" +
(this.id != null ? this.id : "NOID") + "|" +
(this.getExplicitId() != null ? this.getExplicitId() : "NOID") + "|" +
this.pluginDefinition.getName() + "|" +
this.pluginDefinition.getType().toString() + "|" +
objectMapper.writeValueAsString(this.pluginDefinition.getArguments()));
@ -78,4 +68,10 @@ public class PluginVertex extends Vertex {
}
return false;
}
@Override
public void clearCache() {
super.clearCache();
this.generatedId = null;
}
}

38
logstash-core/src/main/java/org/logstash/config/ir/graph/Vertex.java
View file

@ -1,7 +1,7 @@
package org.logstash.config.ir.graph;
import org.logstash.common.Util;
import org.logstash.config.ir.Hashable;
import org.logstash.config.ir.HashableWithSource;
import org.logstash.config.ir.SourceComponent;
import org.logstash.config.ir.InvalidIRException;
import org.logstash.common.SourceWithMetadata;
@ -17,19 +17,26 @@ import java.util.stream.Stream;
/**
* Created by andrewvc on 9/15/16.
*/
public abstract class Vertex implements SourceComponent, Hashable {
public abstract class Vertex implements SourceComponent, HashableWithSource {
private final SourceWithMetadata sourceWithMetadata;
private Graph graph = this.getGraph();
private volatile String contextualHashCache;
private volatile String hashCache;
private volatile String individualHashSourceCache;
private final String explicitId;
private volatile String generatedId;
public Vertex() {
this.sourceWithMetadata = null;
this(null);
}
public Vertex(SourceWithMetadata sourceWithMetadata) {
this(sourceWithMetadata, null);
}
public Vertex(SourceWithMetadata sourceWithMetadata, String explicitId) {
this.sourceWithMetadata = sourceWithMetadata;
this.explicitId = explicitId;
}
public abstract Vertex copy();
@ -217,7 +224,30 @@ public abstract class Vertex implements SourceComponent, Hashable {
return true;
}
public abstract String getId();
public String getExplicitId() {
return this.explicitId;
}
public String getId() {
if (explicitId != null) return explicitId;
if (generatedId != null) return generatedId;
if (this.getGraph() == null) {
throw new RuntimeException("Attempted to get ID from PluginVertex before attaching it to a graph!");
}
// Generating unique hashes for vertices is very slow!
// We try to avoid this where possible, which means that generally only tests hit the path with hashes, since
// they have no source metadata. This might also be used in the future by alternate config languages which are
// willing to take the hit.
if (this.getSourceWithMetadata() != null) {
generatedId = this.getGraph().uniqueHash() + "|" + this.getSourceWithMetadata().uniqueHash();
} else {
generatedId = this.uniqueHash();
}
return generatedId;
}
public void clearCache() {
this.hashCache = null;

64
logstash-core/src/test/java/org/logstash/common/SourceWithMetadataTest.java Normal file
View file

@ -0,0 +1,64 @@
package org.logstash.common;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import java.util.Arrays;
import java.util.Collection;
/**
* Created by andrewvc on 6/12/17.
*/
@RunWith(Parameterized.class)
public class SourceWithMetadataTest {
private final ParameterGroup parameterGroup;
public static class ParameterGroup {
public final String protocol;
public final String path;
public final Integer line;
public final Integer column;
public final String text;
public ParameterGroup(String protocol, String path, Integer line, Integer column, String text) {
this.protocol = protocol;
this.path = path;
this.line = line;
this.column = column;
this.text = text;
}
}
@Parameterized.Parameters
public static Iterable<ParameterGroup> data() {
return Arrays.asList(
new ParameterGroup(null, "path", 1, 1, "foo"),
new ParameterGroup("proto", null, 1, 1, "foo"),
new ParameterGroup("proto", "path", null, 1, "foo"),
new ParameterGroup("proto", "path", 1, null, "foo"),
new ParameterGroup("proto", "path", 1, 1, null),
new ParameterGroup("", "path", 1, 1, "foo"),
new ParameterGroup("proto", "", 1, 1, "foo"),
new ParameterGroup("proto", "path", 1, 1, ""),
new ParameterGroup(" ", "path", 1, 1, "foo"),
new ParameterGroup("proto", " ", 1, 1, "foo"),
new ParameterGroup("proto", "path", 1, 1, " ")
);
}
public SourceWithMetadataTest(ParameterGroup parameterGroup) {
this.parameterGroup = parameterGroup;
}
// So, this really isn't parameterized, but it isn't worth making a separate class for this
@Test
public void itShouldInstantiateCleanlyWhenParamsAreGood() throws IncompleteSourceWithMetadataException {
new SourceWithMetadata("proto", "path", 1, 1, "text");
}
@Test(expected = IncompleteSourceWithMetadataException.class)
public void itShouldThrowWhenMissingAField() throws IncompleteSourceWithMetadataException {
new SourceWithMetadata(parameterGroup.protocol, parameterGroup.path, parameterGroup.line, parameterGroup.column, parameterGroup.text);
}
}

5
logstash-core/src/test/java/org/logstash/config/ir/IRHelpers.java
View file

@ -1,6 +1,7 @@
package org.logstash.config.ir;
import org.hamcrest.MatcherAssert;
import org.logstash.common.IncompleteSourceWithMetadataException;
import org.logstash.common.SourceWithMetadata;
import org.logstash.config.ir.expression.BooleanExpression;
import org.logstash.config.ir.expression.ValueExpression;
@ -116,8 +117,8 @@ public class IRHelpers {
return new Truthy(null, new ValueExpression(null, 1));
}
public static SourceWithMetadata testMetadata() {
return new SourceWithMetadata("/fake/file", 1, 2, "<fakesource>");
public static SourceWithMetadata testMetadata() throws IncompleteSourceWithMetadataException {
return new SourceWithMetadata("file", "/fake/file", 1, 2, "<fakesource>");
}
public static PluginDefinition testPluginDefinition() {

1
logstash-core/src/test/java/org/logstash/config/ir/graph/GraphTest.java
View file

@ -9,6 +9,7 @@ import org.logstash.config.ir.imperative.IfStatement;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.CoreMatchers.is;

4
logstash-core/src/test/java/org/logstash/config/ir/graph/PluginVertexTest.java
View file

@ -1,6 +1,8 @@
package org.logstash.config.ir.graph;
import org.junit.Test;
import org.logstash.common.IncompleteSourceWithMetadataException;
import org.logstash.common.SourceWithMetadata;
import org.logstash.config.ir.InvalidIRException;
import org.logstash.config.ir.PluginDefinition;
@ -25,7 +27,7 @@ public class PluginVertexTest {
}
@Test
public void testConstructionIdHandlingWhenExplicitId() {
public void testConstructionIdHandlingWhenExplicitId() throws IncompleteSourceWithMetadataException {
String customId = "mycustomid";
Map<String, Object> pluginArguments = new HashMap<>();
pluginArguments.put("id", customId);