github-mirrors/logstash
1
0
Fork 0
mirror of https://github.com/elastic/logstash.git synced 2025-04-24 06:37:19 -04:00
Code Issues Projects Releases Packages Wiki Activity

Backport of Accessors performance improvements to 6.0

Browse source 
Fixes #7910
This commit is contained in:
Armin 2017-08-04 17:07:01 +02:00 • committed by Armin Braun
parent ee7e6fb932
commit b0fb18ce01
9 changed files with 460 additions and 433 deletions
Download patch file Download diff file Expand all files Collapse all files

269
logstash-core/src/main/java/org/logstash/Accessors.java
View file

@ -1,208 +1,155 @@
package org.logstash;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class Accessors {
public final class Accessors {
private Map<String, Object> data;
protected Map<String, Object> lut;
public Accessors(Map<String, Object> data) {
this.data = data;
this.lut = new HashMap<>(); // reference -> target LUT
private Accessors() {
//Utility Class
}
public Object get(String reference) {
FieldReference field = PathCache.cache(reference);
Object target = findTarget(field);
return (target == null) ? null : fetch(target, field.getKey());
public static Object get(final ConvertedMap data, final FieldReference field) {
final Object target = findParent(data, field);
return target == null ? null : fetch(target, field.getKey());
}
public Object set(String reference, Object value) {
final FieldReference field = PathCache.cache(reference);
final Object target = findCreateTarget(field);
final String key = field.getKey();
if (target instanceof Map) {
((Map<String, Object>) target).put(key, value);
} else if (target instanceof List) {
int i;
try {
i = Integer.parseInt(key);
} catch (NumberFormatException e) {
return null;
}
int size = ((List<Object>) target).size();
if (i >= size) {
// grow array by adding trailing null items
// this strategy reflects legacy Ruby impl behaviour and is backed by specs
// TODO: (colin) this is potentially dangerous, and could produce OOM using arbitrary big numbers
// TODO: (colin) should be guard against this?
for (int j = size; j < i; j++) {
((List<Object>) target).add(null);
}
((List<Object>) target).add(value);
} else {
int offset = listIndex(i, ((List) target).size());
((List<Object>) target).set(offset, value);
}
public static Object set(final ConvertedMap data, final FieldReference field,
final Object value) {
return setChild(findCreateTarget(data, field), field.getKey(), value);
}
public static Object del(final ConvertedMap data, final FieldReference field) {
final Object target = findParent(data, field);
if (target instanceof ConvertedMap) {
return ((ConvertedMap) target).remove(field.getKey());
} else {
throw newCollectionException(target);
return target == null ? null : delFromList((ConvertedList) target, field.getKey());
}
}
public static boolean includes(final ConvertedMap data, final FieldReference field) {
final Object target = findParent(data, field);
final String key = field.getKey();
return target instanceof ConvertedMap && ((ConvertedMap) target).containsKey(key) ||
target instanceof ConvertedList && foundInList(key, (ConvertedList) target);
}
private static Object delFromList(final ConvertedList list, final String key) {
try {
return list.remove(listIndex(key, list.size()));
} catch (IndexOutOfBoundsException | NumberFormatException e) {
return null;
}
}
private static Object setOnList(final String key, final Object value, final ConvertedList list) {
final int index;
try {
index = Integer.parseInt(key);
} catch (NumberFormatException e) {
return null;
}
final int size = list.size();
if (index >= size) {
appendAtIndex(list, value, index, size);
} else {
list.set(listIndex(index, size), value);
}
return value;
}
public Object del(String reference) {
FieldReference field = PathCache.cache(reference);
Object target = findTarget(field);
if (target != null) {
if (target instanceof Map) {
return ((Map<String, Object>) target).remove(field.getKey());
} else if (target instanceof List) {
try {
int i = Integer.parseInt(field.getKey());
int offset = listIndex(i, ((List) target).size());
return ((List)target).remove(offset);
} catch (IndexOutOfBoundsException|NumberFormatException e) {
return null;
}
} else {
throw newCollectionException(target);
}
private static void appendAtIndex(final ConvertedList list, final Object value, final int index,
final int size) {
// grow array by adding trailing null items
// this strategy reflects legacy Ruby impl behaviour and is backed by specs
// TODO: (colin) this is potentially dangerous, and could produce OOM using arbitrary big numbers
// TODO: (colin) should be guard against this?
for (int i = size; i < index; i++) {
list.add(null);
}
return null;
list.add(value);
}
public boolean includes(String reference) {
final FieldReference field = PathCache.cache(reference);
final Object target = findTarget(field);
final String key = field.getKey();
return target instanceof Map && ((Map<String, Object>) target).containsKey(key) ||
target instanceof List && foundInList(key, (List<Object>) target);
}
private static boolean foundInList(final String key, final List<Object> target) {
try {
return foundInList(target, Integer.parseInt(key));
} catch (NumberFormatException e) {
return false;
}
}
private Object findTarget(FieldReference field) {
Object target;
if ((target = this.lut.get(field.getReference())) != null) {
return target;
}
target = this.data;
for (String key : field.getPath()) {
private static Object findParent(final ConvertedMap data, final FieldReference field) {
Object target = data;
for (final String key : field.getPath()) {
target = fetch(target, key);
if (! isCollection(target)) {
if (!(target instanceof ConvertedMap || target instanceof ConvertedList)) {
return null;
}
}
this.lut.put(field.getReference(), target);
return target;
}
private Object findCreateTarget(FieldReference field) {
Object target;
// flush the @lut to prevent stale cached fieldref which may point to an old target
// which was overwritten with a new value. for example, if "[a][b]" is cached and we
// set a new value for "[a]" then reading again "[a][b]" would point in a stale target.
// flushing the complete @lut is suboptimal, but a hierarchical lut would be required
// to be able to invalidate fieldrefs from a common root.
// see https://github.com/elastic/logstash/pull/5132
this.lut.clear();
target = this.data;
for (String key : field.getPath()) {
Object result = fetch(target, key);
if (result == null) {
result = new HashMap<String, Object>();
if (target instanceof Map) {
((Map<String, Object>)target).put(key, result);
} else if (target instanceof List) {
try {
int i = Integer.parseInt(key);
// TODO: what about index out of bound?
((List<Object>)target).set(i, result);
} catch (NumberFormatException e) {
continue;
}
} else if (target != null) {
throw newCollectionException(target);
private static Object findCreateTarget(final ConvertedMap data, final FieldReference field) {
Object target = data;
boolean create = false;
for (final String key : field.getPath()) {
Object result;
if (create) {
result = createChild((ConvertedMap) target, key);
} else {
result = fetch(target, key);
create = result == null;
if (create) {
result = new ConvertedMap(1);
setChild(target, key, result);
}
}
target = result;
}
this.lut.put(field.getReference(), target);
return target;
}
private static boolean foundInList(List<Object> target, int index) {
try {
int offset = listIndex(index, target.size());
return target.get(offset) != null;
} catch (IndexOutOfBoundsException e) {
return false;
private static Object setChild(final Object target, final String key, final Object value) {
if (target instanceof Map) {
((ConvertedMap) target).put(key, value);
return value;
} else {
return setOnList(key, value, (ConvertedList) target);
}
}
private static Object createChild(final ConvertedMap target, final String key) {
final Object result = new ConvertedMap(1);
target.put(key, result);
return result;
}
private static Object fetch(Object target, String key) {
if (target instanceof Map) {
Object result = ((Map<String, Object>) target).get(key);
return result;
} else if (target instanceof List) {
try {
int offset = listIndex(Integer.parseInt(key), ((List) target).size());
return ((List<Object>) target).get(offset);
} catch (IndexOutOfBoundsException|NumberFormatException e) {
return null;
}
} else if (target == null) {
return target instanceof ConvertedMap
? ((ConvertedMap) target).get(key) : fetchFromList((ConvertedList) target, key);
}
private static Object fetchFromList(final ConvertedList list, final String key) {
try {
return list.get(listIndex(key, list.size()));
} catch (IndexOutOfBoundsException | NumberFormatException e) {
return null;
} else {
throw newCollectionException(target);
}
}
private static boolean isCollection(Object target) {
if (target == null) {
return false;
}
return (target instanceof Map || target instanceof List);
private static boolean foundInList(final String key, final ConvertedList target) {
return fetchFromList(target, key) != null;
}
private static ClassCastException newCollectionException(Object target) {
return new ClassCastException("expecting List or Map, found " + target.getClass());
}
/*
* Returns a positive integer offset for a list of known size.
*
* @param i if positive, and offset from the start of the list. If negative, the offset from the end of the list, where -1 means the last element.
* @param size the size of the list.
* @return the positive integer offset for the list given by index i.
/**
* Returns a positive integer offset from a Ruby style positive or negative list index.
* @param i List index
* @param size the size of the list
* @return the positive integer offset for the list given by index i
*/
public static int listIndex(int i, int size) {
if (i >= size || i < -size) {
throw new IndexOutOfBoundsException("Index " + i + " is out of bounds for a list with size " + size);
}
return i < 0 ? size + i : i;
}
if (i < 0) { // Offset from the end of the array.
return size + i;
} else {
return i;
}
/**
* Returns a positive integer offset for a list of known size.
* @param key List index (String matching /[0-9]+/)
* @param size the size of the list
* @return the positive integer offset for the list given by index i
*/
private static int listIndex(final String key, final int size) {
return listIndex(Integer.parseInt(key), size);
}
}

2
logstash-core/src/main/java/org/logstash/ConvertedList.java
View file

@ -9,7 +9,7 @@ import static org.logstash.Valuefier.convert;
public final class ConvertedList extends ArrayList<Object> {
private ConvertedList(final int size) {
ConvertedList(final int size) {
super(size);
}

2
logstash-core/src/main/java/org/logstash/ConvertedMap.java
View file

@ -8,7 +8,7 @@ import org.jruby.runtime.builtin.IRubyObject;
public final class ConvertedMap extends HashMap<String, Object> {
private ConvertedMap(final int size) {
ConvertedMap(final int size) {
super((size << 2) / 3 + 2);
}

181
logstash-core/src/main/java/org/logstash/Event.java
View file

@ -2,7 +2,6 @@ package org.logstash;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
@ -12,6 +11,7 @@ import org.apache.logging.log4j.Logger;
import org.joda.time.DateTime;
import org.jruby.RubySymbol;
import org.logstash.ackedqueue.Queueable;
import org.logstash.bivalues.BiValues;
import org.logstash.bivalues.NullBiValue;
import org.logstash.bivalues.StringBiValue;
import org.logstash.bivalues.TimeBiValue;
@ -24,11 +24,9 @@ import static org.logstash.ObjectMappers.JSON_MAPPER;
public final class Event implements Cloneable, Queueable {
private boolean cancelled;
private Map<String, Object> data;
private Map<String, Object> metadata;
private ConvertedMap data;
private ConvertedMap metadata;
private Timestamp timestamp;
private Accessors accessors;
private Accessors metadata_accessors;
public static final String METADATA = "@metadata";
public static final String METADATA_BRACKETS = "[" + METADATA + "]";
@ -40,18 +38,18 @@ public final class Event implements Cloneable, Queueable {
private static final String DATA_MAP_KEY = "DATA";
private static final String META_MAP_KEY = "META";
private static final FieldReference TAGS_FIELD = PathCache.cache("tags");
private static final Logger logger = LogManager.getLogger(Event.class);
public Event()
{
this.metadata = new HashMap<>();
this.data = new HashMap<>();
this.metadata = new ConvertedMap(10);
this.data = new ConvertedMap(10);
this.data.put(VERSION, VERSION_ONE);
this.cancelled = false;
this.timestamp = new Timestamp();
this.data.put(TIMESTAMP, this.timestamp);
this.accessors = new Accessors(this.data);
this.metadata_accessors = new Accessors(this.metadata);
}
/**
@ -76,22 +74,17 @@ public final class Event implements Cloneable, Queueable {
}
if (this.data.containsKey(METADATA)) {
this.metadata = (Map<String, Object>) this.data.remove(METADATA);
this.metadata = ConvertedMap.newFromMap((Map) this.data.remove(METADATA));
} else {
this.metadata = new HashMap<>();
this.metadata = new ConvertedMap(10);
}
this.metadata_accessors = new Accessors(this.metadata);
this.cancelled = false;
Object providedTimestamp = data.get(TIMESTAMP);
// keep reference to the parsedTimestamp for tagging below
Timestamp parsedTimestamp = initTimestamp(providedTimestamp);
this.timestamp = (parsedTimestamp == null) ? Timestamp.now() : parsedTimestamp;
this.data.put(TIMESTAMP, this.timestamp);
this.accessors = new Accessors(this.data);
Accessors.set(data, FieldReference.TIMESTAMP_REFERENCE, timestamp);
// the tag() method has to be called after the Accessors initialization
if (parsedTimestamp == null) {
tag(TIMESTAMP_FAILURE_TAG);
@ -99,18 +92,14 @@ public final class Event implements Cloneable, Queueable {
}
}
public Map<String, Object> getData() {
public ConvertedMap getData() {
return this.data;
}
public Map<String, Object> getMetadata() {
public ConvertedMap getMetadata() {
return this.metadata;
}
private Accessors getAccessors() {
return this.accessors;
}
public void cancel() {
this.cancelled = true;
}
@ -136,42 +125,55 @@ public final class Event implements Cloneable, Queueable {
this.data.put(TIMESTAMP, this.timestamp);
}
public Object getField(String reference) {
Object val = getUnconvertedField(reference);
return Javafier.deep(val);
public Object getField(final String reference) {
final Object unconverted = getUnconvertedField(PathCache.cache(reference));
return unconverted == null ? null : Javafier.deep(unconverted);
}
public Object getUnconvertedField(String reference) {
if (reference.equals(METADATA)) {
return this.metadata;
} else if (reference.startsWith(METADATA_BRACKETS)) {
return this.metadata_accessors.get(reference.substring(METADATA_BRACKETS.length()));
} else {
return this.accessors.get(reference);
public Object getUnconvertedField(final String reference) {
return getUnconvertedField(PathCache.cache(reference));
}
public Object getUnconvertedField(final FieldReference field) {
switch (field.type()) {
case FieldReference.META_PARENT:
return this.metadata;
case FieldReference.META_CHILD:
return Accessors.get(metadata, field);
default:
return Accessors.get(data, field);
}
}
public void setField(String reference, Object value) {
if (reference.equals(TIMESTAMP)) {
// TODO(talevy): check type of timestamp
this.accessors.set(reference, value);
} else if (reference.equals(METADATA_BRACKETS) || reference.equals(METADATA)) {
this.metadata = (Map<String, Object>) value;
this.metadata_accessors = new Accessors(this.metadata);
} else if (reference.startsWith(METADATA_BRACKETS)) {
this.metadata_accessors.set(reference.substring(METADATA_BRACKETS.length()), value);
} else {
this.accessors.set(reference, Valuefier.convert(value));
public void setField(final String reference, final Object value) {
setField(PathCache.cache(reference), value);
}
public void setField(final FieldReference field, final Object value) {
switch (field.type()) {
case FieldReference.META_PARENT:
this.metadata = ConvertedMap.newFromMap((Map) value);
break;
case FieldReference.META_CHILD:
Accessors.set(metadata, field, value);
break;
default:
Accessors.set(data, field, Valuefier.convert(value));
}
}
public boolean includes(String reference) {
if (reference.equals(METADATA_BRACKETS) || reference.equals(METADATA)) {
return true;
} else if (reference.startsWith(METADATA_BRACKETS)) {
return this.metadata_accessors.includes(reference.substring(METADATA_BRACKETS.length()));
} else {
return this.accessors.includes(reference);
public boolean includes(final String field) {
return includes(PathCache.cache(field));
}
public boolean includes(final FieldReference field) {
switch (field.type()) {
case FieldReference.META_PARENT:
return true;
case FieldReference.META_CHILD:
return Accessors.includes(metadata, field);
default:
return Accessors.includes(data, field);
}
}
@ -244,7 +246,6 @@ public final class Event implements Cloneable, Queueable {
public Event overwrite(Event e) {
this.data = e.getData();
this.accessors = e.getAccessors();
this.cancelled = e.isCancelled();
try {
this.timestamp = e.getTimestamp();
@ -261,8 +262,12 @@ public final class Event implements Cloneable, Queueable {
return this;
}
public Object remove(String path) {
return this.accessors.del(path);
public Object remove(final String path) {
return remove(PathCache.cache(path));
}
public Object remove(final FieldReference field) {
return Accessors.del(data, field);
}
public String sprintf(String s) throws IOException {
@ -271,7 +276,7 @@ public final class Event implements Cloneable, Queueable {
@Override
public Event clone() {
return new Event(Cloner.deep(this.data));
return new Event(Cloner.<Map>deep(this.data));
}
public String toString() {
@ -331,33 +336,63 @@ public final class Event implements Cloneable, Queueable {
return null;
}
public void tag(String tag) {
List<Object> tags;
Object _tags = this.getField("tags");
public void tag(final String tag) {
final Object tags = Accessors.get(data, TAGS_FIELD);
// short circuit the null case where we know we won't need deduplication step below at the end
if (_tags == null) {
setField("tags", Arrays.asList(tag));
return;
}
// assign to tags var the proper List of either the existing _tags List or a new List containing whatever non-List item was in the tags field
if (_tags instanceof List) {
tags = (List<Object>) _tags;
if (tags == null) {
initTag(tag);
} else {
// tags field has a value but not in a List, convert in into a List
tags = new ArrayList<>();
tags.add(_tags);
existingTag(Javafier.deep(tags), tag);
}
}
// now make sure the tags list does not already contain the tag
/**
* Branch of {@link Event#tag(String)} that handles adding the first tag to this event.
* @param tag Tag to add
*/
private void initTag(final String tag) {
final ConvertedList list = new ConvertedList(1);
list.add(BiValues.newBiValue(tag));
Accessors.set(data, TAGS_FIELD, list);
}
/**
* Branch of {@link Event#tag(String)} that handles adding to existing tags.
* @param tags Existing Tag(s)
* @param tag Tag to add
*/
private void existingTag(final Object tags, final String tag) {
if (tags instanceof List) {
appendTag((List<String>) tags, tag);
} else {
scalarTagFallback((String) tags, tag);
}
}
/**
* Merge the given tag into the given list of existing tags if the list doesn't already contain
* the tag.
* @param tags Existing tag list
* @param tag Tag to add
*/
private void appendTag(final List<String> tags, final String tag) {
// TODO: we should eventually look into using alternate data structures to do more efficient dedup but that will require properly defining the tagging API too
if (!tags.contains(tag)) {
tags.add(tag);
Accessors.set(data, TAGS_FIELD, ConvertedList.newFromList((List) tags));
}
}
// set that back as a proper BiValue
this.setField("tags", tags);
/**
* Fallback for {@link Event#tag(String)} in case "tags" was populated by just a String value
* and needs to be converted to a list before appending to it.
* @param existing Existing Tag
* @param tag Tag to add
*/
private void scalarTagFallback(final String existing, final String tag) {
final List<String> tags = new ArrayList<>(2);
tags.add(existing);
appendTag(tags, tag);
}
@Override

144
logstash-core/src/main/java/org/logstash/FieldReference.java
View file

@ -1,25 +1,99 @@
package org.logstash;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
// TODO: implement thread-safe path cache singleton to avoid parsing
public class FieldReference {
public final class FieldReference {
/**
* This type indicates that the referenced that is the metadata of an {@link Event} found in
* {@link Event#metadata}.
*/
public static final int META_PARENT = 0;
/**
* This type indicates that the referenced data must be looked up from {@link Event#metadata}.
*/
public static final int META_CHILD = 1;
/**
* This type indicates that the referenced data must be looked up from {@link Event#data}.
*/
private static final int DATA_CHILD = -1;
private static final String[] EMPTY_STRING_ARRAY = new String[0];
private static final Pattern SPLIT_PATTERN = Pattern.compile("[\\[\\]]");
private List<String> path;
private String key;
private String reference;
/**
* Holds all existing {@link FieldReference} instances for de-duplication.
*/
private static final Map<FieldReference, FieldReference> DEDUP = new HashMap<>(64);
public FieldReference(List<String> path, String key, String reference) {
this.path = path;
/**
* Unique {@link FieldReference} pointing at the timestamp field in a {@link Event}.
*/
public static final FieldReference TIMESTAMP_REFERENCE =
deduplicate(new FieldReference(EMPTY_STRING_ARRAY, Event.TIMESTAMP, DATA_CHILD));
private static final FieldReference METADATA_PARENT_REFERENCE =
new FieldReference(EMPTY_STRING_ARRAY, Event.METADATA, META_PARENT);
private final String[] path;
private final String key;
private final int hash;
/**
* Either {@link FieldReference#META_PARENT}, {@link FieldReference#META_CHILD} or
* {@link FieldReference#DATA_CHILD}.
*/
private final int type;
private FieldReference(final String[] path, final String key, final int type) {
this.key = key;
this.reference = reference;
this.type = type;
this.path = path;
hash = calculateHash(this.key, this.path, this.type);
}
public List<String> getPath() {
public static FieldReference parse(final CharSequence reference) {
final String[] parts = SPLIT_PATTERN.split(reference);
final List<String> path = new ArrayList<>(parts.length);
for (final String part : parts) {
if (!part.isEmpty()) {
path.add(part.intern());
}
}
final String key = path.remove(path.size() - 1).intern();
final boolean empty = path.isEmpty();
if (empty && key.equals(Event.METADATA)) {
return METADATA_PARENT_REFERENCE;
} else if (!empty && path.get(0).equals(Event.METADATA)) {
return deduplicate(new FieldReference(
path.subList(1, path.size()).toArray(EMPTY_STRING_ARRAY), key, META_CHILD));
} else {
return deduplicate(
new FieldReference(path.toArray(EMPTY_STRING_ARRAY), key, DATA_CHILD));
}
}
/**
* Returns the type of this instance to allow for fast switch operations in
* {@link Event#getUnconvertedField(FieldReference)} and
* {@link Event#setField(FieldReference, Object)}.
* @return Type of the FieldReference
*/
public int type() {
return type;
}
public String[] getPath() {
return path;
}
@ -27,19 +101,49 @@ public class FieldReference {
return key;
}
public String getReference() {
return reference;
@Override
public boolean equals(final Object that) {
if (this == that) return true;
if (!(that instanceof FieldReference)) return false;
final FieldReference other = (FieldReference) that;
return type == other.type && key.equals(other.key) && Arrays.equals(path, other.path);
}
public static FieldReference parse(String reference) {
final String[] parts = SPLIT_PATTERN.split(reference);
List<String> path = new ArrayList<>(parts.length);
for (final String part : parts) {
if (!part.isEmpty()) {
path.add(part);
}
@Override
public int hashCode() {
return hash;
}
/**
* De-duplicates instances using {@link FieldReference#DEDUP}. This method must be
* {@code synchronized} since we are running non-atomic get-put sequence on
* {@link FieldReference#DEDUP}.
* @param parsed FieldReference to de-duplicate
* @return De-duplicated FieldReference
*/
private static synchronized FieldReference deduplicate(final FieldReference parsed) {
FieldReference ret = DEDUP.get(parsed);
if (ret == null) {
DEDUP.put(parsed, parsed);
ret = parsed;
}
String key = path.remove(path.size() - 1);
return new FieldReference(path, key, reference);
return ret;
}
/**
* Effective hashcode implementation using knowledge of field types.
* @param key Key Field
* @param path Path Field
* @param type Type Field
* @return Hash Code
*/
private static int calculateHash(final String key, final String[] path, final int type) {
final int prime = 31;
int hash = prime;
for (final String element : path) {
hash = prime * hash + element.hashCode();
}
hash = prime * hash + key.hashCode();
return prime * hash + type;
}
}

36
logstash-core/src/main/java/org/logstash/PathCache.java
View file

@ -1,36 +1,28 @@
package org.logstash;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
public final class PathCache {
private static final ConcurrentHashMap<String, FieldReference> cache = new ConcurrentHashMap<>();
private static final Map<CharSequence, FieldReference> CACHE =
new ConcurrentHashMap<>(64, 0.2F, 1);
private static final FieldReference timestamp = cache(Event.TIMESTAMP);
private static final String BRACKETS_TIMESTAMP = "[" + Event.TIMESTAMP + "]";
static {
// inject @timestamp
cache(BRACKETS_TIMESTAMP, timestamp);
private PathCache() {
}
public static boolean isTimestamp(String reference) {
return cache(reference) == timestamp;
}
public static FieldReference cache(String reference) {
public static FieldReference cache(final CharSequence reference) {
// atomicity between the get and put is not important
FieldReference result = cache.get(reference);
if (result == null) {
result = FieldReference.parse(reference);
cache.put(reference, result);
final FieldReference result = CACHE.get(reference);
if (result != null) {
return result;
}
return parseToCache(reference);
}
private static FieldReference parseToCache(final CharSequence reference) {
final FieldReference result = FieldReference.parse(reference);
CACHE.put(reference, result);
return result;
}
public static FieldReference cache(String reference, FieldReference field) {
cache.put(reference, field);
return field;
}
}

27
logstash-core/src/main/java/org/logstash/ext/JrubyEventExtLibrary.java
View file

@ -22,6 +22,7 @@ import org.jruby.runtime.builtin.IRubyObject;
import org.jruby.runtime.load.Library;
import org.logstash.ConvertedMap;
import org.logstash.Event;
import org.logstash.FieldReference;
import org.logstash.PathCache;
import org.logstash.Rubyfier;
import org.logstash.Valuefier;
@ -112,16 +113,17 @@ public class JrubyEventExtLibrary implements Library {
@JRubyMethod(name = "get", required = 1)
public IRubyObject ruby_get_field(ThreadContext context, RubyString reference)
{
Object value = this.event.getUnconvertedField(reference.asJavaString());
return Rubyfier.deep(context.runtime, value);
return Rubyfier.deep(
context.runtime,
this.event.getUnconvertedField(PathCache.cache(reference.getByteList()))
);
}
@JRubyMethod(name = "set", required = 2)
public IRubyObject ruby_set_field(ThreadContext context, RubyString reference, IRubyObject value)
{
String r = reference.asJavaString();
if (PathCache.isTimestamp(r)) {
final FieldReference r = PathCache.cache(reference.getByteList());
if (r == FieldReference.TIMESTAMP_REFERENCE) {
if (!(value instanceof JrubyTimestampExtLibrary.RubyTimestamp)) {
throw context.runtime.newTypeError("wrong argument type " + value.getMetaClass() + " (expected LogStash::Timestamp)");
}
@ -153,15 +155,18 @@ public class JrubyEventExtLibrary implements Library {
}
@JRubyMethod(name = "include?", required = 1)
public IRubyObject ruby_includes(ThreadContext context, RubyString reference)
{
return RubyBoolean.newBoolean(context.runtime, this.event.includes(reference.asJavaString()));
public IRubyObject ruby_includes(ThreadContext context, RubyString reference) {
return RubyBoolean.newBoolean(
context.runtime, this.event.includes(PathCache.cache(reference.getByteList()))
);
}
@JRubyMethod(name = "remove", required = 1)
public IRubyObject ruby_remove(ThreadContext context, RubyString reference)
{
return Rubyfier.deep(context.runtime, this.event.remove(reference.asJavaString()));
public IRubyObject ruby_remove(ThreadContext context, RubyString reference) {
return Rubyfier.deep(
context.runtime,
this.event.remove(PathCache.cache(reference.getByteList()))
);
}
@JRubyMethod(name = "clone")

211
logstash-core/src/test/java/org/logstash/AccessorsTest.java
View file

@ -1,16 +1,12 @@
package org.logstash;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.theories.DataPoint;
import org.junit.experimental.theories.Theories;
import org.junit.experimental.theories.Theory;
import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
import org.logstash.bivalues.BiValues;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@ -19,111 +15,76 @@ import static org.junit.Assert.assertTrue;
public class AccessorsTest {
public class TestableAccessors extends Accessors {
public TestableAccessors(Map<String, Object> data) {
super(data);
}
public Object lutGet(String reference) {
return this.lut.get(reference);
}
}
@Test
public void testBareGet() throws Exception {
Map<String, Object> data = new HashMap<>();
Map<Serializable, Object> data = new HashMap<>();
data.put("foo", "bar");
String reference = "foo";
TestableAccessors accessors = new TestableAccessors(data);
assertNull(accessors.lutGet(reference));
assertEquals("bar", accessors.get(reference));
assertEquals(data, accessors.lutGet(reference));
assertEquals(
BiValues.newBiValue("bar"), get(ConvertedMap.newFromMap(data), reference)
);
}
@Test
public void testAbsentBareGet() throws Exception {
Map<String, Object> data = new HashMap<>();
Map<Serializable, Object> data = new HashMap<>();
data.put("foo", "bar");
String reference = "baz";
TestableAccessors accessors = new TestableAccessors(data);
assertNull(accessors.lutGet(reference));
assertNull(accessors.get(reference));
assertEquals(data, accessors.lutGet(reference));
assertNull(get(ConvertedMap.newFromMap(data), reference));
}
@Test
public void testBareBracketsGet() throws Exception {
Map<String, Object> data = new HashMap<>();
Map<Serializable, Object> data = new HashMap<>();
data.put("foo", "bar");
String reference = "[foo]";
TestableAccessors accessors = new TestableAccessors(data);
assertNull(accessors.lutGet(reference));
assertEquals("bar", accessors.get(reference));
assertEquals(data, accessors.lutGet(reference));
assertEquals(
BiValues.newBiValue("bar"), get(ConvertedMap.newFromMap(data), reference)
);
}
@Test
public void testDeepMapGet() throws Exception {
Map<String, Object> data = new HashMap<>();
Map<String, Object> inner = new HashMap<>();
Map<Serializable, Object> data = new HashMap<>();
Map<Serializable, Object> inner = new HashMap<>();
data.put("foo", inner);
inner.put("bar", "baz");
String reference = "[foo][bar]";
TestableAccessors accessors = new TestableAccessors(data);
assertNull(accessors.lutGet(reference));
assertEquals("baz", accessors.get(reference));
assertEquals(inner, accessors.lutGet(reference));
assertEquals(
BiValues.newBiValue("baz"), get(ConvertedMap.newFromMap(data), reference)
);
}
@Test
public void testAbsentDeepMapGet() throws Exception {
Map<String, Object> data = new HashMap<>();
Map<String, Object> inner = new HashMap<>();
Map<Serializable, Object> data = new HashMap<>();
Map<Serializable, Object> inner = new HashMap<>();
data.put("foo", inner);
inner.put("bar", "baz");
String reference = "[foo][foo]";
TestableAccessors accessors = new TestableAccessors(data);
assertNull(accessors.lutGet(reference));
assertNull(accessors.get(reference));
assertEquals(inner, accessors.lutGet(reference));
assertNull(get(ConvertedMap.newFromMap(data), reference));
}
@Test
public void testDeepListGet() throws Exception {
Map<String, Object> data = new HashMap<>();
Map<Serializable, Object> data = new HashMap<>();
List inner = new ArrayList();
data.put("foo", inner);
inner.add("bar");
String reference = "[foo][0]";
TestableAccessors accessors = new TestableAccessors(data);
assertNull(accessors.lutGet(reference));
assertEquals("bar", accessors.get(reference));
assertEquals(inner, accessors.lutGet(reference));
assertEquals(
BiValues.newBiValue("bar"), get(ConvertedMap.newFromMap(data), reference)
);
}
@Test
public void testAbsentDeepListGet() throws Exception {
Map<String, Object> data = new HashMap<>();
Map<Serializable, Object> data = new HashMap<>();
List inner = new ArrayList();
data.put("foo", inner);
inner.add("bar");
String reference = "[foo][1]";
TestableAccessors accessors = new TestableAccessors(data);
assertNull(accessors.lutGet(reference));
assertNull(accessors.get(reference));
assertEquals(inner, accessors.lutGet(reference));
assertNull(get(ConvertedMap.newFromMap(data), reference));
}
/*
* Check if accessors are able to recovery from
@ -133,105 +94,88 @@ public class AccessorsTest {
*/
@Test
public void testInvalidIdList() throws Exception {
Map<String, Object> data = new HashMap<>();
List inner = new ArrayList();
final ConvertedMap data = new ConvertedMap(1);
List inner = new ConvertedList(2);
data.put("map1", inner);
inner.add("obj1");
inner.add("obj2");
String reference = "[map1][IdNonNumeric]";
TestableAccessors accessors = new TestableAccessors(data);
assertNull(accessors.lutGet(reference));
assertNull(accessors.get(reference));
assertNull(accessors.set(reference, "obj3"));
assertEquals(inner, accessors.lutGet(reference));
assertFalse(accessors.includes(reference));
assertNull(accessors.del(reference));
assertNull(get(data, reference));
assertNull(set(data, reference, "obj3"));
assertFalse(includes(data, reference));
assertNull(del(data, reference));
}
@Test
public void testBarePut() throws Exception {
Map<String, Object> data = new HashMap<>();
final ConvertedMap data = new ConvertedMap(1);
String reference = "foo";
TestableAccessors accessors = new TestableAccessors(data);
assertNull(accessors.lutGet(reference));
assertEquals("bar", accessors.set(reference, "bar"));
assertEquals(data, accessors.lutGet(reference));
assertEquals("bar", accessors.get(reference));
assertEquals("bar", set(data, reference, "bar"));
assertEquals("bar", get(data, reference));
}
@Test
public void testBareBracketsPut() throws Exception {
Map<String, Object> data = new HashMap<>();
final ConvertedMap data = new ConvertedMap(1);
String reference = "[foo]";
TestableAccessors accessors = new TestableAccessors(data);
assertNull(accessors.lutGet(reference));
assertEquals("bar", accessors.set(reference, "bar"));
assertEquals(data, accessors.lutGet(reference));
assertEquals("bar", accessors.get(reference));
assertEquals("bar", set(data, reference, "bar"));
assertEquals("bar", get(data, reference));
}
@Test
public void testDeepMapSet() throws Exception {
Map<String, Object> data = new HashMap<>();
final ConvertedMap data = new ConvertedMap(1);
String reference = "[foo][bar]";
TestableAccessors accessors = new TestableAccessors(data);
assertNull(accessors.lutGet(reference));
assertEquals("baz", accessors.set(reference, "baz"));
assertEquals(accessors.lutGet(reference), data.get("foo"));
assertEquals("baz", accessors.get(reference));
assertEquals("baz", set(data, reference, "baz"));
assertEquals("baz", get(data, reference));
}
@Test
public void testDel() throws Exception {
Map<String, Object> data = new HashMap<>();
List inner = new ArrayList();
final ConvertedMap data = new ConvertedMap(1);
List inner = new ConvertedList(1);
data.put("foo", inner);
inner.add("bar");
data.put("bar", "baz");
TestableAccessors accessors = new TestableAccessors(data);
assertEquals("bar", accessors.del("[foo][0]"));
assertNull(accessors.del("[foo][0]"));
assertEquals(new ArrayList<>(), accessors.get("[foo]"));
assertEquals("baz", accessors.del("[bar]"));
assertNull(accessors.get("[bar]"));
assertEquals("bar", del(data, "[foo][0]"));
assertNull(del(data, "[foo][0]"));
assertEquals(new ConvertedList(0), get(data,"[foo]"));
assertEquals("baz", del(data, "[bar]"));
assertNull(get(data, "[bar]"));
}
@Test
public void testNilInclude() throws Exception {
Map<String, Object> data = new HashMap<>();
final ConvertedMap data = new ConvertedMap(1);
data.put("nilfield", null);
TestableAccessors accessors = new TestableAccessors(data);
assertTrue(accessors.includes("nilfield"));
assertTrue(includes(data, "nilfield"));
}
@Test
public void testInvalidPath() throws Exception {
Map<String, Object> data = new HashMap<>();
Accessors accessors = new Accessors(data);
final ConvertedMap data = new ConvertedMap(1);
assertEquals(1, accessors.set("[foo]", 1));
assertNull(accessors.get("[foo][bar]"));
assertEquals(1, set(data, "[foo]", 1));
assertNull(get(data, "[foo][bar]"));
}
@Test
public void testStaleTargetCache() throws Exception {
Map<String, Object> data = new HashMap<>();
final ConvertedMap data = new ConvertedMap(1);
Accessors accessors = new Accessors(data);
assertNull(accessors.get("[foo][bar]"));
assertEquals("baz", accessors.set("[foo][bar]", "baz"));
assertEquals("baz", accessors.get("[foo][bar]"));
assertNull(get(data,"[foo][bar]"));
assertEquals("baz", set(data,"[foo][bar]", "baz"));
assertEquals("baz", get(data, "[foo][bar]"));
assertEquals("boom", accessors.set("[foo]", "boom"));
assertNull(accessors.get("[foo][bar]"));
assertEquals("boom", accessors.get("[foo]"));
assertEquals("boom", set(data, "[foo]", "boom"));
assertNull(get(data, "[foo][bar]"));
assertEquals("boom", get(data,"[foo]"));
}
@Test
@ -244,27 +188,20 @@ public class AccessorsTest {
assertEquals(0, Accessors.listIndex(-10, 10));
}
@RunWith(Theories.class)
public static class TestListIndexFailureCases {
private static final int size = 10;
@DataPoint
public static final int tooLarge = size;
@DataPoint
public static final int tooLarge1 = size+1;
@DataPoint
public static final int tooLargeNegative = -size - 1;
@Rule
public ExpectedException exception = ExpectedException.none();
@Theory
public void testListIndexOutOfBounds(int i) {
exception.expect(IndexOutOfBoundsException.class);
Accessors.listIndex(i, size);
}
private static Object get(final ConvertedMap data, final CharSequence reference) {
return Accessors.get(data, PathCache.cache(reference));
}
private static Object set(final ConvertedMap data, final CharSequence reference,
final Object value) {
return Accessors.set(data, PathCache.cache(reference), value);
}
private static Object del(final ConvertedMap data, final CharSequence reference) {
return Accessors.del(data, PathCache.cache(reference));
}
private static boolean includes(final ConvertedMap data, final CharSequence reference) {
return Accessors.includes(data, PathCache.cache(reference));
}
}

21
logstash-core/src/test/java/org/logstash/FieldReferenceTest.java
View file

@ -2,35 +2,42 @@ package org.logstash;
import org.junit.Test;
import static org.junit.Assert.*;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
public class FieldReferenceTest {
public final class FieldReferenceTest {
@Test
public void testParseSingleBareField() throws Exception {
FieldReference f = FieldReference.parse("foo");
assertTrue(f.getPath().isEmpty());
assertEquals(0, f.getPath().length);
assertEquals(f.getKey(), "foo");
}
@Test
public void testParseSingleFieldPath() throws Exception {
FieldReference f = FieldReference.parse("[foo]");
assertTrue(f.getPath().isEmpty());
assertEquals(0, f.getPath().length);
assertEquals(f.getKey(), "foo");
}
@Test
public void testParse2FieldsPath() throws Exception {
FieldReference f = FieldReference.parse("[foo][bar]");
assertArrayEquals(f.getPath().toArray(), new String[]{"foo"});
assertArrayEquals(f.getPath(), new String[]{"foo"});
assertEquals(f.getKey(), "bar");
}
@Test
public void testParse3FieldsPath() throws Exception {
FieldReference f = FieldReference.parse("[foo][bar]]baz]");
assertArrayEquals(f.getPath().toArray(), new String[]{"foo", "bar"});
assertArrayEquals(f.getPath(), new String[]{"foo", "bar"});
assertEquals(f.getKey(), "baz");
}
}
@Test
public void deduplicatesTimestamp() throws Exception {
assertTrue(FieldReference.parse("@timestamp") == FieldReference.parse("[@timestamp]"));
}
}