#8128 Fix timestamp not getting properly deserialized under custom key

Fixes #8130
This commit is contained in:
Armin 2017-09-03 09:46:44 +02:00 committed by Armin Braun
parent 6cf0091ce5
commit cfddcf61bc
13 changed files with 135 additions and 83 deletions

View file

@ -12,6 +12,10 @@ public final class ConvertedList extends ArrayList<Object> {
private static final long serialVersionUID = 1396291343595074238L;
ConvertedList() {
super(10);
}
ConvertedList(final int size) {
super(size);
}

View file

@ -38,6 +38,10 @@ public final class ConvertedMap extends IdentityHashMap<String, Object> {
}
};
ConvertedMap() {
super(10);
}
ConvertedMap(final int size) {
super(size);
}

View file

@ -273,7 +273,7 @@ public final class Event implements Cloneable, Queueable {
try {
// getTimestamp throws an IOException if there is no @timestamp field, see #7613
return getTimestamp().toIso8601() + " " + hostMessageString;
return getTimestamp().toString() + " " + hostMessageString;
} catch (IOException e) {
return hostMessageString;
}

View file

@ -1,11 +1,16 @@
package org.logstash;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.JavaType;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
import com.fasterxml.jackson.databind.jsontype.TypeSerializer;
import com.fasterxml.jackson.databind.module.SimpleModule;
import com.fasterxml.jackson.databind.ser.std.NonTypedScalarSerializerBase;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import com.fasterxml.jackson.dataformat.cbor.CBORFactory;
import com.fasterxml.jackson.dataformat.cbor.CBORGenerator;
import java.io.IOException;
@ -15,6 +20,7 @@ import org.jruby.RubyFixnum;
import org.jruby.RubyFloat;
import org.jruby.RubyString;
import org.jruby.RubySymbol;
import org.logstash.ext.JrubyTimestampExtLibrary;
public final class ObjectMappers {
@ -31,7 +37,7 @@ public final class ObjectMappers {
public static final ObjectMapper CBOR_MAPPER = new ObjectMapper(
new CBORFactory().configure(CBORGenerator.Feature.WRITE_MINIMAL_INTS, false)
).registerModule(RUBY_SERIALIZERS);
).registerModule(RUBY_SERIALIZERS).enableDefaultTyping(ObjectMapper.DefaultTyping.NON_FINAL);
/**
* {@link JavaType} for the {@link HashMap} that {@link Event} is serialized as.
@ -88,7 +94,7 @@ public final class ObjectMappers {
extends NonTypedScalarSerializerBase<RubyFloat> {
RubyFloatSerializer() {
super(RubyFloat.class, true);
super(RubyFloat.class);
}
@Override
@ -106,7 +112,7 @@ public final class ObjectMappers {
extends NonTypedScalarSerializerBase<RubyBoolean> {
RubyBooleanSerializer() {
super(RubyBoolean.class, true);
super(RubyBoolean.class);
}
@Override
@ -133,4 +139,73 @@ public final class ObjectMappers {
generator.writeNumber(value.getLongValue());
}
}
/**
* Serializer for {@link Timestamp} since Jackson can't handle that type natively, so we
* simply serialize it as if it were a {@code String} and wrap it in type arguments, so that
* deserialization happens via {@link ObjectMappers.TimestampDeserializer}.
*/
public static final class TimestampSerializer extends StdSerializer<Timestamp> {
TimestampSerializer() {
super(Timestamp.class);
}
@Override
public void serialize(final Timestamp value, final JsonGenerator jgen,
final SerializerProvider provider) throws IOException {
jgen.writeString(value.toString());
}
@Override
public void serializeWithType(final Timestamp value, final JsonGenerator jgen,
final SerializerProvider serializers, final TypeSerializer typeSer) throws IOException {
typeSer.writeTypePrefixForScalar(value, jgen, Timestamp.class);
jgen.writeString(value.toString());
typeSer.writeTypeSuffixForScalar(value, jgen);
}
}
public static final class TimestampDeserializer extends StdDeserializer<Timestamp> {
TimestampDeserializer() {
super(Timestamp.class);
}
@Override
public Timestamp deserialize(final JsonParser p, final DeserializationContext ctxt)
throws IOException {
return new Timestamp(p.getText());
}
}
/**
* Serializer for {@link JrubyTimestampExtLibrary.RubyTimestamp} that serializes it exactly the
* same way {@link ObjectMappers.TimestampSerializer} serializes
* {@link Timestamp} to ensure consistent serialization across Java and Ruby
* representation of {@link Timestamp}.
*/
public static final class RubyTimestampSerializer
extends StdSerializer<JrubyTimestampExtLibrary.RubyTimestamp> {
RubyTimestampSerializer() {
super(JrubyTimestampExtLibrary.RubyTimestamp.class);
}
@Override
public void serialize(final JrubyTimestampExtLibrary.RubyTimestamp value,
final JsonGenerator jgen, final SerializerProvider provider) throws IOException {
jgen.writeString(value.getTimestamp().toString());
}
@Override
public void serializeWithType(final JrubyTimestampExtLibrary.RubyTimestamp value,
final JsonGenerator jgen, final SerializerProvider serializers,
final TypeSerializer typeSer)
throws IOException {
typeSer.writeTypePrefixForScalar(value, jgen, Timestamp.class);
jgen.writeObject(value.getTimestamp());
typeSer.writeTypeSuffixForScalar(value, jgen);
}
}
}

View file

@ -1,5 +1,6 @@
package org.logstash;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import java.util.Date;
import org.joda.time.DateTime;
@ -9,14 +10,14 @@ import org.joda.time.LocalDateTime;
import org.joda.time.format.DateTimeFormatter;
import org.joda.time.format.ISODateTimeFormat;
import org.logstash.ackedqueue.Queueable;
import org.logstash.json.TimestampSerializer;
/**
* Wrapper around a {@link DateTime} with Logstash specific serialization behaviour.
* This class is immutable and thread-safe since its only state is held in a final {@link DateTime}
* reference and {@link DateTime} which itself is immutable and thread-safe.
*/
@JsonSerialize(using = TimestampSerializer.class)
@JsonSerialize(using = ObjectMappers.TimestampSerializer.class)
@JsonDeserialize(using = ObjectMappers.TimestampDeserializer.class)
public final class Timestamp implements Comparable<Timestamp>, Queueable {
// all methods setting the time object must set it in the UTC timezone
@ -54,12 +55,8 @@ public final class Timestamp implements Comparable<Timestamp>, Queueable {
return new Timestamp();
}
public String toIso8601() {
return iso8601Formatter.print(this.time);
}
public String toString() {
return toIso8601();
return iso8601Formatter.print(this.time);
}
public long usec() {
@ -72,6 +69,11 @@ public final class Timestamp implements Comparable<Timestamp>, Queueable {
public int compareTo(Timestamp other) {
return time.compareTo(other.time);
}
@Override
public boolean equals(final Object other) {
return other instanceof Timestamp && time.equals(((Timestamp) other).time);
}
@Override
public byte[] serialize() {

View file

@ -19,9 +19,9 @@ import org.jruby.runtime.ObjectAllocator;
import org.jruby.runtime.ThreadContext;
import org.jruby.runtime.builtin.IRubyObject;
import org.jruby.runtime.load.Library;
import org.logstash.ObjectMappers;
import org.logstash.RubyUtil;
import org.logstash.Timestamp;
import org.logstash.json.RubyTimestampSerializer;
public class JrubyTimestampExtLibrary implements Library {
@ -43,7 +43,7 @@ public class JrubyTimestampExtLibrary implements Library {
}
@JRubyClass(name = "Timestamp")
@JsonSerialize(using = RubyTimestampSerializer.class)
@JsonSerialize(using = ObjectMappers.RubyTimestampSerializer.class)
public static class RubyTimestamp extends RubyObject {
private Timestamp timestamp;
@ -148,7 +148,7 @@ public class JrubyTimestampExtLibrary implements Library {
@JRubyMethod(name = "to_iso8601")
public IRubyObject ruby_to_iso8601(ThreadContext context)
{
return RubyString.newString(context.runtime, this.timestamp.toIso8601());
return RubyString.newString(context.runtime, this.timestamp.toString());
}
@JRubyMethod(name = "to_java")
@ -160,7 +160,7 @@ public class JrubyTimestampExtLibrary implements Library {
@JRubyMethod(name = "to_json", rest = true)
public IRubyObject ruby_to_json(ThreadContext context, IRubyObject[] args)
{
return RubyString.newString(context.runtime, "\"" + this.timestamp.toIso8601() + "\"");
return RubyString.newString(context.runtime, "\"" + this.timestamp.toString() + "\"");
}
@JRubyMethod(name = "coerce", required = 1, meta = true)

View file

@ -1,22 +0,0 @@
package org.logstash.json;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.JsonSerializer;
import com.fasterxml.jackson.databind.SerializerProvider;
import java.io.IOException;
import org.logstash.ext.JrubyTimestampExtLibrary;
/**
* Serializer for {@link JrubyTimestampExtLibrary.RubyTimestamp} that serializes it exactly the same
* way {@link TimestampSerializer} serializes {@link org.logstash.Timestamp} to ensure consistent
* serialization across Java and Ruby representation of {@link org.logstash.Timestamp}.
*/
public final class RubyTimestampSerializer
extends JsonSerializer<JrubyTimestampExtLibrary.RubyTimestamp> {
@Override
public void serialize(final JrubyTimestampExtLibrary.RubyTimestamp value,
final JsonGenerator jgen, final SerializerProvider provider) throws IOException {
jgen.writeString(value.getTimestamp().toIso8601());
}
}

View file

@ -1,18 +0,0 @@
package org.logstash.json;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.JsonSerializer;
import com.fasterxml.jackson.databind.SerializerProvider;
import org.logstash.Timestamp;
import java.io.IOException;
public class TimestampSerializer extends JsonSerializer<Timestamp> {
@Override
public void serialize(Timestamp value, JsonGenerator jgen, SerializerProvider provider)
throws IOException
{
jgen.writeString(value.toIso8601());
}
}

View file

@ -47,7 +47,7 @@ public class DLQEntryTest {
byte[] bytes = expected.serialize();
DLQEntry actual = DLQEntry.deserialize(bytes);
assertJsonEquals(actual.getEvent().toJson(), event.toJson());
assertThat(actual.getEntryTime().toIso8601(), equalTo(expected.getEntryTime().toIso8601()));
assertThat(actual.getEntryTime().toString(), equalTo(expected.getEntryTime().toString()));
assertThat(actual.getPluginType(), equalTo("type"));
assertThat(actual.getPluginId(), equalTo("id"));
assertThat(actual.getReason(), equalTo("reason"));

View file

@ -3,6 +3,7 @@ package org.logstash;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -37,7 +38,7 @@ public final class EventTest {
assertEquals(42.42, er.getField("[baz][innerQuux]"));
assertEquals(42L, er.getField("[@metadata][foo]"));
assertEquals(e.getTimestamp().toIso8601(), er.getTimestamp().toIso8601());
assertEquals(e.getTimestamp().toString(), er.getTimestamp().toString());
}
@Test
@ -50,20 +51,26 @@ public final class EventTest {
inner.put("innerQuux", 42.42);
e.setField("baz", inner);
e.setField("[@metadata][foo]", 42L);
final Timestamp timestamp = new Timestamp();
e.setField("time", timestamp);
final Collection<Object> list = new ConvertedList(1);
list.add("foo");
e.setField("list", list);
Event er = Event.deserialize(e.serialize());
assertEquals(42L, er.getField("foo"));
assertEquals(42L, er.getField("bar"));
assertEquals(42L, er.getField("[baz][innerFoo]"));
assertEquals(42.42, er.getField("[baz][innerQuux]"));
assertEquals(42L, er.getField("[@metadata][foo]"));
assertEquals(e.getTimestamp().toIso8601(), er.getTimestamp().toIso8601());
assertEquals(timestamp, er.getField("time"));
assertEquals(list, er.getField("list"));
assertEquals(e.getTimestamp().toString(), er.getTimestamp().toString());
}
@Test
public void testBareToJson() throws Exception {
Event e = new Event();
assertJsonEquals("{\"@timestamp\":\"" + e.getTimestamp().toIso8601() + "\",\"@version\":\"1\"}", e.toJson());
assertJsonEquals("{\"@timestamp\":\"" + e.getTimestamp().toString() + "\",\"@version\":\"1\"}", e.toJson());
}
@Test
@ -71,7 +78,7 @@ public final class EventTest {
Map<String, Object> data = new HashMap<>();
data.put("foo", "bar");
Event e = new Event(data);
assertJsonEquals("{\"@timestamp\":\"" + e.getTimestamp().toIso8601() + "\",\"foo\":\"bar\",\"@version\":\"1\"}", e.toJson());
assertJsonEquals("{\"@timestamp\":\"" + e.getTimestamp().toString() + "\",\"foo\":\"bar\",\"@version\":\"1\"}", e.toJson());
}
@Test
@ -79,7 +86,7 @@ public final class EventTest {
Map<String, Object> data = new HashMap<>();
data.put("foo", 1);
Event e = new Event(data);
assertJsonEquals("{\"@timestamp\":\"" + e.getTimestamp().toIso8601() + "\",\"foo\":1,\"@version\":\"1\"}", e.toJson());
assertJsonEquals("{\"@timestamp\":\"" + e.getTimestamp().toString() + "\",\"foo\":1,\"@version\":\"1\"}", e.toJson());
}
@Test
@ -87,7 +94,7 @@ public final class EventTest {
Map<String, Object> data = new HashMap<>();
data.put("foo", 1L);
Event e = new Event(data);
assertJsonEquals("{\"@timestamp\":\"" + e.getTimestamp().toIso8601() + "\",\"foo\":1,\"@version\":\"1\"}", e.toJson());
assertJsonEquals("{\"@timestamp\":\"" + e.getTimestamp().toString() + "\",\"foo\":1,\"@version\":\"1\"}", e.toJson());
}
@Test
@ -95,7 +102,7 @@ public final class EventTest {
Map<String, Object> data = new HashMap<>();
data.put("foo", 1.0);
Event e = new Event(data);
assertJsonEquals("{\"@timestamp\":\"" + e.getTimestamp().toIso8601() + "\",\"foo\":1.0,\"@version\":\"1\"}", e.toJson());
assertJsonEquals("{\"@timestamp\":\"" + e.getTimestamp().toString() + "\",\"foo\":1.0,\"@version\":\"1\"}", e.toJson());
}
@Test
@ -105,18 +112,18 @@ public final class EventTest {
data.put("bar", "bar");
data.put("baz", 1);
Event e = new Event(data);
assertJsonEquals("{\"bar\":\"bar\",\"@timestamp\":\"" + e.getTimestamp().toIso8601() + "\",\"foo\":1.0,\"@version\":\"1\",\"baz\":1}", e.toJson());
assertJsonEquals("{\"bar\":\"bar\",\"@timestamp\":\"" + e.getTimestamp().toString() + "\",\"foo\":1.0,\"@version\":\"1\",\"baz\":1}", e.toJson());
}
@Test
public void testDeepMapFieldToJson() throws Exception {
Event e = new Event();
e.setField("[foo][bar][baz]", 1);
assertJsonEquals("{\"@timestamp\":\"" + e.getTimestamp().toIso8601() + "\",\"foo\":{\"bar\":{\"baz\":1}},\"@version\":\"1\"}", e.toJson());
assertJsonEquals("{\"@timestamp\":\"" + e.getTimestamp().toString() + "\",\"foo\":{\"bar\":{\"baz\":1}},\"@version\":\"1\"}", e.toJson());
e = new Event();
e.setField("[foo][0][baz]", 1);
assertJsonEquals("{\"@timestamp\":\"" + e.getTimestamp().toIso8601() + "\",\"foo\":{\"0\":{\"baz\":1}},\"@version\":\"1\"}", e.toJson());
assertJsonEquals("{\"@timestamp\":\"" + e.getTimestamp().toString() + "\",\"foo\":{\"0\":{\"baz\":1}},\"@version\":\"1\"}", e.toJson());
}
@Test
@ -127,7 +134,7 @@ public final class EventTest {
assertJsonEquals(
String.format(
"{\"@timestamp\":\"%s\",\"foo\":{\"bar\":{\"baz\":\"%s\"}},\"@version\":\"1\"}",
e.getTimestamp().toIso8601(), new Timestamp(time.getDateTime()).toIso8601()
e.getTimestamp().toString(), new Timestamp(time.getDateTime()).toString()
), e.toJson()
);
}
@ -139,7 +146,7 @@ public final class EventTest {
assertJsonEquals(
String.format(
"{\"@timestamp\":\"%s\",\"foo\":{\"bar\":{\"baz\":true}},\"@version\":\"1\"}",
e.getTimestamp().toIso8601()
e.getTimestamp().toString()
), e.toJson()
);
}
@ -185,7 +192,7 @@ public final class EventTest {
Event f = e.clone();
assertJsonEquals("{\"bar\":\"bar\",\"@timestamp\":\"" + e.getTimestamp().toIso8601() + "\",\"array\":[{\"foo\":\"bar\"}],\"foo\":1.0,\"@version\":\"1\",\"baz\":1}", f.toJson());
assertJsonEquals("{\"bar\":\"bar\",\"@timestamp\":\"" + e.getTimestamp().toString() + "\",\"array\":[{\"foo\":\"bar\"}],\"foo\":1.0,\"@version\":\"1\",\"baz\":1}", f.toJson());
assertJsonEquals(f.toJson(), e.toJson());
}
@ -257,7 +264,7 @@ public final class EventTest {
Event e = Event.fromJson("{\"@timestamp\":\"2015-05-28T23:02:05.350Z\",\"foo\":\"bar\"}")[0];
assertEquals("bar", e.getField("[foo]"));
assertEquals("2015-05-28T23:02:05.350Z", e.getTimestamp().toIso8601());
assertEquals("2015-05-28T23:02:05.350Z", e.getTimestamp().toString());
}
@Test
@ -266,7 +273,7 @@ public final class EventTest {
assertEquals(1, l.length);
assertEquals("bar", l[0].getField("[foo]"));
assertEquals("2015-05-28T23:02:05.350Z", l[0].getTimestamp().toIso8601());
assertEquals("2015-05-28T23:02:05.350Z", l[0].getTimestamp().toString());
l = Event.fromJson("[{}]");
@ -277,9 +284,9 @@ public final class EventTest {
assertEquals(2, l.length);
assertEquals("bar", l[0].getField("[foo]"));
assertEquals("2015-05-28T23:02:05.350Z", l[0].getTimestamp().toIso8601());
assertEquals("2015-05-28T23:02:05.350Z", l[0].getTimestamp().toString());
assertEquals("baz", l[1].getField("[foo]"));
assertEquals("2016-05-28T23:02:05.350Z", l[1].getTimestamp().toIso8601());
assertEquals("2016-05-28T23:02:05.350Z", l[1].getTimestamp().toString());
}
@Test(expected=IOException.class)
@ -331,7 +338,7 @@ public final class EventTest {
data.put("host", "foo");
data.put("message", "bar");
Event e = new Event(data);
assertEquals(e.toString(), e.getTimestamp().toIso8601() + " foo bar");
assertEquals(e.toString(), e.getTimestamp().toString() + " foo bar");
}
@Test

View file

@ -12,14 +12,14 @@ public class TimestampTest {
@Test
public void testCircularIso8601() throws Exception {
Timestamp t1 = new Timestamp();
Timestamp t2 = new Timestamp(t1.toIso8601());
Timestamp t2 = new Timestamp(t1.toString());
assertEquals(t1.getTime(), t2.getTime());
}
@Test
public void testToIso8601() throws Exception {
Timestamp t = new Timestamp("2014-09-23T00:00:00-0800");
assertEquals("2014-09-23T08:00:00.000Z", t.toIso8601());
assertEquals("2014-09-23T08:00:00.000Z", t.toString());
}
// Timestamp should always be in a UTC representation

View file

@ -209,7 +209,7 @@ public class DeadLetterQueueReaderTest {
@Test
public void testBlockBoundary() throws Exception {
final int PAD_FOR_BLOCK_SIZE_EVENT = 32616;
final int PAD_FOR_BLOCK_SIZE_EVENT = 32513;
Event event = new Event();
char[] field = new char[PAD_FOR_BLOCK_SIZE_EVENT];
Arrays.fill(field, 'e');
@ -234,7 +234,7 @@ public class DeadLetterQueueReaderTest {
@Test
public void testBlockBoundaryMultiple() throws Exception {
Event event = new Event(Collections.emptyMap());
char[] field = new char[8053];
char[] field = new char[7952];
Arrays.fill(field, 'x');
event.setField("message", new String(field));
long startTime = System.currentTimeMillis();
@ -260,7 +260,7 @@ public class DeadLetterQueueReaderTest {
// This test tests for a single event that ends on a block and segment boundary
@Test
public void testBlockAndSegmentBoundary() throws Exception {
final int PAD_FOR_BLOCK_SIZE_EVENT = 32616;
final int PAD_FOR_BLOCK_SIZE_EVENT = 32513;
Event event = new Event();
event.setField("T", generateMessageContent(PAD_FOR_BLOCK_SIZE_EVENT));
Timestamp timestamp = new Timestamp();
@ -354,7 +354,7 @@ public class DeadLetterQueueReaderTest {
readManager.seekToNextEvent(seekTarget);
DLQEntry readEntry = readManager.pollEntry(100);
assertThat(readEntry.getReason(), equalTo(expectedValue));
assertThat(readEntry.getEntryTime().toIso8601(), equalTo(seekTarget.toIso8601()));
assertThat(readEntry.getEntryTime().toString(), equalTo(seekTarget.toString()));
}
}

View file

@ -74,7 +74,7 @@ public class ReloadWitnessTest {
witness.success();
witness.lastSuccessTimestamp(rubyTimestamp);
String json = witness.asJson();
assertThat(json).isEqualTo("{\"reloads\":{\"last_error\":{\"message\":null,\"backtrace\":null},\"successes\":1,\"last_success_timestamp\":\"" + timestamp.toIso8601() +
assertThat(json).isEqualTo("{\"reloads\":{\"last_error\":{\"message\":null,\"backtrace\":null},\"successes\":1,\"last_success_timestamp\":\"" + timestamp.toString() +
"\",\"last_failure_timestamp\":null,\"failures\":0}}");
}
@ -84,7 +84,7 @@ public class ReloadWitnessTest {
witness.lastFailureTimestamp(rubyTimestamp);
String json = witness.asJson();
assertThat(json).isEqualTo("{\"reloads\":{\"last_error\":{\"message\":null,\"backtrace\":null},\"successes\":0,\"last_success_timestamp\":null," +
"\"last_failure_timestamp\":\"" + timestamp.toIso8601() + "\",\"failures\":1}}");
"\"last_failure_timestamp\":\"" + timestamp.toString() + "\",\"failures\":1}}");
}
@Test