mirror of
https://github.com/elastic/logstash.git
synced 2025-04-25 15:17:52 -04:00
logstash-core & logstash-core-event extraction to support logstash-core-event-java impl, relates to #4191
fixed timezone issue extracted logstash-core and reorganized specs extracted logstash-core-event extract java Event into logstash-core-event-java in a proper gem remove obsolete jruby_event bootstrapping fix require path add java code bootstrap use logstash-core-event/logstash-core-event.rb remove obsolete files basic instructions LogStash::Json need to be initialized from event update jruby and gradle versions update compile:logstash-core-event-java rake task WIP tasks refactor fix gem.files skip test if class is not defined fix gem related tasks for new structure add gem spec dirs in core tests bootstrap java implementation when requiring timestamp new Cloner class and Event clone impl fix array fields assignments, see #4140 don't rely on json implementation ordering fix skipped last iterpolation char remove implementation specific unnecessary check also require ruby classes define error class in ruby raise exception on invalid format remove implementation specific tests and extract and put logger related test in pending missing bits for having all core timestamp specs pass run all core specs remove leftover comment regex missing encoding header revert to logstash-core-event by default finished proper gemification useless require dynamically pick specs depending on logstash-core-event-* implementation logstash root package version missing file for proper gemification do not build java event by default always check for root logstash lib dir fix concurrent-ruby version confict fix rebase conflict re-enable specs user vars instead of constants move non core code in bootstrap document version files move version file remove useless code use version in logstash-core fix gem files list put back concurrent-ruby version constrain as in master add dependency on logstash-core-event remove dependency on logstash-core to avoid circular dependency fix rebase conflict remove circular dependency fix specs update README
This commit is contained in:
Colin Surprenant
parent
e28f188e12
commit
d74d41cb30
153 changed files with 1041 additions and 508 deletions
9
logstash-core-event-java/.gitignore
vendored
Normal file
9
logstash-core-event-java/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
*.class
|
||||
|
||||
# build dirs
|
||||
build
|
||||
.gradle
|
||||
|
||||
# Intellij
|
||||
.idea
|
||||
*.iml
|
||||
63
logstash-core-event-java/README.md
Normal file
63
logstash-core-event-java/README.md
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
# logstash-core-event-java
|
||||
|
||||
## dev install
|
||||
|
||||
1- build code with
|
||||
|
||||
```
|
||||
$ cd logstash-core-event-java
|
||||
$ gradle build
|
||||
```
|
||||
|
||||
A bunch of warning are expected, it should end with:
|
||||
|
||||
```
|
||||
BUILD SUCCESSFUL
|
||||
```
|
||||
|
||||
2- update root logstash `Gemfile` to use this gem with:
|
||||
|
||||
```
|
||||
# gem "logstash-core-event", "x.y.z", :path => "./logstash-core-event"
|
||||
gem "logstash-core-event-java", "x.y.z", :path => "./logstash-core-event-java"
|
||||
```
|
||||
|
||||
3- update `logstash-core/logstash-core.gemspec` with:
|
||||
|
||||
```
|
||||
# gem.add_runtime_dependency "logstash-core-event", "x.y.z"
|
||||
gem.add_runtime_dependency "logstash-core-event-java", "x.y.z"
|
||||
```
|
||||
|
||||
4- and install:
|
||||
|
||||
```
|
||||
$ bin/bundle
|
||||
```
|
||||
|
||||
- install core plugins for tests
|
||||
|
||||
```
|
||||
$ rake test:install-core
|
||||
```
|
||||
|
||||
## specs
|
||||
|
||||
```
|
||||
$ bin/rspec spec
|
||||
$ bin/rspec logstash-core/spec
|
||||
$ bin/rspec logstash-core-event/spec
|
||||
$ bin/rspec logstash-core-event-java/spec
|
||||
```
|
||||
|
||||
or
|
||||
|
||||
```
|
||||
$ rake test:core
|
||||
```
|
||||
|
||||
also
|
||||
|
||||
```
|
||||
$ rake test:plugins
|
||||
```
|
||||
104
logstash-core-event-java/build.gradle
Normal file
104
logstash-core-event-java/build.gradle
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
buildscript {
|
||||
repositories {
|
||||
mavenLocal()
|
||||
mavenCentral()
|
||||
jcenter()
|
||||
}
|
||||
dependencies {
|
||||
classpath 'net.saliman:gradle-cobertura-plugin:2.2.8'
|
||||
classpath 'com.github.jengelman.gradle.plugins:shadow:1.2.2'
|
||||
}
|
||||
}
|
||||
|
||||
//allprojects {
|
||||
|
||||
repositories {
|
||||
mavenLocal()
|
||||
mavenCentral()
|
||||
}
|
||||
|
||||
gradle.projectsEvaluated {
|
||||
tasks.withType(JavaCompile) {
|
||||
options.compilerArgs << "-Xlint:unchecked" << "-Xlint:deprecation"
|
||||
}
|
||||
}
|
||||
|
||||
//}
|
||||
|
||||
//subprojects { project ->
|
||||
|
||||
apply plugin: 'java'
|
||||
apply plugin: 'idea'
|
||||
apply plugin: 'com.github.johnrengelman.shadow'
|
||||
|
||||
group = 'org.logstash'
|
||||
|
||||
project.sourceCompatibility = 1.7
|
||||
|
||||
task sourcesJar(type: Jar, dependsOn:classes) {
|
||||
from sourceSets.main.allSource
|
||||
classifier 'sources'
|
||||
extension 'jar'
|
||||
}
|
||||
|
||||
task javadocJar(type: Jar, dependsOn:javadoc) {
|
||||
from javadoc.destinationDir
|
||||
classifier 'javadoc'
|
||||
extension 'jar'
|
||||
}
|
||||
|
||||
configurations.create('sources')
|
||||
configurations.create('javadoc')
|
||||
configurations.archives {
|
||||
extendsFrom configurations.sources
|
||||
extendsFrom configurations.javadoc
|
||||
}
|
||||
|
||||
artifacts {
|
||||
sources(sourcesJar) {
|
||||
// Weird Gradle quirk where type will be used for the extension, but only for sources
|
||||
type 'jar'
|
||||
}
|
||||
|
||||
javadoc(javadocJar) {
|
||||
type 'javadoc'
|
||||
}
|
||||
}
|
||||
|
||||
configurations {
|
||||
provided
|
||||
}
|
||||
|
||||
project.sourceSets {
|
||||
main.compileClasspath += project.configurations.provided
|
||||
main.runtimeClasspath += project.configurations.provided
|
||||
test.compileClasspath += project.configurations.provided
|
||||
test.runtimeClasspath += project.configurations.provided
|
||||
}
|
||||
project.javadoc.classpath += project.configurations.provided
|
||||
|
||||
idea {
|
||||
module {
|
||||
scopes.PROVIDED.plus += [ project.configurations.provided ]
|
||||
}
|
||||
}
|
||||
|
||||
dependencies {
|
||||
compile 'org.codehaus.jackson:jackson-mapper-asl:1.9.13'
|
||||
compile 'org.codehaus.jackson:jackson-core-asl:1.9.13'
|
||||
compile 'joda-time:joda-time:2.8.2'
|
||||
compile 'com.google.guava:guava:18.0'
|
||||
compile 'org.slf4j:slf4j-api:1.7.12'
|
||||
provided 'org.jruby:jruby-core:1.7.22'
|
||||
testCompile 'org.testng:testng:6.9.6'
|
||||
testCompile 'org.mockito:mockito-core:1.10.19'
|
||||
}
|
||||
|
||||
//}
|
||||
|
||||
// See http://www.gradle.org/docs/current/userguide/gradle_wrapper.html
|
||||
task wrapper(type: Wrapper) {
|
||||
description = 'Install Gradle wrapper'
|
||||
gradleVersion = '2.7'
|
||||
}
|
||||
|
||||
1
logstash-core-event-java/gradle.properties
Normal file
1
logstash-core-event-java/gradle.properties
Normal file
|
|
@ -0,0 +1 @@
|
|||
VERSION=0.0.1-SNAPSHOT
|
||||
BIN
logstash-core-event-java/gradle/wrapper/gradle-wrapper.jar
vendored
Normal file
BIN
logstash-core-event-java/gradle/wrapper/gradle-wrapper.jar
vendored
Normal file
Binary file not shown.
6
logstash-core-event-java/gradle/wrapper/gradle-wrapper.properties
vendored
Normal file
6
logstash-core-event-java/gradle/wrapper/gradle-wrapper.properties
vendored
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
#Tue Mar 17 11:58:32 PDT 2015
|
||||
distributionBase=GRADLE_USER_HOME
|
||||
distributionPath=wrapper/dists
|
||||
zipStoreBase=GRADLE_USER_HOME
|
||||
zipStorePath=wrapper/dists
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-2.3-bin.zip
|
||||
164
logstash-core-event-java/gradlew
vendored
Executable file
164
logstash-core-event-java/gradlew
vendored
Executable file
|
|
@ -0,0 +1,164 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
##############################################################################
|
||||
##
|
||||
## Gradle start up script for UN*X
|
||||
##
|
||||
##############################################################################
|
||||
|
||||
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
||||
DEFAULT_JVM_OPTS=""
|
||||
|
||||
APP_NAME="Gradle"
|
||||
APP_BASE_NAME=`basename "$0"`
|
||||
|
||||
# Use the maximum available, or set MAX_FD != -1 to use that value.
|
||||
MAX_FD="maximum"
|
||||
|
||||
warn ( ) {
|
||||
echo "$*"
|
||||
}
|
||||
|
||||
die ( ) {
|
||||
echo
|
||||
echo "$*"
|
||||
echo
|
||||
exit 1
|
||||
}
|
||||
|
||||
# OS specific support (must be 'true' or 'false').
|
||||
cygwin=false
|
||||
msys=false
|
||||
darwin=false
|
||||
case "`uname`" in
|
||||
CYGWIN* )
|
||||
cygwin=true
|
||||
;;
|
||||
Darwin* )
|
||||
darwin=true
|
||||
;;
|
||||
MINGW* )
|
||||
msys=true
|
||||
;;
|
||||
esac
|
||||
|
||||
# For Cygwin, ensure paths are in UNIX format before anything is touched.
|
||||
if $cygwin ; then
|
||||
[ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
|
||||
fi
|
||||
|
||||
# Attempt to set APP_HOME
|
||||
# Resolve links: $0 may be a link
|
||||
PRG="$0"
|
||||
# Need this for relative symlinks.
|
||||
while [ -h "$PRG" ] ; do
|
||||
ls=`ls -ld "$PRG"`
|
||||
link=`expr "$ls" : '.*-> \(.*\)$'`
|
||||
if expr "$link" : '/.*' > /dev/null; then
|
||||
PRG="$link"
|
||||
else
|
||||
PRG=`dirname "$PRG"`"/$link"
|
||||
fi
|
||||
done
|
||||
SAVED="`pwd`"
|
||||
cd "`dirname \"$PRG\"`/" >&-
|
||||
APP_HOME="`pwd -P`"
|
||||
cd "$SAVED" >&-
|
||||
|
||||
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
|
||||
|
||||
# Determine the Java command to use to start the JVM.
|
||||
if [ -n "$JAVA_HOME" ] ; then
|
||||
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
|
||||
# IBM's JDK on AIX uses strange locations for the executables
|
||||
JAVACMD="$JAVA_HOME/jre/sh/java"
|
||||
else
|
||||
JAVACMD="$JAVA_HOME/bin/java"
|
||||
fi
|
||||
if [ ! -x "$JAVACMD" ] ; then
|
||||
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
|
||||
|
||||
Please set the JAVA_HOME variable in your environment to match the
|
||||
location of your Java installation."
|
||||
fi
|
||||
else
|
||||
JAVACMD="java"
|
||||
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
||||
|
||||
Please set the JAVA_HOME variable in your environment to match the
|
||||
location of your Java installation."
|
||||
fi
|
||||
|
||||
# Increase the maximum file descriptors if we can.
|
||||
if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
|
||||
MAX_FD_LIMIT=`ulimit -H -n`
|
||||
if [ $? -eq 0 ] ; then
|
||||
if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
|
||||
MAX_FD="$MAX_FD_LIMIT"
|
||||
fi
|
||||
ulimit -n $MAX_FD
|
||||
if [ $? -ne 0 ] ; then
|
||||
warn "Could not set maximum file descriptor limit: $MAX_FD"
|
||||
fi
|
||||
else
|
||||
warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
|
||||
fi
|
||||
fi
|
||||
|
||||
# For Darwin, add options to specify how the application appears in the dock
|
||||
if $darwin; then
|
||||
GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
|
||||
fi
|
||||
|
||||
# For Cygwin, switch paths to Windows format before running java
|
||||
if $cygwin ; then
|
||||
APP_HOME=`cygpath --path --mixed "$APP_HOME"`
|
||||
CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
|
||||
|
||||
# We build the pattern for arguments to be converted via cygpath
|
||||
ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
|
||||
SEP=""
|
||||
for dir in $ROOTDIRSRAW ; do
|
||||
ROOTDIRS="$ROOTDIRS$SEP$dir"
|
||||
SEP="|"
|
||||
done
|
||||
OURCYGPATTERN="(^($ROOTDIRS))"
|
||||
# Add a user-defined pattern to the cygpath arguments
|
||||
if [ "$GRADLE_CYGPATTERN" != "" ] ; then
|
||||
OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
|
||||
fi
|
||||
# Now convert the arguments - kludge to limit ourselves to /bin/sh
|
||||
i=0
|
||||
for arg in "$@" ; do
|
||||
CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
|
||||
CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
|
||||
|
||||
if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
|
||||
eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
|
||||
else
|
||||
eval `echo args$i`="\"$arg\""
|
||||
fi
|
||||
i=$((i+1))
|
||||
done
|
||||
case $i in
|
||||
(0) set -- ;;
|
||||
(1) set -- "$args0" ;;
|
||||
(2) set -- "$args0" "$args1" ;;
|
||||
(3) set -- "$args0" "$args1" "$args2" ;;
|
||||
(4) set -- "$args0" "$args1" "$args2" "$args3" ;;
|
||||
(5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
|
||||
(6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
|
||||
(7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
|
||||
(8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
|
||||
(9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
|
||||
function splitJvmOpts() {
|
||||
JVM_OPTS=("$@")
|
||||
}
|
||||
eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
|
||||
JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
|
||||
|
||||
exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
|
||||
90
logstash-core-event-java/gradlew.bat
vendored
Normal file
90
logstash-core-event-java/gradlew.bat
vendored
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
@if "%DEBUG%" == "" @echo off
|
||||
@rem ##########################################################################
|
||||
@rem
|
||||
@rem Gradle startup script for Windows
|
||||
@rem
|
||||
@rem ##########################################################################
|
||||
|
||||
@rem Set local scope for the variables with windows NT shell
|
||||
if "%OS%"=="Windows_NT" setlocal
|
||||
|
||||
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
||||
set DEFAULT_JVM_OPTS=
|
||||
|
||||
set DIRNAME=%~dp0
|
||||
if "%DIRNAME%" == "" set DIRNAME=.
|
||||
set APP_BASE_NAME=%~n0
|
||||
set APP_HOME=%DIRNAME%
|
||||
|
||||
@rem Find java.exe
|
||||
if defined JAVA_HOME goto findJavaFromJavaHome
|
||||
|
||||
set JAVA_EXE=java.exe
|
||||
%JAVA_EXE% -version >NUL 2>&1
|
||||
if "%ERRORLEVEL%" == "0" goto init
|
||||
|
||||
echo.
|
||||
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
||||
echo.
|
||||
echo Please set the JAVA_HOME variable in your environment to match the
|
||||
echo location of your Java installation.
|
||||
|
||||
goto fail
|
||||
|
||||
:findJavaFromJavaHome
|
||||
set JAVA_HOME=%JAVA_HOME:"=%
|
||||
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
||||
|
||||
if exist "%JAVA_EXE%" goto init
|
||||
|
||||
echo.
|
||||
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
|
||||
echo.
|
||||
echo Please set the JAVA_HOME variable in your environment to match the
|
||||
echo location of your Java installation.
|
||||
|
||||
goto fail
|
||||
|
||||
:init
|
||||
@rem Get command-line arguments, handling Windowz variants
|
||||
|
||||
if not "%OS%" == "Windows_NT" goto win9xME_args
|
||||
if "%@eval[2+2]" == "4" goto 4NT_args
|
||||
|
||||
:win9xME_args
|
||||
@rem Slurp the command line arguments.
|
||||
set CMD_LINE_ARGS=
|
||||
set _SKIP=2
|
||||
|
||||
:win9xME_args_slurp
|
||||
if "x%~1" == "x" goto execute
|
||||
|
||||
set CMD_LINE_ARGS=%*
|
||||
goto execute
|
||||
|
||||
:4NT_args
|
||||
@rem Get arguments from the 4NT Shell from JP Software
|
||||
set CMD_LINE_ARGS=%$
|
||||
|
||||
:execute
|
||||
@rem Setup the command line
|
||||
|
||||
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
|
||||
|
||||
@rem Execute Gradle
|
||||
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
|
||||
|
||||
:end
|
||||
@rem End local scope for the variables with windows NT shell
|
||||
if "%ERRORLEVEL%"=="0" goto mainEnd
|
||||
|
||||
:fail
|
||||
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
|
||||
rem the _cmd.exe /c_ return code!
|
||||
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
|
||||
exit /b 1
|
||||
|
||||
:mainEnd
|
||||
if "%OS%"=="Windows_NT" endlocal
|
||||
|
||||
:omega
|
||||
1
logstash-core-event-java/lib/logstash-core-event-java.rb
Normal file
1
logstash-core-event-java/lib/logstash-core-event-java.rb
Normal file
|
|
@ -0,0 +1 @@
|
|||
require "logstash-core-event-java/logstash-core-event-java"
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
# encoding: utf-8
|
||||
|
||||
require "java"
|
||||
|
||||
module LogStash
|
||||
end
|
||||
|
||||
# TODO: (colin) integrate jar loading with gradle and verify dev vs prod environment setups
|
||||
|
||||
# insert all jars in this directory into CLASSPATH
|
||||
Dir.glob(File.join(File.expand_path("..", __FILE__), "*.jar")).each do |jar|
|
||||
$CLASSPATH << jar unless $CLASSPATH.include?(jar)
|
||||
end
|
||||
|
||||
# TODO: (colin) correctly handle dev env build/ dir and local jar
|
||||
|
||||
# local dev setup
|
||||
classes_dir = File.expand_path("../../../build/classes/main", __FILE__)
|
||||
|
||||
if File.directory?(classes_dir)
|
||||
# if in local dev setup, add target to classpath
|
||||
$CLASSPATH << classes_dir unless $CLASSPATH.include?(classes_dir)
|
||||
else
|
||||
# otherwise use included jar
|
||||
raise("TODO build dir not found and no jar file")
|
||||
end
|
||||
|
||||
require "jruby_event_ext"
|
||||
require "jruby_timestamp_ext"
|
||||
require "logstash/event"
|
||||
require "logstash/timestamp"
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
# encoding: utf-8
|
||||
|
||||
# The version of logstash core event java gem.
|
||||
#
|
||||
# Note to authors: this should not include dashes because 'gem' barfs if
|
||||
# you include a dash in the version string.
|
||||
|
||||
LOGSTASH_CORE_EVENT_JAVA_VERSION = "3.0.0.dev"
|
||||
1
logstash-core-event-java/lib/logstash-core-event.rb
Normal file
1
logstash-core-event-java/lib/logstash-core-event.rb
Normal file
|
|
@ -0,0 +1 @@
|
|||
require "logstash-core-event-java/logstash-core-event-java"
|
||||
24
logstash-core-event-java/lib/logstash/event.rb
Normal file
24
logstash-core-event-java/lib/logstash/event.rb
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
# encoding: utf-8
|
||||
|
||||
require "logstash/namespace"
|
||||
require "logstash/json"
|
||||
|
||||
# transcient pipeline events for normal in-flow signaling as opposed to
|
||||
# flow altering exceptions. for now having base classes is adequate and
|
||||
# in the future it might be necessary to refactor using like a BaseEvent
|
||||
# class to have a common interface for all pileline events to support
|
||||
# eventual queueing persistence for example, TBD.
|
||||
class LogStash::ShutdownEvent; end
|
||||
class LogStash::FlushEvent; end
|
||||
|
||||
module LogStash
|
||||
FLUSH = LogStash::FlushEvent.new
|
||||
|
||||
# LogStash::SHUTDOWN is used by plugins
|
||||
SHUTDOWN = LogStash::ShutdownEvent.new
|
||||
end
|
||||
|
||||
# for backward compatibility, require "logstash/event" is used a lots of places so let's bootstrap the
|
||||
# Java code loading from here.
|
||||
# TODO: (colin) I think we should mass replace require "logstash/event" with require "logstash-core-event"
|
||||
require "logstash-core-event"
|
||||
28
logstash-core-event-java/lib/logstash/timestamp.rb
Normal file
28
logstash-core-event-java/lib/logstash/timestamp.rb
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
# encoding: utf-8
|
||||
|
||||
require "logstash/namespace"
|
||||
require "logstash-core-event"
|
||||
|
||||
module LogStash
|
||||
class TimestampParserError < StandardError; end
|
||||
|
||||
class Timestamp
|
||||
include Comparable
|
||||
|
||||
# TODO (colin) implement in Java
|
||||
def <=>(other)
|
||||
self.time <=> other.time
|
||||
end
|
||||
|
||||
# TODO (colin) implement in Java
|
||||
def +(other)
|
||||
self.time + other
|
||||
end
|
||||
|
||||
# TODO (colin) implement in Java
|
||||
def -(value)
|
||||
self.time - (value.is_a?(Timestamp) ? value.time : value)
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
23
logstash-core-event-java/logstash-core-event-java.gemspec
Normal file
23
logstash-core-event-java/logstash-core-event-java.gemspec
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
# -*- encoding: utf-8 -*-
|
||||
lib = File.expand_path('../lib', __FILE__)
|
||||
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
||||
require 'logstash-core-event-java/version'
|
||||
|
||||
Gem::Specification.new do |gem|
|
||||
gem.authors = ["Jordan Sissel", "Pete Fritchman", "Elasticsearch"]
|
||||
gem.email = ["jls@semicomplete.com", "petef@databits.net", "info@elasticsearch.com"]
|
||||
gem.description = %q{The core event component of logstash, the scalable log and event management tool}
|
||||
gem.summary = %q{logstash-core-event-java - The core event component of logstash}
|
||||
gem.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
|
||||
gem.license = "Apache License (2.0)"
|
||||
|
||||
gem.files = Dir.glob(["logstash-core-event-java.gemspec", "lib/**/*.rb", "spec/**/*.rb"])
|
||||
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
||||
gem.name = "logstash-core-event-java"
|
||||
gem.require_paths = ["lib"]
|
||||
gem.version = LOGSTASH_CORE_EVENT_JAVA_VERSION
|
||||
|
||||
if RUBY_PLATFORM == 'java'
|
||||
gem.platform = RUBY_PLATFORM
|
||||
end
|
||||
end
|
||||
2
logstash-core-event-java/settings.gradle
Normal file
2
logstash-core-event-java/settings.gradle
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
rootProject.name = 'logstash-core-event-java'
|
||||
|
||||
138
logstash-core-event-java/spec/event_spec.rb
Normal file
138
logstash-core-event-java/spec/event_spec.rb
Normal file
|
|
@ -0,0 +1,138 @@
|
|||
# encoding: utf-8
|
||||
|
||||
require "spec_helper"
|
||||
require "logstash/util"
|
||||
require "logstash/event"
|
||||
require "json"
|
||||
|
||||
TIMESTAMP = "@timestamp"
|
||||
|
||||
describe LogStash::Event do
|
||||
context "to_json" do
|
||||
it "should serialize simple values" do
|
||||
e = LogStash::Event.new({"foo" => "bar", "bar" => 1, "baz" => 1.0, TIMESTAMP => "2015-05-28T23:02:05.350Z"})
|
||||
expect(JSON.parse(e.to_json)).to eq(JSON.parse("{\"foo\":\"bar\",\"bar\":1,\"baz\":1.0,\"@timestamp\":\"2015-05-28T23:02:05.350Z\",\"@version\":\"1\"}"))
|
||||
end
|
||||
|
||||
it "should serialize deep hash values" do
|
||||
e = LogStash::Event.new({"foo" => {"bar" => 1, "baz" => 1.0, "biz" => "boz"}, TIMESTAMP => "2015-05-28T23:02:05.350Z"})
|
||||
expect(JSON.parse(e.to_json)).to eq(JSON.parse("{\"foo\":{\"bar\":1,\"baz\":1.0,\"biz\":\"boz\"},\"@timestamp\":\"2015-05-28T23:02:05.350Z\",\"@version\":\"1\"}"))
|
||||
end
|
||||
|
||||
it "should serialize deep array values" do
|
||||
e = LogStash::Event.new({"foo" => ["bar", 1, 1.0], TIMESTAMP => "2015-05-28T23:02:05.350Z"})
|
||||
expect(JSON.parse(e.to_json)).to eq(JSON.parse("{\"foo\":[\"bar\",1,1.0],\"@timestamp\":\"2015-05-28T23:02:05.350Z\",\"@version\":\"1\"}"))
|
||||
end
|
||||
|
||||
it "should serialize deep hash from field reference assignments" do
|
||||
e = LogStash::Event.new({TIMESTAMP => "2015-05-28T23:02:05.350Z"})
|
||||
e["foo"] = "bar"
|
||||
e["bar"] = 1
|
||||
e["baz"] = 1.0
|
||||
e["[fancy][pants][socks]"] = "shoes"
|
||||
expect(JSON.parse(e.to_json)).to eq(JSON.parse("{\"@timestamp\":\"2015-05-28T23:02:05.350Z\",\"@version\":\"1\",\"foo\":\"bar\",\"bar\":1,\"baz\":1.0,\"fancy\":{\"pants\":{\"socks\":\"shoes\"}}}"))
|
||||
end
|
||||
end
|
||||
|
||||
context "[]" do
|
||||
it "should get simple values" do
|
||||
e = LogStash::Event.new({"foo" => "bar", "bar" => 1, "baz" => 1.0, TIMESTAMP => "2015-05-28T23:02:05.350Z"})
|
||||
expect(e["foo"]).to eq("bar")
|
||||
expect(e["[foo]"]).to eq("bar")
|
||||
expect(e["bar"]).to eq(1)
|
||||
expect(e["[bar]"]).to eq(1)
|
||||
expect(e["baz"]).to eq(1.0)
|
||||
expect(e["[baz]"]).to eq(1.0)
|
||||
expect(e[TIMESTAMP].to_s).to eq("2015-05-28T23:02:05.350Z")
|
||||
expect(e["[#{TIMESTAMP}]"].to_s).to eq("2015-05-28T23:02:05.350Z")
|
||||
end
|
||||
|
||||
it "should get deep hash values" do
|
||||
e = LogStash::Event.new({"foo" => {"bar" => 1, "baz" => 1.0}})
|
||||
expect(e["[foo][bar]"]).to eq(1)
|
||||
expect(e["[foo][baz]"]).to eq(1.0)
|
||||
end
|
||||
|
||||
it "should get deep array values" do
|
||||
e = LogStash::Event.new({"foo" => ["bar", 1, 1.0]})
|
||||
expect(e["[foo][0]"]).to eq("bar")
|
||||
expect(e["[foo][1]"]).to eq(1)
|
||||
expect(e["[foo][2]"]).to eq(1.0)
|
||||
expect(e["[foo][3]"]).to be_nil
|
||||
end
|
||||
end
|
||||
|
||||
context "[]=" do
|
||||
it "should set simple values" do
|
||||
e = LogStash::Event.new()
|
||||
expect(e["foo"] = "bar").to eq("bar")
|
||||
expect(e["foo"]).to eq("bar")
|
||||
|
||||
e = LogStash::Event.new({"foo" => "test"})
|
||||
expect(e["foo"] = "bar").to eq("bar")
|
||||
expect(e["foo"]).to eq("bar")
|
||||
end
|
||||
|
||||
it "should set deep hash values" do
|
||||
e = LogStash::Event.new()
|
||||
expect(e["[foo][bar]"] = "baz").to eq("baz")
|
||||
expect(e["[foo][bar]"]).to eq("baz")
|
||||
expect(e["[foo][baz]"]).to be_nil
|
||||
end
|
||||
|
||||
it "should set deep array values" do
|
||||
e = LogStash::Event.new()
|
||||
expect(e["[foo][0]"] = "bar").to eq("bar")
|
||||
expect(e["[foo][0]"]).to eq("bar")
|
||||
expect(e["[foo][1]"] = 1).to eq(1)
|
||||
expect(e["[foo][1]"]).to eq(1)
|
||||
expect(e["[foo][2]"] = 1.0 ).to eq(1.0)
|
||||
expect(e["[foo][2]"]).to eq(1.0)
|
||||
expect(e["[foo][3]"]).to be_nil
|
||||
end
|
||||
end
|
||||
|
||||
context "timestamp" do
|
||||
it "getters should present a Ruby LogStash::Timestamp" do
|
||||
e = LogStash::Event.new()
|
||||
expect(e.timestamp.class).to eq(LogStash::Timestamp)
|
||||
expect(e[TIMESTAMP].class).to eq(LogStash::Timestamp)
|
||||
end
|
||||
|
||||
it "to_hash should inject a Ruby LogStash::Timestamp" do
|
||||
e = LogStash::Event.new()
|
||||
|
||||
expect(e.to_java).to be_kind_of(Java::ComLogstash::Event)
|
||||
expect(e.to_java.get_field(TIMESTAMP)).to be_kind_of(Java::ComLogstash::Timestamp)
|
||||
|
||||
expect(e.to_hash[TIMESTAMP]).to be_kind_of(LogStash::Timestamp)
|
||||
# now make sure the original map was not touched
|
||||
expect(e.to_java.get_field(TIMESTAMP)).to be_kind_of(Java::ComLogstash::Timestamp)
|
||||
end
|
||||
|
||||
it "should set timestamp" do
|
||||
e = LogStash::Event.new
|
||||
now = Time.now
|
||||
e["@timestamp"] = LogStash::Timestamp.at(now.to_i)
|
||||
expect(e.timestamp.to_i).to eq(now.to_i)
|
||||
expect(e["@timestamp"].to_i).to eq(now.to_i)
|
||||
end
|
||||
end
|
||||
|
||||
context "append" do
|
||||
it "should append" do
|
||||
event = LogStash::Event.new("message" => "hello world")
|
||||
event.append(LogStash::Event.new("message" => "another thing"))
|
||||
expect(event["message"]).to eq(["hello world", "another thing"])
|
||||
end
|
||||
end
|
||||
|
||||
context "tags" do
|
||||
it "should tag" do
|
||||
event = LogStash::Event.new("message" => "hello world")
|
||||
expect(event["tags"]).to be_nil
|
||||
event["tags"] = ["foo"]
|
||||
expect(event["tags"]).to eq(["foo"])
|
||||
end
|
||||
end
|
||||
end
|
||||
29
logstash-core-event-java/spec/timestamp_spec.rb
Normal file
29
logstash-core-event-java/spec/timestamp_spec.rb
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
# encoding: utf-8
|
||||
|
||||
require "spec_helper"
|
||||
require "logstash/timestamp"
|
||||
|
||||
describe LogStash::Timestamp do
|
||||
context "constructors" do
|
||||
it "should work" do
|
||||
t = LogStash::Timestamp.new
|
||||
expect(t.time.to_i).to be_within(1).of Time.now.to_i
|
||||
|
||||
t = LogStash::Timestamp.now
|
||||
expect(t.time.to_i).to be_within(1).of Time.now.to_i
|
||||
|
||||
now = Time.now.utc
|
||||
t = LogStash::Timestamp.new(now)
|
||||
expect(t.time).to eq(now)
|
||||
|
||||
t = LogStash::Timestamp.at(now.to_i)
|
||||
expect(t.time.to_i).to eq(now.to_i)
|
||||
end
|
||||
|
||||
it "should raise exception on invalid format" do
|
||||
expect{LogStash::Timestamp.new("foobar")}.to raise_error
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
end
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
import com.logstash.ext.JrubyEventExtLibrary;
|
||||
import org.jruby.Ruby;
|
||||
import org.jruby.runtime.load.BasicLibraryService;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public class JrubyEventExtService implements BasicLibraryService {
|
||||
public boolean basicLoad(final Ruby runtime)
|
||||
throws IOException
|
||||
{
|
||||
new JrubyEventExtLibrary().load(runtime, false);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
import com.logstash.ext.JrubyEventExtLibrary;
|
||||
import com.logstash.ext.JrubyTimestampExtLibrary;
|
||||
import org.jruby.Ruby;
|
||||
import org.jruby.runtime.load.BasicLibraryService;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public class JrubyTimestampExtService implements BasicLibraryService {
|
||||
public boolean basicLoad(final Ruby runtime)
|
||||
throws IOException
|
||||
{
|
||||
new JrubyTimestampExtLibrary().load(runtime, false);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,153 @@
|
|||
package com.logstash;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.List;
|
||||
|
||||
public class Accessors {
|
||||
|
||||
private Map<String, Object> data;
|
||||
protected Map<String, Object> lut;
|
||||
|
||||
public Accessors(Map<String, Object> data) {
|
||||
this.data = data;
|
||||
this.lut = new HashMap<>(); // reference -> target LUT
|
||||
}
|
||||
|
||||
public Object get(String reference) {
|
||||
FieldReference field = PathCache.getInstance().cache(reference);
|
||||
Object target = findTarget(field);
|
||||
return (target == null) ? null : fetch(target, field.getKey());
|
||||
}
|
||||
|
||||
public Object set(String reference, Object value) {
|
||||
FieldReference field = PathCache.getInstance().cache(reference);
|
||||
Object target = findCreateTarget(field);
|
||||
return store(target, field.getKey(), value);
|
||||
}
|
||||
|
||||
public Object del(String reference) {
|
||||
FieldReference field = PathCache.getInstance().cache(reference);
|
||||
Object target = findTarget(field);
|
||||
if (target != null) {
|
||||
if (target instanceof Map) {
|
||||
return ((Map<String, Object>) target).remove(field.getKey());
|
||||
} else if (target instanceof List) {
|
||||
int i = Integer.parseInt(field.getKey());
|
||||
if (i < 0 || i >= ((List) target).size()) {
|
||||
return null;
|
||||
}
|
||||
return ((List<Object>) target).remove(i);
|
||||
} else {
|
||||
throw new ClassCastException("expecting List or Map");
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public boolean includes(String reference) {
|
||||
FieldReference field = PathCache.getInstance().cache(reference);
|
||||
Object target = findTarget(field);
|
||||
if (target instanceof Map && foundInMap((Map<String, Object>) target, field.getKey())) {
|
||||
return true;
|
||||
} else if (target instanceof List && foundInList((List<Object>) target, Integer.parseInt(field.getKey()))) {
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private Object findTarget(FieldReference field) {
|
||||
Object target;
|
||||
|
||||
if ((target = this.lut.get(field.getReference())) != null) {
|
||||
return target;
|
||||
}
|
||||
|
||||
target = this.data;
|
||||
for (String key : field.getPath()) {
|
||||
target = fetch(target, key);
|
||||
if (target == null) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
this.lut.put(field.getReference(), target);
|
||||
|
||||
return target;
|
||||
}
|
||||
|
||||
private Object findCreateTarget(FieldReference field) {
|
||||
Object target;
|
||||
|
||||
if ((target = this.lut.get(field.getReference())) != null) {
|
||||
return target;
|
||||
}
|
||||
|
||||
target = this.data;
|
||||
for (String key : field.getPath()) {
|
||||
Object result = fetch(target, key);
|
||||
if (result == null) {
|
||||
result = new HashMap<String, Object>();
|
||||
if (target instanceof Map) {
|
||||
((Map<String, Object>)target).put(key, result);
|
||||
} else if (target instanceof List) {
|
||||
int i = Integer.parseInt(key);
|
||||
// TODO: what about index out of bound?
|
||||
((List<Object>)target).set(i, result);
|
||||
} else if (target == null) {
|
||||
// do nothing
|
||||
} else {
|
||||
throw new ClassCastException("expecting List or Map");
|
||||
}
|
||||
}
|
||||
target = result;
|
||||
}
|
||||
|
||||
this.lut.put(field.getReference(), target);
|
||||
|
||||
return target;
|
||||
}
|
||||
|
||||
private boolean foundInList(List<Object> target, int index) {
|
||||
if (index < 0 || index >= target.size()) {
|
||||
return false;
|
||||
}
|
||||
return target.get(index) != null;
|
||||
}
|
||||
|
||||
private boolean foundInMap(Map<String, Object> target, String key) {
|
||||
return target.containsKey(key);
|
||||
}
|
||||
|
||||
private Object fetch(Object target, String key) {
|
||||
if (target instanceof Map) {
|
||||
Object result = ((Map<String, Object>) target).get(key);
|
||||
return result;
|
||||
} else if (target instanceof List) {
|
||||
int i = Integer.parseInt(key);
|
||||
if (i < 0 || i >= ((List) target).size()) {
|
||||
return null;
|
||||
}
|
||||
Object result = ((List<Object>) target).get(i);
|
||||
return result;
|
||||
} else if (target == null) {
|
||||
return null;
|
||||
} {
|
||||
throw new ClassCastException("expecting List or Map");
|
||||
}
|
||||
}
|
||||
|
||||
private Object store(Object target, String key, Object value) {
|
||||
if (target instanceof Map) {
|
||||
((Map<String, Object>) target).put(key, value);
|
||||
} else if (target instanceof List) {
|
||||
int i = Integer.parseInt(key);
|
||||
// TODO: what about index out of bound?
|
||||
((List<Object>) target).set(i, value);
|
||||
} else {
|
||||
throw new ClassCastException("expecting List or Map");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
package com.logstash;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
public final class Cloner {
|
||||
|
||||
private Cloner(){}
|
||||
|
||||
public static <T> T deep(final T input) {
|
||||
if (input instanceof Map<?, ?>) {
|
||||
return (T) deepMap((Map<?, ?>) input);
|
||||
} else if (input instanceof List<?>) {
|
||||
return (T) deepList((List<?>) input);
|
||||
} else if (input instanceof Collection<?>) {
|
||||
throw new ClassCastException("unexpected Collection type " + input.getClass());
|
||||
}
|
||||
|
||||
return input;
|
||||
}
|
||||
|
||||
private static <E> List<E> deepList(final List<E> list) {
|
||||
List<E> clone;
|
||||
if (list instanceof LinkedList<?>) {
|
||||
clone = new LinkedList<E>();
|
||||
} else if (list instanceof ArrayList<?>) {
|
||||
clone = new ArrayList<E>();
|
||||
} else {
|
||||
throw new ClassCastException("unexpected List type " + list.getClass());
|
||||
}
|
||||
|
||||
for (E item : list) {
|
||||
clone.add(deep(item));
|
||||
}
|
||||
|
||||
return clone;
|
||||
}
|
||||
|
||||
private static <K, V> Map<K, V> deepMap(final Map<K, V> map) {
|
||||
Map<K, V> clone;
|
||||
if (map instanceof LinkedHashMap<?, ?>) {
|
||||
clone = new LinkedHashMap<K, V>();
|
||||
} else if (map instanceof TreeMap<?, ?>) {
|
||||
clone = new TreeMap<K, V>();
|
||||
} else if (map instanceof HashMap<?, ?>) {
|
||||
clone = new HashMap<K, V>();
|
||||
} else {
|
||||
throw new ClassCastException("unexpected Map type " + map.getClass());
|
||||
}
|
||||
|
||||
for (Map.Entry<K, V> entry : map.entrySet()) {
|
||||
clone.put(entry.getKey(), deep(entry.getValue()));
|
||||
}
|
||||
|
||||
return clone;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
package com.logstash;
|
||||
|
||||
import org.joda.time.DateTimeZone;
|
||||
import org.joda.time.format.DateTimeFormat;
|
||||
import org.joda.time.format.DateTimeFormatter;
|
||||
|
||||
import java.io.IOError;
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* Created by ph on 15-05-22.
|
||||
*/
|
||||
public class DateNode implements TemplateNode {
|
||||
private DateTimeFormatter formatter;
|
||||
|
||||
public DateNode(String format) {
|
||||
this.formatter = DateTimeFormat.forPattern(format).withZone(DateTimeZone.UTC);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String evaluate(Event event) throws IOException {
|
||||
return event.getTimestamp().getTime().toString(this.formatter);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
package com.logstash;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* Created by ph on 15-05-22.
|
||||
*/
|
||||
public class EpochNode implements TemplateNode {
|
||||
public EpochNode(){ }
|
||||
|
||||
@Override
|
||||
public String evaluate(Event event) throws IOException {
|
||||
return String.valueOf(event.getTimestamp().getTime().getMillis() / 1000);
|
||||
}
|
||||
}
|
||||
253
logstash-core-event-java/src/main/java/com/logstash/Event.java
Normal file
253
logstash-core-event-java/src/main/java/com/logstash/Event.java
Normal file
|
|
@ -0,0 +1,253 @@
|
|||
package com.logstash;
|
||||
|
||||
import com.logstash.ext.JrubyTimestampExtLibrary;
|
||||
import org.codehaus.jackson.map.ObjectMapper;
|
||||
import org.joda.time.DateTime;
|
||||
import org.jruby.RubyHash;
|
||||
import org.jruby.RubySymbol;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Serializable;
|
||||
import java.util.*;
|
||||
|
||||
|
||||
public class Event implements Cloneable, Serializable {
|
||||
|
||||
private boolean cancelled;
|
||||
private Map<String, Object> data;
|
||||
private Map<String, Object> metadata;
|
||||
private Timestamp timestamp;
|
||||
private Accessors accessors;
|
||||
private Accessors metadata_accessors;
|
||||
|
||||
public static final String METADATA = "@metadata";
|
||||
public static final String METADATA_BRACKETS = "[" + METADATA + "]";
|
||||
public static final String TIMESTAMP = "@timestamp";
|
||||
public static final String TIMESTAMP_FAILURE_TAG = "_timestampparsefailure";
|
||||
public static final String TIMESTAMP_FAILURE_FIELD = "_@timestamp";
|
||||
public static final String VERSION = "@version";
|
||||
public static final String VERSION_ONE = "1";
|
||||
|
||||
private static final ObjectMapper mapper = new ObjectMapper();
|
||||
|
||||
// TODO: add metadata support
|
||||
|
||||
public Event()
|
||||
{
|
||||
this.metadata = new HashMap<String, Object>();
|
||||
this.data = new HashMap<String, Object>();
|
||||
this.data.put(VERSION, VERSION_ONE);
|
||||
this.cancelled = false;
|
||||
this.timestamp = new Timestamp();
|
||||
this.data.put(TIMESTAMP, this.timestamp);
|
||||
this.accessors = new Accessors(this.data);
|
||||
this.metadata_accessors = new Accessors(this.metadata);
|
||||
}
|
||||
|
||||
public Event(Map data)
|
||||
{
|
||||
this.data = data;
|
||||
this.data.putIfAbsent(VERSION, VERSION_ONE);
|
||||
|
||||
if (this.data.containsKey(METADATA)) {
|
||||
this.metadata = (HashMap<String, Object>) this.data.remove(METADATA);
|
||||
} else {
|
||||
this.metadata = new HashMap<String, Object>();
|
||||
}
|
||||
this.metadata_accessors = new Accessors(this.metadata);
|
||||
|
||||
this.cancelled = false;
|
||||
this.timestamp = initTimestamp(data.get(TIMESTAMP));
|
||||
this.data.put(TIMESTAMP, this.timestamp);
|
||||
this.accessors = new Accessors(this.data);
|
||||
}
|
||||
|
||||
public Map<String, Object> getData() {
|
||||
return this.data;
|
||||
}
|
||||
|
||||
public Map<String, Object> getMetadata() {
|
||||
return this.metadata;
|
||||
}
|
||||
|
||||
public void setData(Map<String, Object> data) {
|
||||
this.data = data;
|
||||
}
|
||||
|
||||
public Accessors getAccessors() {
|
||||
return this.accessors;
|
||||
}
|
||||
|
||||
public Accessors getMetadataAccessors() {
|
||||
return this.metadata_accessors;
|
||||
}
|
||||
|
||||
public void setAccessors(Accessors accessors) {
|
||||
this.accessors = accessors;
|
||||
}
|
||||
|
||||
public void setMetadataAccessors(Accessors accessors) {
|
||||
this.metadata_accessors = accessors;
|
||||
}
|
||||
|
||||
public void cancel() {
|
||||
this.cancelled = true;
|
||||
}
|
||||
|
||||
public void uncancel() {
|
||||
this.cancelled = false;
|
||||
}
|
||||
|
||||
public boolean isCancelled() {
|
||||
return this.cancelled;
|
||||
}
|
||||
|
||||
public Timestamp getTimestamp() throws IOException {
|
||||
if (this.data.containsKey(TIMESTAMP)) {
|
||||
return this.timestamp;
|
||||
} else {
|
||||
throw new IOException("fails");
|
||||
}
|
||||
}
|
||||
|
||||
public void setTimestamp(Timestamp t) {
|
||||
this.timestamp = t;
|
||||
this.data.put(TIMESTAMP, this.timestamp);
|
||||
}
|
||||
|
||||
public Object getField(String reference) {
|
||||
if (reference.equals(METADATA)) {
|
||||
return this.metadata;
|
||||
} else if (reference.startsWith(METADATA_BRACKETS)) {
|
||||
return this.metadata_accessors.get(reference.substring(METADATA_BRACKETS.length()));
|
||||
} else {
|
||||
return this.accessors.get(reference);
|
||||
}
|
||||
}
|
||||
|
||||
public void setField(String reference, Object value) {
|
||||
if (reference.equals(TIMESTAMP)) {
|
||||
// TODO(talevy): check type of timestamp
|
||||
this.accessors.set(reference, value);
|
||||
} else if (reference.equals(METADATA_BRACKETS) || reference.equals(METADATA)) {
|
||||
this.metadata = (HashMap<String, Object>) value;
|
||||
this.metadata_accessors = new Accessors(this.metadata);
|
||||
} else if (reference.startsWith(METADATA_BRACKETS)) {
|
||||
this.metadata_accessors.set(reference.substring(METADATA_BRACKETS.length()), value);
|
||||
} else {
|
||||
this.accessors.set(reference, value);
|
||||
}
|
||||
}
|
||||
|
||||
public boolean includes(String reference) {
|
||||
if (reference.equals(METADATA_BRACKETS) || reference.equals(METADATA)) {
|
||||
return true;
|
||||
} else if (reference.startsWith(METADATA_BRACKETS)) {
|
||||
return this.metadata_accessors.includes(reference.substring(METADATA_BRACKETS.length()));
|
||||
} else {
|
||||
return this.accessors.includes(reference);
|
||||
}
|
||||
}
|
||||
|
||||
public String toJson() throws IOException {
|
||||
return mapper.writeValueAsString((Map<String, Object>)this.data);
|
||||
}
|
||||
|
||||
public Map toMap() {
|
||||
return this.data;
|
||||
}
|
||||
|
||||
public Event overwrite(Event e) {
|
||||
this.data = e.getData();
|
||||
this.accessors = e.getAccessors();
|
||||
this.cancelled = e.isCancelled();
|
||||
try {
|
||||
this.timestamp = e.getTimestamp();
|
||||
} catch (IOException exception) {
|
||||
this.timestamp = new Timestamp();
|
||||
}
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
|
||||
public Event append(Event e) {
|
||||
Util.mapMerge(this.data, e.data);
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
public Object remove(String path) {
|
||||
return this.accessors.del(path);
|
||||
}
|
||||
|
||||
public String sprintf(String s) throws IOException {
|
||||
return StringInterpolation.getInstance().evaluate(this, s);
|
||||
}
|
||||
|
||||
public Event clone()
|
||||
throws CloneNotSupportedException
|
||||
{
|
||||
// Event clone = (Event)super.clone();
|
||||
// clone.setAccessors(new Accessors(clone.getData()));
|
||||
|
||||
Event clone = new Event(Cloner.deep(getData()));
|
||||
return clone;
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
// TODO: until we have sprintf
|
||||
String host = (String)this.data.getOrDefault("host", "%{host}");
|
||||
String message = (String)this.data.getOrDefault("message", "%{message}");
|
||||
try {
|
||||
return getTimestamp().toIso8601() + " " + host + " " + message;
|
||||
} catch (IOException e) {
|
||||
return host + " " + message;
|
||||
}
|
||||
}
|
||||
|
||||
private Timestamp initTimestamp(Object o) {
|
||||
try {
|
||||
if (o == null) {
|
||||
// most frequent
|
||||
return new Timestamp();
|
||||
} else if (o instanceof String) {
|
||||
// second most frequent
|
||||
return new Timestamp((String) o);
|
||||
} else if (o instanceof JrubyTimestampExtLibrary.RubyTimestamp) {
|
||||
return new Timestamp(((JrubyTimestampExtLibrary.RubyTimestamp) o).getTimestamp());
|
||||
} else if (o instanceof Timestamp) {
|
||||
return new Timestamp((Timestamp) o);
|
||||
} else if (o instanceof DateTime) {
|
||||
return new Timestamp((DateTime) o);
|
||||
} else if (o instanceof Date) {
|
||||
return new Timestamp((Date) o);
|
||||
} else if (o instanceof RubySymbol) {
|
||||
return new Timestamp(((RubySymbol) o).asJavaString());
|
||||
} else {
|
||||
// TODO: add logging
|
||||
//return Timestamp.now();
|
||||
throw new IllegalArgumentException();
|
||||
}
|
||||
} catch (IllegalArgumentException e) {
|
||||
// TODO: add error logging
|
||||
tag(TIMESTAMP_FAILURE_TAG);
|
||||
|
||||
this.data.put(TIMESTAMP_FAILURE_FIELD, o);
|
||||
|
||||
return Timestamp.now();
|
||||
}
|
||||
}
|
||||
|
||||
public void tag(String tag) {
|
||||
List<Object> tags = (List<Object>) this.data.get("tags");
|
||||
if (tags == null) {
|
||||
tags = new ArrayList<>();
|
||||
this.data.put("tags", tags);
|
||||
}
|
||||
|
||||
if (!tags.contains(tag)) {
|
||||
tags.add(tag);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
package com.logstash;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
// TODO: implement thread-safe path cache singleton to avoid parsing
|
||||
|
||||
public class FieldReference {
|
||||
|
||||
private List<String> path;
|
||||
private String key;
|
||||
private String reference;
|
||||
private static List<String> EMPTY_STRINGS = new ArrayList(Arrays.asList(new String[]{""}));
|
||||
|
||||
public FieldReference(List<String> path, String key, String reference) {
|
||||
this.path = path;
|
||||
this.key = key;
|
||||
this.reference = reference;
|
||||
}
|
||||
|
||||
public List<String> getPath() {
|
||||
return path;
|
||||
}
|
||||
|
||||
public String getKey() {
|
||||
return key;
|
||||
}
|
||||
|
||||
public String getReference() {
|
||||
return reference;
|
||||
}
|
||||
|
||||
public static FieldReference parse(String reference) {
|
||||
List<String> path = new ArrayList(Arrays.asList(reference.split("[\\[\\]]")));
|
||||
path.removeAll(EMPTY_STRINGS);
|
||||
String key = path.remove(path.size() - 1);
|
||||
return new FieldReference(path, key, reference);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
package com.logstash;
|
||||
|
||||
import org.codehaus.jackson.JsonGenerationException;
|
||||
import org.codehaus.jackson.map.ObjectMapper;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Created by ph on 15-05-22.
|
||||
*/
|
||||
public class KeyNode implements TemplateNode {
|
||||
private String key;
|
||||
|
||||
public KeyNode(String key) {
|
||||
this.key = key;
|
||||
}
|
||||
|
||||
/**
|
||||
This will be more complicated with hash and array.
|
||||
leverage jackson lib to do the actual.
|
||||
*/
|
||||
@Override
|
||||
public String evaluate(Event event) throws IOException {
|
||||
Object value = event.getField(this.key);
|
||||
|
||||
if (value != null) {
|
||||
if (value instanceof List) {
|
||||
return String.join(",", (List) value);
|
||||
} else if (value instanceof Map) {
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
return mapper.writeValueAsString((Map<String, Object>)value);
|
||||
} else {
|
||||
return event.getField(this.key).toString();
|
||||
}
|
||||
|
||||
} else {
|
||||
return "%{" + this.key + "}";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
package com.logstash;
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
public class PathCache {
|
||||
|
||||
private static PathCache instance = null;
|
||||
private static ConcurrentHashMap<String, FieldReference> cache = new ConcurrentHashMap<>();
|
||||
|
||||
private FieldReference timestamp;
|
||||
|
||||
// TODO: dry with Event
|
||||
public static final String TIMESTAMP = "@timestamp";
|
||||
public static final String BRACKETS_TIMESTAMP = "[" + TIMESTAMP + "]";
|
||||
|
||||
protected PathCache() {
|
||||
// inject @timestamp
|
||||
this.timestamp = cache(TIMESTAMP);
|
||||
cache(BRACKETS_TIMESTAMP, this.timestamp);
|
||||
}
|
||||
|
||||
public static PathCache getInstance() {
|
||||
if (instance == null) {
|
||||
instance = new PathCache();
|
||||
}
|
||||
return instance;
|
||||
}
|
||||
|
||||
public boolean isTimestamp(String reference) {
|
||||
return (cache(reference) == this.timestamp);
|
||||
}
|
||||
|
||||
public FieldReference cache(String reference) {
|
||||
// atomicity between the get and put is not important
|
||||
FieldReference result = cache.get(reference);
|
||||
if (result == null) {
|
||||
result = FieldReference.parse(reference);
|
||||
cache.put(reference, result);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public FieldReference cache(String reference, FieldReference field) {
|
||||
cache.put(reference, field);
|
||||
return field;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
package com.logstash;
|
||||
|
||||
import org.jruby.RubyArray;
|
||||
import org.jruby.RubyHash;
|
||||
import org.jruby.RubyString;
|
||||
import org.jruby.runtime.builtin.IRubyObject;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
public class RubyToJavaConverter {
|
||||
|
||||
public static Object convert(IRubyObject obj) {
|
||||
if (obj instanceof RubyArray) {
|
||||
return convertToList((RubyArray) obj);
|
||||
} else if (obj instanceof RubyHash) {
|
||||
return convertToMap((RubyHash) obj);
|
||||
} else if (obj instanceof RubyString) {
|
||||
return convertToString((RubyString) obj);
|
||||
}
|
||||
|
||||
return obj.toJava(obj.getJavaClass());
|
||||
}
|
||||
|
||||
public static HashMap<String, Object> convertToMap(RubyHash hash) {
|
||||
HashMap<String, Object> hashMap = new HashMap();
|
||||
Set<RubyHash.RubyHashEntry> entries = hash.directEntrySet();
|
||||
for (RubyHash.RubyHashEntry e : entries) {
|
||||
hashMap.put(e.getJavaifiedKey().toString(), convert((IRubyObject) e.getValue()));
|
||||
}
|
||||
return hashMap;
|
||||
}
|
||||
|
||||
public static List<Object> convertToList(RubyArray array) {
|
||||
ArrayList<Object> list = new ArrayList();
|
||||
for (IRubyObject obj : array.toJavaArray()) {
|
||||
list.add(convert(obj));
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
public static String convertToString(RubyString string) {
|
||||
return string.decodeString();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
package com.logstash;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* Created by ph on 15-05-22.
|
||||
*/
|
||||
public class StaticNode implements TemplateNode {
|
||||
private String content;
|
||||
|
||||
public StaticNode(String content) {
|
||||
this.content = content;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String evaluate(Event event) throws IOException {
|
||||
return this.content;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
package com.logstash;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
public class StringInterpolation {
|
||||
static Pattern TEMPLATE_TAG = Pattern.compile("%\\{([^}]+)\\}");
|
||||
static Map cache;
|
||||
|
||||
protected static class HoldCurrent {
|
||||
private static final StringInterpolation INSTANCE = new StringInterpolation();
|
||||
}
|
||||
|
||||
private StringInterpolation() {
|
||||
// TODO:
|
||||
// This may need some tweaking for the concurrency level to get better memory usage.
|
||||
// The current implementation doesn't allow the keys to expire, I think under normal usage
|
||||
// the keys will converge to a fixed number.
|
||||
//
|
||||
// If this code make logstash goes OOM, we have the following options:
|
||||
// - If the key doesn't contains a `%` do not cache it, this will reduce the key size at a performance cost.
|
||||
// - Use some kind LRU cache
|
||||
// - Create a new data structure that use weakref or use Google Guava for the cache https://code.google.com/p/guava-libraries/
|
||||
this.cache = new ConcurrentHashMap<>();
|
||||
}
|
||||
|
||||
public String evaluate(Event event, String template) throws IOException {
|
||||
TemplateNode compiledTemplate = (TemplateNode) this.cache.get(template);
|
||||
|
||||
if(compiledTemplate == null) {
|
||||
compiledTemplate = this.compile(template);
|
||||
TemplateNode set = (TemplateNode) this.cache.putIfAbsent(template, compiledTemplate);
|
||||
compiledTemplate = (set != null) ? set : compiledTemplate;
|
||||
}
|
||||
|
||||
return compiledTemplate.evaluate(event);
|
||||
}
|
||||
|
||||
public TemplateNode compile(String template) {
|
||||
Template compiledTemplate = new Template();
|
||||
|
||||
if (template.indexOf('%') == -1) {
|
||||
// Move the nodes to a custom instance
|
||||
// so we can remove the iterator and do one `.evaluate`
|
||||
compiledTemplate.add(new StaticNode(template));
|
||||
} else {
|
||||
Matcher matcher = TEMPLATE_TAG.matcher(template);
|
||||
String tag;
|
||||
int pos = 0;
|
||||
|
||||
while (matcher.find()) {
|
||||
if (matcher.start() > 0) {
|
||||
compiledTemplate.add(new StaticNode(template.substring(pos, matcher.start())));
|
||||
}
|
||||
|
||||
tag = matcher.group(1);
|
||||
compiledTemplate.add(identifyTag(tag));
|
||||
pos = matcher.end();
|
||||
}
|
||||
|
||||
if(pos <= template.length() - 1) {
|
||||
compiledTemplate.add(new StaticNode(template.substring(pos)));
|
||||
}
|
||||
}
|
||||
|
||||
// if we only have one node return the node directly
|
||||
// and remove the need to loop.
|
||||
if(compiledTemplate.size() == 1) {
|
||||
return compiledTemplate.get(0);
|
||||
} else {
|
||||
return compiledTemplate;
|
||||
}
|
||||
}
|
||||
|
||||
public TemplateNode identifyTag(String tag) {
|
||||
if(tag.equals("+%s")) {
|
||||
return new EpochNode();
|
||||
} else if(tag.charAt(0) == '+') {
|
||||
return new DateNode(tag.substring(1));
|
||||
|
||||
} else {
|
||||
return new KeyNode(tag);
|
||||
}
|
||||
}
|
||||
|
||||
static StringInterpolation getInstance() {
|
||||
return HoldCurrent.INSTANCE;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
package com.logstash;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class Template implements TemplateNode {
|
||||
public List nodes = new ArrayList<>();
|
||||
public Template() {}
|
||||
|
||||
public void add(TemplateNode node) {
|
||||
nodes.add(node);
|
||||
}
|
||||
|
||||
public int size() {
|
||||
return nodes.size();
|
||||
}
|
||||
|
||||
public TemplateNode get(int index) {
|
||||
return (TemplateNode) nodes.get(index);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String evaluate(Event event) throws IOException {
|
||||
StringBuffer results = new StringBuffer();
|
||||
|
||||
for (int i = 0; i < nodes.size(); i++) {
|
||||
results.append(((TemplateNode) nodes.get(i)).evaluate(event));
|
||||
}
|
||||
return results.toString();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
package com.logstash;
|
||||
|
||||
import org.codehaus.jackson.JsonGenerationException;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* Created by ph on 15-05-22.
|
||||
*/
|
||||
public interface TemplateNode {
|
||||
String evaluate(Event event) throws IOException;
|
||||
}
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
package com.logstash;
|
||||
|
||||
import org.codehaus.jackson.map.annotate.JsonSerialize;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.DateTimeZone;
|
||||
import org.joda.time.format.DateTimeFormatter;
|
||||
import org.joda.time.format.ISODateTimeFormat;
|
||||
import org.jruby.Ruby;
|
||||
import org.jruby.RubyString;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
@JsonSerialize(using = TimestampSerializer.class)
|
||||
public class Timestamp implements Cloneable {
|
||||
|
||||
private DateTime time;
|
||||
// TODO: is this DateTimeFormatter thread safe?
|
||||
private static DateTimeFormatter iso8601Formatter = ISODateTimeFormat.dateTime();
|
||||
|
||||
public Timestamp() {
|
||||
this.time = new DateTime(DateTimeZone.UTC);
|
||||
}
|
||||
|
||||
public Timestamp(String iso8601) {
|
||||
this.time = ISODateTimeFormat.dateTimeParser().parseDateTime(iso8601).toDateTime(DateTimeZone.UTC);
|
||||
}
|
||||
|
||||
public Timestamp(Timestamp t) {
|
||||
this.time = t.getTime();
|
||||
}
|
||||
|
||||
public Timestamp(long epoch_milliseconds) {
|
||||
this.time = new DateTime(epoch_milliseconds, DateTimeZone.UTC);
|
||||
}
|
||||
|
||||
public Timestamp(Long epoch_milliseconds) {
|
||||
this.time = new DateTime(epoch_milliseconds, DateTimeZone.UTC);
|
||||
}
|
||||
|
||||
public Timestamp(Date date) {
|
||||
this.time = new DateTime(date, DateTimeZone.UTC);
|
||||
}
|
||||
|
||||
public Timestamp(DateTime date) {
|
||||
this.time = date.toDateTime(DateTimeZone.UTC);
|
||||
}
|
||||
|
||||
public DateTime getTime() {
|
||||
return time;
|
||||
}
|
||||
|
||||
public void setTime(DateTime time) {
|
||||
this.time = time;
|
||||
}
|
||||
|
||||
public static Timestamp now() {
|
||||
return new Timestamp();
|
||||
}
|
||||
|
||||
public String toIso8601() {
|
||||
return this.iso8601Formatter.print(this.time);
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
return toIso8601();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Timestamp clone() throws CloneNotSupportedException {
|
||||
Timestamp clone = (Timestamp)super.clone();
|
||||
clone.setTime(this.getTime());
|
||||
return clone;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
package com.logstash;
|
||||
|
||||
import org.codehaus.jackson.JsonGenerator;
|
||||
import org.codehaus.jackson.map.JsonSerializer;
|
||||
import org.codehaus.jackson.map.SerializerProvider;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public class TimestampSerializer extends JsonSerializer<Timestamp> {
|
||||
|
||||
@Override
|
||||
public void serialize(Timestamp value, JsonGenerator jgen, SerializerProvider provider)
|
||||
throws IOException
|
||||
{
|
||||
jgen.writeString(value.toIso8601());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
package com.logstash;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import org.jruby.RubyHash;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class Util {
|
||||
private Util() {}
|
||||
|
||||
public static void mapMerge(Map<String, Object> target, Map<String, Object> add) {
|
||||
for (Map.Entry<String, Object> e : add.entrySet()) {
|
||||
if (target.containsKey(e.getKey())) {
|
||||
if (target.get(e.getKey()) instanceof Map && e.getValue() instanceof Map) {
|
||||
mapMerge((Map<String, Object>) target.get(e.getKey()), (Map<String, Object>) e.getValue());
|
||||
} else if (e.getValue() instanceof List) {
|
||||
if (target.get(e.getKey()) instanceof List) {
|
||||
// needs optimizing
|
||||
List targetList = (List) target.get(e.getKey());
|
||||
targetList.addAll((List) e.getValue());
|
||||
target.put(e.getKey(), new ArrayList<Object>(new LinkedHashSet<Object>(targetList)));
|
||||
} else {
|
||||
Object targetValue = target.get(e.getKey());
|
||||
List targetValueList = Lists.newArrayList(targetValue);
|
||||
for (Object o : (List) e.getValue()) {
|
||||
if (!targetValue.equals(o)) {
|
||||
targetValueList.add(o);
|
||||
}
|
||||
}
|
||||
target.put(e.getKey(), targetValueList);
|
||||
}
|
||||
} else if (target.get(e.getKey()) instanceof List) {
|
||||
List t = ((List) target.get(e.getKey()));
|
||||
if (!t.contains(e.getValue())) {
|
||||
t.add(e.getValue());
|
||||
}
|
||||
} else if (!target.get(e.getKey()).equals(e.getValue())) {
|
||||
Object targetValue = target.get(e.getKey());
|
||||
targetValue = Lists.newArrayList(targetValue);
|
||||
((List) targetValue).add(e.getValue());
|
||||
target.put(e.getKey(), targetValue);
|
||||
}
|
||||
} else {
|
||||
target.put(e.getKey(), e.getValue());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,278 @@
|
|||
package com.logstash.ext;
|
||||
|
||||
import com.logstash.Event;
|
||||
import com.logstash.PathCache;
|
||||
import com.logstash.RubyToJavaConverter;
|
||||
import com.logstash.Timestamp;
|
||||
import org.jruby.*;
|
||||
import org.jruby.anno.JRubyClass;
|
||||
import org.jruby.anno.JRubyConstant;
|
||||
import org.jruby.anno.JRubyMethod;
|
||||
import org.jruby.exceptions.RaiseException;
|
||||
import org.jruby.java.proxies.MapJavaProxy;
|
||||
import org.jruby.javasupport.JavaUtil;
|
||||
import org.jruby.runtime.Arity;
|
||||
import org.jruby.runtime.ObjectAllocator;
|
||||
import org.jruby.runtime.ThreadContext;
|
||||
import org.jruby.runtime.builtin.IRubyObject;
|
||||
import org.jruby.runtime.load.Library;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.*;
|
||||
|
||||
|
||||
public class JrubyEventExtLibrary implements Library {
|
||||
|
||||
public void load(Ruby runtime, boolean wrap) throws IOException {
|
||||
RubyModule module = runtime.defineModule("LogStash");
|
||||
RubyClass clazz = runtime.defineClassUnder("Event", runtime.getObject(), new ObjectAllocator() {
|
||||
public IRubyObject allocate(Ruby runtime, RubyClass rubyClass) {
|
||||
return new RubyEvent(runtime, rubyClass);
|
||||
}
|
||||
}, module);
|
||||
clazz.setConstant("LOGGER", runtime.getModule("Cabin").getClass("Channel")
|
||||
.callMethod("get", runtime.getModule("LogStash")));
|
||||
clazz.setConstant("TIMESTAMP", runtime.newString(Event.TIMESTAMP));
|
||||
clazz.setConstant("TIMESTAMP_FAILURE_TAG", runtime.newString(Event.TIMESTAMP_FAILURE_TAG));
|
||||
clazz.setConstant("TIMESTAMP_FAILURE_FIELD", runtime.newString(Event.TIMESTAMP_FAILURE_FIELD));
|
||||
clazz.defineAnnotatedMethods(RubyEvent.class);
|
||||
clazz.defineAnnotatedConstants(RubyEvent.class);
|
||||
}
|
||||
|
||||
@JRubyClass(name = "Event", parent = "Object")
|
||||
public static class RubyEvent extends RubyObject {
|
||||
private Event event;
|
||||
|
||||
public RubyEvent(Ruby runtime, RubyClass klass) {
|
||||
super(runtime, klass);
|
||||
}
|
||||
|
||||
public RubyEvent(Ruby runtime) {
|
||||
this(runtime, runtime.getModule("LogStash").getClass("Event"));
|
||||
}
|
||||
|
||||
public RubyEvent(Ruby runtime, Event event) {
|
||||
this(runtime);
|
||||
this.event = event;
|
||||
}
|
||||
|
||||
public static RubyEvent newRubyEvent(Ruby runtime, Event event) {
|
||||
return new RubyEvent(runtime, event);
|
||||
}
|
||||
|
||||
public Event getEvent() {
|
||||
return event;
|
||||
}
|
||||
|
||||
public void setEvent(Event event) {
|
||||
this.event = event;
|
||||
}
|
||||
|
||||
// def initialize(data = {})
|
||||
@JRubyMethod(name = "initialize", optional = 1)
|
||||
public IRubyObject ruby_initialize(ThreadContext context, IRubyObject[] args)
|
||||
{
|
||||
args = Arity.scanArgs(context.runtime, args, 0, 1);
|
||||
IRubyObject data = args[0];
|
||||
|
||||
if (data.isNil()) {
|
||||
this.event = new Event();
|
||||
} else if (data instanceof RubyHash) {
|
||||
HashMap<String, Object> newObj = RubyToJavaConverter.convertToMap((RubyHash) data);
|
||||
this.event = new Event(newObj);
|
||||
} else if (data instanceof Map) {
|
||||
this.event = new Event((Map) data);
|
||||
} else if (Map.class.isAssignableFrom(data.getJavaClass())) {
|
||||
this.event = new Event((Map)data.toJava(Map.class));
|
||||
} else {
|
||||
throw context.runtime.newTypeError("wrong argument type " + data.getMetaClass() + " (expected Hash)");
|
||||
}
|
||||
|
||||
return context.nil;
|
||||
}
|
||||
|
||||
@JRubyMethod(name = "[]", required = 1)
|
||||
public IRubyObject ruby_get_field(ThreadContext context, RubyString reference)
|
||||
{
|
||||
String r = reference.asJavaString();
|
||||
Object value = this.event.getField(r);
|
||||
if (value instanceof Timestamp) {
|
||||
return JrubyTimestampExtLibrary.RubyTimestamp.newRubyTimestamp(context.runtime, (Timestamp)value);
|
||||
} else if (value instanceof List) {
|
||||
IRubyObject obj = JavaUtil.convertJavaToRuby(context.runtime, value);
|
||||
return obj.callMethod(context, "to_a");
|
||||
} else {
|
||||
return JavaUtil.convertJavaToRuby(context.runtime, value);
|
||||
}
|
||||
}
|
||||
|
||||
@JRubyMethod(name = "[]=", required = 2)
|
||||
public IRubyObject ruby_set_field(ThreadContext context, RubyString reference, IRubyObject value)
|
||||
{
|
||||
String r = reference.asJavaString();
|
||||
if (PathCache.getInstance().isTimestamp(r)) {
|
||||
if (!(value instanceof JrubyTimestampExtLibrary.RubyTimestamp)) {
|
||||
throw context.runtime.newTypeError("wrong argument type " + value.getMetaClass() + " (expected LogStash::Timestamp)");
|
||||
}
|
||||
this.event.setTimestamp(((JrubyTimestampExtLibrary.RubyTimestamp)value).getTimestamp());
|
||||
} else {
|
||||
if (value instanceof RubyString) {
|
||||
String val = ((RubyString) value).asJavaString();
|
||||
this.event.setField(r, val);
|
||||
} else if (value instanceof RubyInteger) {
|
||||
this.event.setField(r, ((RubyInteger) value).getLongValue());
|
||||
} else if (value instanceof RubyFloat) {
|
||||
this.event.setField(r, ((RubyFloat) value).getDoubleValue());
|
||||
} else if (value instanceof JrubyTimestampExtLibrary.RubyTimestamp) {
|
||||
// RubyTimestamp could be assigned in another field thant @timestamp
|
||||
this.event.setField(r, ((JrubyTimestampExtLibrary.RubyTimestamp) value).getTimestamp());
|
||||
} else if (value instanceof RubyArray) {
|
||||
this.event.setField(r, RubyToJavaConverter.convertToList((RubyArray) value));
|
||||
} else if (value instanceof RubyHash) {
|
||||
this.event.setField(r, RubyToJavaConverter.convertToMap((RubyHash) value));
|
||||
} else {
|
||||
throw context.runtime.newTypeError("wrong argument type " + value.getMetaClass());
|
||||
}
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
@JRubyMethod(name = "cancel")
|
||||
public IRubyObject ruby_cancel(ThreadContext context)
|
||||
{
|
||||
this.event.cancel();
|
||||
return RubyBoolean.createTrueClass(context.runtime);
|
||||
}
|
||||
|
||||
@JRubyMethod(name = "uncancel")
|
||||
public IRubyObject ruby_uncancel(ThreadContext context)
|
||||
{
|
||||
this.event.uncancel();
|
||||
return RubyBoolean.createFalseClass(context.runtime);
|
||||
}
|
||||
|
||||
@JRubyMethod(name = "cancelled?")
|
||||
public IRubyObject ruby_cancelled(ThreadContext context)
|
||||
{
|
||||
return RubyBoolean.newBoolean(context.runtime, this.event.isCancelled());
|
||||
}
|
||||
|
||||
@JRubyMethod(name = "include?", required = 1)
|
||||
public IRubyObject ruby_includes(ThreadContext context, RubyString reference)
|
||||
{
|
||||
return RubyBoolean.newBoolean(context.runtime, this.event.includes(reference.asJavaString()));
|
||||
}
|
||||
|
||||
@JRubyMethod(name = "remove", required = 1)
|
||||
public IRubyObject ruby_remove(ThreadContext context, RubyString reference)
|
||||
{
|
||||
return JavaUtil.convertJavaToRuby(context.runtime, this.event.remove(reference.asJavaString()));
|
||||
}
|
||||
|
||||
@JRubyMethod(name = "clone")
|
||||
public IRubyObject ruby_clone(ThreadContext context)
|
||||
{
|
||||
try {
|
||||
return RubyEvent.newRubyEvent(context.runtime, this.event.clone());
|
||||
} catch (CloneNotSupportedException e) {
|
||||
throw context.runtime.newRuntimeError(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@JRubyMethod(name = "overwrite", required = 1)
|
||||
public IRubyObject ruby_overwrite(ThreadContext context, IRubyObject value)
|
||||
{
|
||||
if (!(value instanceof RubyEvent)) {
|
||||
throw context.runtime.newTypeError("wrong argument type " + value.getMetaClass() + " (expected LogStash::Event)");
|
||||
}
|
||||
|
||||
return RubyEvent.newRubyEvent(context.runtime, this.event.overwrite(((RubyEvent) value).event));
|
||||
}
|
||||
|
||||
@JRubyMethod(name = "append", required = 1)
|
||||
public IRubyObject ruby_append(ThreadContext context, IRubyObject value)
|
||||
{
|
||||
if (!(value instanceof RubyEvent)) {
|
||||
throw context.runtime.newTypeError("wrong argument type " + value.getMetaClass() + " (expected LogStash::Event)");
|
||||
}
|
||||
|
||||
this.event.append(((RubyEvent) value).getEvent());
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
@JRubyMethod(name = "sprintf", required = 1)
|
||||
public IRubyObject ruby_sprintf(ThreadContext context, IRubyObject format) throws IOException {
|
||||
try {
|
||||
return RubyString.newString(context.runtime, event.sprintf(format.toString()));
|
||||
} catch (IOException e) {
|
||||
throw new RaiseException(getRuntime(),
|
||||
(RubyClass) getRuntime().getModule("LogStash").getClass("Error"),
|
||||
"timestamp field is missing", true);
|
||||
}
|
||||
}
|
||||
|
||||
@JRubyMethod(name = "to_s")
|
||||
public IRubyObject ruby_to_s(ThreadContext context)
|
||||
{
|
||||
return RubyString.newString(context.runtime, event.toString());
|
||||
}
|
||||
|
||||
@JRubyMethod(name = "to_hash")
|
||||
public IRubyObject ruby_to_hash(ThreadContext context) throws IOException
|
||||
{
|
||||
// TODO: is this the most efficient?
|
||||
RubyHash hash = JavaUtil.convertJavaToUsableRubyObject(context.runtime, this.event.toMap()).convertToHash();
|
||||
// inject RubyTimestamp in new hash
|
||||
hash.put(PathCache.TIMESTAMP, JrubyTimestampExtLibrary.RubyTimestamp.newRubyTimestamp(context.runtime, this.event.getTimestamp()));
|
||||
return hash;
|
||||
}
|
||||
|
||||
@JRubyMethod(name = "to_hash_with_metadata")
|
||||
public IRubyObject ruby_to_hash_with_metadata(ThreadContext context) throws IOException
|
||||
{
|
||||
HashMap<String, Object> dataAndMetadata = new HashMap<String, Object>(this.event.getData());
|
||||
if (!this.event.getMetadata().isEmpty()) {
|
||||
dataAndMetadata.put(Event.METADATA, this.event.getMetadata());
|
||||
}
|
||||
|
||||
RubyHash hash = JavaUtil.convertJavaToUsableRubyObject(context.runtime, dataAndMetadata).convertToHash();
|
||||
|
||||
// inject RubyTimestamp in new hash
|
||||
hash.put(PathCache.TIMESTAMP, JrubyTimestampExtLibrary.RubyTimestamp.newRubyTimestamp(context.runtime, this.event.getTimestamp()));
|
||||
return hash;
|
||||
}
|
||||
|
||||
@JRubyMethod(name = "to_java")
|
||||
public IRubyObject ruby_to_java(ThreadContext context)
|
||||
{
|
||||
return JavaUtil.convertJavaToUsableRubyObject(context.runtime, this.event);
|
||||
}
|
||||
|
||||
@JRubyMethod(name = "to_json", rest = true)
|
||||
public IRubyObject ruby_to_json(ThreadContext context, IRubyObject[] args)
|
||||
throws IOException
|
||||
{
|
||||
return RubyString.newString(context.runtime, event.toJson());
|
||||
}
|
||||
|
||||
@JRubyMethod(name = "validate_value", required = 1, meta = true)
|
||||
public static IRubyObject ruby_validate_value(ThreadContext context, IRubyObject recv, IRubyObject value)
|
||||
{
|
||||
// TODO: add UTF-8 validation
|
||||
return value;
|
||||
}
|
||||
|
||||
@JRubyMethod(name = "tag", required = 1)
|
||||
public IRubyObject ruby_tag(ThreadContext context, RubyString value)
|
||||
{
|
||||
this.event.tag(((RubyString) value).asJavaString());
|
||||
return context.runtime.getNil();
|
||||
}
|
||||
|
||||
@JRubyMethod(name = "timestamp")
|
||||
public IRubyObject ruby_timestamp(ThreadContext context) throws IOException {
|
||||
return new JrubyTimestampExtLibrary.RubyTimestamp(context.getRuntime(), this.event.getTimestamp());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,208 @@
|
|||
package com.logstash.ext;
|
||||
|
||||
import com.logstash.*;
|
||||
import org.codehaus.jackson.map.annotate.JsonSerialize;
|
||||
import org.jruby.*;
|
||||
import org.jruby.anno.JRubyClass;
|
||||
import org.jruby.anno.JRubyMethod;
|
||||
import org.jruby.exceptions.RaiseException;
|
||||
import org.jruby.javasupport.JavaUtil;
|
||||
import org.jruby.runtime.Arity;
|
||||
import org.jruby.runtime.ObjectAllocator;
|
||||
import org.jruby.runtime.ThreadContext;
|
||||
import org.jruby.runtime.builtin.IRubyObject;
|
||||
import org.jruby.runtime.load.Library;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public class JrubyTimestampExtLibrary implements Library {
|
||||
public void load(Ruby runtime, boolean wrap) throws IOException {
|
||||
RubyModule module = runtime.defineModule("LogStash");
|
||||
RubyClass clazz = runtime.defineClassUnder("Timestamp", runtime.getObject(), new ObjectAllocator() {
|
||||
public IRubyObject allocate(Ruby runtime, RubyClass rubyClass) {
|
||||
return new RubyTimestamp(runtime, rubyClass);
|
||||
}
|
||||
}, module);
|
||||
clazz.defineAnnotatedMethods(RubyTimestamp.class);
|
||||
}
|
||||
|
||||
@JRubyClass(name = "Timestamp", parent = "Object")
|
||||
public static class RubyTimestamp extends RubyObject {
|
||||
|
||||
private Timestamp timestamp;
|
||||
|
||||
public RubyTimestamp(Ruby runtime, RubyClass klass) {
|
||||
super(runtime, klass);
|
||||
}
|
||||
|
||||
public RubyTimestamp(Ruby runtime, RubyClass klass, Timestamp timestamp) {
|
||||
this(runtime, klass);
|
||||
this.timestamp = timestamp;
|
||||
}
|
||||
|
||||
public RubyTimestamp(Ruby runtime, Timestamp timestamp) {
|
||||
this(runtime, runtime.getModule("LogStash").getClass("Timestamp"), timestamp);
|
||||
}
|
||||
|
||||
public RubyTimestamp(Ruby runtime) {
|
||||
this(runtime, new Timestamp());
|
||||
}
|
||||
|
||||
public static RubyTimestamp newRubyTimestamp(Ruby runtime) {
|
||||
return new RubyTimestamp(runtime);
|
||||
}
|
||||
|
||||
public static RubyTimestamp newRubyTimestamp(Ruby runtime, long epoch) {
|
||||
// Ruby epoch is in seconds, Java in milliseconds
|
||||
return new RubyTimestamp(runtime, new Timestamp(epoch * 1000));
|
||||
}
|
||||
|
||||
public static RubyTimestamp newRubyTimestamp(Ruby runtime, Timestamp timestamp) {
|
||||
return new RubyTimestamp(runtime, timestamp);
|
||||
}
|
||||
|
||||
public Timestamp getTimestamp() {
|
||||
return timestamp;
|
||||
}
|
||||
|
||||
public void setTimestamp(Timestamp timestamp) {
|
||||
this.timestamp = timestamp;
|
||||
}
|
||||
|
||||
// def initialize(time = Time.new)
|
||||
@JRubyMethod(name = "initialize", optional = 1)
|
||||
public IRubyObject ruby_initialize(ThreadContext context, IRubyObject[] args)
|
||||
{
|
||||
args = Arity.scanArgs(context.runtime, args, 0, 1);
|
||||
IRubyObject time = args[0];
|
||||
|
||||
if (time.isNil()) {
|
||||
this.timestamp = new Timestamp();
|
||||
} else if (time instanceof RubyTime) {
|
||||
this.timestamp = new Timestamp(((RubyTime)time).getDateTime());
|
||||
} else if (time instanceof RubyString) {
|
||||
try {
|
||||
this.timestamp = new Timestamp(((RubyString) time).toString());
|
||||
} catch (IllegalArgumentException e) {
|
||||
throw new RaiseException(
|
||||
getRuntime(),
|
||||
getRuntime().getModule("LogStash").getClass("TimestampParserError"),
|
||||
"invalid timestamp string format " + time,
|
||||
true
|
||||
);
|
||||
|
||||
}
|
||||
} else {
|
||||
throw context.runtime.newTypeError("wrong argument type " + time.getMetaClass() + " (expected Time)");
|
||||
}
|
||||
return context.nil;
|
||||
}
|
||||
|
||||
@JRubyMethod(name = "time")
|
||||
public IRubyObject ruby_time(ThreadContext context)
|
||||
{
|
||||
return RubyTime.newTime(context.runtime, this.timestamp.getTime());
|
||||
}
|
||||
|
||||
@JRubyMethod(name = "to_i")
|
||||
public IRubyObject ruby_to_i(ThreadContext context)
|
||||
{
|
||||
return RubyFixnum.newFixnum(context.runtime, this.timestamp.getTime().getMillis() / 1000);
|
||||
}
|
||||
|
||||
@JRubyMethod(name = "to_s")
|
||||
public IRubyObject ruby_to_s(ThreadContext context)
|
||||
{
|
||||
return ruby_to_iso8601(context);
|
||||
}
|
||||
|
||||
@JRubyMethod(name = "to_iso8601")
|
||||
public IRubyObject ruby_to_iso8601(ThreadContext context)
|
||||
{
|
||||
return RubyString.newString(context.runtime, this.timestamp.toIso8601());
|
||||
}
|
||||
|
||||
@JRubyMethod(name = "to_java")
|
||||
public IRubyObject ruby_to_java(ThreadContext context)
|
||||
{
|
||||
return JavaUtil.convertJavaToUsableRubyObject(context.runtime, this.timestamp);
|
||||
}
|
||||
|
||||
@JRubyMethod(name = "to_json", rest = true)
|
||||
public IRubyObject ruby_to_json(ThreadContext context, IRubyObject[] args)
|
||||
{
|
||||
return RubyString.newString(context.runtime, "\"" + this.timestamp.toIso8601() + "\"");
|
||||
}
|
||||
|
||||
public static Timestamp newTimetsamp(IRubyObject time)
|
||||
{
|
||||
if (time.isNil()) {
|
||||
return new Timestamp();
|
||||
} else if (time instanceof RubyTime) {
|
||||
return new Timestamp(((RubyTime)time).getDateTime());
|
||||
} else if (time instanceof RubyString) {
|
||||
return new Timestamp(((RubyString) time).toString());
|
||||
} else if (time instanceof RubyTimestamp) {
|
||||
return new Timestamp(((RubyTimestamp) time).timestamp);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@JRubyMethod(name = "coerce", required = 1, meta = true)
|
||||
public static IRubyObject ruby_coerce(ThreadContext context, IRubyObject recv, IRubyObject time)
|
||||
{
|
||||
try {
|
||||
Timestamp ts = newTimetsamp(time);
|
||||
return (ts == null) ? context.runtime.getNil() : RubyTimestamp.newRubyTimestamp(context.runtime, ts);
|
||||
} catch (IllegalArgumentException e) {
|
||||
throw new RaiseException(
|
||||
context.runtime,
|
||||
context.runtime.getModule("LogStash").getClass("TimestampParserError"),
|
||||
"invalid timestamp format " + e.getMessage(),
|
||||
true
|
||||
);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@JRubyMethod(name = "parse_iso8601", required = 1, meta = true)
|
||||
public static IRubyObject ruby_parse_iso8601(ThreadContext context, IRubyObject recv, IRubyObject time)
|
||||
{
|
||||
if (time instanceof RubyString) {
|
||||
try {
|
||||
return RubyTimestamp.newRubyTimestamp(context.runtime, newTimetsamp(time));
|
||||
} catch (IllegalArgumentException e) {
|
||||
throw new RaiseException(
|
||||
context.runtime,
|
||||
context.runtime.getModule("LogStash").getClass("TimestampParserError"),
|
||||
"invalid timestamp format " + e.getMessage(),
|
||||
true
|
||||
);
|
||||
|
||||
}
|
||||
} else {
|
||||
throw context.runtime.newTypeError("wrong argument type " + time.getMetaClass() + " (expected String)");
|
||||
}
|
||||
}
|
||||
|
||||
@JRubyMethod(name = "at", required = 1, optional = 1, meta = true)
|
||||
public static IRubyObject ruby_at(ThreadContext context, IRubyObject recv, IRubyObject[] args)
|
||||
{
|
||||
RubyTime t;
|
||||
if (args.length == 1) {
|
||||
t = (RubyTime)RubyTime.at(context, context.runtime.getTime(), args[0]);
|
||||
} else {
|
||||
t = (RubyTime)RubyTime.at(context, context.runtime.getTime(), args[0], args[1]);
|
||||
}
|
||||
return RubyTimestamp.newRubyTimestamp(context.runtime, new Timestamp(t.getDateTime()));
|
||||
}
|
||||
|
||||
@JRubyMethod(name = "now", meta = true)
|
||||
public static IRubyObject ruby_now(ThreadContext context, IRubyObject recv)
|
||||
{
|
||||
return RubyTimestamp.newRubyTimestamp(context.runtime);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,185 @@
|
|||
package com.logstash;
|
||||
|
||||
import org.junit.Test;
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class AccessorsTest {
|
||||
|
||||
public class TestableAccessors extends Accessors {
|
||||
|
||||
public TestableAccessors(Map data) {
|
||||
super(data);
|
||||
}
|
||||
|
||||
public Map<String, Object> getLut() {
|
||||
return lut;
|
||||
}
|
||||
|
||||
public Object lutGet(String reference) {
|
||||
return this.lut.get(reference);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBareGet() throws Exception {
|
||||
Map data = new HashMap();
|
||||
data.put("foo", "bar");
|
||||
String reference = "foo";
|
||||
|
||||
TestableAccessors accessors = new TestableAccessors(data);
|
||||
assertEquals(accessors.lutGet(reference), null);
|
||||
assertEquals(accessors.get(reference), "bar");
|
||||
assertEquals(accessors.lutGet(reference), data);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAbsentBareGet() throws Exception {
|
||||
Map data = new HashMap();
|
||||
data.put("foo", "bar");
|
||||
String reference = "baz";
|
||||
|
||||
TestableAccessors accessors = new TestableAccessors(data);
|
||||
assertEquals(accessors.lutGet(reference), null);
|
||||
assertEquals(accessors.get(reference), null);
|
||||
assertEquals(accessors.lutGet(reference), data);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBareBracketsGet() throws Exception {
|
||||
Map data = new HashMap();
|
||||
data.put("foo", "bar");
|
||||
String reference = "[foo]";
|
||||
|
||||
TestableAccessors accessors = new TestableAccessors(data);
|
||||
assertEquals(accessors.lutGet(reference), null);
|
||||
assertEquals(accessors.get(reference), "bar");
|
||||
assertEquals(accessors.lutGet(reference), data);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDeepMapGet() throws Exception {
|
||||
Map data = new HashMap();
|
||||
Map inner = new HashMap();
|
||||
data.put("foo", inner);
|
||||
inner.put("bar", "baz");
|
||||
|
||||
String reference = "[foo][bar]";
|
||||
|
||||
TestableAccessors accessors = new TestableAccessors(data);
|
||||
assertEquals(accessors.lutGet(reference), null);
|
||||
assertEquals(accessors.get(reference), "baz");
|
||||
assertEquals(accessors.lutGet(reference), inner);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAbsentDeepMapGet() throws Exception {
|
||||
Map data = new HashMap();
|
||||
Map inner = new HashMap();
|
||||
data.put("foo", inner);
|
||||
inner.put("bar", "baz");
|
||||
|
||||
String reference = "[foo][foo]";
|
||||
|
||||
TestableAccessors accessors = new TestableAccessors(data);
|
||||
assertEquals(accessors.lutGet(reference), null);
|
||||
assertEquals(accessors.get(reference), null);
|
||||
assertEquals(accessors.lutGet(reference), inner);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDeepListGet() throws Exception {
|
||||
Map data = new HashMap();
|
||||
List inner = new ArrayList();
|
||||
data.put("foo", inner);
|
||||
inner.add("bar");
|
||||
|
||||
String reference = "[foo][0]";
|
||||
|
||||
TestableAccessors accessors = new TestableAccessors(data);
|
||||
assertEquals(accessors.lutGet(reference), null);
|
||||
assertEquals(accessors.get(reference), "bar");
|
||||
assertEquals(accessors.lutGet(reference), inner);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAbsentDeepListGet() throws Exception {
|
||||
Map data = new HashMap();
|
||||
List inner = new ArrayList();
|
||||
data.put("foo", inner);
|
||||
inner.add("bar");
|
||||
|
||||
String reference = "[foo][1]";
|
||||
|
||||
TestableAccessors accessors = new TestableAccessors(data);
|
||||
assertEquals(accessors.lutGet(reference), null);
|
||||
assertEquals(accessors.get(reference), null);
|
||||
assertEquals(accessors.lutGet(reference), inner);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBarePut() throws Exception {
|
||||
Map data = new HashMap();
|
||||
String reference = "foo";
|
||||
|
||||
TestableAccessors accessors = new TestableAccessors(data);
|
||||
assertEquals(accessors.lutGet(reference), null);
|
||||
assertEquals(accessors.set(reference, "bar"), "bar");
|
||||
assertEquals(accessors.lutGet(reference), data);
|
||||
assertEquals(accessors.get(reference), "bar");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBareBracketsPut() throws Exception {
|
||||
Map data = new HashMap();
|
||||
String reference = "[foo]";
|
||||
|
||||
TestableAccessors accessors = new TestableAccessors(data);
|
||||
assertEquals(accessors.lutGet(reference), null);
|
||||
assertEquals(accessors.set(reference, "bar"), "bar");
|
||||
assertEquals(accessors.lutGet(reference), data);
|
||||
assertEquals(accessors.get(reference), "bar");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDeepMapSet() throws Exception {
|
||||
Map data = new HashMap();
|
||||
|
||||
String reference = "[foo][bar]";
|
||||
|
||||
TestableAccessors accessors = new TestableAccessors(data);
|
||||
assertEquals(accessors.lutGet(reference), null);
|
||||
assertEquals(accessors.set(reference, "baz"), "baz");
|
||||
assertEquals(accessors.lutGet(reference), data.get("foo"));
|
||||
assertEquals(accessors.get(reference), "baz");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDel() throws Exception {
|
||||
Map data = new HashMap();
|
||||
List inner = new ArrayList();
|
||||
data.put("foo", inner);
|
||||
inner.add("bar");
|
||||
data.put("bar", "baz");
|
||||
TestableAccessors accessors = new TestableAccessors(data);
|
||||
|
||||
assertEquals(accessors.del("[foo][0]"), "bar");
|
||||
assertEquals(accessors.del("[foo][0]"), null);
|
||||
assertEquals(accessors.get("[foo]"), new ArrayList<>());
|
||||
assertEquals(accessors.del("[bar]"), "baz");
|
||||
assertEquals(accessors.get("[bar]"), null);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNilInclude() throws Exception {
|
||||
Map data = new HashMap();
|
||||
data.put("nilfield", null);
|
||||
TestableAccessors accessors = new TestableAccessors(data);
|
||||
|
||||
assertEquals(accessors.includes("nilfield"), true);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,124 @@
|
|||
package com.logstash;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.Maps;
|
||||
import org.jruby.RubyHash;
|
||||
import org.jruby.ir.operands.Hash;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
public class EventTest {
|
||||
|
||||
@Test
|
||||
public void testBareToJson() throws Exception {
|
||||
Event e = new Event();
|
||||
assertEquals("{\"@timestamp\":\"" + e.getTimestamp().toIso8601() + "\",\"@version\":\"1\"}", e.toJson());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSimpleStringFieldToJson() throws Exception {
|
||||
Map data = new HashMap();
|
||||
data.put("foo", "bar");
|
||||
Event e = new Event(data);
|
||||
assertEquals("{\"@timestamp\":\"" + e.getTimestamp().toIso8601() + "\",\"foo\":\"bar\",\"@version\":\"1\"}", e.toJson());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSimpleIntegerFieldToJson() throws Exception {
|
||||
Map data = new HashMap();
|
||||
data.put("foo", 1);
|
||||
Event e = new Event(data);
|
||||
assertEquals("{\"@timestamp\":\"" + e.getTimestamp().toIso8601() + "\",\"foo\":1,\"@version\":\"1\"}", e.toJson());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSimpleDecimalFieldToJson() throws Exception {
|
||||
Map data = new HashMap();
|
||||
data.put("foo", 1.0);
|
||||
Event e = new Event(data);
|
||||
assertEquals("{\"@timestamp\":\"" + e.getTimestamp().toIso8601() + "\",\"foo\":1.0,\"@version\":\"1\"}", e.toJson());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSimpleMultipleFieldToJson() throws Exception {
|
||||
Map data = new HashMap();
|
||||
data.put("foo", 1.0);
|
||||
data.put("bar", "bar");
|
||||
data.put("baz", 1);
|
||||
Event e = new Event(data);
|
||||
assertEquals("{\"bar\":\"bar\",\"@timestamp\":\"" + e.getTimestamp().toIso8601() + "\",\"foo\":1.0,\"@version\":\"1\",\"baz\":1}", e.toJson());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDeepMapFieldToJson() throws Exception {
|
||||
Event e = new Event();
|
||||
e.setField("[foo][bar][baz]", 1);
|
||||
assertEquals("{\"@timestamp\":\"" + e.getTimestamp().toIso8601() + "\",\"foo\":{\"bar\":{\"baz\":1}},\"@version\":\"1\"}", e.toJson());
|
||||
|
||||
e = new Event();
|
||||
e.setField("[foo][0][baz]", 1);
|
||||
assertEquals("{\"@timestamp\":\"" + e.getTimestamp().toIso8601() + "\",\"foo\":{\"0\":{\"baz\":1}},\"@version\":\"1\"}", e.toJson());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetFieldList() throws Exception {
|
||||
Map data = new HashMap();
|
||||
List l = new ArrayList();
|
||||
data.put("foo", l);
|
||||
l.add(1);
|
||||
Event e = new Event(data);
|
||||
assertEquals(1, e.getField("[foo][0]"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDeepGetField() throws Exception {
|
||||
Map data = new HashMap();
|
||||
List l = new ArrayList();
|
||||
data.put("foo", l);
|
||||
Map m = new HashMap();
|
||||
m.put("bar", "baz");
|
||||
l.add(m);
|
||||
Event e = new Event(data);
|
||||
assertEquals("baz", e.getField("[foo][0][bar]"));
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testClone() throws Exception {
|
||||
Map data = new HashMap();
|
||||
List l = new ArrayList();
|
||||
data.put("array", l);
|
||||
|
||||
Map m = new HashMap();
|
||||
m.put("foo", "bar");
|
||||
l.add(m);
|
||||
|
||||
data.put("foo", 1.0);
|
||||
data.put("bar", "bar");
|
||||
data.put("baz", 1);
|
||||
|
||||
Event e = new Event(data);
|
||||
|
||||
Event f = e.clone();
|
||||
assertEquals("{\"bar\":\"bar\",\"@timestamp\":\"" + e.getTimestamp().toIso8601() + "\",\"array\":[{\"foo\":\"bar\"}],\"foo\":1.0,\"@version\":\"1\",\"baz\":1}", f.toJson());
|
||||
assertEquals(f.toJson(), e.toJson());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAppend() throws Exception {
|
||||
Map data1 = Maps.newHashMap(ImmutableMap.of("field1", Lists.newArrayList("original1", "original2")));
|
||||
Map data2 = Maps.newHashMap(ImmutableMap.of("field1", "original1"));
|
||||
Event e = new Event(data1);
|
||||
Event e2 = new Event(data2);
|
||||
e.append(e2);
|
||||
|
||||
assertEquals(Lists.newArrayList("original1", "original2"), e.getField("field1"));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
package com.logstash;
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
public class FieldReferenceTest {
|
||||
|
||||
@Test
|
||||
public void testParseSingleBareField() throws Exception {
|
||||
FieldReference f = FieldReference.parse("foo");
|
||||
assertTrue(f.getPath().isEmpty());
|
||||
assertEquals(f.getKey(), "foo");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testParseSingleFieldPath() throws Exception {
|
||||
FieldReference f = FieldReference.parse("[foo]");
|
||||
assertTrue(f.getPath().isEmpty());
|
||||
assertEquals(f.getKey(), "foo");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testParse2FieldsPath() throws Exception {
|
||||
FieldReference f = FieldReference.parse("[foo][bar]");
|
||||
assertEquals(f.getPath().toArray(), new String[]{"foo"});
|
||||
assertEquals(f.getKey(), "bar");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testParse3FieldsPath() throws Exception {
|
||||
FieldReference f = FieldReference.parse("[foo][bar]]baz]");
|
||||
assertEquals(f.getPath().toArray(), new String[]{"foo", "bar"});
|
||||
assertEquals(f.getKey(), "baz");
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,143 @@
|
|||
package com.logstash;
|
||||
|
||||
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.DateTimeZone;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
|
||||
public class StringInterpolationTest {
|
||||
@Test
|
||||
public void testCompletelyStaticTemplate() throws IOException {
|
||||
Event event = getTestEvent();
|
||||
String path = "/full/path/awesome";
|
||||
StringInterpolation si = StringInterpolation.getInstance();
|
||||
|
||||
assertEquals(path, si.evaluate(event, path));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testOneLevelField() throws IOException {
|
||||
Event event = getTestEvent();
|
||||
String path = "/full/%{bar}/awesome";
|
||||
StringInterpolation si = StringInterpolation.getInstance();
|
||||
|
||||
assertEquals("/full/foo/awesome", si.evaluate(event, path));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMultipleLevelField() throws IOException {
|
||||
Event event = getTestEvent();
|
||||
String path = "/full/%{bar}/%{awesome}";
|
||||
StringInterpolation si = StringInterpolation.getInstance();
|
||||
|
||||
assertEquals("/full/foo/logstash", si.evaluate(event, path));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMissingKey() throws IOException {
|
||||
Event event = getTestEvent();
|
||||
String path = "/full/%{do-not-exist}";
|
||||
StringInterpolation si = StringInterpolation.getInstance();
|
||||
|
||||
assertEquals("/full/%{do-not-exist}", si.evaluate(event, path));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDateFormater() throws IOException {
|
||||
Event event = getTestEvent();
|
||||
String path = "/full/%{+YYYY}";
|
||||
StringInterpolation si = StringInterpolation.getInstance();
|
||||
|
||||
assertEquals("/full/2015", si.evaluate(event, path));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void TestMixDateAndFields() throws IOException {
|
||||
Event event = getTestEvent();
|
||||
String path = "/full/%{+YYYY}/weeee/%{bar}";
|
||||
StringInterpolation si = StringInterpolation.getInstance();
|
||||
|
||||
assertEquals("/full/2015/weeee/foo", si.evaluate(event, path));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUnclosedTag() throws IOException {
|
||||
Event event = getTestEvent();
|
||||
String path = "/full/%{+YYY/web";
|
||||
StringInterpolation si = StringInterpolation.getInstance();
|
||||
|
||||
assertEquals("/full/%{+YYY/web", si.evaluate(event, path));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void TestStringIsOneDateTag() throws IOException {
|
||||
Event event = getTestEvent();
|
||||
String path = "%{+YYYY}";
|
||||
StringInterpolation si = StringInterpolation.getInstance();
|
||||
assertEquals("2015", si.evaluate(event, path));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void TestFieldRef() throws IOException {
|
||||
Event event = getTestEvent();
|
||||
String path = "%{[j][k1]}";
|
||||
StringInterpolation si = StringInterpolation.getInstance();
|
||||
assertEquals("v", si.evaluate(event, path));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void TestEpoch() throws IOException {
|
||||
Event event = getTestEvent();
|
||||
String path = "%{+%s}";
|
||||
StringInterpolation si = StringInterpolation.getInstance();
|
||||
assertEquals("1443657600", si.evaluate(event, path));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void TestValueIsArray() throws IOException {
|
||||
ArrayList l = new ArrayList();
|
||||
l.add("Hello");
|
||||
l.add("world");
|
||||
|
||||
Event event = getTestEvent();
|
||||
event.setField("message", l);
|
||||
|
||||
String path = "%{message}";
|
||||
StringInterpolation si = StringInterpolation.getInstance();
|
||||
assertEquals("Hello,world", si.evaluate(event, path));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void TestValueIsHash() throws IOException {
|
||||
Event event = getTestEvent();
|
||||
|
||||
String path = "%{j}";
|
||||
StringInterpolation si = StringInterpolation.getInstance();
|
||||
assertEquals("{\"k1\":\"v\"}", si.evaluate(event, path));
|
||||
}
|
||||
|
||||
public Event getTestEvent() {
|
||||
Map data = new HashMap();
|
||||
Map inner = new HashMap();
|
||||
|
||||
inner.put("k1", "v");
|
||||
|
||||
data.put("bar", "foo");
|
||||
data.put("awesome", "logstash");
|
||||
data.put("j", inner);
|
||||
data.put("@timestamp", new DateTime(2015, 10, 1, 0, 0, 0, DateTimeZone.UTC));
|
||||
|
||||
|
||||
Event event = new Event(data);
|
||||
|
||||
return event;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
package com.logstash;
|
||||
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.DateTimeZone;
|
||||
import org.junit.Test;
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
public class TimestampTest {
|
||||
|
||||
|
||||
@Test
|
||||
public void testCircularIso8601() throws Exception {
|
||||
Timestamp t1 = new Timestamp();
|
||||
Timestamp t2 = new Timestamp(t1.toIso8601());
|
||||
assertEquals(t1.getTime(), t2.getTime());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testToIso8601() throws Exception {
|
||||
Timestamp t = new Timestamp("2014-09-23T00:00:00-0800");
|
||||
assertEquals("2014-09-23T08:00:00.000Z", t.toIso8601());
|
||||
}
|
||||
|
||||
// Timestamp should always be in a UTC representation
|
||||
@Test
|
||||
public void testUTC() throws Exception {
|
||||
Timestamp t;
|
||||
|
||||
t = new Timestamp();
|
||||
assertEquals(DateTimeZone.UTC, t.getTime().getZone());
|
||||
|
||||
t = new Timestamp("2014-09-23T00:00:00-0800");
|
||||
assertEquals(DateTimeZone.UTC, t.getTime().getZone());
|
||||
|
||||
t = new Timestamp("2014-09-23T08:00:00.000Z");
|
||||
assertEquals(DateTimeZone.UTC, t.getTime().getZone());
|
||||
|
||||
t = new Timestamp(new Timestamp());
|
||||
assertEquals(DateTimeZone.UTC, t.getTime().getZone());
|
||||
|
||||
long ms = DateTime.now(DateTimeZone.forID("EST")).getMillis();
|
||||
t = new Timestamp(ms);
|
||||
assertEquals(DateTimeZone.UTC, t.getTime().getZone());
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue