mirror of
https://github.com/elastic/logstash.git
synced 2025-04-24 06:37:19 -04:00
Add Logstash configs for filebeat modules (#6791)
* Add Logstash configs for filebeat modules Hand transpiled Logstash configs for FB modules to be included in the docs for 5.3. See FB modules for more details: https://github.com/elastic/beats/tree/master/filebeat/module * Fix newlines * Remove multiline, fix bugs
This commit is contained in:
Suyog Rao
Suyog Rao
parent
86f5070a27
commit
397f005766
8 changed files with 271 additions and 0 deletions
39
docs/static/filebeat_modules/apache2/access/pipeline.conf
vendored
Normal file
39
docs/static/filebeat_modules/apache2/access/pipeline.conf
vendored
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
input {
|
||||
beats {
|
||||
# The port to listen on for filebeat connections.
|
||||
port => 5044
|
||||
# The IP address to listen for filebeat connections.
|
||||
host => "0.0.0.0"
|
||||
}
|
||||
}
|
||||
filter {
|
||||
grok {
|
||||
match => { "message" => ["%{IPORHOST:[apache2][access][remote_ip]} - %{DATA:[apache2][access][user_name]} \[%{HTTPDATE:[apache2][access][time]}\] \"%{WORD:[apache2][access][method]} %{DATA:[apache2][access][url]} HTTP/%{NUMBER:[apache2][access][http_version]}\" %{NUMBER:[apache2][access][response_code]} %{NUMBER:[apache2][access][body_sent][bytes]}( \"%{DATA:[apache2][access][referrer]}\")?( \"%{DATA:[apache2][access][agent]}\")?",
|
||||
"%{IPORHOST:[apache2][access][remote_ip]} - %{DATA:[apache2][access][user_name]} \\[%{HTTPDATE:[apache2][access][time]}\\] \"-\" %{NUMBER:[apache2][access][response_code]} -" ] }
|
||||
remove_field => "message"
|
||||
}
|
||||
mutate {
|
||||
add_field => { "read_timestamp" => "%{@timestamp}" }
|
||||
}
|
||||
date {
|
||||
match => [ "[apache2][access][time]", "dd/MMM/YYYY:H:m:s Z" ]
|
||||
remove_field => "[apache2][access][time]"
|
||||
}
|
||||
useragent {
|
||||
source => "[apache2][access][agent]"
|
||||
target => "[apache2][access][user_agent]"
|
||||
remove_field => "[apache2][access][agent]"
|
||||
}
|
||||
geoip {
|
||||
source => "[apache2][access][remote_ip]"
|
||||
target => "[apache2][access][geoip]"
|
||||
}
|
||||
}
|
||||
output {
|
||||
elasticsearch {
|
||||
hosts => localhost
|
||||
manage_template => false
|
||||
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
|
||||
document_type => "%{[@metadata][type]}"
|
||||
}
|
||||
}
|
||||
33
docs/static/filebeat_modules/apache2/error/pipeline.conf
vendored
Normal file
33
docs/static/filebeat_modules/apache2/error/pipeline.conf
vendored
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
input {
|
||||
beats {
|
||||
# The port to listen on for filebeat connections.
|
||||
port => 5044
|
||||
# The IP address to listen for filebeat connections.
|
||||
host => "0.0.0.0"
|
||||
}
|
||||
}
|
||||
filter {
|
||||
grok {
|
||||
match => { "message" => ["\[%{APACHE_TIME:[apache2][error][timestamp]}\] \[%{LOGLEVEL:[apache2][error][level]}\]( \[client %{IPORHOST:[apache2][error][client]}\])? %{GREEDYDATA:[apache2][error][message]}",
|
||||
"\[%{APACHE_TIME:[apache2][error][timestamp]}\] \[%{DATA:[apache2][error][module]}:%{LOGLEVEL:[apache2][error][level]}\] \[pid %{NUMBER:[apache2][error][pid]}(:tid %{NUMBER:[apache2][error][tid]})?\]( \[client %{IPORHOST:[apache2][error][client]}\])? %{GREEDYDATA:[apache2][error][message1]}" ] }
|
||||
pattern_definitions => {
|
||||
"APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"
|
||||
}
|
||||
remove_field => "message"
|
||||
}
|
||||
mutate {
|
||||
rename => { "[apache2][error][message1]" => "[apache2][error][message]" }
|
||||
}
|
||||
date {
|
||||
match => [ "[apache2][error][timestamp]", "EEE MMM dd H:m:s YYYY", "EEE MMM dd H:m:s.SSSSSS YYYY" ]
|
||||
remove_field => "[apache2][error][timestamp]"
|
||||
}
|
||||
}
|
||||
output {
|
||||
elasticsearch {
|
||||
hosts => localhost
|
||||
manage_template => false
|
||||
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
|
||||
document_type => "%{[@metadata][type]}"
|
||||
}
|
||||
}
|
||||
37
docs/static/filebeat_modules/mysql/error/pipeline.conf
vendored
Normal file
37
docs/static/filebeat_modules/mysql/error/pipeline.conf
vendored
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
input {
|
||||
beats {
|
||||
# The port to listen on for filebeat connections.
|
||||
port => 5044
|
||||
# The IP address to listen for filebeat connections.
|
||||
host => "0.0.0.0"
|
||||
}
|
||||
}
|
||||
filter {
|
||||
grok {
|
||||
match => { "message" => ["%{LOCALDATETIME:[mysql][error][timestamp]} (\[%{DATA:[mysql][error][level]}\] )?%{GREEDYDATA:[mysql][error][message]}",
|
||||
"%{TIMESTAMP_ISO8601:[mysql][error][timestamp]} %{NUMBER:[mysql][error][thread_id]} \[%{DATA:[mysql][error][level]}\] %{GREEDYDATA:[mysql][error][message1]}",
|
||||
"%{GREEDYDATA:[mysql][error][message2]}"] }
|
||||
pattern_definitions => {
|
||||
"LOCALDATETIME" => "[0-9]+ %{TIME}"
|
||||
}
|
||||
remove_field => "message"
|
||||
}
|
||||
mutate {
|
||||
rename => { "[mysql][error][message1]" => "[mysql][error][message]" }
|
||||
}
|
||||
mutate {
|
||||
rename => { "[mysql][error][message2]" => "[mysql][error][message]" }
|
||||
}
|
||||
date {
|
||||
match => [ "[mysql][error][timestamp]", "ISO8601", "YYMMdd H:m:s" ]
|
||||
remove_field => "[apache2][access][time]"
|
||||
}
|
||||
}
|
||||
output {
|
||||
elasticsearch {
|
||||
hosts => localhost
|
||||
manage_template => false
|
||||
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
|
||||
document_type => "%{[@metadata][type]}"
|
||||
}
|
||||
}
|
||||
31
docs/static/filebeat_modules/mysql/slowlog/pipeline.conf
vendored
Normal file
31
docs/static/filebeat_modules/mysql/slowlog/pipeline.conf
vendored
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
input {
|
||||
beats {
|
||||
# The port to listen on for filebeat connections.
|
||||
port => 5044
|
||||
# The IP address to listen for filebeat connections.
|
||||
host => "0.0.0.0"
|
||||
}
|
||||
}
|
||||
filter {
|
||||
grok {
|
||||
match => { "message" => ["^# User@Host: %{USER:[mysql][slowlog][user]}(\[[^\]]+\])? @ %{HOSTNAME:[mysql][slowlog][host]} \[(IP:[mysql][slowlog][ip])?\](\s*Id:\s* %{NUMBER:[mysql][slowlog][id]})?\n# Query_time: %{NUMBER:[mysql][slowlog][query_time][sec]}\s* Lock_time: %{NUMBER:[mysql][slowlog][lock_time][sec]}\s* Rows_sent: %{NUMBER:[mysql][slowlog][rows_sent]}\s* Rows_examined: %{NUMBER:[mysql][slowlog][rows_examined]}\n(SET timestamp=%{NUMBER:[mysql][slowlog][timestamp]};\n)?%{GREEDYMULTILINE:[mysql][slowlog][query]}"] }
|
||||
pattern_definitions => {
|
||||
"GREEDYMULTILINE" => "(.|\n)*"
|
||||
}
|
||||
remove_field => "message"
|
||||
}
|
||||
date {
|
||||
match => [ "[mysql][slowlog][timestamp]", "UNIX" ]
|
||||
}
|
||||
mutate {
|
||||
gsub => ["[mysql][slowlog][query]", "\n# Time: [0-9]+ [0-9][0-9]:[0-9][0-9]:[0-9][0-9](\\.[0-9]+)?$", ""]
|
||||
}
|
||||
}
|
||||
output {
|
||||
elasticsearch {
|
||||
hosts => localhost
|
||||
manage_template => false
|
||||
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
|
||||
document_type => "%{[@metadata][type]}"
|
||||
}
|
||||
}
|
||||
38
docs/static/filebeat_modules/nginx/access/pipeline.conf
vendored
Normal file
38
docs/static/filebeat_modules/nginx/access/pipeline.conf
vendored
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
input {
|
||||
beats {
|
||||
# The port to listen on for filebeat connections.
|
||||
port => 5044
|
||||
# The IP address to listen for filebeat connections.
|
||||
host => "0.0.0.0"
|
||||
}
|
||||
}
|
||||
filter {
|
||||
grok {
|
||||
match => { "message" => ["%{IPORHOST:[nginx][access][remote_ip]} - %{DATA:[nginx][access][user_name]} \[%{HTTPDATE:[nginx][access][time]}\] \"%{WORD:[nginx][access][method]} %{DATA:[nginx][access][url]} HTTP/%{NUMBER:[nginx][access][http_version]}\" %{NUMBER:[nginx][access][response_code]} %{NUMBER:[nginx][access][body_sent][bytes]} \"%{DATA:[nginx][access][referrer]}\" \"%{DATA:[nginx][access][agent]}\""] }
|
||||
remove_field => "message"
|
||||
}
|
||||
mutate {
|
||||
rename => { "@timestamp" => "read_timestamp" }
|
||||
}
|
||||
date {
|
||||
match => [ "[nginx][access][time]", "dd/MMM/YYYY:H:m:s Z" ]
|
||||
remove_field => "[nginx][access][time]"
|
||||
}
|
||||
useragent {
|
||||
source => "[nginx][access][agent]"
|
||||
target => "[nginx][access][user_agent]"
|
||||
remove_field => "[nginx][access][agent]"
|
||||
}
|
||||
geoip {
|
||||
source => "[nginx][access][remote_ip]"
|
||||
target => "[nginx][access][geoip]"
|
||||
}
|
||||
}
|
||||
output {
|
||||
elasticsearch {
|
||||
hosts => localhost
|
||||
manage_template => false
|
||||
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
|
||||
document_type => "%{[@metadata][type]}"
|
||||
}
|
||||
}
|
||||
29
docs/static/filebeat_modules/nginx/error/pipeline.conf
vendored
Normal file
29
docs/static/filebeat_modules/nginx/error/pipeline.conf
vendored
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
input {
|
||||
beats {
|
||||
# The port to listen on for filebeat connections.
|
||||
port => 5044
|
||||
# The IP address to listen for filebeat connections.
|
||||
host => "0.0.0.0"
|
||||
}
|
||||
}
|
||||
filter {
|
||||
grok {
|
||||
match => { "message" => ["%{DATA:[nginx][error][time]} \[%{DATA:[nginx][error][level]}\] %{NUMBER:[nginx][error][pid]}#%{NUMBER:[nginx][error][tid]}: (\*%{NUMBER:[nginx][error][connection_id]} )?%{GREEDYDATA:[nginx][error][message]}"] }
|
||||
remove_field => "message"
|
||||
}
|
||||
mutate {
|
||||
rename => { "@timestamp" => "read_timestamp" }
|
||||
}
|
||||
date {
|
||||
match => [ "[nginx][error][time]", "YYYY/MM/dd H:m:s" ]
|
||||
remove_field => "[nginx][error][time]"
|
||||
}
|
||||
}
|
||||
output {
|
||||
elasticsearch {
|
||||
hosts => localhost
|
||||
manage_template => false
|
||||
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
|
||||
document_type => "%{[@metadata][type]}"
|
||||
}
|
||||
}
|
||||
38
docs/static/filebeat_modules/system/auth/pipeline.conf
vendored
Normal file
38
docs/static/filebeat_modules/system/auth/pipeline.conf
vendored
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
input {
|
||||
beats {
|
||||
# The port to listen on for filebeat connections.
|
||||
port => 5044
|
||||
# The IP address to listen for filebeat connections.
|
||||
host => "0.0.0.0"
|
||||
}
|
||||
}
|
||||
filter {
|
||||
grok {
|
||||
match => { "message" => ["%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} sshd(?:\[%{POSINT:[system][auth][pid]}\])?: %{DATA:[system][auth][ssh][event]} %{DATA:[system][auth][ssh][method]} for (invalid user )?%{DATA:[system][auth][user]} from %{IPORHOST:[system][auth][ssh][ip]} port %{NUMBER:[system][auth][ssh][port]} ssh2(: %{GREEDYDATA:[system][auth][ssh][signature]})?",
|
||||
"%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} sshd(?:\[%{POSINT:[system][auth][pid]}\])?: %{DATA:[system][auth][ssh][event]} user %{DATA:[system][auth][user]} from %{IPORHOST:[system][auth][ssh][ip]}",
|
||||
"%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} sshd(?:\[%{POSINT:[system][auth][pid]}\])?: Did not receive identification string from %{IPORHOST:[system][auth][ssh][dropped_ip]}",
|
||||
"%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} sudo(?:\[%{POSINT:[system][auth][pid]}\])?: \s*%{DATA:[system][auth][user]} :( %{DATA:[system][auth][sudo][error]} ;)? TTY=%{DATA:[system][auth][sudo][tty]} ; PWD=%{DATA:[system][auth][sudo][pwd]} ; USER=%{DATA:[system][auth][sudo][user]} ; COMMAND=%{GREEDYDATA:[system][auth][sudo][command]}",
|
||||
"%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} groupadd(?:\[%{POSINT:[system][auth][pid]}\])?: new group: name=%{DATA:system.auth.groupadd.name}, GID=%{NUMBER:system.auth.groupadd.gid}",
|
||||
"%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} useradd(?:\[%{POSINT:[system][auth][pid]}\])?: new user: name=%{DATA:[system][auth][user][add][name]}, UID=%{NUMBER:[system][auth][user][add][uid]}, GID=%{NUMBER:[system][auth][user][add][gid]}, home=%{DATA:[system][auth][user][add][home]}, shell=%{DATA:[system][auth][user][add][shell]}$",
|
||||
"%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} %{DATA:[system][auth][program]}(?:\[%{POSINT:[system][auth][pid]}\])?: %{GREEDYMULTILINE:[system][auth][message]}"] }
|
||||
pattern_definitions => {
|
||||
"GREEDYMULTILINE"=> "(.|\n)*"
|
||||
}
|
||||
remove_field => "message"
|
||||
}
|
||||
date {
|
||||
match => [ "[system][auth][timestamp]", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
|
||||
}
|
||||
geoip {
|
||||
source => "[system][auth][ssh][ip]"
|
||||
target => "[system][auth][ssh][geoip]"
|
||||
}
|
||||
}
|
||||
output {
|
||||
elasticsearch {
|
||||
hosts => localhost
|
||||
manage_template => false
|
||||
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
|
||||
document_type => "%{[@metadata][type]}"
|
||||
}
|
||||
}
|
||||
26
docs/static/filebeat_modules/system/syslog/pipeline.conf
vendored
Normal file
26
docs/static/filebeat_modules/system/syslog/pipeline.conf
vendored
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
input {
|
||||
beats {
|
||||
# The port to listen on for filebeat connections.
|
||||
port => 5044
|
||||
# The IP address to listen for filebeat connections.
|
||||
host => "0.0.0.0"
|
||||
}
|
||||
}
|
||||
filter {
|
||||
grok {
|
||||
match => { "message" => ["%{SYSLOGTIMESTAMP:[system][syslog][timestamp]} %{SYSLOGHOST:[system][syslog][hostname]} %{DATA:[system][syslog][program]}(?:\[%{POSINT:[system][syslog][pid]}\])?: %{GREEDYMULTILINE:[system][syslog][message]}"] }
|
||||
pattern_definitions => { "GREEDYMULTILINE" => "(.|\n)*" }
|
||||
remove_field => "message"
|
||||
}
|
||||
date {
|
||||
match => [ "[system][syslog][timestamp]", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
|
||||
}
|
||||
}
|
||||
output {
|
||||
elasticsearch {
|
||||
hosts => localhost
|
||||
manage_template => false
|
||||
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
|
||||
document_type => "%{[@metadata][type]}"
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue