mirror of
https://github.com/elastic/logstash.git
synced 2025-04-24 14:47:19 -04:00
DeDe Morton
Jordan Sissel
parent
dd8ebbec0a
commit
f75799844d
21 changed files with 315 additions and 526 deletions
283
docs/static/filebeat-modules.asciidoc
vendored
283
docs/static/filebeat-modules.asciidoc
vendored
|
|
@ -2,12 +2,11 @@
|
|||
|
||||
== Working with Filebeat Modules
|
||||
|
||||
Starting with version 5.3, Filebeat comes packaged with pre-built
|
||||
{filebeat}filebeat-modules.html[modules] that contain the configurations needed
|
||||
to collect, parse, enrich, and visualize data from various log file formats.
|
||||
Each Filebeat module consists of one or more filesets that contain ingest node
|
||||
pipelines, Elasticsearch templates, Filebeat prospector configurations, and
|
||||
Kibana dashboards.
|
||||
Filebeat comes packaged with pre-built {filebeat}filebeat-modules.html[modules]
|
||||
that contain the configurations needed to collect, parse, enrich, and visualize
|
||||
data from various log file formats. Each Filebeat module consists of one or more
|
||||
filesets that contain ingest node pipelines, Elasticsearch templates, Filebeat
|
||||
prospector configurations, and Kibana dashboards.
|
||||
|
||||
Filebeat modules are a great way to get started, but you might find that ingest
|
||||
pipelines don't offer the processing power that you require. If that's the case,
|
||||
|
|
@ -20,57 +19,21 @@ you'll need to use Logstash.
|
|||
Logstash provides an <<ingest-converter,ingest pipeline conversion tool>>
|
||||
to help you migrate ingest pipeline definitions to Logstash configs. However,
|
||||
the tool does not currently support all the processors that are available for
|
||||
ingest node. You can follow the steps in this section to configure Filebeat and
|
||||
build Logstash pipeline configurations that are equivalent to the ingest node
|
||||
pipelines available with the Filebeat modules. Then you'll be able to use the
|
||||
same dashboards available with Filebeat to visualize your data in Kibana.
|
||||
ingest node.
|
||||
|
||||
Follow the steps in this section to build and run Logstash configurations that
|
||||
provide capabilities similar to Filebeat modules.
|
||||
You can follow the steps in this section to build and run Logstash
|
||||
configurations that parse the data collected by Filebeat modules. Then you'll be
|
||||
able to use the same dashboards available with Filebeat to visualize your data
|
||||
in Kibana.
|
||||
|
||||
. Load the Filebeat index pattern and sample Kibana dashboards. To do this, you
|
||||
need to run the Filebeat module with the Elasticsearch output enabled and
|
||||
specify the `-setup` flag.
|
||||
+
|
||||
For example, to load the sample dashboards for Nginx, run:
|
||||
+
|
||||
[source,shell]
|
||||
----------------------------------------------------------------------
|
||||
./filebeat -e -modules=nginx -setup -E "output.elasticsearch.hosts=["http://localhost:9200"]"
|
||||
----------------------------------------------------------------------
|
||||
+
|
||||
A connection to Elasticsearch is required for this one-time setup step because
|
||||
Filebeat needs to create the index pattern and load the sample dashboards into the
|
||||
Kibana index.
|
||||
+
|
||||
After the template and dashboards are loaded, you'll see the message
|
||||
`INFO Elasticsearch template with name 'filebeat' loaded`. You can shut
|
||||
down Filebeat.
|
||||
|
||||
. Configure Filebeat to send log lines to Logstash.
|
||||
+
|
||||
See <<logstash-config-for-filebeat-modules>> for detailed examples.
|
||||
[float]
|
||||
==== Create and start the Logstash pipeline
|
||||
|
||||
. Create a Logstash pipeline configuration that reads from the Beats input and
|
||||
parses the log events.
|
||||
parses the events.
|
||||
+
|
||||
See <<logstash-config-for-filebeat-modules>> for detailed examples.
|
||||
|
||||
. Start Filebeat. For example, to start Filebeat in the foreground, use:
|
||||
+
|
||||
[source,shell]
|
||||
----------------------------------------------------------------------
|
||||
sudo ./filebeat -e -c filebeat.yml -d "publish"
|
||||
----------------------------------------------------------------------
|
||||
+
|
||||
NOTE: Depending on how you've installed Filebeat, you might see errors
|
||||
related to file ownership or permissions when you try to run Filebeat modules.
|
||||
See {libbeat}/config-file-permissions.html[Config File Ownership and Permissions]
|
||||
in the _Beats Platform Reference_ if you encounter errors related to file
|
||||
ownership or permissions.
|
||||
+
|
||||
See {filebeat}/filebeat-starting.html[Starting Filebeat] for more info.
|
||||
|
||||
. Start Logstash, passing in the pipeline configuration file that parses the
|
||||
log. For example:
|
||||
+
|
||||
|
|
@ -84,185 +47,139 @@ input from Beats:
|
|||
+
|
||||
[source,shell]
|
||||
----------------------------------------------------------------------
|
||||
[2017-03-17T16:31:40,319][INFO ][logstash.inputs.beats ] Beats inputs: Starting input listener {:address=>"127.0.0.1:5044"}
|
||||
[2017-03-17T16:31:40,350][INFO ][logstash.pipeline ] Pipeline main started
|
||||
[2017-10-13T00:01:15,413][INFO ][logstash.inputs.beats ] Beats inputs: Starting input listener {:address=>"127.0.0.1:5044"}
|
||||
[2017-10-13T00:01:15,443][INFO ][logstash.pipeline ] Pipeline started {"pipeline.id"=>"main"}
|
||||
----------------------------------------------------------------------
|
||||
|
||||
. To visualize the data in Kibana, launch the Kibana web interface by pointing
|
||||
|
||||
The Logstash pipeline is now ready to receive events from Filebeat. Next, you
|
||||
set up and run Filebeat.
|
||||
|
||||
[float]
|
||||
==== Set up and run Filebeat
|
||||
|
||||
. If you haven't already set up the Filebeat index template and sample Kibana
|
||||
dashboards, run the Filebeat `setup` command to do that now:
|
||||
+
|
||||
[source,shell]
|
||||
----------------------------------------------------------------------
|
||||
./filebeat -e setup
|
||||
----------------------------------------------------------------------
|
||||
+
|
||||
The `-e` flag is optional and sends output to standard error instead of syslog.
|
||||
+
|
||||
A connection to Elasticsearch and Kibana is required for this one-time setup
|
||||
step because Filebeat needs to create the index template in Elasticsearch and
|
||||
load the sample dashboards into Kibana.
|
||||
+
|
||||
After the template and dashboards are loaded, you'll see the message `INFO
|
||||
Kibana dashboards successfully loaded. Loaded dashboards`.
|
||||
|
||||
. Configure Filebeat to send log lines to Logstash. To do this, in the
|
||||
+filebeat.yml+ config file, disable the Elasticsearch output, and enable the
|
||||
Logstash output. For example:
|
||||
+
|
||||
[source,yaml]
|
||||
-----
|
||||
#output.elasticsearch:
|
||||
#hosts: ["localhost:9200"]
|
||||
output.logstash:
|
||||
hosts: ["localhost:5044"]
|
||||
-----
|
||||
|
||||
. Run the `modules enable` command to enable the modules that you want to run.
|
||||
For example:
|
||||
+
|
||||
[source,shell]
|
||||
----------------------------------------------------------------------
|
||||
./filebeat modules enable nginx
|
||||
----------------------------------------------------------------------
|
||||
+
|
||||
You can further configure the module by editing the config file under the
|
||||
Filebeat `modules.d` directory. For example, if the log files are not in the
|
||||
location expected by the module, you can set the `var.paths` option.
|
||||
|
||||
. Start Filebeat. For example, to start Filebeat in the foreground, use:
|
||||
+
|
||||
[source,shell]
|
||||
----------------------------------------------------------------------
|
||||
./filebeat -e
|
||||
----------------------------------------------------------------------
|
||||
+
|
||||
NOTE: Depending on how you've installed Filebeat, you might see errors
|
||||
related to file ownership or permissions when you try to run Filebeat modules.
|
||||
See {libbeat}/config-file-permissions.html[Config File Ownership and Permissions]
|
||||
in the _Beats Platform Reference_ if you encounter errors related to file
|
||||
ownership or permissions.
|
||||
+
|
||||
See {filebeat}/filebeat-starting.html[Starting Filebeat] for more info.
|
||||
|
||||
[float]
|
||||
==== Visualize the data
|
||||
|
||||
To visualize the data in Kibana, launch the Kibana web interface by pointing
|
||||
your browser to port 5601. For example,
|
||||
http://127.0.0.1:5601[http://127.0.0.1:5601].
|
||||
|
||||
[[logstash-config-for-filebeat-modules]]
|
||||
=== Configuration Examples
|
||||
|
||||
The examples in this section show you how to configure Filebeat and build
|
||||
Logstash pipelines that parse:
|
||||
The examples in this section show you how to build Logstash pipelines that parse
|
||||
data sent collected by Filebeat modules:
|
||||
|
||||
* <<parsing-apache2>>
|
||||
* <<parsing-mysql>>
|
||||
* <<parsing-nginx>>
|
||||
* <<parsing-system>>
|
||||
|
||||
Of course, the paths that you specify in the Filebeat config depend on the location
|
||||
of the logs you are harvesting. The examples show common default locations.
|
||||
|
||||
[[parsing-apache2]]
|
||||
==== Apache 2 Logs
|
||||
|
||||
Here are some configuration examples for shipping and parsing Apache 2 access and
|
||||
error logs.
|
||||
|
||||
===== Apache 2 Access Logs
|
||||
|
||||
Example Filebeat config:
|
||||
|
||||
[source,yml]
|
||||
----------------------------------------------------------------------
|
||||
include::filebeat_modules/apache2/access/filebeat.yml[]
|
||||
----------------------------------------------------------------------
|
||||
|
||||
|
||||
Example Logstash pipeline config:
|
||||
The Logstash pipeline configuration in this example shows how to ship and parse
|
||||
access and error logs collected by the
|
||||
{filebeat}filebeat-module-apache2.html[`apache2` Filebeat module].
|
||||
|
||||
[source,json]
|
||||
----------------------------------------------------------------------------
|
||||
include::filebeat_modules/apache2/access/pipeline.conf[]
|
||||
include::filebeat_modules/apache2/pipeline.conf[]
|
||||
----------------------------------------------------------------------------
|
||||
|
||||
===== Apache 2 Error Logs
|
||||
|
||||
Example Filebeat config:
|
||||
|
||||
[source,yml]
|
||||
----------------------------------------------------------------------
|
||||
include::filebeat_modules/apache2/error/filebeat.yml[]
|
||||
----------------------------------------------------------------------
|
||||
|
||||
|
||||
Example Logstash pipeline config:
|
||||
|
||||
[source,json]
|
||||
----------------------------------------------------------------------------
|
||||
include::filebeat_modules/apache2/error/pipeline.conf[]
|
||||
----------------------------------------------------------------------------
|
||||
|
||||
[[parsing-mysql]]
|
||||
==== MySQL Logs
|
||||
|
||||
Here are some configuration examples for shipping and parsing MySQL error and
|
||||
slowlog logs.
|
||||
|
||||
===== MySQL Error Logs
|
||||
|
||||
Example Filebeat config:
|
||||
|
||||
[source,yml]
|
||||
----------------------------------------------------------------------
|
||||
include::filebeat_modules/mysql/error/filebeat.yml[]
|
||||
----------------------------------------------------------------------
|
||||
|
||||
|
||||
Example Logstash pipeline config:
|
||||
The Logstash pipeline configuration in this example shows how to ship and parse
|
||||
error and slowlog logs collected by the
|
||||
{filebeat}filebeat-module-mysql.html[`mysql` Filebeat module].
|
||||
|
||||
[source,json]
|
||||
----------------------------------------------------------------------------
|
||||
include::filebeat_modules/mysql/error/pipeline.conf[]
|
||||
include::filebeat_modules/mysql/pipeline.conf[]
|
||||
----------------------------------------------------------------------------
|
||||
|
||||
===== MySQL Slowlog
|
||||
|
||||
Example Filebeat config:
|
||||
|
||||
[source,yml]
|
||||
----------------------------------------------------------------------
|
||||
include::filebeat_modules/mysql/slowlog/filebeat.yml[]
|
||||
----------------------------------------------------------------------
|
||||
|
||||
|
||||
Example Logstash pipeline config:
|
||||
|
||||
[source,json]
|
||||
----------------------------------------------------------------------------
|
||||
include::filebeat_modules/mysql/slowlog/pipeline.conf[]
|
||||
----------------------------------------------------------------------------
|
||||
|
||||
[[parsing-nginx]]
|
||||
==== Nginx Logs
|
||||
|
||||
Here are some configuration examples for shipping and parsing Nginx access and
|
||||
error logs.
|
||||
|
||||
===== Nginx Access Logs
|
||||
|
||||
Example Filebeat config:
|
||||
|
||||
[source,yml]
|
||||
----------------------------------------------------------------------
|
||||
include::filebeat_modules/nginx/access/filebeat.yml[]
|
||||
----------------------------------------------------------------------
|
||||
|
||||
|
||||
Example Logstash pipeline config:
|
||||
The Logstash pipeline configuration in this example shows how to ship and parse
|
||||
access and error logs collected by the
|
||||
{filebeat}filebeat-module-nginx.html[`nginx` Filebeat module].
|
||||
|
||||
[source,json]
|
||||
----------------------------------------------------------------------------
|
||||
include::filebeat_modules/nginx/access/pipeline.conf[]
|
||||
include::filebeat_modules/nginx/pipeline.conf[]
|
||||
----------------------------------------------------------------------------
|
||||
|
||||
|
||||
===== Nginx Error Logs
|
||||
|
||||
Example Filebeat config:
|
||||
|
||||
[source,yml]
|
||||
----------------------------------------------------------------------
|
||||
include::filebeat_modules/nginx/error/filebeat.yml[]
|
||||
----------------------------------------------------------------------
|
||||
|
||||
|
||||
Example Logstash pipeline config:
|
||||
|
||||
[source,json]
|
||||
----------------------------------------------------------------------------
|
||||
include::filebeat_modules/nginx/error/pipeline.conf[]
|
||||
----------------------------------------------------------------------------
|
||||
|
||||
[[parsing-system]]
|
||||
==== System Logs
|
||||
|
||||
Here are some configuration examples for shipping and parsing system
|
||||
logs.
|
||||
|
||||
===== System Authorization Logs
|
||||
|
||||
Example Filebeat config:
|
||||
|
||||
[source,yml]
|
||||
----------------------------------------------------------------------
|
||||
include::filebeat_modules/system/auth/filebeat.yml[]
|
||||
----------------------------------------------------------------------
|
||||
|
||||
|
||||
Example Logstash pipeline config:
|
||||
The Logstash pipeline configuration in this example shows how to ship and parse
|
||||
system logs collected by the
|
||||
{filebeat}filebeat-module-system.html[`system` Filebeat module].
|
||||
|
||||
[source,json]
|
||||
----------------------------------------------------------------------------
|
||||
include::filebeat_modules/system/auth/pipeline.conf[]
|
||||
include::filebeat_modules/system/pipeline.conf[]
|
||||
----------------------------------------------------------------------------
|
||||
|
||||
===== Syslog
|
||||
|
||||
Example Filebeat config:
|
||||
|
||||
[source,yml]
|
||||
----------------------------------------------------------------------
|
||||
include::filebeat_modules/system/syslog/filebeat.yml[]
|
||||
----------------------------------------------------------------------
|
||||
|
||||
|
||||
Example Logstash pipeline config:
|
||||
|
||||
[source,json]
|
||||
----------------------------------------------------------------------------
|
||||
include::filebeat_modules/system/syslog/pipeline.conf[]
|
||||
----------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -1,8 +0,0 @@
|
|||
filebeat.prospectors:
|
||||
- input_type: log
|
||||
paths:
|
||||
- /var/log/apache2/access.log*
|
||||
- /var/log/apache2/other_vhosts_access.log*
|
||||
exclude_files: [".gz$"]
|
||||
output.logstash:
|
||||
hosts: ["localhost:5044"]
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
input {
|
||||
beats {
|
||||
# The port to listen on for filebeat connections.
|
||||
port => 5044
|
||||
# The IP address to listen for filebeat connections.
|
||||
host => "0.0.0.0"
|
||||
}
|
||||
}
|
||||
filter {
|
||||
grok {
|
||||
match => { "message" => ["%{IPORHOST:[apache2][access][remote_ip]} - %{DATA:[apache2][access][user_name]} \[%{HTTPDATE:[apache2][access][time]}\] \"%{WORD:[apache2][access][method]} %{DATA:[apache2][access][url]} HTTP/%{NUMBER:[apache2][access][http_version]}\" %{NUMBER:[apache2][access][response_code]} %{NUMBER:[apache2][access][body_sent][bytes]}( \"%{DATA:[apache2][access][referrer]}\")?( \"%{DATA:[apache2][access][agent]}\")?",
|
||||
"%{IPORHOST:[apache2][access][remote_ip]} - %{DATA:[apache2][access][user_name]} \\[%{HTTPDATE:[apache2][access][time]}\\] \"-\" %{NUMBER:[apache2][access][response_code]} -" ] }
|
||||
remove_field => "message"
|
||||
}
|
||||
mutate {
|
||||
add_field => { "read_timestamp" => "%{@timestamp}" }
|
||||
}
|
||||
date {
|
||||
match => [ "[apache2][access][time]", "dd/MMM/YYYY:H:m:s Z" ]
|
||||
remove_field => "[apache2][access][time]"
|
||||
}
|
||||
useragent {
|
||||
source => "[apache2][access][agent]"
|
||||
target => "[apache2][access][user_agent]"
|
||||
remove_field => "[apache2][access][agent]"
|
||||
}
|
||||
geoip {
|
||||
source => "[apache2][access][remote_ip]"
|
||||
target => "[apache2][access][geoip]"
|
||||
}
|
||||
}
|
||||
output {
|
||||
elasticsearch {
|
||||
hosts => localhost
|
||||
manage_template => false
|
||||
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
|
||||
document_type => "%{[@metadata][type]}"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
filebeat.prospectors:
|
||||
- input_type: log
|
||||
paths:
|
||||
- /var/log/apache2/error.log*
|
||||
exclude_files: [".gz$"]
|
||||
output.logstash:
|
||||
hosts: ["localhost:5044"]
|
||||
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
input {
|
||||
beats {
|
||||
# The port to listen on for filebeat connections.
|
||||
port => 5044
|
||||
# The IP address to listen for filebeat connections.
|
||||
host => "0.0.0.0"
|
||||
}
|
||||
}
|
||||
filter {
|
||||
grok {
|
||||
match => { "message" => ["\[%{APACHE_TIME:[apache2][error][timestamp]}\] \[%{LOGLEVEL:[apache2][error][level]}\]( \[client %{IPORHOST:[apache2][error][client]}\])? %{GREEDYDATA:[apache2][error][message]}",
|
||||
"\[%{APACHE_TIME:[apache2][error][timestamp]}\] \[%{DATA:[apache2][error][module]}:%{LOGLEVEL:[apache2][error][level]}\] \[pid %{NUMBER:[apache2][error][pid]}(:tid %{NUMBER:[apache2][error][tid]})?\]( \[client %{IPORHOST:[apache2][error][client]}\])? %{GREEDYDATA:[apache2][error][message1]}" ] }
|
||||
pattern_definitions => {
|
||||
"APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"
|
||||
}
|
||||
remove_field => "message"
|
||||
}
|
||||
mutate {
|
||||
rename => { "[apache2][error][message1]" => "[apache2][error][message]" }
|
||||
}
|
||||
date {
|
||||
match => [ "[apache2][error][timestamp]", "EEE MMM dd H:m:s YYYY", "EEE MMM dd H:m:s.SSSSSS YYYY" ]
|
||||
remove_field => "[apache2][error][timestamp]"
|
||||
}
|
||||
}
|
||||
output {
|
||||
elasticsearch {
|
||||
hosts => localhost
|
||||
manage_template => false
|
||||
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
|
||||
document_type => "%{[@metadata][type]}"
|
||||
}
|
||||
}
|
||||
58
docs/static/filebeat_modules/apache2/pipeline.conf
vendored
Normal file
58
docs/static/filebeat_modules/apache2/pipeline.conf
vendored
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
input {
|
||||
beats {
|
||||
port => 5044
|
||||
host => "0.0.0.0"
|
||||
}
|
||||
}
|
||||
filter {
|
||||
if [fileset][module] == "apache2" {
|
||||
if [fileset][name] == "access" {
|
||||
grok {
|
||||
match => { "message" => ["%{IPORHOST:[apache2][access][remote_ip]} - %{DATA:[apache2][access][user_name]} \[%{HTTPDATE:[apache2][access][time]}\] \"%{WORD:[apache2][access][method]} %{DATA:[apache2][access][url]} HTTP/%{NUMBER:[apache2][access][http_version]}\" %{NUMBER:[apache2][access][response_code]} %{NUMBER:[apache2][access][body_sent][bytes]}( \"%{DATA:[apache2][access][referrer]}\")?( \"%{DATA:[apache2][access][agent]}\")?",
|
||||
"%{IPORHOST:[apache2][access][remote_ip]} - %{DATA:[apache2][access][user_name]} \\[%{HTTPDATE:[apache2][access][time]}\\] \"-\" %{NUMBER:[apache2][access][response_code]} -" ] }
|
||||
remove_field => "message"
|
||||
}
|
||||
mutate {
|
||||
add_field => { "read_timestamp" => "%{@timestamp}" }
|
||||
}
|
||||
date {
|
||||
match => [ "[apache2][access][time]", "dd/MMM/YYYY:H:m:s Z" ]
|
||||
remove_field => "[apache2][access][time]"
|
||||
}
|
||||
useragent {
|
||||
source => "[apache2][access][agent]"
|
||||
target => "[apache2][access][user_agent]"
|
||||
remove_field => "[apache2][access][agent]"
|
||||
}
|
||||
geoip {
|
||||
source => "[apache2][access][remote_ip]"
|
||||
target => "[apache2][access][geoip]"
|
||||
}
|
||||
}
|
||||
else if [fileset][name] == "error" {
|
||||
grok {
|
||||
match => { "message" => ["\[%{APACHE_TIME:[apache2][error][timestamp]}\] \[%{LOGLEVEL:[apache2][error][level]}\]( \[client %{IPORHOST:[apache2][error][client]}\])? %{GREEDYDATA:[apache2][error][message]}",
|
||||
"\[%{APACHE_TIME:[apache2][error][timestamp]}\] \[%{DATA:[apache2][error][module]}:%{LOGLEVEL:[apache2][error][level]}\] \[pid %{NUMBER:[apache2][error][pid]}(:tid %{NUMBER:[apache2][error][tid]})?\]( \[client %{IPORHOST:[apache2][error][client]}\])? %{GREEDYDATA:[apache2][error][message1]}" ] }
|
||||
pattern_definitions => {
|
||||
"APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"
|
||||
}
|
||||
remove_field => "message"
|
||||
}
|
||||
mutate {
|
||||
rename => { "[apache2][error][message1]" => "[apache2][error][message]" }
|
||||
}
|
||||
date {
|
||||
match => [ "[apache2][error][timestamp]", "EEE MMM dd H:m:s YYYY", "EEE MMM dd H:m:s.SSSSSS YYYY" ]
|
||||
remove_field => "[apache2][error][timestamp]"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
output {
|
||||
elasticsearch {
|
||||
hosts => localhost
|
||||
manage_template => false
|
||||
index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
|
||||
document_type => "%{[@metadata][type]}"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
filebeat.prospectors:
|
||||
- input_type: log
|
||||
paths:
|
||||
- /var/log/mysql/error.log*
|
||||
- /var/log/mysqld.log*
|
||||
exclude_files: [".gz$"]
|
||||
output.logstash:
|
||||
hosts: ["localhost:5044"]
|
||||
|
|
@ -1,37 +0,0 @@
|
|||
input {
|
||||
beats {
|
||||
# The port to listen on for filebeat connections.
|
||||
port => 5044
|
||||
# The IP address to listen for filebeat connections.
|
||||
host => "0.0.0.0"
|
||||
}
|
||||
}
|
||||
filter {
|
||||
grok {
|
||||
match => { "message" => ["%{LOCALDATETIME:[mysql][error][timestamp]} (\[%{DATA:[mysql][error][level]}\] )?%{GREEDYDATA:[mysql][error][message]}",
|
||||
"%{TIMESTAMP_ISO8601:[mysql][error][timestamp]} %{NUMBER:[mysql][error][thread_id]} \[%{DATA:[mysql][error][level]}\] %{GREEDYDATA:[mysql][error][message1]}",
|
||||
"%{GREEDYDATA:[mysql][error][message2]}"] }
|
||||
pattern_definitions => {
|
||||
"LOCALDATETIME" => "[0-9]+ %{TIME}"
|
||||
}
|
||||
remove_field => "message"
|
||||
}
|
||||
mutate {
|
||||
rename => { "[mysql][error][message1]" => "[mysql][error][message]" }
|
||||
}
|
||||
mutate {
|
||||
rename => { "[mysql][error][message2]" => "[mysql][error][message]" }
|
||||
}
|
||||
date {
|
||||
match => [ "[mysql][error][timestamp]", "ISO8601", "YYMMdd H:m:s" ]
|
||||
remove_field => "[apache2][access][time]"
|
||||
}
|
||||
}
|
||||
output {
|
||||
elasticsearch {
|
||||
hosts => localhost
|
||||
manage_template => false
|
||||
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
|
||||
document_type => "%{[@metadata][type]}"
|
||||
}
|
||||
}
|
||||
54
docs/static/filebeat_modules/mysql/pipeline.conf
vendored
Normal file
54
docs/static/filebeat_modules/mysql/pipeline.conf
vendored
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
input {
|
||||
beats {
|
||||
port => 5044
|
||||
host => "0.0.0.0"
|
||||
}
|
||||
}
|
||||
filter {
|
||||
if [fileset][module] == "mysql" {
|
||||
if [fileset][name] == "error" {
|
||||
grok {
|
||||
match => { "message" => ["%{LOCALDATETIME:[mysql][error][timestamp]} (\[%{DATA:[mysql][error][level]}\] )?%{GREEDYDATA:[mysql][error][message]}",
|
||||
"%{TIMESTAMP_ISO8601:[mysql][error][timestamp]} %{NUMBER:[mysql][error][thread_id]} \[%{DATA:[mysql][error][level]}\] %{GREEDYDATA:[mysql][error][message1]}",
|
||||
"%{GREEDYDATA:[mysql][error][message2]}"] }
|
||||
pattern_definitions => {
|
||||
"LOCALDATETIME" => "[0-9]+ %{TIME}"
|
||||
}
|
||||
remove_field => "message"
|
||||
}
|
||||
mutate {
|
||||
rename => { "[mysql][error][message1]" => "[mysql][error][message]" }
|
||||
}
|
||||
mutate {
|
||||
rename => { "[mysql][error][message2]" => "[mysql][error][message]" }
|
||||
}
|
||||
date {
|
||||
match => [ "[mysql][error][timestamp]", "ISO8601", "YYMMdd H:m:s" ]
|
||||
remove_field => "[mysql][error][time]"
|
||||
}
|
||||
}
|
||||
else if [fileset][name] == "slowlog" {
|
||||
grok {
|
||||
match => { "message" => ["^# User@Host: %{USER:[mysql][slowlog][user]}(\[[^\]]+\])? @ %{HOSTNAME:[mysql][slowlog][host]} \[(IP:[mysql][slowlog][ip])?\](\s*Id:\s* %{NUMBER:[mysql][slowlog][id]})?\n# Query_time: %{NUMBER:[mysql][slowlog][query_time][sec]}\s* Lock_time: %{NUMBER:[mysql][slowlog][lock_time][sec]}\s* Rows_sent: %{NUMBER:[mysql][slowlog][rows_sent]}\s* Rows_examined: %{NUMBER:[mysql][slowlog][rows_examined]}\n(SET timestamp=%{NUMBER:[mysql][slowlog][timestamp]};\n)?%{GREEDYMULTILINE:[mysql][slowlog][query]}"] }
|
||||
pattern_definitions => {
|
||||
"GREEDYMULTILINE" => "(.|\n)*"
|
||||
}
|
||||
remove_field => "message"
|
||||
}
|
||||
date {
|
||||
match => [ "[mysql][slowlog][timestamp]", "UNIX" ]
|
||||
}
|
||||
mutate {
|
||||
gsub => ["[mysql][slowlog][query]", "\n# Time: [0-9]+ [0-9][0-9]:[0-9][0-9]:[0-9][0-9](\\.[0-9]+)?$", ""]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
output {
|
||||
elasticsearch {
|
||||
hosts => localhost
|
||||
manage_template => false
|
||||
index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
|
||||
document_type => "%{[@metadata][type]}"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
filebeat.prospectors:
|
||||
- input_type: log
|
||||
paths:
|
||||
- /var/log/mysql/mysql-slow.log*
|
||||
- /var/lib/mysql/hostname-slow.log
|
||||
exclude_files: [".gz$"]
|
||||
multiline:
|
||||
pattern: "^# User@Host: "
|
||||
negate: true
|
||||
match: after
|
||||
output.logstash:
|
||||
hosts: ["localhost:5044"]
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
input {
|
||||
beats {
|
||||
# The port to listen on for filebeat connections.
|
||||
port => 5044
|
||||
# The IP address to listen for filebeat connections.
|
||||
host => "0.0.0.0"
|
||||
}
|
||||
}
|
||||
filter {
|
||||
grok {
|
||||
match => { "message" => ["^# User@Host: %{USER:[mysql][slowlog][user]}(\[[^\]]+\])? @ %{HOSTNAME:[mysql][slowlog][host]} \[(IP:[mysql][slowlog][ip])?\](\s*Id:\s* %{NUMBER:[mysql][slowlog][id]})?\n# Query_time: %{NUMBER:[mysql][slowlog][query_time][sec]}\s* Lock_time: %{NUMBER:[mysql][slowlog][lock_time][sec]}\s* Rows_sent: %{NUMBER:[mysql][slowlog][rows_sent]}\s* Rows_examined: %{NUMBER:[mysql][slowlog][rows_examined]}\n(SET timestamp=%{NUMBER:[mysql][slowlog][timestamp]};\n)?%{GREEDYMULTILINE:[mysql][slowlog][query]}"] }
|
||||
pattern_definitions => {
|
||||
"GREEDYMULTILINE" => "(.|\n)*"
|
||||
}
|
||||
remove_field => "message"
|
||||
}
|
||||
date {
|
||||
match => [ "[mysql][slowlog][timestamp]", "UNIX" ]
|
||||
}
|
||||
mutate {
|
||||
gsub => ["[mysql][slowlog][query]", "\n# Time: [0-9]+ [0-9][0-9]:[0-9][0-9]:[0-9][0-9](\\.[0-9]+)?$", ""]
|
||||
}
|
||||
}
|
||||
output {
|
||||
elasticsearch {
|
||||
hosts => localhost
|
||||
manage_template => false
|
||||
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
|
||||
document_type => "%{[@metadata][type]}"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
filebeat.prospectors:
|
||||
- input_type: log
|
||||
paths:
|
||||
- /var/log/nginx/access.log*
|
||||
exclude_files: [".gz$"]
|
||||
output.logstash:
|
||||
hosts: ["localhost:5044"]
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
input {
|
||||
beats {
|
||||
# The port to listen on for filebeat connections.
|
||||
port => 5044
|
||||
# The IP address to listen for filebeat connections.
|
||||
host => "0.0.0.0"
|
||||
}
|
||||
}
|
||||
filter {
|
||||
grok {
|
||||
match => { "message" => ["%{IPORHOST:[nginx][access][remote_ip]} - %{DATA:[nginx][access][user_name]} \[%{HTTPDATE:[nginx][access][time]}\] \"%{WORD:[nginx][access][method]} %{DATA:[nginx][access][url]} HTTP/%{NUMBER:[nginx][access][http_version]}\" %{NUMBER:[nginx][access][response_code]} %{NUMBER:[nginx][access][body_sent][bytes]} \"%{DATA:[nginx][access][referrer]}\" \"%{DATA:[nginx][access][agent]}\""] }
|
||||
remove_field => "message"
|
||||
}
|
||||
mutate {
|
||||
rename => { "@timestamp" => "read_timestamp" }
|
||||
}
|
||||
date {
|
||||
match => [ "[nginx][access][time]", "dd/MMM/YYYY:H:m:s Z" ]
|
||||
remove_field => "[nginx][access][time]"
|
||||
}
|
||||
useragent {
|
||||
source => "[nginx][access][agent]"
|
||||
target => "[nginx][access][user_agent]"
|
||||
remove_field => "[nginx][access][agent]"
|
||||
}
|
||||
geoip {
|
||||
source => "[nginx][access][remote_ip]"
|
||||
target => "[nginx][access][geoip]"
|
||||
}
|
||||
}
|
||||
output {
|
||||
elasticsearch {
|
||||
hosts => localhost
|
||||
manage_template => false
|
||||
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
|
||||
document_type => "%{[@metadata][type]}"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
filebeat.prospectors:
|
||||
- input_type: log
|
||||
paths:
|
||||
- /var/log/nginx/error.log*
|
||||
exclude_files: [".gz$"]
|
||||
output.logstash:
|
||||
hosts: ["localhost:5044"]
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
input {
|
||||
beats {
|
||||
# The port to listen on for filebeat connections.
|
||||
port => 5044
|
||||
# The IP address to listen for filebeat connections.
|
||||
host => "0.0.0.0"
|
||||
}
|
||||
}
|
||||
filter {
|
||||
grok {
|
||||
match => { "message" => ["%{DATA:[nginx][error][time]} \[%{DATA:[nginx][error][level]}\] %{NUMBER:[nginx][error][pid]}#%{NUMBER:[nginx][error][tid]}: (\*%{NUMBER:[nginx][error][connection_id]} )?%{GREEDYDATA:[nginx][error][message]}"] }
|
||||
remove_field => "message"
|
||||
}
|
||||
mutate {
|
||||
rename => { "@timestamp" => "read_timestamp" }
|
||||
}
|
||||
date {
|
||||
match => [ "[nginx][error][time]", "YYYY/MM/dd H:m:s" ]
|
||||
remove_field => "[nginx][error][time]"
|
||||
}
|
||||
}
|
||||
output {
|
||||
elasticsearch {
|
||||
hosts => localhost
|
||||
manage_template => false
|
||||
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
|
||||
document_type => "%{[@metadata][type]}"
|
||||
}
|
||||
}
|
||||
53
docs/static/filebeat_modules/nginx/pipeline.conf
vendored
Normal file
53
docs/static/filebeat_modules/nginx/pipeline.conf
vendored
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
input {
|
||||
beats {
|
||||
port => 5044
|
||||
host => "0.0.0.0"
|
||||
}
|
||||
}
|
||||
filter {
|
||||
if [fileset][module] == "nginx" {
|
||||
if [fileset][name] == "access" {
|
||||
grok {
|
||||
match => { "message" => ["%{IPORHOST:[nginx][access][remote_ip]} - %{DATA:[nginx][access][user_name]} \[%{HTTPDATE:[nginx][access][time]}\] \"%{WORD:[nginx][access][method]} %{DATA:[nginx][access][url]} HTTP/%{NUMBER:[nginx][access][http_version]}\" %{NUMBER:[nginx][access][response_code]} %{NUMBER:[nginx][access][body_sent][bytes]} \"%{DATA:[nginx][access][referrer]}\" \"%{DATA:[nginx][access][agent]}\""] }
|
||||
remove_field => "message"
|
||||
}
|
||||
mutate {
|
||||
add_field => { "read_timestamp" => "%{@timestamp}" }
|
||||
}
|
||||
date {
|
||||
match => [ "[nginx][access][time]", "dd/MMM/YYYY:H:m:s Z" ]
|
||||
remove_field => "[nginx][access][time]"
|
||||
}
|
||||
useragent {
|
||||
source => "[nginx][access][agent]"
|
||||
target => "[nginx][access][user_agent]"
|
||||
remove_field => "[nginx][access][agent]"
|
||||
}
|
||||
geoip {
|
||||
source => "[nginx][access][remote_ip]"
|
||||
target => "[nginx][access][geoip]"
|
||||
}
|
||||
}
|
||||
else if [fileset][name] == "error" {
|
||||
grok {
|
||||
match => { "message" => ["%{DATA:[nginx][error][time]} \[%{DATA:[nginx][error][level]}\] %{NUMBER:[nginx][error][pid]}#%{NUMBER:[nginx][error][tid]}: (\*%{NUMBER:[nginx][error][connection_id]} )?%{GREEDYDATA:[nginx][error][message]}"] }
|
||||
remove_field => "message"
|
||||
}
|
||||
mutate {
|
||||
rename => { "@timestamp" => "read_timestamp" }
|
||||
}
|
||||
date {
|
||||
match => [ "[nginx][error][time]", "YYYY/MM/dd H:m:s" ]
|
||||
remove_field => "[nginx][error][time]"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
output {
|
||||
elasticsearch {
|
||||
hosts => localhost
|
||||
manage_template => false
|
||||
index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
|
||||
document_type => "%{[@metadata][type]}"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
filebeat.prospectors:
|
||||
- input_type: log
|
||||
paths:
|
||||
- /var/log/auth.log*
|
||||
- /var/log/secure*
|
||||
exclude_files: [".gz$"]
|
||||
multiline:
|
||||
pattern: "^\\s"
|
||||
match: after
|
||||
output.logstash:
|
||||
hosts: ["localhost:5044"]
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
input {
|
||||
beats {
|
||||
# The port to listen on for filebeat connections.
|
||||
port => 5044
|
||||
# The IP address to listen for filebeat connections.
|
||||
host => "0.0.0.0"
|
||||
}
|
||||
}
|
||||
filter {
|
||||
grok {
|
||||
match => { "message" => ["%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} sshd(?:\[%{POSINT:[system][auth][pid]}\])?: %{DATA:[system][auth][ssh][event]} %{DATA:[system][auth][ssh][method]} for (invalid user )?%{DATA:[system][auth][user]} from %{IPORHOST:[system][auth][ssh][ip]} port %{NUMBER:[system][auth][ssh][port]} ssh2(: %{GREEDYDATA:[system][auth][ssh][signature]})?",
|
||||
"%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} sshd(?:\[%{POSINT:[system][auth][pid]}\])?: %{DATA:[system][auth][ssh][event]} user %{DATA:[system][auth][user]} from %{IPORHOST:[system][auth][ssh][ip]}",
|
||||
"%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} sshd(?:\[%{POSINT:[system][auth][pid]}\])?: Did not receive identification string from %{IPORHOST:[system][auth][ssh][dropped_ip]}",
|
||||
"%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} sudo(?:\[%{POSINT:[system][auth][pid]}\])?: \s*%{DATA:[system][auth][user]} :( %{DATA:[system][auth][sudo][error]} ;)? TTY=%{DATA:[system][auth][sudo][tty]} ; PWD=%{DATA:[system][auth][sudo][pwd]} ; USER=%{DATA:[system][auth][sudo][user]} ; COMMAND=%{GREEDYDATA:[system][auth][sudo][command]}",
|
||||
"%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} groupadd(?:\[%{POSINT:[system][auth][pid]}\])?: new group: name=%{DATA:system.auth.groupadd.name}, GID=%{NUMBER:system.auth.groupadd.gid}",
|
||||
"%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} useradd(?:\[%{POSINT:[system][auth][pid]}\])?: new user: name=%{DATA:[system][auth][user][add][name]}, UID=%{NUMBER:[system][auth][user][add][uid]}, GID=%{NUMBER:[system][auth][user][add][gid]}, home=%{DATA:[system][auth][user][add][home]}, shell=%{DATA:[system][auth][user][add][shell]}$",
|
||||
"%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} %{DATA:[system][auth][program]}(?:\[%{POSINT:[system][auth][pid]}\])?: %{GREEDYMULTILINE:[system][auth][message]}"] }
|
||||
pattern_definitions => {
|
||||
"GREEDYMULTILINE"=> "(.|\n)*"
|
||||
}
|
||||
remove_field => "message"
|
||||
}
|
||||
date {
|
||||
match => [ "[system][auth][timestamp]", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
|
||||
}
|
||||
geoip {
|
||||
source => "[system][auth][ssh][ip]"
|
||||
target => "[system][auth][ssh][geoip]"
|
||||
}
|
||||
}
|
||||
output {
|
||||
elasticsearch {
|
||||
hosts => localhost
|
||||
manage_template => false
|
||||
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
|
||||
document_type => "%{[@metadata][type]}"
|
||||
}
|
||||
}
|
||||
50
docs/static/filebeat_modules/system/pipeline.conf
vendored
Normal file
50
docs/static/filebeat_modules/system/pipeline.conf
vendored
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
input {
|
||||
beats {
|
||||
port => 5044
|
||||
host => "0.0.0.0"
|
||||
}
|
||||
}
|
||||
filter {
|
||||
if [fileset][module] == "system" {
|
||||
if [fileset][name] == "auth" {
|
||||
grok {
|
||||
match => { "message" => ["%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} sshd(?:\[%{POSINT:[system][auth][pid]}\])?: %{DATA:[system][auth][ssh][event]} %{DATA:[system][auth][ssh][method]} for (invalid user )?%{DATA:[system][auth][user]} from %{IPORHOST:[system][auth][ssh][ip]} port %{NUMBER:[system][auth][ssh][port]} ssh2(: %{GREEDYDATA:[system][auth][ssh][signature]})?",
|
||||
"%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} sshd(?:\[%{POSINT:[system][auth][pid]}\])?: %{DATA:[system][auth][ssh][event]} user %{DATA:[system][auth][user]} from %{IPORHOST:[system][auth][ssh][ip]}",
|
||||
"%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} sshd(?:\[%{POSINT:[system][auth][pid]}\])?: Did not receive identification string from %{IPORHOST:[system][auth][ssh][dropped_ip]}",
|
||||
"%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} sudo(?:\[%{POSINT:[system][auth][pid]}\])?: \s*%{DATA:[system][auth][user]} :( %{DATA:[system][auth][sudo][error]} ;)? TTY=%{DATA:[system][auth][sudo][tty]} ; PWD=%{DATA:[system][auth][sudo][pwd]} ; USER=%{DATA:[system][auth][sudo][user]} ; COMMAND=%{GREEDYDATA:[system][auth][sudo][command]}",
|
||||
"%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} groupadd(?:\[%{POSINT:[system][auth][pid]}\])?: new group: name=%{DATA:system.auth.groupadd.name}, GID=%{NUMBER:system.auth.groupadd.gid}",
|
||||
"%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} useradd(?:\[%{POSINT:[system][auth][pid]}\])?: new user: name=%{DATA:[system][auth][user][add][name]}, UID=%{NUMBER:[system][auth][user][add][uid]}, GID=%{NUMBER:[system][auth][user][add][gid]}, home=%{DATA:[system][auth][user][add][home]}, shell=%{DATA:[system][auth][user][add][shell]}$",
|
||||
"%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} %{DATA:[system][auth][program]}(?:\[%{POSINT:[system][auth][pid]}\])?: %{GREEDYMULTILINE:[system][auth][message]}"] }
|
||||
pattern_definitions => {
|
||||
"GREEDYMULTILINE"=> "(.|\n)*"
|
||||
}
|
||||
remove_field => "message"
|
||||
}
|
||||
date {
|
||||
match => [ "[system][auth][timestamp]", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
|
||||
}
|
||||
geoip {
|
||||
source => "[system][auth][ssh][ip]"
|
||||
target => "[system][auth][ssh][geoip]"
|
||||
}
|
||||
}
|
||||
else if [fileset][name] == "syslog" {
|
||||
grok {
|
||||
match => { "message" => ["%{SYSLOGTIMESTAMP:[system][syslog][timestamp]} %{SYSLOGHOST:[system][syslog][hostname]} %{DATA:[system][syslog][program]}(?:\[%{POSINT:[system][syslog][pid]}\])?: %{GREEDYMULTILINE:[system][syslog][message]}"] }
|
||||
pattern_definitions => { "GREEDYMULTILINE" => "(.|\n)*" }
|
||||
remove_field => "message"
|
||||
}
|
||||
date {
|
||||
match => [ "[system][syslog][timestamp]", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
output {
|
||||
elasticsearch {
|
||||
hosts => localhost
|
||||
manage_template => false
|
||||
index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
|
||||
document_type => "%{[@metadata][type]}"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
filebeat.prospectors:
|
||||
- input_type: log
|
||||
paths:
|
||||
- /var/log/messages*
|
||||
- /var/log/syslog*
|
||||
exclude_files: [".gz$"]
|
||||
multiline:
|
||||
pattern: "^\\s"
|
||||
match: after
|
||||
output.logstash:
|
||||
hosts: ["localhost:5044"]
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
input {
|
||||
beats {
|
||||
# The port to listen on for filebeat connections.
|
||||
port => 5044
|
||||
# The IP address to listen for filebeat connections.
|
||||
host => "0.0.0.0"
|
||||
}
|
||||
}
|
||||
filter {
|
||||
grok {
|
||||
match => { "message" => ["%{SYSLOGTIMESTAMP:[system][syslog][timestamp]} %{SYSLOGHOST:[system][syslog][hostname]} %{DATA:[system][syslog][program]}(?:\[%{POSINT:[system][syslog][pid]}\])?: %{GREEDYMULTILINE:[system][syslog][message]}"] }
|
||||
pattern_definitions => { "GREEDYMULTILINE" => "(.|\n)*" }
|
||||
remove_field => "message"
|
||||
}
|
||||
date {
|
||||
match => [ "[system][syslog][timestamp]", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
|
||||
}
|
||||
}
|
||||
output {
|
||||
elasticsearch {
|
||||
hosts => localhost
|
||||
manage_template => false
|
||||
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
|
||||
document_type => "%{[@metadata][type]}"
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue