mirror of
https://github.com/elastic/logstash.git
synced 2025-04-24 22:57:16 -04:00
DeDe Morton
parent
78ca2b18ce
commit
1213c56f37
2 changed files with 140 additions and 134 deletions
270
docs/static/advanced-pipeline.asciidoc
vendored
270
docs/static/advanced-pipeline.asciidoc
vendored
|
|
@ -161,18 +161,20 @@ If your pipeline is working correctly, you should see a series of events like th
|
||||||
[source,json]
|
[source,json]
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
{
|
{
|
||||||
"@timestamp" => 2016-10-11T20:54:06.733Z,
|
"@timestamp" => 2017-11-09T01:44:20.071Z,
|
||||||
"offset" => 325,
|
"offset" => 325,
|
||||||
"@version" => "1",
|
"@version" => "1",
|
||||||
"beat" => {
|
"beat" => {
|
||||||
|
"name" => "My-MacBook-Pro.local",
|
||||||
"hostname" => "My-MacBook-Pro.local",
|
"hostname" => "My-MacBook-Pro.local",
|
||||||
"name" => "My-MacBook-Pro.local"
|
"version" => "6.0.0"
|
||||||
},
|
},
|
||||||
"type" => "log",
|
|
||||||
"host" => "My-MacBook-Pro.local",
|
"host" => "My-MacBook-Pro.local",
|
||||||
|
"prospector" => {
|
||||||
|
"type" => "log"
|
||||||
|
},
|
||||||
"source" => "/path/to/file/logstash-tutorial.log",
|
"source" => "/path/to/file/logstash-tutorial.log",
|
||||||
"message" => "83.149.9.216 - - [04/Jan/2015:05:13:42 +0000] \"GET /presentations/logstash-monitorama-2013/images/kibana-search.png HTTP/1.1\" 200 203023 \"http://semicomplete.com/presentations/logstash-monitorama-2013/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\"",
|
"message" => "83.149.9.216 - - [04/Jan/2015:05:13:42 +0000] \"GET /presentations/logstash-monitorama-2013/images/kibana-search.png HTTP/1.1\" 200 203023 \"http://semicomplete.com/presentations/logstash-monitorama-2013/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\"",
|
||||||
"type" => "log",
|
|
||||||
"tags" => [
|
"tags" => [
|
||||||
[0] "beats_input_codec_plain_applied"
|
[0] "beats_input_codec_plain_applied"
|
||||||
]
|
]
|
||||||
|
|
@ -284,29 +286,30 @@ After Logstash applies the grok pattern, the events will have the following JSON
|
||||||
|
|
||||||
[source,json]
|
[source,json]
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
{
|
|
||||||
"request" => "/presentations/logstash-monitorama-2013/images/kibana-search.png",
|
"request" => "/presentations/logstash-monitorama-2013/images/kibana-search.png",
|
||||||
"agent" => "\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\"",
|
"agent" => "\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\"",
|
||||||
"offset" => 325,
|
"offset" => 325,
|
||||||
"auth" => "-",
|
"auth" => "-",
|
||||||
"ident" => "-",
|
"ident" => "-",
|
||||||
"type" => "log",
|
|
||||||
"verb" => "GET",
|
"verb" => "GET",
|
||||||
|
"prospector" => {
|
||||||
|
"type" => "log"
|
||||||
|
},
|
||||||
"source" => "/path/to/file/logstash-tutorial.log",
|
"source" => "/path/to/file/logstash-tutorial.log",
|
||||||
"message" => "83.149.9.216 - - [04/Jan/2015:05:13:42 +0000] \"GET /presentations/logstash-monitorama-2013/images/kibana-search.png HTTP/1.1\" 200 203023 \"http://semicomplete.com/presentations/logstash-monitorama-2013/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\"",
|
"message" => "83.149.9.216 - - [04/Jan/2015:05:13:42 +0000] \"GET /presentations/logstash-monitorama-2013/images/kibana-search.png HTTP/1.1\" 200 203023 \"http://semicomplete.com/presentations/logstash-monitorama-2013/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\"",
|
||||||
"type" => "log",
|
|
||||||
"tags" => [
|
"tags" => [
|
||||||
[0] "beats_input_codec_plain_applied"
|
[0] "beats_input_codec_plain_applied"
|
||||||
],
|
],
|
||||||
"referrer" => "\"http://semicomplete.com/presentations/logstash-monitorama-2013/\"",
|
"referrer" => "\"http://semicomplete.com/presentations/logstash-monitorama-2013/\"",
|
||||||
"@timestamp" => 2016-10-11T21:04:36.167Z,
|
"@timestamp" => 2017-11-09T02:51:12.416Z,
|
||||||
"response" => "200",
|
"response" => "200",
|
||||||
"bytes" => "203023",
|
"bytes" => "203023",
|
||||||
"clientip" => "83.149.9.216",
|
"clientip" => "83.149.9.216",
|
||||||
"@version" => "1",
|
"@version" => "1",
|
||||||
"beat" => {
|
"beat" => {
|
||||||
|
"name" => "My-MacBook-Pro.local",
|
||||||
"hostname" => "My-MacBook-Pro.local",
|
"hostname" => "My-MacBook-Pro.local",
|
||||||
"name" => "My-MacBook-Pro.local"
|
"version" => "6.0.0"
|
||||||
},
|
},
|
||||||
"host" => "My-MacBook-Pro.local",
|
"host" => "My-MacBook-Pro.local",
|
||||||
"httpversion" => "1.1",
|
"httpversion" => "1.1",
|
||||||
|
|
@ -379,21 +382,20 @@ Notice that the event now contains geographic location information:
|
||||||
"geoip" => {
|
"geoip" => {
|
||||||
"timezone" => "Europe/Moscow",
|
"timezone" => "Europe/Moscow",
|
||||||
"ip" => "83.149.9.216",
|
"ip" => "83.149.9.216",
|
||||||
"latitude" => 55.7522,
|
"latitude" => 55.7485,
|
||||||
"continent_code" => "EU",
|
"continent_code" => "EU",
|
||||||
"city_name" => "Moscow",
|
"city_name" => "Moscow",
|
||||||
"country_code2" => "RU",
|
|
||||||
"country_name" => "Russia",
|
"country_name" => "Russia",
|
||||||
"dma_code" => nil,
|
"country_code2" => "RU",
|
||||||
"country_code3" => "RU",
|
"country_code3" => "RU",
|
||||||
"region_name" => "Moscow",
|
"region_name" => "Moscow",
|
||||||
"location" => [
|
"location" => {
|
||||||
[0] 37.6156,
|
"lon" => 37.6184,
|
||||||
[1] 55.7522
|
"lat" => 55.7485
|
||||||
],
|
},
|
||||||
"postal_code" => "101194",
|
"postal_code" => "101194",
|
||||||
"longitude" => 37.6156,
|
"region_code" => "MOW",
|
||||||
"region_code" => "MOW"
|
"longitude" => 37.6184
|
||||||
},
|
},
|
||||||
...
|
...
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
@ -477,71 +479,72 @@ You should get multiple hits back. For example:
|
||||||
[source,json]
|
[source,json]
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
{
|
{
|
||||||
"took" : 21,
|
"took": 50,
|
||||||
"timed_out" : false,
|
"timed_out": false,
|
||||||
"_shards" : {
|
"_shards": {
|
||||||
"total" : 5,
|
"total": 5,
|
||||||
"successful" : 5,
|
"successful": 5,
|
||||||
"failed" : 0
|
"skipped": 0,
|
||||||
|
"failed": 0
|
||||||
},
|
},
|
||||||
"hits" : {
|
"hits": {
|
||||||
"total" : 98,
|
"total": 98,
|
||||||
"max_score" : 3.745223,
|
"max_score": 2.793642,
|
||||||
"hits" : [
|
"hits": [
|
||||||
{
|
{
|
||||||
"_index" : "logstash-2016.10.11",
|
"_index": "logstash-2017.11.09",
|
||||||
"_type" : "log",
|
"_type": "doc",
|
||||||
"_id" : "AVe14gMiYMkU36o_eVsA",
|
"_id": "3IzDnl8BW52sR0fx5wdV",
|
||||||
"_score" : 3.745223,
|
"_score": 2.793642,
|
||||||
"_source" : {
|
"_source": {
|
||||||
"request" : "/presentations/logstash-monitorama-2013/images/frontend-response-codes.png",
|
"request": "/presentations/logstash-monitorama-2013/images/frontend-response-codes.png",
|
||||||
"agent" : "\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\"",
|
"agent": """"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"""",
|
||||||
"geoip" : {
|
"geoip": {
|
||||||
"timezone" : "Europe/Moscow",
|
"timezone": "Europe/Moscow",
|
||||||
"ip" : "83.149.9.216",
|
"ip": "83.149.9.216",
|
||||||
"latitude" : 55.7522,
|
"latitude": 55.7485,
|
||||||
"continent_code" : "EU",
|
"continent_code": "EU",
|
||||||
"city_name" : "Moscow",
|
"city_name": "Moscow",
|
||||||
"country_code2" : "RU",
|
"country_name": "Russia",
|
||||||
"country_name" : "Russia",
|
"country_code2": "RU",
|
||||||
"dma_code" : null,
|
"country_code3": "RU",
|
||||||
"country_code3" : "RU",
|
"region_name": "Moscow",
|
||||||
"region_name" : "Moscow",
|
"location": {
|
||||||
"location" : [
|
"lon": 37.6184,
|
||||||
37.6156,
|
"lat": 55.7485
|
||||||
55.7522
|
},
|
||||||
],
|
"postal_code": "101194",
|
||||||
"postal_code" : "101194",
|
"region_code": "MOW",
|
||||||
"longitude" : 37.6156,
|
"longitude": 37.6184
|
||||||
"region_code" : "MOW"
|
|
||||||
},
|
},
|
||||||
"offset" : 2932,
|
"offset": 2932,
|
||||||
"auth" : "-",
|
"auth": "-",
|
||||||
"ident" : "-",
|
"ident": "-",
|
||||||
"type" : "log",
|
"verb": "GET",
|
||||||
"verb" : "GET",
|
"prospector": {
|
||||||
"source" : "/path/to/file/logstash-tutorial.log",
|
"type": "log"
|
||||||
"message" : "83.149.9.216 - - [04/Jan/2015:05:13:45 +0000] \"GET /presentations/logstash-monitorama-2013/images/frontend-response-codes.png HTTP/1.1\" 200 52878 \"http://semicomplete.com/presentations/logstash-monitorama-2013/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\"",
|
},
|
||||||
"type" : "log",
|
"source": "/path/to/file/logstash-tutorial.log",
|
||||||
"tags" : [
|
"message": """83.149.9.216 - - [04/Jan/2015:05:13:45 +0000] "GET /presentations/logstash-monitorama-2013/images/frontend-response-codes.png HTTP/1.1" 200 52878 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"""",
|
||||||
|
"tags": [
|
||||||
"beats_input_codec_plain_applied"
|
"beats_input_codec_plain_applied"
|
||||||
],
|
],
|
||||||
"referrer" : "\"http://semicomplete.com/presentations/logstash-monitorama-2013/\"",
|
"referrer": """"http://semicomplete.com/presentations/logstash-monitorama-2013/"""",
|
||||||
"@timestamp" : "2016-10-11T22:34:25.317Z",
|
"@timestamp": "2017-11-09T03:11:35.304Z",
|
||||||
"response" : "200",
|
"response": "200",
|
||||||
"bytes" : "52878",
|
"bytes": "52878",
|
||||||
"clientip" : "83.149.9.216",
|
"clientip": "83.149.9.216",
|
||||||
"@version" : "1",
|
"@version": "1",
|
||||||
"beat" : {
|
"beat": {
|
||||||
"hostname" : "My-MacBook-Pro.local",
|
"name": "My-MacBook-Pro.local",
|
||||||
"name" : "My-MacBook-Pro.local"
|
"hostname": "My-MacBook-Pro.local",
|
||||||
|
"version": "6.0.0"
|
||||||
},
|
},
|
||||||
"host" : "My-MacBook-Pro.local",
|
"host": "My-MacBook-Pro.local",
|
||||||
"httpversion" : "1.1",
|
"httpversion": "1.1",
|
||||||
"timestamp" : "04/Jan/2015:05:13:45 +0000"
|
"timestamp": "04/Jan/2015:05:13:45 +0000"
|
||||||
}
|
}
|
||||||
}
|
},
|
||||||
},
|
|
||||||
...
|
...
|
||||||
|
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
@ -559,68 +562,71 @@ A few log entries come from Buffalo, so the query produces the following respons
|
||||||
[source,json]
|
[source,json]
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
{
|
{
|
||||||
"took" : 3,
|
"took": 9,
|
||||||
"timed_out" : false,
|
"timed_out": false,
|
||||||
"_shards" : {
|
"_shards": {
|
||||||
"total" : 5,
|
"total": 5,
|
||||||
"successful" : 5,
|
"successful": 5,
|
||||||
"failed" : 0
|
"skipped": 0,
|
||||||
|
"failed": 0
|
||||||
},
|
},
|
||||||
"hits" : {
|
"hits": {
|
||||||
"total" : 3,
|
"total": 2,
|
||||||
"max_score" : 2.6390574,
|
"max_score": 2.6390574,
|
||||||
"hits" : [
|
"hits": [
|
||||||
{
|
{
|
||||||
"_index" : "logstash-2016.10.11",
|
"_index": "logstash-2017.11.09",
|
||||||
"_type" : "log",
|
"_type": "doc",
|
||||||
"_id" : "AVe14gMjYMkU36o_eVtO",
|
"_id": "L4zDnl8BW52sR0fx5whY",
|
||||||
"_score" : 2.6390574,
|
"_score": 2.6390574,
|
||||||
"_source" : {
|
"_source": {
|
||||||
"request" : "/?flav=rss20",
|
"request": "/blog/geekery/disabling-battery-in-ubuntu-vms.html?utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+semicomplete%2Fmain+%28semicomplete.com+-+Jordan+Sissel%29",
|
||||||
"agent" : "\"-\"",
|
"agent": """"Tiny Tiny RSS/1.11 (http://tt-rss.org/)"""",
|
||||||
"geoip" : {
|
"geoip": {
|
||||||
"timezone" : "America/New_York",
|
"timezone": "America/New_York",
|
||||||
"ip" : "108.174.55.234",
|
"ip": "198.46.149.143",
|
||||||
"latitude" : 42.9864,
|
"latitude": 42.8864,
|
||||||
"continent_code" : "NA",
|
"continent_code": "NA",
|
||||||
"city_name" : "Buffalo",
|
"city_name": "Buffalo",
|
||||||
"country_code2" : "US",
|
"country_name": "United States",
|
||||||
"country_name" : "United States",
|
"country_code2": "US",
|
||||||
"dma_code" : 514,
|
"dma_code": 514,
|
||||||
"country_code3" : "US",
|
"country_code3": "US",
|
||||||
"region_name" : "New York",
|
"region_name": "New York",
|
||||||
"location" : [
|
"location": {
|
||||||
-78.7279,
|
"lon": -78.8781,
|
||||||
42.9864
|
"lat": 42.8864
|
||||||
],
|
},
|
||||||
"postal_code" : "14221",
|
"postal_code": "14202",
|
||||||
"longitude" : -78.7279,
|
"region_code": "NY",
|
||||||
"region_code" : "NY"
|
"longitude": -78.8781
|
||||||
},
|
},
|
||||||
"offset" : 21471,
|
"offset": 22795,
|
||||||
"auth" : "-",
|
"auth": "-",
|
||||||
"ident" : "-",
|
"ident": "-",
|
||||||
"type" : "log",
|
"verb": "GET",
|
||||||
"verb" : "GET",
|
"prospector": {
|
||||||
"source" : "/path/to/file/logstash-tutorial.log",
|
"type": "log"
|
||||||
"message" : "108.174.55.234 - - [04/Jan/2015:05:27:45 +0000] \"GET /?flav=rss20 HTTP/1.1\" 200 29941 \"-\" \"-\"",
|
},
|
||||||
"type" : "log",
|
"source": "/path/to/file/logstash-tutorial.log",
|
||||||
"tags" : [
|
"message": """198.46.149.143 - - [04/Jan/2015:05:29:13 +0000] "GET /blog/geekery/disabling-battery-in-ubuntu-vms.html?utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+semicomplete%2Fmain+%28semicomplete.com+-+Jordan+Sissel%29 HTTP/1.1" 200 9316 "-" "Tiny Tiny RSS/1.11 (http://tt-rss.org/)"""",
|
||||||
|
"tags": [
|
||||||
"beats_input_codec_plain_applied"
|
"beats_input_codec_plain_applied"
|
||||||
],
|
],
|
||||||
"referrer" : "\"-\"",
|
"referrer": """"-"""",
|
||||||
"@timestamp" : "2016-10-11T22:34:25.318Z",
|
"@timestamp": "2017-11-09T03:11:35.321Z",
|
||||||
"response" : "200",
|
"response": "200",
|
||||||
"bytes" : "29941",
|
"bytes": "9316",
|
||||||
"clientip" : "108.174.55.234",
|
"clientip": "198.46.149.143",
|
||||||
"@version" : "1",
|
"@version": "1",
|
||||||
"beat" : {
|
"beat": {
|
||||||
"hostname" : "My-MacBook-Pro.local",
|
"name": "My-MacBook-Pro.local",
|
||||||
"name" : "My-MacBook-Pro.local"
|
"hostname": "My-MacBook-Pro.local",
|
||||||
|
"version": "6.0.0"
|
||||||
},
|
},
|
||||||
"host" : "My-MacBook-Pro.local",
|
"host": "My-MacBook-Pro.local",
|
||||||
"httpversion" : "1.1",
|
"httpversion": "1.1",
|
||||||
"timestamp" : "04/Jan/2015:05:27:45 +0000"
|
"timestamp": "04/Jan/2015:05:29:13 +0000"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
...
|
...
|
||||||
|
|
|
||||||
4
docs/static/dead-letter-queues.asciidoc
vendored
4
docs/static/dead-letter-queues.asciidoc
vendored
|
|
@ -178,7 +178,7 @@ failure:
|
||||||
"entry_time" => #<Java::OrgLogstash::Timestamp:0x5b5dacd5>,
|
"entry_time" => #<Java::OrgLogstash::Timestamp:0x5b5dacd5>,
|
||||||
"plugin_id" => "fb80f1925088497215b8d037e622dec5819b503e-4",
|
"plugin_id" => "fb80f1925088497215b8d037e622dec5819b503e-4",
|
||||||
"plugin_type" => "elasticsearch",
|
"plugin_type" => "elasticsearch",
|
||||||
"reason" => "Could not index event to Elasticsearch. status: 400, action: [\"index\", {:_id=>nil, :_index=>\"logstash-2017.06.22\", :_type=>\"logs\", :_routing=>nil}, 2017-06-22T01:29:29.804Z Suyogs-MacBook-Pro-2.local {\"geoip\":{\"location\":\"home\"}}], response: {\"index\"=>{\"_index\"=>\"logstash-2017.06.22\", \"_type\"=>\"logs\", \"_id\"=>\"AVzNayPze1iR9yDdI2MD\", \"status\"=>400, \"error\"=>{\"type\"=>\"mapper_parsing_exception\", \"reason\"=>\"failed to parse\", \"caused_by\"=>{\"type\"=>\"illegal_argument_exception\", \"reason\"=>\"illegal latitude value [266.30859375] for geoip.location\"}}}}"
|
"reason" => "Could not index event to Elasticsearch. status: 400, action: [\"index\", {:_id=>nil, :_index=>\"logstash-2017.06.22\", :_type=>\"doc\", :_routing=>nil}, 2017-06-22T01:29:29.804Z My-MacBook-Pro-2.local {\"geoip\":{\"location\":\"home\"}}], response: {\"index\"=>{\"_index\"=>\"logstash-2017.06.22\", \"_type\"=>\"doc\", \"_id\"=>\"AVzNayPze1iR9yDdI2MD\", \"status\"=>400, \"error\"=>{\"type\"=>\"mapper_parsing_exception\", \"reason\"=>\"failed to parse\", \"caused_by\"=>{\"type\"=>\"illegal_argument_exception\", \"reason\"=>\"illegal latitude value [266.30859375] for geoip.location\"}}}}"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"@timestamp" => 2017-06-22T01:29:29.804Z,
|
"@timestamp" => 2017-06-22T01:29:29.804Z,
|
||||||
|
|
@ -186,7 +186,7 @@ failure:
|
||||||
"geoip" => {
|
"geoip" => {
|
||||||
"location" => "home"
|
"location" => "home"
|
||||||
},
|
},
|
||||||
"host" => "Suyogs-MacBook-Pro-2.local",
|
"host" => "My-MacBook-Pro-2.local",
|
||||||
"message" => "{\"geoip\":{\"location\":\"home\"}}"
|
"message" => "{\"geoip\":{\"location\":\"home\"}}"
|
||||||
}
|
}
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue