diff --git a/docs/static/advanced-pipeline.asciidoc b/docs/static/advanced-pipeline.asciidoc index f704f6ad5..6d670fac0 100644 --- a/docs/static/advanced-pipeline.asciidoc +++ b/docs/static/advanced-pipeline.asciidoc @@ -161,18 +161,20 @@ If your pipeline is working correctly, you should see a series of events like th [source,json] -------------------------------------------------------------------------------- { - "@timestamp" => 2016-10-11T20:54:06.733Z, + "@timestamp" => 2017-11-09T01:44:20.071Z, "offset" => 325, "@version" => "1", "beat" => { + "name" => "My-MacBook-Pro.local", "hostname" => "My-MacBook-Pro.local", - "name" => "My-MacBook-Pro.local" + "version" => "6.0.0" }, - "type" => "log", "host" => "My-MacBook-Pro.local", + "prospector" => { + "type" => "log" + }, "source" => "/path/to/file/logstash-tutorial.log", "message" => "83.149.9.216 - - [04/Jan/2015:05:13:42 +0000] \"GET /presentations/logstash-monitorama-2013/images/kibana-search.png HTTP/1.1\" 200 203023 \"http://semicomplete.com/presentations/logstash-monitorama-2013/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\"", - "type" => "log", "tags" => [ [0] "beats_input_codec_plain_applied" ] @@ -284,29 +286,30 @@ After Logstash applies the grok pattern, the events will have the following JSON [source,json] -------------------------------------------------------------------------------- -{ "request" => "/presentations/logstash-monitorama-2013/images/kibana-search.png", "agent" => "\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\"", "offset" => 325, "auth" => "-", "ident" => "-", - "type" => "log", "verb" => "GET", + "prospector" => { + "type" => "log" + }, "source" => "/path/to/file/logstash-tutorial.log", "message" => "83.149.9.216 - - [04/Jan/2015:05:13:42 +0000] \"GET /presentations/logstash-monitorama-2013/images/kibana-search.png HTTP/1.1\" 200 203023 \"http://semicomplete.com/presentations/logstash-monitorama-2013/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\"", - "type" => "log", "tags" => [ [0] "beats_input_codec_plain_applied" ], "referrer" => "\"http://semicomplete.com/presentations/logstash-monitorama-2013/\"", - "@timestamp" => 2016-10-11T21:04:36.167Z, + "@timestamp" => 2017-11-09T02:51:12.416Z, "response" => "200", "bytes" => "203023", "clientip" => "83.149.9.216", "@version" => "1", "beat" => { + "name" => "My-MacBook-Pro.local", "hostname" => "My-MacBook-Pro.local", - "name" => "My-MacBook-Pro.local" + "version" => "6.0.0" }, "host" => "My-MacBook-Pro.local", "httpversion" => "1.1", @@ -379,21 +382,20 @@ Notice that the event now contains geographic location information: "geoip" => { "timezone" => "Europe/Moscow", "ip" => "83.149.9.216", - "latitude" => 55.7522, + "latitude" => 55.7485, "continent_code" => "EU", "city_name" => "Moscow", - "country_code2" => "RU", "country_name" => "Russia", - "dma_code" => nil, + "country_code2" => "RU", "country_code3" => "RU", "region_name" => "Moscow", - "location" => [ - [0] 37.6156, - [1] 55.7522 - ], + "location" => { + "lon" => 37.6184, + "lat" => 55.7485 + }, "postal_code" => "101194", - "longitude" => 37.6156, - "region_code" => "MOW" + "region_code" => "MOW", + "longitude" => 37.6184 }, ... -------------------------------------------------------------------------------- @@ -477,71 +479,72 @@ You should get multiple hits back. For example: [source,json] -------------------------------------------------------------------------------- { - "took" : 21, - "timed_out" : false, - "_shards" : { - "total" : 5, - "successful" : 5, - "failed" : 0 + "took": 50, + "timed_out": false, + "_shards": { + "total": 5, + "successful": 5, + "skipped": 0, + "failed": 0 }, - "hits" : { - "total" : 98, - "max_score" : 3.745223, - "hits" : [ + "hits": { + "total": 98, + "max_score": 2.793642, + "hits": [ { - "_index" : "logstash-2016.10.11", - "_type" : "log", - "_id" : "AVe14gMiYMkU36o_eVsA", - "_score" : 3.745223, - "_source" : { - "request" : "/presentations/logstash-monitorama-2013/images/frontend-response-codes.png", - "agent" : "\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\"", - "geoip" : { - "timezone" : "Europe/Moscow", - "ip" : "83.149.9.216", - "latitude" : 55.7522, - "continent_code" : "EU", - "city_name" : "Moscow", - "country_code2" : "RU", - "country_name" : "Russia", - "dma_code" : null, - "country_code3" : "RU", - "region_name" : "Moscow", - "location" : [ - 37.6156, - 55.7522 - ], - "postal_code" : "101194", - "longitude" : 37.6156, - "region_code" : "MOW" + "_index": "logstash-2017.11.09", + "_type": "doc", + "_id": "3IzDnl8BW52sR0fx5wdV", + "_score": 2.793642, + "_source": { + "request": "/presentations/logstash-monitorama-2013/images/frontend-response-codes.png", + "agent": """"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"""", + "geoip": { + "timezone": "Europe/Moscow", + "ip": "83.149.9.216", + "latitude": 55.7485, + "continent_code": "EU", + "city_name": "Moscow", + "country_name": "Russia", + "country_code2": "RU", + "country_code3": "RU", + "region_name": "Moscow", + "location": { + "lon": 37.6184, + "lat": 55.7485 + }, + "postal_code": "101194", + "region_code": "MOW", + "longitude": 37.6184 }, - "offset" : 2932, - "auth" : "-", - "ident" : "-", - "type" : "log", - "verb" : "GET", - "source" : "/path/to/file/logstash-tutorial.log", - "message" : "83.149.9.216 - - [04/Jan/2015:05:13:45 +0000] \"GET /presentations/logstash-monitorama-2013/images/frontend-response-codes.png HTTP/1.1\" 200 52878 \"http://semicomplete.com/presentations/logstash-monitorama-2013/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\"", - "type" : "log", - "tags" : [ + "offset": 2932, + "auth": "-", + "ident": "-", + "verb": "GET", + "prospector": { + "type": "log" + }, + "source": "/path/to/file/logstash-tutorial.log", + "message": """83.149.9.216 - - [04/Jan/2015:05:13:45 +0000] "GET /presentations/logstash-monitorama-2013/images/frontend-response-codes.png HTTP/1.1" 200 52878 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"""", + "tags": [ "beats_input_codec_plain_applied" ], - "referrer" : "\"http://semicomplete.com/presentations/logstash-monitorama-2013/\"", - "@timestamp" : "2016-10-11T22:34:25.317Z", - "response" : "200", - "bytes" : "52878", - "clientip" : "83.149.9.216", - "@version" : "1", - "beat" : { - "hostname" : "My-MacBook-Pro.local", - "name" : "My-MacBook-Pro.local" + "referrer": """"http://semicomplete.com/presentations/logstash-monitorama-2013/"""", + "@timestamp": "2017-11-09T03:11:35.304Z", + "response": "200", + "bytes": "52878", + "clientip": "83.149.9.216", + "@version": "1", + "beat": { + "name": "My-MacBook-Pro.local", + "hostname": "My-MacBook-Pro.local", + "version": "6.0.0" }, - "host" : "My-MacBook-Pro.local", - "httpversion" : "1.1", - "timestamp" : "04/Jan/2015:05:13:45 +0000" + "host": "My-MacBook-Pro.local", + "httpversion": "1.1", + "timestamp": "04/Jan/2015:05:13:45 +0000" } - } - }, + }, ... -------------------------------------------------------------------------------- @@ -559,68 +562,71 @@ A few log entries come from Buffalo, so the query produces the following respons [source,json] -------------------------------------------------------------------------------- { - "took" : 3, - "timed_out" : false, - "_shards" : { - "total" : 5, - "successful" : 5, - "failed" : 0 + "took": 9, + "timed_out": false, + "_shards": { + "total": 5, + "successful": 5, + "skipped": 0, + "failed": 0 }, - "hits" : { - "total" : 3, - "max_score" : 2.6390574, - "hits" : [ + "hits": { + "total": 2, + "max_score": 2.6390574, + "hits": [ { - "_index" : "logstash-2016.10.11", - "_type" : "log", - "_id" : "AVe14gMjYMkU36o_eVtO", - "_score" : 2.6390574, - "_source" : { - "request" : "/?flav=rss20", - "agent" : "\"-\"", - "geoip" : { - "timezone" : "America/New_York", - "ip" : "108.174.55.234", - "latitude" : 42.9864, - "continent_code" : "NA", - "city_name" : "Buffalo", - "country_code2" : "US", - "country_name" : "United States", - "dma_code" : 514, - "country_code3" : "US", - "region_name" : "New York", - "location" : [ - -78.7279, - 42.9864 - ], - "postal_code" : "14221", - "longitude" : -78.7279, - "region_code" : "NY" + "_index": "logstash-2017.11.09", + "_type": "doc", + "_id": "L4zDnl8BW52sR0fx5whY", + "_score": 2.6390574, + "_source": { + "request": "/blog/geekery/disabling-battery-in-ubuntu-vms.html?utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+semicomplete%2Fmain+%28semicomplete.com+-+Jordan+Sissel%29", + "agent": """"Tiny Tiny RSS/1.11 (http://tt-rss.org/)"""", + "geoip": { + "timezone": "America/New_York", + "ip": "198.46.149.143", + "latitude": 42.8864, + "continent_code": "NA", + "city_name": "Buffalo", + "country_name": "United States", + "country_code2": "US", + "dma_code": 514, + "country_code3": "US", + "region_name": "New York", + "location": { + "lon": -78.8781, + "lat": 42.8864 + }, + "postal_code": "14202", + "region_code": "NY", + "longitude": -78.8781 }, - "offset" : 21471, - "auth" : "-", - "ident" : "-", - "type" : "log", - "verb" : "GET", - "source" : "/path/to/file/logstash-tutorial.log", - "message" : "108.174.55.234 - - [04/Jan/2015:05:27:45 +0000] \"GET /?flav=rss20 HTTP/1.1\" 200 29941 \"-\" \"-\"", - "type" : "log", - "tags" : [ + "offset": 22795, + "auth": "-", + "ident": "-", + "verb": "GET", + "prospector": { + "type": "log" + }, + "source": "/path/to/file/logstash-tutorial.log", + "message": """198.46.149.143 - - [04/Jan/2015:05:29:13 +0000] "GET /blog/geekery/disabling-battery-in-ubuntu-vms.html?utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+semicomplete%2Fmain+%28semicomplete.com+-+Jordan+Sissel%29 HTTP/1.1" 200 9316 "-" "Tiny Tiny RSS/1.11 (http://tt-rss.org/)"""", + "tags": [ "beats_input_codec_plain_applied" ], - "referrer" : "\"-\"", - "@timestamp" : "2016-10-11T22:34:25.318Z", - "response" : "200", - "bytes" : "29941", - "clientip" : "108.174.55.234", - "@version" : "1", - "beat" : { - "hostname" : "My-MacBook-Pro.local", - "name" : "My-MacBook-Pro.local" + "referrer": """"-"""", + "@timestamp": "2017-11-09T03:11:35.321Z", + "response": "200", + "bytes": "9316", + "clientip": "198.46.149.143", + "@version": "1", + "beat": { + "name": "My-MacBook-Pro.local", + "hostname": "My-MacBook-Pro.local", + "version": "6.0.0" }, - "host" : "My-MacBook-Pro.local", - "httpversion" : "1.1", - "timestamp" : "04/Jan/2015:05:27:45 +0000" + "host": "My-MacBook-Pro.local", + "httpversion": "1.1", + "timestamp": "04/Jan/2015:05:29:13 +0000" } }, ... diff --git a/docs/static/dead-letter-queues.asciidoc b/docs/static/dead-letter-queues.asciidoc index 8ffb68378..5b63cd7c5 100644 --- a/docs/static/dead-letter-queues.asciidoc +++ b/docs/static/dead-letter-queues.asciidoc @@ -178,7 +178,7 @@ failure: "entry_time" => #, "plugin_id" => "fb80f1925088497215b8d037e622dec5819b503e-4", "plugin_type" => "elasticsearch", - "reason" => "Could not index event to Elasticsearch. status: 400, action: [\"index\", {:_id=>nil, :_index=>\"logstash-2017.06.22\", :_type=>\"logs\", :_routing=>nil}, 2017-06-22T01:29:29.804Z Suyogs-MacBook-Pro-2.local {\"geoip\":{\"location\":\"home\"}}], response: {\"index\"=>{\"_index\"=>\"logstash-2017.06.22\", \"_type\"=>\"logs\", \"_id\"=>\"AVzNayPze1iR9yDdI2MD\", \"status\"=>400, \"error\"=>{\"type\"=>\"mapper_parsing_exception\", \"reason\"=>\"failed to parse\", \"caused_by\"=>{\"type\"=>\"illegal_argument_exception\", \"reason\"=>\"illegal latitude value [266.30859375] for geoip.location\"}}}}" + "reason" => "Could not index event to Elasticsearch. status: 400, action: [\"index\", {:_id=>nil, :_index=>\"logstash-2017.06.22\", :_type=>\"doc\", :_routing=>nil}, 2017-06-22T01:29:29.804Z My-MacBook-Pro-2.local {\"geoip\":{\"location\":\"home\"}}], response: {\"index\"=>{\"_index\"=>\"logstash-2017.06.22\", \"_type\"=>\"doc\", \"_id\"=>\"AVzNayPze1iR9yDdI2MD\", \"status\"=>400, \"error\"=>{\"type\"=>\"mapper_parsing_exception\", \"reason\"=>\"failed to parse\", \"caused_by\"=>{\"type\"=>\"illegal_argument_exception\", \"reason\"=>\"illegal latitude value [266.30859375] for geoip.location\"}}}}" } }, "@timestamp" => 2017-06-22T01:29:29.804Z, @@ -186,7 +186,7 @@ failure: "geoip" => { "location" => "home" }, - "host" => "Suyogs-MacBook-Pro-2.local", + "host" => "My-MacBook-Pro-2.local", "message" => "{\"geoip\":{\"location\":\"home\"}}" } --------------------------------------------------------------------------------