mirror of
https://github.com/elastic/logstash.git
synced 2025-04-23 22:27:21 -04:00
Remove old tutorials now replaced by Kurt's new one.
This commit is contained in:
Jordan Sissel
parent
3c1dd49671
commit
7a0be0274b
4 changed files with 0 additions and 418 deletions
Binary file not shown.
|
Before Width: | Height: | Size: 35 KiB |
|
|
@ -1 +0,0 @@
|
|||
<mxGraphModel dx="800" dy="800" grid="1" guides="1" tooltips="1" connect="1" fold="1" page="1" pageScale="1" pageWidth="826" pageHeight="1169" style="default-style2"><root><mxCell id="0"/><mxCell id="1" parent="0"/><mxCell id="2" value="Shipper" style="icon;image=http://logstash.net/images/logstash.png;fontSize=18" parent="1" vertex="1"><mxGeometry x="50" y="110" width="60" height="60" as="geometry"/></mxCell><mxCell id="5" value="Broker" style="icon;image=http://www.java-freelance.fr/wp-content/uploads/2012/06/banner_redis-300dpi-0315a8013afee137cce47b474541d7f1.png;fontSize=18" parent="1" vertex="1"><mxGeometry x="180" y="161" width="60" height="60" as="geometry"/></mxCell><mxCell id="7" value="Storage & Search" style="icon;image=http://www.elasticsearch.org/images/set3/bonsai2.png;fontSize=18" parent="1" vertex="1"><mxGeometry x="470" y="161" width="120" height="60" as="geometry"/></mxCell><mxCell id="8" value="" style="endArrow=open;entryX=0;entryY=0.5;fontSize=18" parent="1" source="5" target="11" edge="1"><mxGeometry x="-40" as="geometry"><mxPoint x="520" y="160" as="targetPoint"/></mxGeometry></mxCell><mxCell id="11" value="Indexer" style="icon;image=http://logstash.net/images/logstash.png;fontSize=18" parent="1" vertex="1"><mxGeometry x="280" y="161" width="140" height="60" as="geometry"/></mxCell><mxCell id="13" value="Shipper" style="icon;image=http://logstash.net/images/logstash.png;fontSize=18" parent="1" vertex="1"><mxGeometry x="50" y="240" width="60" height="60" as="geometry"/></mxCell><mxCell id="16" value="...
" style="text;fontSize=36" parent="1" vertex="1"><mxGeometry x="60" y="178" width="40" height="52" as="geometry"/></mxCell><mxCell id="17" value="" style="endArrow=none;exitX=1;exitY=0.5;entryX=1;entryY=0.5;fontSize=18" parent="1" edge="1"><mxGeometry x="50" y="-7" as="geometry"><mxPoint x="260" y="250" as="sourcePoint"/><mxPoint x="260" y="250" as="targetPoint"/></mxGeometry></mxCell><mxCell id="19" value="" style="endArrow=open;entryX=0;entryY=0.5;exitX=1;exitY=0.5;fontSize=18" parent="1" source="2" target="5" edge="1"><mxGeometry x="-40" as="geometry"><mxPoint x="520" y="160" as="targetPoint"/></mxGeometry></mxCell><mxCell id="20" value="" style="endArrow=open;entryX=0;entryY=0.5;exitX=1;exitY=0.5;fontSize=18" parent="1" source="13" target="5" edge="1"><mxGeometry x="-40" as="geometry"><mxPoint x="520" y="160" as="targetPoint"/></mxGeometry></mxCell><mxCell id="21" value="" style="endArrow=open;exitX=1;exitY=0.5;fontSize=18" parent="1" source="11" target="7" edge="1"><mxGeometry x="-40" as="geometry"><mxPoint x="490" y="200" as="sourcePoint"/><mxPoint x="520" y="160" as="targetPoint"/></mxGeometry></mxCell><mxCell id="26" value="Logstash" style="label;image=http://logstash.net/images/logstash.png;fontSize=12" parent="1" vertex="1"><mxGeometry x="210" y="10" width="140" height="40" as="geometry"/></mxCell><mxCell id="28" value="Redis" style="label;image=http://www.java-freelance.fr/wp-content/uploads/2012/06/banner_redis-300dpi-0315a8013afee137cce47b474541d7f1.png" parent="1" vertex="1"><mxGeometry x="350" y="10" width="110" height="40" as="geometry"/></mxCell><mxCell id="32" value="ElasticSearch" style="label;image=http://www.elasticsearch.org/images/set3/bonsai2.png;fontSize=12" parent="1" vertex="1"><mxGeometry x="460" y="10" width="210" height="40" as="geometry"/></mxCell><mxCell id="34" value="Web Interface" style="icon;image=http://logstash.net/images/logstash.png;fontSize=18" parent="1" vertex="1"><mxGeometry x="630" y="161" width="140" height="60" as="geometry"/></mxCell><mxCell id="35" value="" style="endArrow=open;entryX=0;entryY=0.5;exitX=1;exitY=0.5;fontSize=18" parent="1" source="7" target="34" edge="1"><mxGeometry x="-40" as="geometry"><mxPoint x="520" y="160" as="targetPoint"/></mxGeometry></mxCell><mxCell id="36" value="Legend:" style="text;align=center;verticalAlign=middle;fontSize=18" parent="1" vertex="1"><mxGeometry x="130" y="10" width="70" height="40" as="geometry"/></mxCell><mxCell id="37" value="" style="endArrow=open;entryX=0;entryY=0.5;exitX=1;exitY=0.75" parent="1" source="16" target="5" edge="1"><mxGeometry as="geometry"><mxPoint x="560" y="160" as="targetPoint"/></mxGeometry></mxCell></root></mxGraphModel>
|
||||
|
|
@ -1,217 +0,0 @@
|
|||
---
|
||||
title: Getting Started (Centralized Server) - logstash
|
||||
layout: content_right
|
||||
---
|
||||
|
||||
# Getting Started
|
||||
|
||||
## Centralized Setup with Event Parsing
|
||||
|
||||
This guide shows how to get you going quickly with logstash with multiple
|
||||
servers. This guide is for folks who want to ship all their logstash logs to a
|
||||
central location for indexing and search.
|
||||
|
||||
We'll have two classes of server. First, one that ships logs. Second, one that
|
||||
collects and indexes logs.
|
||||
|
||||
It's important to note that logstash itself has no concept of "shipper" and
|
||||
"collector" - the behavior of an agent depends entirely on how you configure
|
||||
it.
|
||||
|
||||
This diagram gives you an overview of the architecture:
|
||||
|
||||
|
||||
|
||||
On servers shipping logs:
|
||||
|
||||
* Download and run logstash (See section 'logstash log shipper' below)
|
||||
|
||||
On the server collecting and indexing your logs:
|
||||
|
||||
* Download and run Elasticsearch
|
||||
* Download and run Redis
|
||||
* Download and run Logstash
|
||||
|
||||
## ElasticSearch
|
||||
|
||||
Requirements: java
|
||||
|
||||
You'll most likely want the version of ElasticSearch specified by the
|
||||
[elasticsearch output](../outputs/elasticsearch) docs. Modify this in your shell
|
||||
for easy downloading of ElasticSearch:
|
||||
|
||||
ES_PACKAGE=elasticsearch-%ELASTICSEARCH_VERSION%.zip
|
||||
ES_DIR=${ES_PACKAGE%%.zip}
|
||||
SITE=https://download.elasticsearch.org/elasticsearch/elasticsearch
|
||||
if [ ! -d "$ES_DIR" ] ; then
|
||||
wget --no-check-certificate $SITE/$ES_PACKAGE
|
||||
unzip $ES_PACKAGE
|
||||
fi
|
||||
|
||||
ElasticSearch requires Java (uses Lucene on the backend; if you want to know
|
||||
more read the elasticsearch docs).
|
||||
|
||||
To start the service, run `bin/elasticsearch -f`. This will run it in the foreground. We want to keep it this way for debugging for now.
|
||||
|
||||
## Redis
|
||||
|
||||
Previous versions of this guide used AMQP via RabbitMQ. Due to the complexity of AMQP as well as performance issues related to the Bunny driver we use, we're now recommending Redis instead.
|
||||
|
||||
Redis has no external dependencies and has a much simpler configuration in Logstash.
|
||||
|
||||
Building and installing Redis is fairly straightforward. While normally this would be out of the scope of this document, as the instructions are so simple we'll include them here:
|
||||
|
||||
- Download Redis from http://redis.io/download (The latest stable release is likely what you want)
|
||||
- Extract the source, change to the directory and run `make`
|
||||
- Run Redis with `src/redis-server --loglevel verbose`
|
||||
|
||||
That's it.
|
||||
|
||||
## logstash
|
||||
|
||||
Once you have elasticsearch and redis running, you're
|
||||
ready to configure logstash.
|
||||
|
||||
Download the logstash release jar file. The package contains all
|
||||
required dependencies to save you time chasing down requirements.
|
||||
|
||||
Follow [this link to download logstash-%VERSION%](https://download.elasticsearch.org/logstash/logstash/logstash-%VERSION%-flatjar.jar).
|
||||
|
||||
Since we're doing a centralized configuration, you'll have two main
|
||||
logstash agent roles: a shipper and an indexer. You will ship logs from
|
||||
all servers via Redis and have another agent receive those messages,
|
||||
parse them, and index them in elasticsearch.
|
||||
|
||||
### logstash log shipper
|
||||
|
||||
As with the simple example, we're going to start simple to ensure that events are flowing
|
||||
|
||||
input {
|
||||
stdin {
|
||||
type => "example"
|
||||
}
|
||||
}
|
||||
|
||||
output {
|
||||
stdout { codec => rubydebug }
|
||||
redis { host => "127.0.0.1" data_type => "list" key => "logstash" }
|
||||
}
|
||||
|
||||
Put this in a file and call it 'shipper.conf' (or anything, really), and run:
|
||||
|
||||
java -jar logstash-%VERSION%-flatjar.jar agent -f shipper.conf
|
||||
|
||||
This will take anything you type into this console and display it on the console. Additionally it will save events to Redis in a `list` named after the `key` value you provided.
|
||||
|
||||
### Testing the Redis output
|
||||
|
||||
To verify that the message made it into Redis, check your Redis window. You should see something like the following:
|
||||
|
||||
[83019] 02 Jul 12:51:02 - Accepted 127.0.0.1:58312
|
||||
[83019] 02 Jul 12:51:06 - Client closed connection
|
||||
[83019] 02 Jul 12:51:06 - DB 0: 1 keys (0 volatile) in 4 slots HT.
|
||||
|
||||
The redis application ships with a CLI application that you can use to query the data. From your Redis source directory, run the following:
|
||||
|
||||
`src/redis-cli`
|
||||
|
||||
Once connected, run the following commands:
|
||||
|
||||
redis 127.0.0.1:6379> llen logstash
|
||||
(integer) 1
|
||||
redis 127.0.0.1:6379> lpop logstash
|
||||
"{\"message\":\"hello\",\"@timestamp\":\"2013-09-07T00:59:28.383Z\",\"@version\":\"1\",\"type\":\"stdin\",\"host\":\"pork\"}"
|
||||
redis 127.0.0.1:6379> llen logstash
|
||||
(integer) 0
|
||||
|
||||
What we've just done is check the length of the list, read and removed the oldest item in the list, and checked the length again.
|
||||
|
||||
This behavior is what Logstash does when it reads from a Redis input (technically logstash performs a blocking lpop). We're essentially using Redis to simulate a queue via the `list` data type.
|
||||
|
||||
Go ahead and type a few more entries in the agent window:
|
||||
|
||||
- test 1
|
||||
- test 2
|
||||
- test 3
|
||||
|
||||
As you `lpop` you should get them in the correct order of insertion.
|
||||
|
||||
### logstash indexer
|
||||
|
||||
This agent will parse and index your logs as they come in over Redis. Here's a
|
||||
sample config based on the previous section. Save this as `indexer.conf`
|
||||
|
||||
input {
|
||||
redis {
|
||||
host => "127.0.0.1"
|
||||
# these settings should match the output of the agent
|
||||
data_type => "list"
|
||||
key => "logstash"
|
||||
|
||||
# We use the 'json' codec here because we expect to read
|
||||
# json events from redis.
|
||||
codec => json
|
||||
}
|
||||
}
|
||||
|
||||
output {
|
||||
stdout { debug => true debug_format => "json"}
|
||||
|
||||
elasticsearch {
|
||||
host => "127.0.0.1"
|
||||
}
|
||||
}
|
||||
|
||||
The above configuration will attach to Redis and issue a `BLPOP` against the `logstash` list. When an event is recieved, it will be pulled off and sent to Elasticsearch for indexing.
|
||||
|
||||
Start the indexer the same way as the agent but specifying the `indexer.conf` file:
|
||||
|
||||
`java -jar logstash-%VERSION%-flatjar.jar agent -f indexer.conf`
|
||||
|
||||
To verify that your Logstash indexer is connecting to Elasticsearch properly, you should see a message in your Elasticsearch window similar to the following:
|
||||
|
||||
`[2012-07-02 13:14:27,008][INFO ][cluster.service ] [Baron Samedi] added {[Bes][JZQBMR21SUWRNtTMsDV3_g][inet[/192.168.1.194:9301]]{client=true, data=false},}`
|
||||
|
||||
The names `Bes` and `Baron Samedi` may differ as ES uses random names for nodes.
|
||||
|
||||
### Testing the flow
|
||||
Now we want to test the flow. In your agent window, type something to generate an event.
|
||||
The indexer should read this and persist it to Elasticsearch. It will also display the event to stdout.
|
||||
|
||||
In your Elasticsearch window, you should see something like the following:
|
||||
|
||||
[2012-07-02 13:21:58,982][INFO ][cluster.metadata ] [Baron Samedi] [logstash-2012.07.02] creating index, cause [auto(index api)], shards [5]/[1], mappings []
|
||||
[2012-07-02 13:21:59,495][INFO ][cluster.metadata ] [Baron Samedi] [logstash-2012.07.02] update_mapping [stdin-type] (dynamic)
|
||||
|
||||
Since indexes are created dynamically, this is the first sign that Logstash was able to write to ES. Let's use curl to verify our data is there:
|
||||
Using our curl command from the simple tutorial should let us see the data:
|
||||
|
||||
`curl -gs -XGET http://localhost:9200/logstash-*/_search?q=type:example`
|
||||
|
||||
You may need to modify the date as this is based on the date this guide was written.
|
||||
|
||||
Now we can move on to the final step...
|
||||
## logstash web interface
|
||||
|
||||
Run this on the same server as your elasticsearch server.
|
||||
|
||||
To run the logstash web server, just run the jar with 'web' as the first
|
||||
argument.
|
||||
|
||||
java -jar logstash-%VERSION%-flatjar.jar web
|
||||
|
||||
Just point your browser at the http://127.0.0.1:9292/ and start searching
|
||||
logs!
|
||||
|
||||
The web interface is called 'kibana' - you can learn more about kibana at <http://kibana.org>
|
||||
|
||||
# Distributing the load
|
||||
At this point we've been simulating a distributed environment on a single machine. If only the world were so easy.
|
||||
In all of the example configurations, we've been explicitly setting the connection to connect to `127.0.0.1` despite the fact in most network-related plugins, that's the default host.
|
||||
|
||||
Since Logstash is so modular, you can install the various components on different systems.
|
||||
|
||||
- If you want to give Redis a dedicated host, simply ensure that the `host` attribute in configurations points to that host.
|
||||
- If you want to give Elasticsearch a dedicated host, simple ensure that the `host` attribute is correct as well (in both web and indexer).
|
||||
|
||||
As with the simple input example, reading from stdin is fairly useless. Check the Logstash documentation for the various inputs offered and mix and match to taste!
|
||||
|
|
@ -1,200 +0,0 @@
|
|||
---
|
||||
title: Getting Started (Standalone server) - logstash
|
||||
layout: content_right
|
||||
---
|
||||
# Getting started with logstash (standalone server example)
|
||||
|
||||
This guide shows how to get you going quickly with logstash on a single,
|
||||
standalone server. We'll begin by showing you how to read events from standard
|
||||
input (your keyboard) and emit them to standard output. After that, we'll start
|
||||
collecting actual log files.
|
||||
|
||||
By standalone, I mean that everything happens on a single server: log collection, indexing, and the web interface.
|
||||
|
||||
logstash can be run on multiple servers (collect from many servers to a single
|
||||
indexer) if you want, but this example shows simply a standalone configuration.
|
||||
|
||||
Steps detailed in this guide:
|
||||
|
||||
* Download and run logstash
|
||||
|
||||
## Problems?
|
||||
|
||||
If you have problems, feel free to email the users list
|
||||
(logstash-users@googlegroups.com) or join IRC (#logstash on irc.freenode.org)
|
||||
|
||||
## logstash
|
||||
|
||||
You should download the logstash jar file - if you haven't yet,
|
||||
[download it
|
||||
now](https://download.elasticsearch.org/logstash/logstash/logstash-%VERSION%-flatjar.jar).
|
||||
This package includes most of the dependencies for logstash in it and
|
||||
helps you get started quicker.
|
||||
|
||||
The configuration of any logstash agent consists of specifying inputs, filters,
|
||||
and outputs. For this example, we will not configure any filters.
|
||||
|
||||
The inputs are your log files. The output will be elasticsearch. The config
|
||||
format should be simple to read and write. The bottom of this document includes
|
||||
links for further reading (config, etc) if you want to learn more.
|
||||
|
||||
Here is a simple Logstash configuration:
|
||||
|
||||
input { stdin { } }
|
||||
output { stdout { codec => rubydebug } }
|
||||
|
||||
Save this to a file called `logstash-simple.conf` and run it like so:
|
||||
|
||||
java -jar logstash-%VERSION%-flatjar.jar agent -f logstash-simple.conf
|
||||
|
||||
After a few seconds, type something in the console where you started logstash.
|
||||
Maybe `hello`. You should get some output like so:
|
||||
|
||||
{
|
||||
"message" => "hello",
|
||||
"@timestamp" => "2013-09-04T00:24:21.707Z",
|
||||
"@version" => "1",
|
||||
"host" => "pork"
|
||||
}
|
||||
|
||||
If everything is okay, let's move on to a more complex version:
|
||||
|
||||
### Saving to Elasticsearch
|
||||
|
||||
The recommended storage engine for Logstash is Elasticsearch. If you're running
|
||||
Logstash from the jar file or via jruby, you can use an embedded version of
|
||||
Elasticsearch for storage.
|
||||
|
||||
Using our configuration above, let's change it to look like so:
|
||||
|
||||
input { stdin { type => example } }
|
||||
output {
|
||||
stdout { codec => rubydebug }
|
||||
elasticsearch { embedded => true }
|
||||
}
|
||||
|
||||
We're going to KEEP the existing configuration but add a second output -
|
||||
embedded Elasticsearch. Restart your Logstash (CTRL-C and rerun the java
|
||||
command). Depending on the horsepower of your machine, this could take some
|
||||
time. Logstash needs to extract the jar contents to a working directory AND
|
||||
start an instance of Elasticsearch.
|
||||
|
||||
Let's do our test again by simply typing `test`. You should get the same output to the console.
|
||||
Now let's verify that Logstash stored the message in Elasticsearch:
|
||||
|
||||
curl -s http://127.0.0.1:9200/_status?pretty=true | grep logstash
|
||||
|
||||
_This assumes you have the `curl` command installed._
|
||||
|
||||
You should get back some output like so:
|
||||
|
||||
"logstash-2012.07.02" : {
|
||||
"index" : "logstash-2012.07.02"
|
||||
|
||||
This means Logstash created a new index based on today's date. Likely your data is in there as well:
|
||||
|
||||
`curl -gs -XGET "http://localhost:9200/logstash-*/_search?pretty&q=type:example"`
|
||||
|
||||
This will return a rather large JSON output. We're only concerned with a subset:
|
||||
|
||||
"_index" : "logstash-2013.09.07",
|
||||
"_type" : "logs",
|
||||
"_id" : "iARTN3MtQ-Kaf_x0fZaFwQ",
|
||||
"_score" : 1.4054651, "_source" : {
|
||||
"message": "fizzle",
|
||||
"@timestamp": "2013-09-07T00:42:23.453Z",
|
||||
"@version": "1",
|
||||
"type": "example",
|
||||
"host": "pork"
|
||||
}
|
||||
|
||||
Your output may look a little different.
|
||||
The reason we're going about it this way is to make absolutely sure that we have all the bits working before adding more complexity.
|
||||
|
||||
If you are unable to get these steps working, you likely have something interfering with multicast traffic. This has been known to happen when connected to VPNs for instance.
|
||||
For best results, test on a Linux VM or system with less complicated networking. If in doubt, rerun the command with the options `-vv` and paste the output to Github Gist or Pastie.
|
||||
Hop on the logstash IRC channel or mailing list and ask for help with that output as reference.
|
||||
|
||||
Obviously this is fairly useless this way. Let's add the final step and test with the builtin logstash web ui:
|
||||
|
||||
### Testing the webui
|
||||
|
||||
We've already proven that events can make it into Elasticsearch. However using
|
||||
curl for everything is less than ideal.
|
||||
|
||||
Logstash ships with a built-in web interface (called Kibana). Let's restart our
|
||||
logstash process with an additional option:
|
||||
|
||||
java -jar logstash-%VERSION%-flatjar.jar agent -f logstash-simple.conf -- web
|
||||
|
||||
One important thing to note is that the `web` option is actually its own set of
|
||||
commmand-line options. We're essentially starting two programs in one. This is
|
||||
worth remembering as you move to an external Elasticsearch server. The options
|
||||
you specify in your logstash.conf have no bearing on the web ui. It has its own
|
||||
options.
|
||||
|
||||
Again, the reason for testing without the web interface is to ensure that the
|
||||
logstash agent itself is getting events into Elasticsearch. This is different
|
||||
than the Logstash web ui being able to read them. As before, we'll need to
|
||||
wait a bit for everything to spin up. You can verify that everything is running
|
||||
(assuming you aren't running with any `-v` options) by checking the output of
|
||||
`netstat`:
|
||||
|
||||
netstat -napt | grep -i LISTEN
|
||||
|
||||
What's interesting is that you should see the following ports in use:
|
||||
|
||||
- 9200
|
||||
- 9300
|
||||
- 9301
|
||||
- 9302
|
||||
- 9292
|
||||
|
||||
The `9200` and `9300` ports are the embedded ES listening. The `9301` and `9302` ports are the agent and web interfaces talking to ES. `9292` is the port the web ui listens on.
|
||||
|
||||
If you open a browser to http://localhost:9292/ and click on the link in the body, you should see results. If not, switch back to your console, type some test and hit return.
|
||||
Refresh the browser page and you should have results!
|
||||
|
||||
### Continuing on
|
||||
At this point you have a working self-contained Logstash instance. However typing things into stdin is likely not to be what you want.
|
||||
|
||||
Here is a sample config you can start with. It defines some basic inputs
|
||||
grouped by type and two outputs.
|
||||
|
||||
input {
|
||||
stdin {
|
||||
type => "stdin-type"
|
||||
}
|
||||
|
||||
file {
|
||||
type => "syslog"
|
||||
|
||||
# Wildcards work, here :)
|
||||
path => [ "/var/log/*.log", "/var/log/messages", "/var/log/syslog" ]
|
||||
}
|
||||
}
|
||||
|
||||
output {
|
||||
stdout { }
|
||||
elasticsearch { embedded => true }
|
||||
}
|
||||
|
||||
Put this in a file called "logstash-complex.conf"
|
||||
|
||||
Now run it all (again. Be sure to stop your previous Logstash tests!):
|
||||
|
||||
java -jar logstash-%VERSION%-flatjar.jar agent -f logstash-complex.conf -- web
|
||||
|
||||
Point your browser at <http://yourserver:9292> and start searching!
|
||||
|
||||
*Note*: If things are not working, such as you get an error message while
|
||||
searching, like 'SERVICE_UNAVAILABLE' or some other elasticsearch error, you
|
||||
should check that your firewall (local, too) is not blocking multicast.
|
||||
|
||||
## Further reading
|
||||
|
||||
Want to know more about the configuration language? Check out the
|
||||
[configuration](../configuration) documentation.
|
||||
|
||||
You may have logs on many servers you want to centralize through logstash. To
|
||||
learn how to do that, [read this](getting-started-centralized)
|
||||
Loading…
Add table
Add a link
Reference in a new issue