- add a quick walkthrough based on my tutorial presented at dreamforce.

docs/index.html.erb

@@ -16,6 +16,7 @@ layout: content_right
   <h3> use case tutorials </h3>
 
   <ul>
+    <li> <a href="tutorials/10-minute-walkthrough"> 10-minute walkthrough</a> - a simple walkthrough to show you how to configure the logstash agent to process events and even old logs. </li>
     <li> <a href="tutorials/metrics-from-logs"> Gathering metrics from logs </a> - take metrics from logs and ship them to graphite, ganglia, and more. </li>
   </ul>
 

docs/tutorials/10-minute-walkthrough/apache-elasticsearch.conf

@@ -0,0 +1,35 @@
+input {
+  tcp { 
+    type => "apache"
+    port => 3333
+  } 
+}
+
+filter {
+  grok {
+    type => "apache"
+    # See the following URL for a complete list of named patterns
+    # logstash/grok ships with by default:
+    # https://github.com/logstash/logstash/tree/master/patterns
+    #
+    # The grok filter will use the below pattern and on successful match use
+    # any captured values as new fields in the event.
+    pattern => "%{COMBINEDAPACHELOG}"
+  }
+
+  date {
+    type => "apache"
+    # Try to pull the timestamp from the 'timestamp' field (parsed above with
+    # grok). The apache time format looks like: "18/Aug/2011:05:44:34 -0700"
+    timestamp => "dd/MMM/yyyy:HH:mm:ss Z"
+  }
+}
+
+output {
+  elasticsearch {
+    # Setting 'embedded' will run  a real elasticsearch server inside logstash.
+    # This option below saves you from having to run a separate process just
+    # for ElasticSearch, so you can get started quicker!
+    embedded => true
+  }
+}

docs/tutorials/10-minute-walkthrough/apache-parse.conf

@@ -0,0 +1,33 @@
+input {
+  tcp { 
+    type => "apache"
+    port => 3333
+  } 
+}
+
+filter {
+  grok {
+    type => "apache"
+    # See the following URL for a complete list of named patterns
+    # logstash/grok ships with by default:
+    # https://github.com/logstash/logstash/tree/master/patterns
+    #
+    # The grok filter will use the below pattern and on successful match use
+    # any captured values as new fields in the event.
+    pattern => "%{COMBINEDAPACHELOG}"
+  }
+
+  date {
+    type => "apache"
+    # Try to pull the timestamp from the 'timestamp' field (parsed above with
+    # grok). The apache time format looks like: "18/Aug/2011:05:44:34 -0700"
+    timestamp => "dd/MMM/yyyy:HH:mm:ss Z"
+  }
+}
+
+output {
+  # Use stdout in debug mode again to see what logstash makes of the event.
+  stdout {
+    debug => true
+  }
+}

docs/tutorials/10-minute-walkthrough/apache_log.1

@@ -0,0 +1 @@
+129.92.249.70 - - [18/Aug/2011:06:00:14 -0700] "GET /style2.css HTTP/1.1" 200 1820 "http://www.semicomplete.com/blog/geekery/bypassing-captive-portals.html" "Mozilla/5.0 (iPad; U; CPU OS 4_3_5 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8L1 Safari/6533.18.5"

docs/tutorials/10-minute-walkthrough/hello-search.conf

@@ -0,0 +1,25 @@
+input {
+  stdin { 
+    # A type is a label applied to an event. It is used later with filters
+    # to restrict what filters are run against each event.
+    type => "human"
+  } 
+}
+
+output {
+  # Print each event to stdout.
+  stdout {
+    # Enabling 'debug' on the stdout output will make logstash pretty-print the
+    # entire event as something similar to a JSON representation.
+    debug => true
+  }
+  
+  # You can have multiple outputs. All events generally to all outputs.
+  # Output events to elasticsearch
+  elasticsearch {
+    # Setting 'embedded' will run  a real elasticsearch server inside logstash.
+    # This option below saves you from having to run a separate process just
+    # for ElasticSearch, so you can get started quicker!
+    embedded => true
+  }
+}

docs/tutorials/10-minute-walkthrough/hello.conf

@@ -0,0 +1,16 @@
+input {
+  stdin { 
+    # A type is a label applied to an event. It is used later with filters
+    # to restrict what filters are run against each event.
+    type => "human"
+  } 
+}
+
+output {
+  # Print each event to stdout.
+  stdout {
+    # Enabling 'debug' on the stdout output will make logstash pretty-print the
+    # entire event as something similar to a JSON representation.
+    debug => true
+  }
+}

docs/tutorials/10-minute-walkthrough/index.md

@@ -0,0 +1,127 @@
+---
+title: Logstash tutorial @ Dreamforce
+layout: content_right
+---
+# Logstash tutorial @ Dreamforce
+
+## Step 1 - Download
+
+### Download logstash:
+
+* [logstash-1.0.17-monolithic.jar](http://semicomplete.com/files/logstash/logstash-1.0.17-monolithic.jar)
+
+### Requirements:
+
+* java
+
+### The Secret:
+
+logstash is written in JRuby but I release standalone jar files for easy
+deployment, so you don't need to download JRuby or most any other dependencies.
+I bake as much as possible into the single release file.
+
+## Step 2 - A hello world.
+
+### Download this config file:
+
+* [hello.conf](hello.conf)
+
+### Run it:
+
+    java -jar logstash-1.0.17-monolithic.jar agent -f hello.conf
+
+Type stuff on standard input. Press enter. Watch what event logstash sees.
+Press ^C to kill it.
+
+## Step 3 - Add ElasticSearch
+
+### Download this config file:
+
+* [hello-search.conf](hello-search.conf)
+
+### Run it:
+
+    java -jar logstash-1.0.17-monolithic.jar agent -f hello-search.conf
+
+Same config as step 2, but now we are also writing events to ElasticSearch. Do
+a search for '*' (all):
+
+    curl http://localhost:9200/_search?pretty=1&q=*
+
+## Step 4 - logstash web
+
+The previous step is good, but a better frontend on elasticsearch would help!
+
+The same config as step 3 is used.
+
+### Run it:
+
+    java -jar logstash-1.0.17-monolithic.jar agent -f hello-search.conf -- web --backend 'elasticsearch:///?local'
+
+The above runs both the agent and the logstash web interface in the same
+process. Useful for simple deploys.
+
+### Use it:
+
+Go to the logstash web interface in browser: <http://localhost:9292/>
+
+Type stuff on stdin on the agent, then search for it in the web interface.
+
+## Step 5 - real world example
+
+Let's backfill some old apache logs.  First, let's use grok.
+
+Requirements:
+
+* libgrok [INSTALL notes here](https://github.com/jordansissel/grok/blob/master/INSTALL)
+
+Use the 'grok' logstash filter to parse logs. Once you have libgrok installed,
+keep reading below.
+
+### Download
+
+* [apache-parse.conf](apache-parse.conf)
+* [apache_log.1](apache_log.1) (a single apache log line)
+
+### Run it
+
+    java -jar logstash-1.0.17-monolithic.jar agent -f apache-parse.conf
+
+Logstash will now be listening on TCP port 3333. Send an apache log message at it:
+
+    nc localhost 3333 < apache_log.1
+
+The expected output can be viewed here: [step-5-output.txt](step-5-output.txt)
+
+## Step 6 - real world example + search
+
+Same as the previous step, but we'll output to ElasticSearch now.
+
+### Download
+
+* [apache-elasticsearch.conf](apache-elasticsearch.conf)
+* [apache_log.2.bz2](apache_log.2.bz2) (2 days of apache logs)
+
+### Run it
+
+    java -jar logstash-1.0.17-monolithic.jar agent -f apache-elasticsearch.conf -- web --backend 'elasticsearch:///?local'
+
+Logstash should be all set for you now. Start feeding it logs:
+
+    bzip2 -d apache_log.2.bz2
+
+    nc localhost 3333 < apache_log.2 
+
+Go to the logstash web interface in browser: <http://localhost:9292/>
+
+Try some search queries. Click on some results.
+
+## Want more?
+
+For further learning, try these:
+
+* [Watch a presentation on logstash](http://blip.tv/carolinacon/logstash-open-source-log-and-event-management-jordan-sissel-5123601)
+* [Getting started 'standalone' guide](http://logstash.net/docs/1.0.17/getting-started-simple)
+* [Getting started 'centralized' guide](http://logstash.net/docs/1.0.17/getting-started-centralized) - 
+  learn how to build out your logstash infrastructure and centralize your logs.
+* [Dive into the docs](http://logstash.net/docs/1.0.17/)

docs/tutorials/10-minute-walkthrough/step-5-output.txt

@@ -0,0 +1,107 @@
+{
+  "@source"      => "tcp://0.0.0.0:3333/client/127.0.0.1:35019",
+  "@type"        => "apache",
+  "@tags"        => [],
+  "@fields"      => {
+    "COMBINEDAPACHELOG" => [
+      [0] "129.92.249.70 - - [18/Aug/2011:06:00:14 -0700] \"GET /style2.css HTTP/1.1\" 200 1820 \"http://www.semicomplete.com/blog/geekery/bypassing-captive-portals.html\" \"Mozilla/5.0 (iPad; U; CPU OS 4_3_5 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8L1 Safari/6533.18.5\""
+    ],
+    "clientip"          => [
+      [0] "129.92.249.70"
+    ],
+    "HOSTNAME"          => [
+      [0] "129.92.249.70",
+      [1] "www.semicomplete.com"
+    ],
+    "IP"                => [],
+    "ident"             => [
+      [0] "-"
+    ],
+    "USERNAME"          => [
+      [0] "-",
+      [1] "-"
+    ],
+    "auth"              => [
+      [0] "-"
+    ],
+    "timestamp"         => [
+      [0] "18/Aug/2011:06:00:14 -0700"
+    ],
+    "MONTHDAY"          => [
+      [0] "18"
+    ],
+    "MONTH"             => [
+      [0] "Aug"
+    ],
+    "YEAR"              => [
+      [0] "2011"
+    ],
+    "TIME"              => [
+      [0] "06:00:14"
+    ],
+    "HOUR"              => [
+      [0] "06"
+    ],
+    "MINUTE"            => [
+      [0] "00"
+    ],
+    "SECOND"            => [
+      [0] "14"
+    ],
+    "ZONE"              => [
+      [0] "-0700"
+    ],
+    "verb"              => [
+      [0] "GET"
+    ],
+    "request"           => [
+      [0] "/style2.css"
+    ],
+    "URIPATH"           => [
+      [0] "/style2.css",
+      [1] "/blog/geekery/bypassing-captive-portals.html"
+    ],
+    "URIPARAM"          => [],
+    "httpversion"       => [
+      [0] "1.1"
+    ],
+    "BASE10NUM"         => [
+      [0] "1.1",
+      [1] "200",
+      [2] "1820"
+    ],
+    "response"          => [
+      [0] "200"
+    ],
+    "bytes"             => [
+      [0] "1820"
+    ],
+    "referrer"          => [
+      [0] "http://www.semicomplete.com/blog/geekery/bypassing-captive-portals.html"
+    ],
+    "URIPROTO"          => [
+      [0] "http"
+    ],
+    "USER"              => [],
+    "URIHOST"           => [
+      [0] "www.semicomplete.com"
+    ],
+    "IPORHOST"          => [
+      [0] "www.semicomplete.com"
+    ],
+    "POSINT"            => [],
+    "URIPATHPARAM"      => [
+      [0] "/blog/geekery/bypassing-captive-portals.html"
+    ],
+    "agent"             => [
+      [0] "\"Mozilla/5.0 (iPad; U; CPU OS 4_3_5 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8L1 Safari/6533.18.5\""
+    ],
+    "QUOTEDSTRING"      => [
+      [0] "\"Mozilla/5.0 (iPad; U; CPU OS 4_3_5 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8L1 Safari/6533.18.5\""
+    ]
+  },
+  "@timestamp"   => "2011-08-18T13:00:14.000Z",
+  "@source_host" => "0.0.0.0",
+  "@source_path" => "/client/127.0.0.1:35019",
+  "@message"     => "129.92.249.70 - - [18/Aug/2011:06:00:14 -0700] \"GET /style2.css HTTP/1.1\" 200 1820 \"http://www.semicomplete.com/blog/geekery/bypassing-captive-portals.html\" \"Mozilla/5.0 (iPad; U; CPU OS 4_3_5 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8L1 Safari/6533.18.5\"\n"
+}