mirror of
https://github.com/elastic/logstash.git
synced 2025-04-23 22:27:21 -04:00
Merge pull request #1070 from jordansissel/prep-for-beta
Preparations for 1.4.0.beta1
This commit is contained in:
Jordan Sissel
commit
1556d1aaa1
16 changed files with 184 additions and 650 deletions
51
Makefile
51
Makefile
|
|
@ -20,7 +20,7 @@ GEOIP_URL=http://logstash.objects.dreamhost.com/maxmind/GeoLiteCity-2013-01-18.d
|
|||
GEOIP_ASN=vendor/geoip/GeoIPASNum.dat
|
||||
GEOIP_ASN_URL=http://logstash.objects.dreamhost.com/maxmind/GeoIPASNum-2014-02-12.dat.gz
|
||||
KIBANA_URL=https://download.elasticsearch.org/kibana/kibana/kibana-3.0.0milestone5.tar.gz
|
||||
PLUGIN_FILES=$(shell git ls-files | egrep '^lib/logstash/(inputs|outputs|filters|codecs)/[^/]+$$' | egrep -v '/(base|threadable).rb$$|/inputs/ganglia/')
|
||||
PLUGIN_FILES=$(shell find lib -type f| egrep '^lib/logstash/(inputs|outputs|filters|codecs)/[^/]+$$' | egrep -v '/(base|threadable).rb$$|/inputs/ganglia/')
|
||||
QUIET=@
|
||||
ifeq (@,$(QUIET))
|
||||
QUIET_OUTPUT=> /dev/null 2>&1
|
||||
|
|
@ -202,7 +202,7 @@ vendor/bundle: | vendor $(JRUBY)
|
|||
@# Purge any rspec or test directories
|
||||
-$(QUIET)rm -rf $@/jruby/1.9/gems/*/spec $@/jruby/1.9/gems/*/test
|
||||
@# Purge any comments in ruby code.
|
||||
@#-find $@/jruby/1.9/gems/ -name '*.rb' | xargs -n1 sed -i -re '/^[ \t]*#/d; /^[ \t]*$$/d'
|
||||
@#-find $@/jruby/1.9/gems/ -name '*.rb' | xargs -n1 sed -i -e '/^[ \t]*#/d; /^[ \t]*$$/d'
|
||||
|
||||
.PHONY: build
|
||||
build:
|
||||
|
|
@ -288,7 +288,7 @@ build/flatgems: | build vendor/bundle
|
|||
$(QUIET)rsync -a $(VENDOR_DIR)/gems/user_agent_parser-*/vendor/ua-parser $@/vendor
|
||||
$(QUIET)rsync -a $(VENDOR_DIR)/gems/aws-sdk-*/ca-bundle.crt $@/root/
|
||||
@# A lame hack to work around the aws-sdk bug (LOGSTASH-1718)
|
||||
sed -i -e "s@SRC = ROOT + '/lib/aws'@SRC = ROOT + 'aws'@" $@/lib/aws/core.rb
|
||||
sed -i "" -e "s@SRC = ROOT + '/lib/aws'@SRC = ROOT + 'aws'@" $@/lib/aws/core.rb
|
||||
|
||||
|
||||
flatjar-test:
|
||||
|
|
@ -322,33 +322,46 @@ test: | $(JRUBY) vendor-elasticsearch vendor-geoip vendor-collectd
|
|||
.PHONY: docs
|
||||
docs: docgen doccopy docindex
|
||||
|
||||
doccopy: $(addprefix build/,$(shell git ls-files | grep '^docs/')) | build/docs
|
||||
doccopy: $(addprefix build/,$(shell find docs -type f | grep '^docs/')) | build/docs
|
||||
docindex: build/docs/index.html
|
||||
|
||||
docgen: $(addprefix build/docs/,$(subst lib/logstash/,,$(subst .rb,.html,$(PLUGIN_FILES))))
|
||||
docgen: build/docs/tutorials/getting-started-with-logstash.md
|
||||
|
||||
build/docs: build
|
||||
-$(QUIET)mkdir $@
|
||||
$(QUIET)-mkdir $@
|
||||
|
||||
build/docs/inputs build/docs/filters build/docs/outputs build/docs/codecs: | build/docs
|
||||
-$(QUIET)mkdir $@
|
||||
$(QUIET)-mkdir $@
|
||||
|
||||
build/docs/tutorials/getting-started-with-logstash.md: build/docs/tutorials/getting-started-with-logstash.xml
|
||||
$(QUIET)( \
|
||||
echo "---"; \
|
||||
echo "title: Metrics from Logs - logstash"; \
|
||||
echo "layout: content_right"; \
|
||||
echo "---"; \
|
||||
pandoc -f docbook -t markdown $< \
|
||||
) > $@
|
||||
|
||||
build/docs/tutorials/getting-started-with-logstash.xml: docs/tutorials/getting-started-with-logstash.asciidoc
|
||||
$(QUIET)asciidoc -b docbook -o $@ $<
|
||||
|
||||
# bluecloth gem doesn't work on jruby. Use ruby.
|
||||
build/docs/inputs/%.html: lib/logstash/inputs/%.rb docs/docgen.rb docs/plugin-doc.html.erb | build/docs/inputs
|
||||
$(QUIET)ruby docs/docgen.rb -o build/docs $<
|
||||
$(QUIET)sed -i -re 's/%VERSION%/$(VERSION)/g' $@
|
||||
$(QUIET)sed -i -re 's/%ELASTICSEARCH_VERSION%/$(ELASTICSEARCH_VERSION)/g' $@
|
||||
$(QUIET)sed -i "" -e 's/%VERSION%/$(VERSION)/g' $@
|
||||
$(QUIET)sed -i "" -e 's/%ELASTICSEARCH_VERSION%/$(ELASTICSEARCH_VERSION)/g' $@
|
||||
build/docs/filters/%.html: lib/logstash/filters/%.rb docs/docgen.rb docs/plugin-doc.html.erb | build/docs/filters
|
||||
$(QUIET)ruby docs/docgen.rb -o build/docs $<
|
||||
$(QUIET)sed -i -re 's/%VERSION%/$(VERSION)/g' $@
|
||||
$(QUIET)sed -i -re 's/%ELASTICSEARCH_VERSION%/$(ELASTICSEARCH_VERSION)/g' $@
|
||||
$(QUIET)sed -i "" -e 's/%VERSION%/$(VERSION)/g' $@
|
||||
$(QUIET)sed -i "" -e 's/%ELASTICSEARCH_VERSION%/$(ELASTICSEARCH_VERSION)/g' $@
|
||||
build/docs/outputs/%.html: lib/logstash/outputs/%.rb docs/docgen.rb docs/plugin-doc.html.erb | build/docs/outputs
|
||||
$(QUIET)ruby docs/docgen.rb -o build/docs $<
|
||||
$(QUIET)sed -i -re 's/%VERSION%/$(VERSION)/g' $@
|
||||
$(QUIET)sed -i -re 's/%ELASTICSEARCH_VERSION%/$(ELASTICSEARCH_VERSION)/g' $@
|
||||
$(QUIET)sed -i "" -e 's/%VERSION%/$(VERSION)/g' $@
|
||||
$(QUIET)sed -i "" -e 's/%ELASTICSEARCH_VERSION%/$(ELASTICSEARCH_VERSION)/g' $@
|
||||
build/docs/codecs/%.html: lib/logstash/codecs/%.rb docs/docgen.rb docs/plugin-doc.html.erb | build/docs/codecs
|
||||
$(QUIET)ruby docs/docgen.rb -o build/docs $<
|
||||
$(QUIET)sed -i -re 's/%VERSION%/$(VERSION)/g' $@
|
||||
$(QUIET)sed -i "" -e 's/%VERSION%/$(VERSION)/g' $@
|
||||
|
||||
build/docs/%: docs/% lib/logstash/version.rb Makefile
|
||||
@echo "Copying $< (to $@)"
|
||||
|
|
@ -357,8 +370,8 @@ build/docs/%: docs/% lib/logstash/version.rb Makefile
|
|||
$(QUIET)case "$(suffix $<)" in \
|
||||
.gz|.bz2|.png|.jpg) ;; \
|
||||
*) \
|
||||
sed -i -re 's/%VERSION%/$(VERSION)/g' $@ ; \
|
||||
sed -i -re 's/%ELASTICSEARCH_VERSION%/$(ELASTICSEARCH_VERSION)/g' $@ ; \
|
||||
sed -i "" -e 's/%VERSION%/$(VERSION)/g' $@ ; \
|
||||
sed -i "" -e 's/%ELASTICSEARCH_VERSION%/$(ELASTICSEARCH_VERSION)/g' $@ ; \
|
||||
;; \
|
||||
esac
|
||||
|
||||
|
|
@ -366,8 +379,8 @@ build/docs/index.html: $(addprefix build/docs/,$(subst lib/logstash/,,$(subst .r
|
|||
build/docs/index.html: docs/generate_index.rb lib/logstash/version.rb docs/index.html.erb Makefile
|
||||
@echo "Building documentation index.html"
|
||||
$(QUIET)ruby $< build/docs > $@
|
||||
$(QUIET)sed -i -re 's/%VERSION%/$(VERSION)/g' $@
|
||||
$(QUIET)sed -i -re 's/%ELASTICSEARCH_VERSION%/$(ELASTICSEARCH_VERSION)/g' $@
|
||||
$(QUIET)sed -i "" -e 's/%VERSION%/$(VERSION)/g' $@
|
||||
$(QUIET)sed -i "" -e 's/%ELASTICSEARCH_VERSION%/$(ELASTICSEARCH_VERSION)/g' $@
|
||||
|
||||
.PHONY: patterns
|
||||
patterns:
|
||||
|
|
@ -438,8 +451,8 @@ prepare-tarball:
|
|||
@echo "=> Preparing tarball"
|
||||
$(QUIET)$(MAKE) $(WORKDIR)
|
||||
$(QUIET)rsync -a --relative bin lib spec locales patterns vendor/bundle/jruby vendor/geoip vendor/jar vendor/kibana vendor/ua-parser vendor/collectd LICENSE README.md --exclude 'vendor/bundle/jruby/1.9/cache' --exclude 'vendor/bundle/jruby/1.9/gems/*/doc' --exclude 'vendor/jar/elasticsearch-$(ELASTICSEARCH_VERSION).tar.gz' $(WORKDIR)
|
||||
$(QUIET)sed -i -e 's/^LOGSTASH_VERSION = .*/LOGSTASH_VERSION = "$(VERSION)"/' $(WORKDIR)/lib/logstash/version.rb
|
||||
$(QUIET)sed -i -e 's/%JRUBY_VERSION%/$(JRUBY_VERSION)/' $(WORKDIR)/bin/logstash.bat
|
||||
$(QUIET)sed -i "" -e 's/^LOGSTASH_VERSION = .*/LOGSTASH_VERSION = "$(VERSION)"/' $(WORKDIR)/lib/logstash/version.rb
|
||||
$(QUIET)sed -i "" -e 's/%JRUBY_VERSION%/$(JRUBY_VERSION)/' $(WORKDIR)/bin/logstash.bat
|
||||
|
||||
.PHONY: tarball
|
||||
tarball: | build/logstash-$(VERSION).tar.gz
|
||||
|
|
|
|||
59
docs/contrib-plugins.md
Normal file
59
docs/contrib-plugins.md
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
---
|
||||
title: Logstash Contrib plugins
|
||||
layout: content_right
|
||||
---
|
||||
|
||||
# contrib plugins
|
||||
|
||||
As logstash has grown, we've accumulated a massive repository of plugins. Well
|
||||
over 100 plugins, it became difficult for the project maintainers to adequately
|
||||
support everything effectively.
|
||||
|
||||
In order to improve the quality of popular plugins, we've moved the
|
||||
less-commonly-used plugins to a separate repository we're calling "contrib".
|
||||
Concentrating common plugin usage into core solves a few problems, most notably
|
||||
user complaints about the size of logstash releases, support/maintenance costs,
|
||||
etc.
|
||||
|
||||
It is our intent that this separation will improve life for users. If it
|
||||
doesn't, please file a bug so we can work to address it!
|
||||
|
||||
If a plugin is available in the 'contrib' package, the documentation for that
|
||||
plugin will note this boldly at the top of that plugin's documentation.
|
||||
|
||||
Contrib plugins reside in a [separate github project](https://github.com/elasticsearch/logstash-contrib).
|
||||
|
||||
# Packaging
|
||||
|
||||
At present, the contrib modules are available as a tarball.
|
||||
|
||||
# Automated Installation
|
||||
|
||||
The `bin/plugin` script will handle the installation for you:
|
||||
|
||||
cd /path/to/logstash
|
||||
bin/plugin install contrib
|
||||
|
||||
# Manual Installation
|
||||
|
||||
The contrib plugins can be extracted on top of an existing Logstash installation.
|
||||
|
||||
For example, if I've extracted `logstash-%VERSION%.tar.gz` into `/path`, e.g.
|
||||
|
||||
cd /path
|
||||
tar zxf ~/logstash-%VERSION%.tar.gz
|
||||
|
||||
It will have a `/path/logstash-%VERSION%` directory, e.g.
|
||||
|
||||
$ ls
|
||||
logstash-%VERSION%
|
||||
|
||||
The method to install the contrib tarball is identical.
|
||||
|
||||
cd /path
|
||||
wget http://download.elasticsearch.org/logstash/logstash/logstash-contrib-%VERSION%.tar.gz
|
||||
tar zxf ~/logstash-contrib-%VERSION%.tar.gz
|
||||
|
||||
This will install the contrib plugins in the same directory as the core
|
||||
install. These plugins will be available to logstash the next time it starts.
|
||||
|
||||
|
|
@ -1,49 +0,0 @@
|
|||
---
|
||||
title: Logstash Core and Contrib
|
||||
layout: content_right
|
||||
---
|
||||
|
||||
# core and contrib plugins
|
||||
|
||||
Starting in version 1.4.0, core and contributed plugins will be separated.
|
||||
Contrib plugins reside in a [separate github project](https://github.com/elasticsearch/logstash-contrib).
|
||||
|
||||
# Packaging
|
||||
At present, the contrib modules are available as a tarball.
|
||||
|
||||
# Automated Installation
|
||||
The `bin/plugin` script will handle the installation for you:
|
||||
|
||||
```
|
||||
cd /path/to/logstash
|
||||
bin/plugin install contrib
|
||||
```
|
||||
|
||||
# Manual Installation (if you're behind a firewall, etc.)
|
||||
The contrib plugins can be extracted on top of an existing Logstash installation.
|
||||
|
||||
For example, if I've extracted `logstash-1.4.0.tar.gz` into `/path`, e.g.
|
||||
|
||||
```
|
||||
cd /path
|
||||
tar zxf ~/logstash-1.4.0.tar.gz
|
||||
```
|
||||
|
||||
It will have a `/path/logstash-1.4.0` directory, e.g.
|
||||
|
||||
```
|
||||
$ ls
|
||||
logstash-1.4.0
|
||||
```
|
||||
|
||||
The method to install the contrib tarball is identical.
|
||||
|
||||
```
|
||||
cd /path
|
||||
tar zxf ~/logstash-contrib-1.4.0.tar.gz
|
||||
```
|
||||
|
||||
This will install the contrib plugins in the same directory as the core install.
|
||||
|
||||
The download link is http://download.elasticsearch.org/logstash/logstash/logstash-contrib-${VERSION}.tar.gz
|
||||
where ${VERSION} is the same version of Logstash you currently have installed, e.g. 1.4.0
|
||||
|
|
@ -27,6 +27,12 @@ class LogStashConfigDocGenerator
|
|||
/^ *flag[( ].*/ => lambda { |m| add_flag(m[0]) },
|
||||
/^ *(class|def|module) / => lambda { |m| clear_comments },
|
||||
}
|
||||
|
||||
if File.exists?("build/contrib_plugins")
|
||||
@contrib_list = File.read("build/contrib_plugins").split("\n")
|
||||
else
|
||||
@contrib_list = []
|
||||
end
|
||||
end
|
||||
|
||||
def parse(string)
|
||||
|
|
@ -193,6 +199,8 @@ class LogStashConfigDocGenerator
|
|||
template_file = File.join(File.dirname(__FILE__), "plugin-doc.html.erb")
|
||||
template = ERB.new(File.new(template_file).read, nil, "-")
|
||||
|
||||
is_contrib_plugin = @contrib_list.include?(file)
|
||||
|
||||
# descriptions are assumed to be markdown
|
||||
description = BlueCloth.new(@class_description).to_html
|
||||
|
||||
|
|
|
|||
|
|
@ -4,27 +4,15 @@ layout: content_right
|
|||
---
|
||||
<h2><%= name %></h2>
|
||||
<h3>Milestone: <a href="../plugin-milestones"><%= @milestone %></a></h3>
|
||||
<% if is_contrib_plugin -%>
|
||||
<div class="community-plugin-notice">
|
||||
<strong>This is a community-contributed plugin!</strong> It does not ship with logstash by default, but it is easy to install!
|
||||
To use this, you must have <a href="../contrib-plugins">installed the contrib plugins package</a>.
|
||||
</div>
|
||||
<% end -%>
|
||||
|
||||
<%= description %>
|
||||
|
||||
<% if !@flags.empty? -%>
|
||||
<!-- Flags are deprecated
|
||||
<h3> Flags </h3>
|
||||
|
||||
This plugin provides the following flags:
|
||||
|
||||
<dl>
|
||||
<% @flags.each do |flag, description| -%>
|
||||
<%# Prefix flag with plugin name. %>
|
||||
<dt> <%= flag.gsub(/^--/, "--#{name}-") %> </dt>
|
||||
<dd> <%= description %> </dd>
|
||||
<% end -%>
|
||||
</dl>
|
||||
|
||||
... flags are deprecated -->
|
||||
|
||||
<% end -%>
|
||||
|
||||
<h3> Synopsis </h3>
|
||||
|
||||
This is what it might look like in your config file:
|
||||
|
|
|
|||
Binary file not shown.
|
Before Width: | Height: | Size: 35 KiB |
|
|
@ -1 +0,0 @@
|
|||
<mxGraphModel dx="800" dy="800" grid="1" guides="1" tooltips="1" connect="1" fold="1" page="1" pageScale="1" pageWidth="826" pageHeight="1169" style="default-style2"><root><mxCell id="0"/><mxCell id="1" parent="0"/><mxCell id="2" value="Shipper" style="icon;image=http://logstash.net/images/logstash.png;fontSize=18" parent="1" vertex="1"><mxGeometry x="50" y="110" width="60" height="60" as="geometry"/></mxCell><mxCell id="5" value="Broker" style="icon;image=http://www.java-freelance.fr/wp-content/uploads/2012/06/banner_redis-300dpi-0315a8013afee137cce47b474541d7f1.png;fontSize=18" parent="1" vertex="1"><mxGeometry x="180" y="161" width="60" height="60" as="geometry"/></mxCell><mxCell id="7" value="Storage & Search" style="icon;image=http://www.elasticsearch.org/images/set3/bonsai2.png;fontSize=18" parent="1" vertex="1"><mxGeometry x="470" y="161" width="120" height="60" as="geometry"/></mxCell><mxCell id="8" value="" style="endArrow=open;entryX=0;entryY=0.5;fontSize=18" parent="1" source="5" target="11" edge="1"><mxGeometry x="-40" as="geometry"><mxPoint x="520" y="160" as="targetPoint"/></mxGeometry></mxCell><mxCell id="11" value="Indexer" style="icon;image=http://logstash.net/images/logstash.png;fontSize=18" parent="1" vertex="1"><mxGeometry x="280" y="161" width="140" height="60" as="geometry"/></mxCell><mxCell id="13" value="Shipper" style="icon;image=http://logstash.net/images/logstash.png;fontSize=18" parent="1" vertex="1"><mxGeometry x="50" y="240" width="60" height="60" as="geometry"/></mxCell><mxCell id="16" value="...
" style="text;fontSize=36" parent="1" vertex="1"><mxGeometry x="60" y="178" width="40" height="52" as="geometry"/></mxCell><mxCell id="17" value="" style="endArrow=none;exitX=1;exitY=0.5;entryX=1;entryY=0.5;fontSize=18" parent="1" edge="1"><mxGeometry x="50" y="-7" as="geometry"><mxPoint x="260" y="250" as="sourcePoint"/><mxPoint x="260" y="250" as="targetPoint"/></mxGeometry></mxCell><mxCell id="19" value="" style="endArrow=open;entryX=0;entryY=0.5;exitX=1;exitY=0.5;fontSize=18" parent="1" source="2" target="5" edge="1"><mxGeometry x="-40" as="geometry"><mxPoint x="520" y="160" as="targetPoint"/></mxGeometry></mxCell><mxCell id="20" value="" style="endArrow=open;entryX=0;entryY=0.5;exitX=1;exitY=0.5;fontSize=18" parent="1" source="13" target="5" edge="1"><mxGeometry x="-40" as="geometry"><mxPoint x="520" y="160" as="targetPoint"/></mxGeometry></mxCell><mxCell id="21" value="" style="endArrow=open;exitX=1;exitY=0.5;fontSize=18" parent="1" source="11" target="7" edge="1"><mxGeometry x="-40" as="geometry"><mxPoint x="490" y="200" as="sourcePoint"/><mxPoint x="520" y="160" as="targetPoint"/></mxGeometry></mxCell><mxCell id="26" value="Logstash" style="label;image=http://logstash.net/images/logstash.png;fontSize=12" parent="1" vertex="1"><mxGeometry x="210" y="10" width="140" height="40" as="geometry"/></mxCell><mxCell id="28" value="Redis" style="label;image=http://www.java-freelance.fr/wp-content/uploads/2012/06/banner_redis-300dpi-0315a8013afee137cce47b474541d7f1.png" parent="1" vertex="1"><mxGeometry x="350" y="10" width="110" height="40" as="geometry"/></mxCell><mxCell id="32" value="ElasticSearch" style="label;image=http://www.elasticsearch.org/images/set3/bonsai2.png;fontSize=12" parent="1" vertex="1"><mxGeometry x="460" y="10" width="210" height="40" as="geometry"/></mxCell><mxCell id="34" value="Web Interface" style="icon;image=http://logstash.net/images/logstash.png;fontSize=18" parent="1" vertex="1"><mxGeometry x="630" y="161" width="140" height="60" as="geometry"/></mxCell><mxCell id="35" value="" style="endArrow=open;entryX=0;entryY=0.5;exitX=1;exitY=0.5;fontSize=18" parent="1" source="7" target="34" edge="1"><mxGeometry x="-40" as="geometry"><mxPoint x="520" y="160" as="targetPoint"/></mxGeometry></mxCell><mxCell id="36" value="Legend:" style="text;align=center;verticalAlign=middle;fontSize=18" parent="1" vertex="1"><mxGeometry x="130" y="10" width="70" height="40" as="geometry"/></mxCell><mxCell id="37" value="" style="endArrow=open;entryX=0;entryY=0.5;exitX=1;exitY=0.75" parent="1" source="16" target="5" edge="1"><mxGeometry as="geometry"><mxPoint x="560" y="160" as="targetPoint"/></mxGeometry></mxCell></root></mxGraphModel>
|
||||
|
|
@ -1,217 +0,0 @@
|
|||
---
|
||||
title: Getting Started (Centralized Server) - logstash
|
||||
layout: content_right
|
||||
---
|
||||
|
||||
# Getting Started
|
||||
|
||||
## Centralized Setup with Event Parsing
|
||||
|
||||
This guide shows how to get you going quickly with logstash with multiple
|
||||
servers. This guide is for folks who want to ship all their logstash logs to a
|
||||
central location for indexing and search.
|
||||
|
||||
We'll have two classes of server. First, one that ships logs. Second, one that
|
||||
collects and indexes logs.
|
||||
|
||||
It's important to note that logstash itself has no concept of "shipper" and
|
||||
"collector" - the behavior of an agent depends entirely on how you configure
|
||||
it.
|
||||
|
||||
This diagram gives you an overview of the architecture:
|
||||
|
||||
|
||||
|
||||
On servers shipping logs:
|
||||
|
||||
* Download and run logstash (See section 'logstash log shipper' below)
|
||||
|
||||
On the server collecting and indexing your logs:
|
||||
|
||||
* Download and run Elasticsearch
|
||||
* Download and run Redis
|
||||
* Download and run Logstash
|
||||
|
||||
## ElasticSearch
|
||||
|
||||
Requirements: java
|
||||
|
||||
You'll most likely want the version of ElasticSearch specified by the
|
||||
[elasticsearch output](../outputs/elasticsearch) docs. Modify this in your shell
|
||||
for easy downloading of ElasticSearch:
|
||||
|
||||
ES_PACKAGE=elasticsearch-%ELASTICSEARCH_VERSION%.zip
|
||||
ES_DIR=${ES_PACKAGE%%.zip}
|
||||
SITE=https://download.elasticsearch.org/elasticsearch/elasticsearch
|
||||
if [ ! -d "$ES_DIR" ] ; then
|
||||
wget --no-check-certificate $SITE/$ES_PACKAGE
|
||||
unzip $ES_PACKAGE
|
||||
fi
|
||||
|
||||
ElasticSearch requires Java (uses Lucene on the backend; if you want to know
|
||||
more read the elasticsearch docs).
|
||||
|
||||
To start the service, run `bin/elasticsearch -f`. This will run it in the foreground. We want to keep it this way for debugging for now.
|
||||
|
||||
## Redis
|
||||
|
||||
Previous versions of this guide used AMQP via RabbitMQ. Due to the complexity of AMQP as well as performance issues related to the Bunny driver we use, we're now recommending Redis instead.
|
||||
|
||||
Redis has no external dependencies and has a much simpler configuration in Logstash.
|
||||
|
||||
Building and installing Redis is fairly straightforward. While normally this would be out of the scope of this document, as the instructions are so simple we'll include them here:
|
||||
|
||||
- Download Redis from http://redis.io/download (The latest stable release is likely what you want)
|
||||
- Extract the source, change to the directory and run `make`
|
||||
- Run Redis with `src/redis-server --loglevel verbose`
|
||||
|
||||
That's it.
|
||||
|
||||
## logstash
|
||||
|
||||
Once you have elasticsearch and redis running, you're
|
||||
ready to configure logstash.
|
||||
|
||||
Download the logstash release jar file. The package contains all
|
||||
required dependencies to save you time chasing down requirements.
|
||||
|
||||
Follow [this link to download logstash-%VERSION%](https://download.elasticsearch.org/logstash/logstash/logstash-%VERSION%-flatjar.jar).
|
||||
|
||||
Since we're doing a centralized configuration, you'll have two main
|
||||
logstash agent roles: a shipper and an indexer. You will ship logs from
|
||||
all servers via Redis and have another agent receive those messages,
|
||||
parse them, and index them in elasticsearch.
|
||||
|
||||
### logstash log shipper
|
||||
|
||||
As with the simple example, we're going to start simple to ensure that events are flowing
|
||||
|
||||
input {
|
||||
stdin {
|
||||
type => "example"
|
||||
}
|
||||
}
|
||||
|
||||
output {
|
||||
stdout { codec => rubydebug }
|
||||
redis { host => "127.0.0.1" data_type => "list" key => "logstash" }
|
||||
}
|
||||
|
||||
Put this in a file and call it 'shipper.conf' (or anything, really), and run:
|
||||
|
||||
java -jar logstash-%VERSION%-flatjar.jar agent -f shipper.conf
|
||||
|
||||
This will take anything you type into this console and display it on the console. Additionally it will save events to Redis in a `list` named after the `key` value you provided.
|
||||
|
||||
### Testing the Redis output
|
||||
|
||||
To verify that the message made it into Redis, check your Redis window. You should see something like the following:
|
||||
|
||||
[83019] 02 Jul 12:51:02 - Accepted 127.0.0.1:58312
|
||||
[83019] 02 Jul 12:51:06 - Client closed connection
|
||||
[83019] 02 Jul 12:51:06 - DB 0: 1 keys (0 volatile) in 4 slots HT.
|
||||
|
||||
The redis application ships with a CLI application that you can use to query the data. From your Redis source directory, run the following:
|
||||
|
||||
`src/redis-cli`
|
||||
|
||||
Once connected, run the following commands:
|
||||
|
||||
redis 127.0.0.1:6379> llen logstash
|
||||
(integer) 1
|
||||
redis 127.0.0.1:6379> lpop logstash
|
||||
"{\"message\":\"hello\",\"@timestamp\":\"2013-09-07T00:59:28.383Z\",\"@version\":\"1\",\"type\":\"stdin\",\"host\":\"pork\"}"
|
||||
redis 127.0.0.1:6379> llen logstash
|
||||
(integer) 0
|
||||
|
||||
What we've just done is check the length of the list, read and removed the oldest item in the list, and checked the length again.
|
||||
|
||||
This behavior is what Logstash does when it reads from a Redis input (technically logstash performs a blocking lpop). We're essentially using Redis to simulate a queue via the `list` data type.
|
||||
|
||||
Go ahead and type a few more entries in the agent window:
|
||||
|
||||
- test 1
|
||||
- test 2
|
||||
- test 3
|
||||
|
||||
As you `lpop` you should get them in the correct order of insertion.
|
||||
|
||||
### logstash indexer
|
||||
|
||||
This agent will parse and index your logs as they come in over Redis. Here's a
|
||||
sample config based on the previous section. Save this as `indexer.conf`
|
||||
|
||||
input {
|
||||
redis {
|
||||
host => "127.0.0.1"
|
||||
# these settings should match the output of the agent
|
||||
data_type => "list"
|
||||
key => "logstash"
|
||||
|
||||
# We use the 'json' codec here because we expect to read
|
||||
# json events from redis.
|
||||
codec => json
|
||||
}
|
||||
}
|
||||
|
||||
output {
|
||||
stdout { debug => true debug_format => "json"}
|
||||
|
||||
elasticsearch {
|
||||
host => "127.0.0.1"
|
||||
}
|
||||
}
|
||||
|
||||
The above configuration will attach to Redis and issue a `BLPOP` against the `logstash` list. When an event is recieved, it will be pulled off and sent to Elasticsearch for indexing.
|
||||
|
||||
Start the indexer the same way as the agent but specifying the `indexer.conf` file:
|
||||
|
||||
`java -jar logstash-%VERSION%-flatjar.jar agent -f indexer.conf`
|
||||
|
||||
To verify that your Logstash indexer is connecting to Elasticsearch properly, you should see a message in your Elasticsearch window similar to the following:
|
||||
|
||||
`[2012-07-02 13:14:27,008][INFO ][cluster.service ] [Baron Samedi] added {[Bes][JZQBMR21SUWRNtTMsDV3_g][inet[/192.168.1.194:9301]]{client=true, data=false},}`
|
||||
|
||||
The names `Bes` and `Baron Samedi` may differ as ES uses random names for nodes.
|
||||
|
||||
### Testing the flow
|
||||
Now we want to test the flow. In your agent window, type something to generate an event.
|
||||
The indexer should read this and persist it to Elasticsearch. It will also display the event to stdout.
|
||||
|
||||
In your Elasticsearch window, you should see something like the following:
|
||||
|
||||
[2012-07-02 13:21:58,982][INFO ][cluster.metadata ] [Baron Samedi] [logstash-2012.07.02] creating index, cause [auto(index api)], shards [5]/[1], mappings []
|
||||
[2012-07-02 13:21:59,495][INFO ][cluster.metadata ] [Baron Samedi] [logstash-2012.07.02] update_mapping [stdin-type] (dynamic)
|
||||
|
||||
Since indexes are created dynamically, this is the first sign that Logstash was able to write to ES. Let's use curl to verify our data is there:
|
||||
Using our curl command from the simple tutorial should let us see the data:
|
||||
|
||||
`curl -gs -XGET http://localhost:9200/logstash-*/_search?q=type:example`
|
||||
|
||||
You may need to modify the date as this is based on the date this guide was written.
|
||||
|
||||
Now we can move on to the final step...
|
||||
## logstash web interface
|
||||
|
||||
Run this on the same server as your elasticsearch server.
|
||||
|
||||
To run the logstash web server, just run the jar with 'web' as the first
|
||||
argument.
|
||||
|
||||
java -jar logstash-%VERSION%-flatjar.jar web
|
||||
|
||||
Just point your browser at the http://127.0.0.1:9292/ and start searching
|
||||
logs!
|
||||
|
||||
The web interface is called 'kibana' - you can learn more about kibana at <http://kibana.org>
|
||||
|
||||
# Distributing the load
|
||||
At this point we've been simulating a distributed environment on a single machine. If only the world were so easy.
|
||||
In all of the example configurations, we've been explicitly setting the connection to connect to `127.0.0.1` despite the fact in most network-related plugins, that's the default host.
|
||||
|
||||
Since Logstash is so modular, you can install the various components on different systems.
|
||||
|
||||
- If you want to give Redis a dedicated host, simply ensure that the `host` attribute in configurations points to that host.
|
||||
- If you want to give Elasticsearch a dedicated host, simple ensure that the `host` attribute is correct as well (in both web and indexer).
|
||||
|
||||
As with the simple input example, reading from stdin is fairly useless. Check the Logstash documentation for the various inputs offered and mix and match to taste!
|
||||
|
|
@ -1,200 +0,0 @@
|
|||
---
|
||||
title: Getting Started (Standalone server) - logstash
|
||||
layout: content_right
|
||||
---
|
||||
# Getting started with logstash (standalone server example)
|
||||
|
||||
This guide shows how to get you going quickly with logstash on a single,
|
||||
standalone server. We'll begin by showing you how to read events from standard
|
||||
input (your keyboard) and emit them to standard output. After that, we'll start
|
||||
collecting actual log files.
|
||||
|
||||
By standalone, I mean that everything happens on a single server: log collection, indexing, and the web interface.
|
||||
|
||||
logstash can be run on multiple servers (collect from many servers to a single
|
||||
indexer) if you want, but this example shows simply a standalone configuration.
|
||||
|
||||
Steps detailed in this guide:
|
||||
|
||||
* Download and run logstash
|
||||
|
||||
## Problems?
|
||||
|
||||
If you have problems, feel free to email the users list
|
||||
(logstash-users@googlegroups.com) or join IRC (#logstash on irc.freenode.org)
|
||||
|
||||
## logstash
|
||||
|
||||
You should download the logstash jar file - if you haven't yet,
|
||||
[download it
|
||||
now](https://download.elasticsearch.org/logstash/logstash/logstash-%VERSION%-flatjar.jar).
|
||||
This package includes most of the dependencies for logstash in it and
|
||||
helps you get started quicker.
|
||||
|
||||
The configuration of any logstash agent consists of specifying inputs, filters,
|
||||
and outputs. For this example, we will not configure any filters.
|
||||
|
||||
The inputs are your log files. The output will be elasticsearch. The config
|
||||
format should be simple to read and write. The bottom of this document includes
|
||||
links for further reading (config, etc) if you want to learn more.
|
||||
|
||||
Here is a simple Logstash configuration:
|
||||
|
||||
input { stdin { } }
|
||||
output { stdout { codec => rubydebug } }
|
||||
|
||||
Save this to a file called `logstash-simple.conf` and run it like so:
|
||||
|
||||
java -jar logstash-%VERSION%-flatjar.jar agent -f logstash-simple.conf
|
||||
|
||||
After a few seconds, type something in the console where you started logstash.
|
||||
Maybe `hello`. You should get some output like so:
|
||||
|
||||
{
|
||||
"message" => "hello",
|
||||
"@timestamp" => "2013-09-04T00:24:21.707Z",
|
||||
"@version" => "1",
|
||||
"host" => "pork"
|
||||
}
|
||||
|
||||
If everything is okay, let's move on to a more complex version:
|
||||
|
||||
### Saving to Elasticsearch
|
||||
|
||||
The recommended storage engine for Logstash is Elasticsearch. If you're running
|
||||
Logstash from the jar file or via jruby, you can use an embedded version of
|
||||
Elasticsearch for storage.
|
||||
|
||||
Using our configuration above, let's change it to look like so:
|
||||
|
||||
input { stdin { type => example } }
|
||||
output {
|
||||
stdout { codec => rubydebug }
|
||||
elasticsearch { embedded => true }
|
||||
}
|
||||
|
||||
We're going to KEEP the existing configuration but add a second output -
|
||||
embedded Elasticsearch. Restart your Logstash (CTRL-C and rerun the java
|
||||
command). Depending on the horsepower of your machine, this could take some
|
||||
time. Logstash needs to extract the jar contents to a working directory AND
|
||||
start an instance of Elasticsearch.
|
||||
|
||||
Let's do our test again by simply typing `test`. You should get the same output to the console.
|
||||
Now let's verify that Logstash stored the message in Elasticsearch:
|
||||
|
||||
curl -s http://127.0.0.1:9200/_status?pretty=true | grep logstash
|
||||
|
||||
_This assumes you have the `curl` command installed._
|
||||
|
||||
You should get back some output like so:
|
||||
|
||||
"logstash-2012.07.02" : {
|
||||
"index" : "logstash-2012.07.02"
|
||||
|
||||
This means Logstash created a new index based on today's date. Likely your data is in there as well:
|
||||
|
||||
`curl -gs -XGET "http://localhost:9200/logstash-*/_search?pretty&q=type:example"`
|
||||
|
||||
This will return a rather large JSON output. We're only concerned with a subset:
|
||||
|
||||
"_index" : "logstash-2013.09.07",
|
||||
"_type" : "logs",
|
||||
"_id" : "iARTN3MtQ-Kaf_x0fZaFwQ",
|
||||
"_score" : 1.4054651, "_source" : {
|
||||
"message": "fizzle",
|
||||
"@timestamp": "2013-09-07T00:42:23.453Z",
|
||||
"@version": "1",
|
||||
"type": "example",
|
||||
"host": "pork"
|
||||
}
|
||||
|
||||
Your output may look a little different.
|
||||
The reason we're going about it this way is to make absolutely sure that we have all the bits working before adding more complexity.
|
||||
|
||||
If you are unable to get these steps working, you likely have something interfering with multicast traffic. This has been known to happen when connected to VPNs for instance.
|
||||
For best results, test on a Linux VM or system with less complicated networking. If in doubt, rerun the command with the options `-vv` and paste the output to Github Gist or Pastie.
|
||||
Hop on the logstash IRC channel or mailing list and ask for help with that output as reference.
|
||||
|
||||
Obviously this is fairly useless this way. Let's add the final step and test with the builtin logstash web ui:
|
||||
|
||||
### Testing the webui
|
||||
|
||||
We've already proven that events can make it into Elasticsearch. However using
|
||||
curl for everything is less than ideal.
|
||||
|
||||
Logstash ships with a built-in web interface (called Kibana). Let's restart our
|
||||
logstash process with an additional option:
|
||||
|
||||
java -jar logstash-%VERSION%-flatjar.jar agent -f logstash-simple.conf -- web
|
||||
|
||||
One important thing to note is that the `web` option is actually its own set of
|
||||
commmand-line options. We're essentially starting two programs in one. This is
|
||||
worth remembering as you move to an external Elasticsearch server. The options
|
||||
you specify in your logstash.conf have no bearing on the web ui. It has its own
|
||||
options.
|
||||
|
||||
Again, the reason for testing without the web interface is to ensure that the
|
||||
logstash agent itself is getting events into Elasticsearch. This is different
|
||||
than the Logstash web ui being able to read them. As before, we'll need to
|
||||
wait a bit for everything to spin up. You can verify that everything is running
|
||||
(assuming you aren't running with any `-v` options) by checking the output of
|
||||
`netstat`:
|
||||
|
||||
netstat -napt | grep -i LISTEN
|
||||
|
||||
What's interesting is that you should see the following ports in use:
|
||||
|
||||
- 9200
|
||||
- 9300
|
||||
- 9301
|
||||
- 9302
|
||||
- 9292
|
||||
|
||||
The `9200` and `9300` ports are the embedded ES listening. The `9301` and `9302` ports are the agent and web interfaces talking to ES. `9292` is the port the web ui listens on.
|
||||
|
||||
If you open a browser to http://localhost:9292/ and click on the link in the body, you should see results. If not, switch back to your console, type some test and hit return.
|
||||
Refresh the browser page and you should have results!
|
||||
|
||||
### Continuing on
|
||||
At this point you have a working self-contained Logstash instance. However typing things into stdin is likely not to be what you want.
|
||||
|
||||
Here is a sample config you can start with. It defines some basic inputs
|
||||
grouped by type and two outputs.
|
||||
|
||||
input {
|
||||
stdin {
|
||||
type => "stdin-type"
|
||||
}
|
||||
|
||||
file {
|
||||
type => "syslog"
|
||||
|
||||
# Wildcards work, here :)
|
||||
path => [ "/var/log/*.log", "/var/log/messages", "/var/log/syslog" ]
|
||||
}
|
||||
}
|
||||
|
||||
output {
|
||||
stdout { }
|
||||
elasticsearch { embedded => true }
|
||||
}
|
||||
|
||||
Put this in a file called "logstash-complex.conf"
|
||||
|
||||
Now run it all (again. Be sure to stop your previous Logstash tests!):
|
||||
|
||||
java -jar logstash-%VERSION%-flatjar.jar agent -f logstash-complex.conf -- web
|
||||
|
||||
Point your browser at <http://yourserver:9292> and start searching!
|
||||
|
||||
*Note*: If things are not working, such as you get an error message while
|
||||
searching, like 'SERVICE_UNAVAILABLE' or some other elasticsearch error, you
|
||||
should check that your firewall (local, too) is not blocking multicast.
|
||||
|
||||
## Further reading
|
||||
|
||||
Want to know more about the configuration language? Check out the
|
||||
[configuration](../configuration) documentation.
|
||||
|
||||
You may have logs on many servers you want to centralize through logstash. To
|
||||
learn how to do that, [read this](getting-started-centralized)
|
||||
|
|
@ -177,7 +177,6 @@ For the complete list of (current) configurations, visit the logstash "plugin co
|
|||
|
||||
Specifying configurations on the command line using '-e' is only so helpful, and more advanced setups will require more lengthy, long-lived configurations. First, let's create a simple configuration file, and invoke logstash using it. Create a file named "logstash-simple.conf" and save it in the same directory as the logstash flatjar.
|
||||
|
||||
http://foo.com[logstash-simple.conf]
|
||||
----
|
||||
input { stdin { } }
|
||||
output {
|
||||
|
|
@ -197,7 +196,6 @@ Et voilà! Logstash will read in the configuration file you just created and run
|
|||
=== Filters
|
||||
Filters are an in-line processing mechanism which provide the flexibility to slice and dice your data to fit your needs. Let's see one in action, namely the *grok filter*.
|
||||
|
||||
http://foo.com[logstash-filter.conf]
|
||||
----
|
||||
input { stdin { } }
|
||||
|
||||
|
|
@ -254,7 +252,6 @@ The other filter used in this example is the *date* filter. This filter parses o
|
|||
=== Apache logs (from files)
|
||||
Now, let's configure something actually *useful*... apache2 access log files! We are going to read the input from a file on the localhost, and use a *conditional* to process the event according to our needs. First, create a file called something like 'logstash-apache.conf' with the following contents (you'll need to change the log's file path to suit your needs):
|
||||
|
||||
http://foo.com[logstash-apache.conf]
|
||||
----
|
||||
input {
|
||||
file {
|
||||
|
|
@ -299,7 +296,6 @@ You should be able to see your apache log data in Elasticsearch now! You'll noti
|
|||
|
||||
In this configuration, logstash is only watching the apache access_log, but it's easy enough to watch both the access_log and the error_log (actually, any file matching '*log'), by changing one line in the above configuration, like this:
|
||||
|
||||
http://foo.com[logstash-apache-wildcard.conf]
|
||||
----
|
||||
input {
|
||||
file {
|
||||
|
|
@ -313,7 +309,6 @@ Also, you might have noticed that logstash did not reprocess the events which we
|
|||
=== Conditionals
|
||||
Now we can build on the previous example, where we introduced the concept of a *conditional*. A conditional should be familiar to most logstash users, in the general sense. You may use 'if', 'else if' and 'else' statements, as in many other programming languages. Let's label each event according to which file it appeared in (access_log, error_log and other random files which end with "log").
|
||||
|
||||
http://foo.com[logstash-apache-error.conf]
|
||||
----
|
||||
input {
|
||||
file {
|
||||
|
|
@ -350,8 +345,6 @@ OK, now we can move on to another incredibly useful example: *syslog*. Syslog is
|
|||
|
||||
First, let's make a simple configuration file for logstash + syslog, called 'logstash-syslog.conf'.
|
||||
|
||||
http://foo.com[logstash-syslog.conf]
|
||||
|
||||
----
|
||||
input {
|
||||
tcp {
|
||||
|
|
@ -432,8 +425,3 @@ Now you should see the output of logstash in your original shell as it processes
|
|||
----
|
||||
|
||||
Congratulations! You're well on your way to being a real Logstash power user. You should be comfortable configuring, running and sending events to Logstash, but there's much more to explore.
|
||||
|
||||
* http://foo.com[Blah 1]
|
||||
* http://foo.com[Blah 2]
|
||||
* http://foo.com[Blah 3]
|
||||
* http://foo.com[Blah 4]
|
||||
|
|
|
|||
|
|
@ -62,4 +62,10 @@ if RUBY_ENGINE == "rbx"
|
|||
end
|
||||
end
|
||||
|
||||
Bundler::CLI.start(["install", "--gemfile=tools/Gemfile", "--path", target, "--clean"])
|
||||
# Try installing a few times in case we hit the "bad_record_mac" ssl error during installation.
|
||||
3.times do
|
||||
begin
|
||||
Bundler::CLI.start(["install", "--gemfile=tools/Gemfile", "--path", target, "--clean"])
|
||||
break
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -147,7 +147,7 @@ class LogStash::Filters::GeoIP < LogStash::Filters::Base
|
|||
# I have found strings coming from GeoIP that are ASCII-8BIT are actually
|
||||
# ISO-8859-1...
|
||||
when Encoding::ASCII_8BIT; value.force_encoding("ISO-8859-1").encode("UTF-8")
|
||||
when Encoding::ISO_8859_1; value.encode("UTF-8")
|
||||
when Encoding::ISO_8859_1, Encoding::US_ASCII; value.encode("UTF-8")
|
||||
else; value
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# encoding: utf-8
|
||||
# The version of logstash.
|
||||
LOGSTASH_VERSION = "1.4.0.dev"
|
||||
LOGSTASH_VERSION = "1.4.0.beta1"
|
||||
|
||||
# Note to authors: this should not include dashes because 'gem' barfs if
|
||||
# you include a dash in the version string.
|
||||
|
|
|
|||
|
|
@ -1,26 +0,0 @@
|
|||
require "test_utils"
|
||||
|
||||
describe "some stuff" do
|
||||
extend LogStash::RSpec
|
||||
|
||||
config <<-'CONFIG'
|
||||
filter {
|
||||
grep {
|
||||
tags => web
|
||||
drop => false
|
||||
add_field => [ "application", "web" ]
|
||||
}
|
||||
|
||||
mutate {
|
||||
tags => web
|
||||
#replace => [ "message", "%{request}" ]
|
||||
add_field => [ "message", "%{request}" ]
|
||||
}
|
||||
}
|
||||
CONFIG
|
||||
|
||||
sample("tags" => [ "web" ], "request" => "hello") do
|
||||
insist { subject["tags"] }.include?("web")
|
||||
insist { subject["message"] } == "hello"
|
||||
end
|
||||
end
|
||||
|
|
@ -5,11 +5,8 @@ GEM
|
|||
i18n (~> 0.6, >= 0.6.4)
|
||||
multi_json (~> 1.0)
|
||||
addressable (2.3.5)
|
||||
atomic (1.1.14)
|
||||
atomic (1.1.14-java)
|
||||
autoparse (0.3.3)
|
||||
addressable (>= 2.3.1)
|
||||
extlib (>= 0.9.15)
|
||||
multi_json (>= 1.0.0)
|
||||
avl_tree (1.1.3)
|
||||
awesome_print (1.2.0)
|
||||
aws-sdk (1.29.0)
|
||||
|
|
@ -21,7 +18,6 @@ GEM
|
|||
bindata (1.6.0)
|
||||
blankslate (2.1.2.4)
|
||||
bouncy-castle-java (1.5.0147)
|
||||
bson (1.9.2-java)
|
||||
buftok (0.1)
|
||||
builder (3.2.2)
|
||||
cabin (0.6.1)
|
||||
|
|
@ -36,23 +32,14 @@ GEM
|
|||
simplecov (>= 0.7)
|
||||
term-ansicolor
|
||||
thor
|
||||
crack (0.1.8)
|
||||
diff-lcs (1.2.5)
|
||||
docile (1.1.1)
|
||||
edn (1.0.2)
|
||||
parslet (~> 1.4.0)
|
||||
elasticsearch (0.4.1)
|
||||
elasticsearch-api (= 0.4.1)
|
||||
elasticsearch-transport (= 0.4.1)
|
||||
elasticsearch-api (0.4.1)
|
||||
multi_json
|
||||
elasticsearch-transport (0.4.1)
|
||||
faraday
|
||||
multi_json
|
||||
excon (0.25.3)
|
||||
extlib (0.9.16)
|
||||
faraday (0.8.8)
|
||||
multipart-post (~> 1.2.0)
|
||||
ffi (1.9.3)
|
||||
ffi (1.9.3-java)
|
||||
ffi-rzmq (1.0.0)
|
||||
ffi
|
||||
|
|
@ -67,43 +54,16 @@ GEM
|
|||
gelfd (0.2.0)
|
||||
geoip (1.3.3)
|
||||
gmetric (0.1.3)
|
||||
google-api-client (0.6.4)
|
||||
addressable (>= 2.3.2)
|
||||
autoparse (>= 0.3.3)
|
||||
extlib (>= 0.9.15)
|
||||
faraday (~> 0.8.4)
|
||||
jwt (>= 0.1.5)
|
||||
launchy (>= 2.1.1)
|
||||
multi_json (>= 1.0.0)
|
||||
signet (~> 0.4.5)
|
||||
uuidtools (>= 2.1.0)
|
||||
haml (4.0.4)
|
||||
tilt
|
||||
hashie (2.0.5)
|
||||
heroku (3.1.0)
|
||||
heroku-api (~> 0.3.7)
|
||||
launchy (>= 0.3.2)
|
||||
netrc (~> 0.7.7)
|
||||
rest-client (~> 1.6.1)
|
||||
rubyzip
|
||||
heroku-api (0.3.15)
|
||||
excon (~> 0.25.1)
|
||||
hitimes (1.2.1)
|
||||
hitimes (1.2.1-java)
|
||||
http (0.5.0)
|
||||
http_parser.rb
|
||||
http_parser.rb (0.5.3)
|
||||
http_parser.rb (0.5.3-java)
|
||||
httparty (0.11.0)
|
||||
multi_json (~> 1.0)
|
||||
multi_xml (>= 0.5.2)
|
||||
i18n (0.6.9)
|
||||
insist (1.0.0)
|
||||
jdbc-mysql (5.1.27)
|
||||
jdbc-sqlite3 (3.7.2.1)
|
||||
jiralicious (0.2.2)
|
||||
crack (~> 0.1.8)
|
||||
hashie (>= 1.1)
|
||||
httparty (>= 0.10, < 0.12.0)
|
||||
json (>= 1.6, < 1.9.0)
|
||||
jls-grok (0.10.12)
|
||||
cabin (>= 0.6.0)
|
||||
jls-lumberjack (0.0.19)
|
||||
|
|
@ -111,17 +71,13 @@ GEM
|
|||
jruby-httpclient (1.1.1-java)
|
||||
jruby-openssl (0.8.7)
|
||||
bouncy-castle-java (>= 1.5.0147)
|
||||
jruby-win32ole (0.8.5)
|
||||
json (1.8.1)
|
||||
json (1.8.1-java)
|
||||
jwt (0.1.8)
|
||||
multi_json (>= 1.5)
|
||||
launchy (2.4.2)
|
||||
addressable (~> 2.3)
|
||||
mail (2.5.3)
|
||||
i18n (>= 0.4.0)
|
||||
mime-types (~> 1.16)
|
||||
treetop (~> 1.4.8)
|
||||
march_hare (2.0.0-java)
|
||||
march_hare (2.1.2-java)
|
||||
metaclass (0.0.1)
|
||||
method_source (0.8.2)
|
||||
metriks (0.9.9.5)
|
||||
|
|
@ -133,46 +89,31 @@ GEM
|
|||
minitest (5.0.8)
|
||||
mocha (0.14.0)
|
||||
metaclass (~> 0.0.1)
|
||||
mongo (1.9.2)
|
||||
bson (~> 1.9.2)
|
||||
msgpack-jruby (1.3.2-java)
|
||||
mtrc (0.0.4)
|
||||
multi_json (1.8.2)
|
||||
multi_xml (0.5.5)
|
||||
multipart-post (1.2.0)
|
||||
murmurhash3 (0.1.3)
|
||||
netrc (0.7.7)
|
||||
nokogiri (1.6.0)
|
||||
mini_portile (~> 0.5.0)
|
||||
nokogiri (1.6.0-java)
|
||||
mini_portile (~> 0.5.0)
|
||||
onstomp (1.0.7)
|
||||
parslet (1.4.0)
|
||||
blankslate (~> 2.0)
|
||||
php-serialize (1.1.0)
|
||||
polyglot (0.3.3)
|
||||
pry (0.9.12.4)
|
||||
coderay (~> 1.0)
|
||||
method_source (~> 0.8)
|
||||
slop (~> 3.4)
|
||||
pry (0.9.12.4-java)
|
||||
coderay (~> 1.0)
|
||||
method_source (~> 0.8)
|
||||
slop (~> 3.4)
|
||||
spoon (~> 0.0)
|
||||
rack (1.5.2)
|
||||
rack-protection (1.5.1)
|
||||
rack
|
||||
rbnacl (2.0.0)
|
||||
ffi
|
||||
redis (3.0.6)
|
||||
rest-client (1.6.7)
|
||||
mime-types (>= 1.16)
|
||||
riak-client (1.0.3)
|
||||
beefcake (~> 0.3.7)
|
||||
builder (>= 2.1.2)
|
||||
i18n (>= 0.4.0)
|
||||
multi_json (~> 1.0)
|
||||
riemann-client (0.2.1)
|
||||
beefcake (>= 0.3.5)
|
||||
mtrc (>= 0.0.4)
|
||||
trollop (>= 1.16.2)
|
||||
rsolr (1.0.9)
|
||||
builder (>= 2.1.2)
|
||||
rspec (2.14.1)
|
||||
rspec-core (~> 2.14.0)
|
||||
rspec-expectations (~> 2.14.0)
|
||||
|
|
@ -181,34 +122,23 @@ GEM
|
|||
rspec-expectations (2.14.4)
|
||||
diff-lcs (>= 1.1.3, < 2.0)
|
||||
rspec-mocks (2.14.4)
|
||||
rubyzip (1.1.0)
|
||||
rufus-scheduler (2.0.24)
|
||||
tzinfo (>= 0.3.22)
|
||||
rumbster (1.1.1)
|
||||
mail (= 2.5.3)
|
||||
sass (3.2.12)
|
||||
sequel (4.5.0)
|
||||
shoulda (3.5.0)
|
||||
shoulda-context (~> 1.0, >= 1.0.1)
|
||||
shoulda-matchers (>= 1.4.1, < 3.0)
|
||||
shoulda-context (1.1.6)
|
||||
shoulda-matchers (2.4.0)
|
||||
activesupport (>= 3.0.0)
|
||||
signet (0.4.5)
|
||||
addressable (>= 2.2.3)
|
||||
faraday (~> 0.8.1)
|
||||
jwt (>= 0.1.5)
|
||||
multi_json (>= 1.0.0)
|
||||
simple_oauth (0.2.0)
|
||||
simplecov (0.8.2)
|
||||
docile (~> 1.1.0)
|
||||
multi_json
|
||||
simplecov-html (~> 0.8.0)
|
||||
simplecov-html (0.8.0)
|
||||
sinatra (1.4.4)
|
||||
rack (~> 1.4)
|
||||
rack-protection (~> 1.4)
|
||||
tilt (~> 1.3, >= 1.3.4)
|
||||
slop (3.4.7)
|
||||
snmp (1.1.1)
|
||||
spoon (0.0.4)
|
||||
|
|
@ -220,6 +150,8 @@ GEM
|
|||
term-ansicolor (1.2.2)
|
||||
tins (~> 0.8)
|
||||
thor (0.18.1)
|
||||
thread_safe (0.1.3)
|
||||
atomic
|
||||
thread_safe (0.1.3-java)
|
||||
atomic
|
||||
tilt (1.4.1)
|
||||
|
|
@ -227,7 +159,6 @@ GEM
|
|||
treetop (1.4.15)
|
||||
polyglot
|
||||
polyglot (>= 0.3.1)
|
||||
trollop (2.0)
|
||||
twitter (5.0.0.rc.1)
|
||||
buftok (~> 0.1.0)
|
||||
faraday (>= 0.8, < 0.10)
|
||||
|
|
@ -239,8 +170,6 @@ GEM
|
|||
thread_safe (~> 0.1)
|
||||
user_agent_parser (2.1.1)
|
||||
uuidtools (2.1.4)
|
||||
varnish-rb (0.2.0)
|
||||
ffi
|
||||
xml-simple (1.1.2)
|
||||
xmpp4r (0.5)
|
||||
|
||||
|
|
@ -260,7 +189,6 @@ DEPENDENCIES
|
|||
clamp
|
||||
coveralls
|
||||
edn
|
||||
elasticsearch
|
||||
extlib (= 0.9.16)
|
||||
ffi
|
||||
ffi-rzmq (= 1.0.0)
|
||||
|
|
@ -270,53 +198,36 @@ DEPENDENCIES
|
|||
gelfd (= 0.2.0)
|
||||
geoip (>= 1.3.2)
|
||||
gmetric (= 0.1.3)
|
||||
google-api-client
|
||||
haml
|
||||
heroku
|
||||
i18n
|
||||
i18n (>= 0.6.6)
|
||||
insist (= 1.0.0)
|
||||
jdbc-mysql
|
||||
jdbc-sqlite3
|
||||
jiralicious (= 0.2.2)
|
||||
jls-grok (= 0.10.12)
|
||||
jls-lumberjack (>= 0.0.19)
|
||||
jruby-elasticsearch (= 0.0.17)
|
||||
jruby-httpclient
|
||||
jruby-openssl (= 0.8.7)
|
||||
jruby-win32ole
|
||||
json
|
||||
mail
|
||||
march_hare (~> 2.0.0)
|
||||
march_hare (~> 2.1.0)
|
||||
metriks
|
||||
mime-types
|
||||
minitest
|
||||
mocha
|
||||
mongo
|
||||
msgpack-jruby
|
||||
murmurhash3
|
||||
onstomp
|
||||
php-serialize
|
||||
pry
|
||||
rack
|
||||
rbnacl
|
||||
redis
|
||||
riak-client (= 1.0.3)
|
||||
riemann-client (= 0.2.1)
|
||||
rsolr
|
||||
rspec
|
||||
rufus-scheduler (~> 2.0.24)
|
||||
rumbster
|
||||
sass
|
||||
sequel
|
||||
shoulda
|
||||
sinatra
|
||||
snmp
|
||||
spoon
|
||||
statsd-ruby (= 1.2.0)
|
||||
stud
|
||||
twitter (= 5.0.0.rc.1)
|
||||
user_agent_parser (>= 2.0.0)
|
||||
uuidtools
|
||||
varnish-rb
|
||||
xml-simple
|
||||
xmpp4r (= 0.5)
|
||||
|
|
|
|||
54
tools/release.sh
Normal file
54
tools/release.sh
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
|
||||
|
||||
logstash=$HOME/projects/logstash
|
||||
contrib=$HOME/projects/logstash-contrib
|
||||
|
||||
workdir="$PWD/build/release/"
|
||||
mkdir -p $workdir
|
||||
|
||||
# circuit breaker to fail if there's something silly wrong.
|
||||
if [ -z "$workdir" ] ; then
|
||||
echo "workdir is empty?!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -d "$contrib" ] ; then
|
||||
echo "Missing: $contrib"
|
||||
echo "Maybe git clone it?"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
set -e
|
||||
|
||||
prepare() {
|
||||
rsync -a --delete $logstash/{bin,docs,lib,spec,Makefile,gembag.rb,logstash.gemspec,tools,locales,patterns,LICENSE,README.md} $contrib/{lib,spec} $workdir
|
||||
rm -f $logstash/.VERSION.mk
|
||||
make -C $logstash .VERSION.mk
|
||||
cp $logstash/.VERSION.mk $workdir
|
||||
}
|
||||
|
||||
docs() {
|
||||
make -C $workdir build
|
||||
(cd $contrib; find lib/logstash -type f -name '*.rb') > $workdir/build/contrib_plugins
|
||||
make -C $workdir -j 4 docs
|
||||
}
|
||||
|
||||
tests() {
|
||||
make -C $workdir test
|
||||
make -C $workdir tarball test
|
||||
}
|
||||
|
||||
packages() {
|
||||
for path in $logstash $contrib ; do
|
||||
rm -f $path/build/*.tar.gz
|
||||
rm -f $path/build/*.zip
|
||||
echo "Building packages: $path"
|
||||
make -C $path tarball package
|
||||
(cd $path/build; cp *.gz *.rpm *.deb *.zip $workdir/build)
|
||||
done
|
||||
}
|
||||
|
||||
prepare
|
||||
tests
|
||||
docs
|
||||
packages
|
||||
Loading…
Add table
Add a link
Reference in a new issue