mirror of
https://github.com/elastic/logstash.git
synced 2025-04-24 22:57:16 -04:00
Doc: Migrate docs from AsciiDoc to Markdown in 9.0 branch (#17289)
* Doc: Delete asciidoc files for 9.0 branch * Add MD files for 9.0 branch
This commit is contained in:
Karen Metts
GitHub
parent
905ff93cec
commit
91927d7450
508 changed files with 67717 additions and 20144 deletions
493
docs/docset.yml
Normal file
493
docs/docset.yml
Normal file
|
|
@ -0,0 +1,493 @@
|
|||
project: 'Logstash'
|
||||
cross_links:
|
||||
- beats
|
||||
- docs-content
|
||||
- ecs
|
||||
- elasticsearch
|
||||
- integration-docs
|
||||
- logstash-docs
|
||||
- search-ui
|
||||
toc:
|
||||
- toc: reference
|
||||
- toc: release-notes
|
||||
- toc: extend
|
||||
subs:
|
||||
ref: "https://www.elastic.co/guide/en/elasticsearch/reference/current"
|
||||
ref-bare: "https://www.elastic.co/guide/en/elasticsearch/reference"
|
||||
ref-8x: "https://www.elastic.co/guide/en/elasticsearch/reference/8.1"
|
||||
ref-80: "https://www.elastic.co/guide/en/elasticsearch/reference/8.0"
|
||||
ref-7x: "https://www.elastic.co/guide/en/elasticsearch/reference/7.17"
|
||||
ref-70: "https://www.elastic.co/guide/en/elasticsearch/reference/7.0"
|
||||
ref-60: "https://www.elastic.co/guide/en/elasticsearch/reference/6.0"
|
||||
ref-64: "https://www.elastic.co/guide/en/elasticsearch/reference/6.4"
|
||||
xpack-ref: "https://www.elastic.co/guide/en/x-pack/6.2"
|
||||
logstash-ref: "https://www.elastic.co/guide/en/logstash/current"
|
||||
kibana-ref: "https://www.elastic.co/guide/en/kibana/current"
|
||||
kibana-ref-all: "https://www.elastic.co/guide/en/kibana"
|
||||
beats-ref-root: "https://www.elastic.co/guide/en/beats"
|
||||
beats-ref: "https://www.elastic.co/guide/en/beats/libbeat/current"
|
||||
beats-ref-60: "https://www.elastic.co/guide/en/beats/libbeat/6.0"
|
||||
beats-ref-63: "https://www.elastic.co/guide/en/beats/libbeat/6.3"
|
||||
beats-devguide: "https://www.elastic.co/guide/en/beats/devguide/current"
|
||||
auditbeat-ref: "https://www.elastic.co/guide/en/beats/auditbeat/current"
|
||||
packetbeat-ref: "https://www.elastic.co/guide/en/beats/packetbeat/current"
|
||||
metricbeat-ref: "https://www.elastic.co/guide/en/beats/metricbeat/current"
|
||||
filebeat-ref: "https://www.elastic.co/guide/en/beats/filebeat/current"
|
||||
functionbeat-ref: "https://www.elastic.co/guide/en/beats/functionbeat/current"
|
||||
winlogbeat-ref: "https://www.elastic.co/guide/en/beats/winlogbeat/current"
|
||||
heartbeat-ref: "https://www.elastic.co/guide/en/beats/heartbeat/current"
|
||||
journalbeat-ref: "https://www.elastic.co/guide/en/beats/journalbeat/current"
|
||||
ingest-guide: "https://www.elastic.co/guide/en/ingest/current"
|
||||
fleet-guide: "https://www.elastic.co/guide/en/fleet/current"
|
||||
apm-guide-ref: "https://www.elastic.co/guide/en/apm/guide/current"
|
||||
apm-guide-7x: "https://www.elastic.co/guide/en/apm/guide/7.17"
|
||||
apm-app-ref: "https://www.elastic.co/guide/en/kibana/current"
|
||||
apm-agents-ref: "https://www.elastic.co/guide/en/apm/agent"
|
||||
apm-android-ref: "https://www.elastic.co/guide/en/apm/agent/android/current"
|
||||
apm-py-ref: "https://www.elastic.co/guide/en/apm/agent/python/current"
|
||||
apm-py-ref-3x: "https://www.elastic.co/guide/en/apm/agent/python/3.x"
|
||||
apm-node-ref-index: "https://www.elastic.co/guide/en/apm/agent/nodejs"
|
||||
apm-node-ref: "https://www.elastic.co/guide/en/apm/agent/nodejs/current"
|
||||
apm-node-ref-1x: "https://www.elastic.co/guide/en/apm/agent/nodejs/1.x"
|
||||
apm-rum-ref: "https://www.elastic.co/guide/en/apm/agent/rum-js/current"
|
||||
apm-ruby-ref: "https://www.elastic.co/guide/en/apm/agent/ruby/current"
|
||||
apm-java-ref: "https://www.elastic.co/guide/en/apm/agent/java/current"
|
||||
apm-go-ref: "https://www.elastic.co/guide/en/apm/agent/go/current"
|
||||
apm-dotnet-ref: "https://www.elastic.co/guide/en/apm/agent/dotnet/current"
|
||||
apm-php-ref: "https://www.elastic.co/guide/en/apm/agent/php/current"
|
||||
apm-ios-ref: "https://www.elastic.co/guide/en/apm/agent/swift/current"
|
||||
apm-lambda-ref: "https://www.elastic.co/guide/en/apm/lambda/current"
|
||||
apm-attacher-ref: "https://www.elastic.co/guide/en/apm/attacher/current"
|
||||
docker-logging-ref: "https://www.elastic.co/guide/en/beats/loggingplugin/current"
|
||||
esf-ref: "https://www.elastic.co/guide/en/esf/current"
|
||||
kinesis-firehose-ref: "https://www.elastic.co/guide/en/kinesis/{{kinesis_version}}"
|
||||
estc-welcome-current: "https://www.elastic.co/guide/en/starting-with-the-elasticsearch-platform-and-its-solutions/current"
|
||||
estc-welcome: "https://www.elastic.co/guide/en/starting-with-the-elasticsearch-platform-and-its-solutions/current"
|
||||
estc-welcome-all: "https://www.elastic.co/guide/en/starting-with-the-elasticsearch-platform-and-its-solutions"
|
||||
hadoop-ref: "https://www.elastic.co/guide/en/elasticsearch/hadoop/current"
|
||||
stack-ref: "https://www.elastic.co/guide/en/elastic-stack/current"
|
||||
stack-ref-67: "https://www.elastic.co/guide/en/elastic-stack/6.7"
|
||||
stack-ref-68: "https://www.elastic.co/guide/en/elastic-stack/6.8"
|
||||
stack-ref-70: "https://www.elastic.co/guide/en/elastic-stack/7.0"
|
||||
stack-ref-80: "https://www.elastic.co/guide/en/elastic-stack/8.0"
|
||||
stack-ov: "https://www.elastic.co/guide/en/elastic-stack-overview/current"
|
||||
stack-gs: "https://www.elastic.co/guide/en/elastic-stack-get-started/current"
|
||||
stack-gs-current: "https://www.elastic.co/guide/en/elastic-stack-get-started/current"
|
||||
javaclient: "https://www.elastic.co/guide/en/elasticsearch/client/java-api/current"
|
||||
java-api-client: "https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/current"
|
||||
java-rest: "https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current"
|
||||
jsclient: "https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current"
|
||||
jsclient-current: "https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current"
|
||||
es-ruby-client: "https://www.elastic.co/guide/en/elasticsearch/client/ruby-api/current"
|
||||
es-dotnet-client: "https://www.elastic.co/guide/en/elasticsearch/client/net-api/current"
|
||||
es-php-client: "https://www.elastic.co/guide/en/elasticsearch/client/php-api/current"
|
||||
es-python-client: "https://www.elastic.co/guide/en/elasticsearch/client/python-api/current"
|
||||
defguide: "https://www.elastic.co/guide/en/elasticsearch/guide/2.x"
|
||||
painless: "https://www.elastic.co/guide/en/elasticsearch/painless/current"
|
||||
plugins: "https://www.elastic.co/guide/en/elasticsearch/plugins/current"
|
||||
plugins-8x: "https://www.elastic.co/guide/en/elasticsearch/plugins/8.1"
|
||||
plugins-7x: "https://www.elastic.co/guide/en/elasticsearch/plugins/7.17"
|
||||
plugins-6x: "https://www.elastic.co/guide/en/elasticsearch/plugins/6.8"
|
||||
glossary: "https://www.elastic.co/guide/en/elastic-stack-glossary/current"
|
||||
upgrade_guide: "https://www.elastic.co/products/upgrade_guide"
|
||||
blog-ref: "https://www.elastic.co/blog/"
|
||||
curator-ref: "https://www.elastic.co/guide/en/elasticsearch/client/curator/current"
|
||||
curator-ref-current: "https://www.elastic.co/guide/en/elasticsearch/client/curator/current"
|
||||
metrics-ref: "https://www.elastic.co/guide/en/metrics/current"
|
||||
metrics-guide: "https://www.elastic.co/guide/en/metrics/guide/current"
|
||||
logs-ref: "https://www.elastic.co/guide/en/logs/current"
|
||||
logs-guide: "https://www.elastic.co/guide/en/logs/guide/current"
|
||||
uptime-guide: "https://www.elastic.co/guide/en/uptime/current"
|
||||
observability-guide: "https://www.elastic.co/guide/en/observability/current"
|
||||
observability-guide-all: "https://www.elastic.co/guide/en/observability"
|
||||
siem-guide: "https://www.elastic.co/guide/en/siem/guide/current"
|
||||
security-guide: "https://www.elastic.co/guide/en/security/current"
|
||||
security-guide-all: "https://www.elastic.co/guide/en/security"
|
||||
endpoint-guide: "https://www.elastic.co/guide/en/endpoint/current"
|
||||
sql-odbc: "https://www.elastic.co/guide/en/elasticsearch/sql-odbc/current"
|
||||
ecs-ref: "https://www.elastic.co/guide/en/ecs/current"
|
||||
ecs-logging-ref: "https://www.elastic.co/guide/en/ecs-logging/overview/current"
|
||||
ecs-logging-go-logrus-ref: "https://www.elastic.co/guide/en/ecs-logging/go-logrus/current"
|
||||
ecs-logging-go-zap-ref: "https://www.elastic.co/guide/en/ecs-logging/go-zap/current"
|
||||
ecs-logging-go-zerolog-ref: "https://www.elastic.co/guide/en/ecs-logging/go-zap/current"
|
||||
ecs-logging-java-ref: "https://www.elastic.co/guide/en/ecs-logging/java/current"
|
||||
ecs-logging-dotnet-ref: "https://www.elastic.co/guide/en/ecs-logging/dotnet/current"
|
||||
ecs-logging-nodejs-ref: "https://www.elastic.co/guide/en/ecs-logging/nodejs/current"
|
||||
ecs-logging-php-ref: "https://www.elastic.co/guide/en/ecs-logging/php/current"
|
||||
ecs-logging-python-ref: "https://www.elastic.co/guide/en/ecs-logging/python/current"
|
||||
ecs-logging-ruby-ref: "https://www.elastic.co/guide/en/ecs-logging/ruby/current"
|
||||
ml-docs: "https://www.elastic.co/guide/en/machine-learning/current"
|
||||
eland-docs: "https://www.elastic.co/guide/en/elasticsearch/client/eland/current"
|
||||
eql-ref: "https://eql.readthedocs.io/en/latest/query-guide"
|
||||
extendtrial: "https://www.elastic.co/trialextension"
|
||||
wikipedia: "https://en.wikipedia.org/wiki"
|
||||
forum: "https://discuss.elastic.co/"
|
||||
xpack-forum: "https://discuss.elastic.co/c/50-x-pack"
|
||||
security-forum: "https://discuss.elastic.co/c/x-pack/shield"
|
||||
watcher-forum: "https://discuss.elastic.co/c/x-pack/watcher"
|
||||
monitoring-forum: "https://discuss.elastic.co/c/x-pack/marvel"
|
||||
graph-forum: "https://discuss.elastic.co/c/x-pack/graph"
|
||||
apm-forum: "https://discuss.elastic.co/c/apm"
|
||||
enterprise-search-ref: "https://www.elastic.co/guide/en/enterprise-search/current"
|
||||
app-search-ref: "https://www.elastic.co/guide/en/app-search/current"
|
||||
workplace-search-ref: "https://www.elastic.co/guide/en/workplace-search/current"
|
||||
enterprise-search-node-ref: "https://www.elastic.co/guide/en/enterprise-search-clients/enterprise-search-node/current"
|
||||
enterprise-search-php-ref: "https://www.elastic.co/guide/en/enterprise-search-clients/php/current"
|
||||
enterprise-search-python-ref: "https://www.elastic.co/guide/en/enterprise-search-clients/python/current"
|
||||
enterprise-search-ruby-ref: "https://www.elastic.co/guide/en/enterprise-search-clients/ruby/current"
|
||||
elastic-maps-service: "https://maps.elastic.co"
|
||||
integrations-docs: "https://docs.elastic.co/en/integrations"
|
||||
integrations-devguide: "https://www.elastic.co/guide/en/integrations-developer/current"
|
||||
time-units: "https://www.elastic.co/guide/en/elasticsearch/reference/current/api-conventions.html#time-units"
|
||||
byte-units: "https://www.elastic.co/guide/en/elasticsearch/reference/current/api-conventions.html#byte-units"
|
||||
apm-py-ref-v: "https://www.elastic.co/guide/en/apm/agent/python/current"
|
||||
apm-node-ref-v: "https://www.elastic.co/guide/en/apm/agent/nodejs/current"
|
||||
apm-rum-ref-v: "https://www.elastic.co/guide/en/apm/agent/rum-js/current"
|
||||
apm-ruby-ref-v: "https://www.elastic.co/guide/en/apm/agent/ruby/current"
|
||||
apm-java-ref-v: "https://www.elastic.co/guide/en/apm/agent/java/current"
|
||||
apm-go-ref-v: "https://www.elastic.co/guide/en/apm/agent/go/current"
|
||||
apm-ios-ref-v: "https://www.elastic.co/guide/en/apm/agent/swift/current"
|
||||
apm-dotnet-ref-v: "https://www.elastic.co/guide/en/apm/agent/dotnet/current"
|
||||
apm-php-ref-v: "https://www.elastic.co/guide/en/apm/agent/php/current"
|
||||
ecloud: "Elastic Cloud"
|
||||
esf: "Elastic Serverless Forwarder"
|
||||
ess: "Elasticsearch Service"
|
||||
ece: "Elastic Cloud Enterprise"
|
||||
eck: "Elastic Cloud on Kubernetes"
|
||||
serverless-full: "Elastic Cloud Serverless"
|
||||
serverless-short: "Serverless"
|
||||
es-serverless: "Elasticsearch Serverless"
|
||||
es3: "Elasticsearch Serverless"
|
||||
obs-serverless: "Elastic Observability Serverless"
|
||||
sec-serverless: "Elastic Security Serverless"
|
||||
serverless-docs: "https://docs.elastic.co/serverless"
|
||||
cloud: "https://www.elastic.co/guide/en/cloud/current"
|
||||
ess-utm-params: "?page=docs&placement=docs-body"
|
||||
ess-baymax: "?page=docs&placement=docs-body"
|
||||
ess-trial: "https://cloud.elastic.co/registration?page=docs&placement=docs-body"
|
||||
ess-product: "https://www.elastic.co/cloud/elasticsearch-service?page=docs&placement=docs-body"
|
||||
ess-console: "https://cloud.elastic.co?page=docs&placement=docs-body"
|
||||
ess-console-name: "Elasticsearch Service Console"
|
||||
ess-deployments: "https://cloud.elastic.co/deployments?page=docs&placement=docs-body"
|
||||
ece-ref: "https://www.elastic.co/guide/en/cloud-enterprise/current"
|
||||
eck-ref: "https://www.elastic.co/guide/en/cloud-on-k8s/current"
|
||||
ess-leadin: "You can run Elasticsearch on your own hardware or use our hosted Elasticsearch Service that is available on AWS, GCP, and Azure. https://cloud.elastic.co/registration{ess-utm-params}[Try the Elasticsearch Service for free]."
|
||||
ess-leadin-short: "Our hosted Elasticsearch Service is available on AWS, GCP, and Azure, and you can https://cloud.elastic.co/registration{ess-utm-params}[try it for free]."
|
||||
ess-icon: "image:https://doc-icons.s3.us-east-2.amazonaws.com/logo_cloud.svg[link=\"https://cloud.elastic.co/registration{ess-utm-params}\", title=\"Supported on Elasticsearch Service\"]"
|
||||
ece-icon: "image:https://doc-icons.s3.us-east-2.amazonaws.com/logo_cloud_ece.svg[link=\"https://cloud.elastic.co/registration{ess-utm-params}\", title=\"Supported on Elastic Cloud Enterprise\"]"
|
||||
cloud-only: "This feature is designed for indirect use by https://cloud.elastic.co/registration{ess-utm-params}[Elasticsearch Service], https://www.elastic.co/guide/en/cloud-enterprise/{ece-version-link}[Elastic Cloud Enterprise], and https://www.elastic.co/guide/en/cloud-on-k8s/current[Elastic Cloud on Kubernetes]. Direct use is not supported."
|
||||
ess-setting-change: "image:https://doc-icons.s3.us-east-2.amazonaws.com/logo_cloud.svg[link=\"{ess-trial}\", title=\"Supported on {ess}\"] indicates a change to a supported https://www.elastic.co/guide/en/cloud/current/ec-add-user-settings.html[user setting] for Elasticsearch Service."
|
||||
ess-skip-section: "If you use Elasticsearch Service, skip this section. Elasticsearch Service handles these changes for you."
|
||||
api-cloud: "https://www.elastic.co/docs/api/doc/cloud"
|
||||
api-ece: "https://www.elastic.co/docs/api/doc/cloud-enterprise"
|
||||
api-kibana-serverless: "https://www.elastic.co/docs/api/doc/serverless"
|
||||
es-feature-flag: "This feature is in development and not yet available for use. This documentation is provided for informational purposes only."
|
||||
es-ref-dir: "'{{elasticsearch-root}}/docs/reference'"
|
||||
apm-app: "APM app"
|
||||
uptime-app: "Uptime app"
|
||||
synthetics-app: "Synthetics app"
|
||||
logs-app: "Logs app"
|
||||
metrics-app: "Metrics app"
|
||||
infrastructure-app: "Infrastructure app"
|
||||
siem-app: "SIEM app"
|
||||
security-app: "Elastic Security app"
|
||||
ml-app: "Machine Learning"
|
||||
dev-tools-app: "Dev Tools"
|
||||
ingest-manager-app: "Ingest Manager"
|
||||
stack-manage-app: "Stack Management"
|
||||
stack-monitor-app: "Stack Monitoring"
|
||||
alerts-ui: "Alerts and Actions"
|
||||
rules-ui: "Rules"
|
||||
rac-ui: "Rules and Connectors"
|
||||
connectors-ui: "Connectors"
|
||||
connectors-feature: "Actions and Connectors"
|
||||
stack-rules-feature: "Stack Rules"
|
||||
user-experience: "User Experience"
|
||||
ems: "Elastic Maps Service"
|
||||
ems-init: "EMS"
|
||||
hosted-ems: "Elastic Maps Server"
|
||||
ipm-app: "Index Pattern Management"
|
||||
ingest-pipelines: "ingest pipelines"
|
||||
ingest-pipelines-app: "Ingest Pipelines"
|
||||
ingest-pipelines-cap: "Ingest pipelines"
|
||||
ls-pipelines: "Logstash pipelines"
|
||||
ls-pipelines-app: "Logstash Pipelines"
|
||||
maint-windows: "maintenance windows"
|
||||
maint-windows-app: "Maintenance Windows"
|
||||
maint-windows-cap: "Maintenance windows"
|
||||
custom-roles-app: "Custom Roles"
|
||||
data-source: "data view"
|
||||
data-sources: "data views"
|
||||
data-source-caps: "Data View"
|
||||
data-sources-caps: "Data Views"
|
||||
data-source-cap: "Data view"
|
||||
data-sources-cap: "Data views"
|
||||
project-settings: "Project settings"
|
||||
manage-app: "Management"
|
||||
index-manage-app: "Index Management"
|
||||
data-views-app: "Data Views"
|
||||
rules-app: "Rules"
|
||||
saved-objects-app: "Saved Objects"
|
||||
tags-app: "Tags"
|
||||
api-keys-app: "API keys"
|
||||
transforms-app: "Transforms"
|
||||
connectors-app: "Connectors"
|
||||
files-app: "Files"
|
||||
reports-app: "Reports"
|
||||
maps-app: "Maps"
|
||||
alerts-app: "Alerts"
|
||||
crawler: "Enterprise Search web crawler"
|
||||
ents: "Enterprise Search"
|
||||
app-search-crawler: "App Search web crawler"
|
||||
agent: "Elastic Agent"
|
||||
agents: "Elastic Agents"
|
||||
fleet: "Fleet"
|
||||
fleet-server: "Fleet Server"
|
||||
integrations-server: "Integrations Server"
|
||||
ingest-manager: "Ingest Manager"
|
||||
ingest-management: "ingest management"
|
||||
package-manager: "Elastic Package Manager"
|
||||
integrations: "Integrations"
|
||||
package-registry: "Elastic Package Registry"
|
||||
artifact-registry: "Elastic Artifact Registry"
|
||||
aws: "AWS"
|
||||
stack: "Elastic Stack"
|
||||
xpack: "X-Pack"
|
||||
es: "Elasticsearch"
|
||||
kib: "Kibana"
|
||||
esms: "Elastic Stack Monitoring Service"
|
||||
esms-init: "ESMS"
|
||||
ls: "Logstash"
|
||||
beats: "Beats"
|
||||
auditbeat: "Auditbeat"
|
||||
filebeat: "Filebeat"
|
||||
heartbeat: "Heartbeat"
|
||||
metricbeat: "Metricbeat"
|
||||
packetbeat: "Packetbeat"
|
||||
winlogbeat: "Winlogbeat"
|
||||
functionbeat: "Functionbeat"
|
||||
journalbeat: "Journalbeat"
|
||||
es-sql: "Elasticsearch SQL"
|
||||
esql: "ES|QL"
|
||||
elastic-agent: "Elastic Agent"
|
||||
k8s: "Kubernetes"
|
||||
log-driver-long: "Elastic Logging Plugin for Docker"
|
||||
security: "X-Pack security"
|
||||
security-features: "security features"
|
||||
operator-feature: "operator privileges feature"
|
||||
es-security-features: "Elasticsearch security features"
|
||||
stack-security-features: "Elastic Stack security features"
|
||||
endpoint-sec: "Endpoint Security"
|
||||
endpoint-cloud-sec: "Endpoint and Cloud Security"
|
||||
elastic-defend: "Elastic Defend"
|
||||
elastic-sec: "Elastic Security"
|
||||
elastic-endpoint: "Elastic Endpoint"
|
||||
swimlane: "Swimlane"
|
||||
sn: "ServiceNow"
|
||||
sn-itsm: "ServiceNow ITSM"
|
||||
sn-itom: "ServiceNow ITOM"
|
||||
sn-sir: "ServiceNow SecOps"
|
||||
jira: "Jira"
|
||||
ibm-r: "IBM Resilient"
|
||||
webhook: "Webhook"
|
||||
webhook-cm: "Webhook - Case Management"
|
||||
opsgenie: "Opsgenie"
|
||||
bedrock: "Amazon Bedrock"
|
||||
gemini: "Google Gemini"
|
||||
hive: "TheHive"
|
||||
monitoring: "X-Pack monitoring"
|
||||
monitor-features: "monitoring features"
|
||||
stack-monitor-features: "Elastic Stack monitoring features"
|
||||
watcher: "Watcher"
|
||||
alert-features: "alerting features"
|
||||
reporting: "X-Pack reporting"
|
||||
report-features: "reporting features"
|
||||
graph: "X-Pack graph"
|
||||
graph-features: "graph analytics features"
|
||||
searchprofiler: "Search Profiler"
|
||||
xpackml: "X-Pack machine learning"
|
||||
ml: "machine learning"
|
||||
ml-cap: "Machine learning"
|
||||
ml-init: "ML"
|
||||
ml-features: "machine learning features"
|
||||
stack-ml-features: "Elastic Stack machine learning features"
|
||||
ccr: "cross-cluster replication"
|
||||
ccr-cap: "Cross-cluster replication"
|
||||
ccr-init: "CCR"
|
||||
ccs: "cross-cluster search"
|
||||
ccs-cap: "Cross-cluster search"
|
||||
ccs-init: "CCS"
|
||||
ilm: "index lifecycle management"
|
||||
ilm-cap: "Index lifecycle management"
|
||||
ilm-init: "ILM"
|
||||
dlm: "data lifecycle management"
|
||||
dlm-cap: "Data lifecycle management"
|
||||
dlm-init: "DLM"
|
||||
search-snap: "searchable snapshot"
|
||||
search-snaps: "searchable snapshots"
|
||||
search-snaps-cap: "Searchable snapshots"
|
||||
slm: "snapshot lifecycle management"
|
||||
slm-cap: "Snapshot lifecycle management"
|
||||
slm-init: "SLM"
|
||||
rollup-features: "data rollup features"
|
||||
ipm: "index pattern management"
|
||||
ipm-cap: "Index pattern"
|
||||
rollup: "rollup"
|
||||
rollup-cap: "Rollup"
|
||||
rollups: "rollups"
|
||||
rollups-cap: "Rollups"
|
||||
rollup-job: "rollup job"
|
||||
rollup-jobs: "rollup jobs"
|
||||
rollup-jobs-cap: "Rollup jobs"
|
||||
dfeed: "datafeed"
|
||||
dfeeds: "datafeeds"
|
||||
dfeed-cap: "Datafeed"
|
||||
dfeeds-cap: "Datafeeds"
|
||||
ml-jobs: "machine learning jobs"
|
||||
ml-jobs-cap: "Machine learning jobs"
|
||||
anomaly-detect: "anomaly detection"
|
||||
anomaly-detect-cap: "Anomaly detection"
|
||||
anomaly-job: "anomaly detection job"
|
||||
anomaly-jobs: "anomaly detection jobs"
|
||||
anomaly-jobs-cap: "Anomaly detection jobs"
|
||||
dataframe: "data frame"
|
||||
dataframes: "data frames"
|
||||
dataframe-cap: "Data frame"
|
||||
dataframes-cap: "Data frames"
|
||||
watcher-transform: "payload transform"
|
||||
watcher-transforms: "payload transforms"
|
||||
watcher-transform-cap: "Payload transform"
|
||||
watcher-transforms-cap: "Payload transforms"
|
||||
transform: "transform"
|
||||
transforms: "transforms"
|
||||
transform-cap: "Transform"
|
||||
transforms-cap: "Transforms"
|
||||
dataframe-transform: "transform"
|
||||
dataframe-transform-cap: "Transform"
|
||||
dataframe-transforms: "transforms"
|
||||
dataframe-transforms-cap: "Transforms"
|
||||
dfanalytics-cap: "Data frame analytics"
|
||||
dfanalytics: "data frame analytics"
|
||||
dataframe-analytics-config: "'{dataframe} analytics config'"
|
||||
dfanalytics-job: "'{dataframe} analytics job'"
|
||||
dfanalytics-jobs: "'{dataframe} analytics jobs'"
|
||||
dfanalytics-jobs-cap: "'{dataframe-cap} analytics jobs'"
|
||||
cdataframe: "continuous data frame"
|
||||
cdataframes: "continuous data frames"
|
||||
cdataframe-cap: "Continuous data frame"
|
||||
cdataframes-cap: "Continuous data frames"
|
||||
cdataframe-transform: "continuous transform"
|
||||
cdataframe-transforms: "continuous transforms"
|
||||
cdataframe-transforms-cap: "Continuous transforms"
|
||||
ctransform: "continuous transform"
|
||||
ctransform-cap: "Continuous transform"
|
||||
ctransforms: "continuous transforms"
|
||||
ctransforms-cap: "Continuous transforms"
|
||||
oldetection: "outlier detection"
|
||||
oldetection-cap: "Outlier detection"
|
||||
olscore: "outlier score"
|
||||
olscores: "outlier scores"
|
||||
fiscore: "feature influence score"
|
||||
evaluatedf-api: "evaluate {dataframe} analytics API"
|
||||
evaluatedf-api-cap: "Evaluate {dataframe} analytics API"
|
||||
binarysc: "binary soft classification"
|
||||
binarysc-cap: "Binary soft classification"
|
||||
regression: "regression"
|
||||
regression-cap: "Regression"
|
||||
reganalysis: "regression analysis"
|
||||
reganalysis-cap: "Regression analysis"
|
||||
depvar: "dependent variable"
|
||||
feature-var: "feature variable"
|
||||
feature-vars: "feature variables"
|
||||
feature-vars-cap: "Feature variables"
|
||||
classification: "classification"
|
||||
classification-cap: "Classification"
|
||||
classanalysis: "classification analysis"
|
||||
classanalysis-cap: "Classification analysis"
|
||||
infer-cap: "Inference"
|
||||
infer: "inference"
|
||||
lang-ident-cap: "Language identification"
|
||||
lang-ident: "language identification"
|
||||
data-viz: "Data Visualizer"
|
||||
file-data-viz: "File Data Visualizer"
|
||||
feat-imp: "feature importance"
|
||||
feat-imp-cap: "Feature importance"
|
||||
nlp: "natural language processing"
|
||||
nlp-cap: "Natural language processing"
|
||||
apm-agent: "APM agent"
|
||||
apm-go-agent: "Elastic APM Go agent"
|
||||
apm-go-agents: "Elastic APM Go agents"
|
||||
apm-ios-agent: "Elastic APM iOS agent"
|
||||
apm-ios-agents: "Elastic APM iOS agents"
|
||||
apm-java-agent: "Elastic APM Java agent"
|
||||
apm-java-agents: "Elastic APM Java agents"
|
||||
apm-dotnet-agent: "Elastic APM .NET agent"
|
||||
apm-dotnet-agents: "Elastic APM .NET agents"
|
||||
apm-node-agent: "Elastic APM Node.js agent"
|
||||
apm-node-agents: "Elastic APM Node.js agents"
|
||||
apm-php-agent: "Elastic APM PHP agent"
|
||||
apm-php-agents: "Elastic APM PHP agents"
|
||||
apm-py-agent: "Elastic APM Python agent"
|
||||
apm-py-agents: "Elastic APM Python agents"
|
||||
apm-ruby-agent: "Elastic APM Ruby agent"
|
||||
apm-ruby-agents: "Elastic APM Ruby agents"
|
||||
apm-rum-agent: "Elastic APM Real User Monitoring (RUM) JavaScript agent"
|
||||
apm-rum-agents: "Elastic APM RUM JavaScript agents"
|
||||
apm-lambda-ext: "Elastic APM AWS Lambda extension"
|
||||
project-monitors: "project monitors"
|
||||
project-monitors-cap: "Project monitors"
|
||||
private-location: "Private Location"
|
||||
private-locations: "Private Locations"
|
||||
pwd: "YOUR_PASSWORD"
|
||||
esh: "ES-Hadoop"
|
||||
default-dist: "default distribution"
|
||||
oss-dist: "OSS-only distribution"
|
||||
observability: "Observability"
|
||||
api-request-title: "Request"
|
||||
api-prereq-title: "Prerequisites"
|
||||
api-description-title: "Description"
|
||||
api-path-parms-title: "Path parameters"
|
||||
api-query-parms-title: "Query parameters"
|
||||
api-request-body-title: "Request body"
|
||||
api-response-codes-title: "Response codes"
|
||||
api-response-body-title: "Response body"
|
||||
api-example-title: "Example"
|
||||
api-examples-title: "Examples"
|
||||
api-definitions-title: "Properties"
|
||||
multi-arg: "†footnoteref:[multi-arg,This parameter accepts multiple arguments.]"
|
||||
multi-arg-ref: "†footnoteref:[multi-arg]"
|
||||
yes-icon: "image:https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png[Yes,20,15]"
|
||||
no-icon: "image:https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png[No,20,15]"
|
||||
es-repo: "https://github.com/elastic/elasticsearch/"
|
||||
es-issue: "https://github.com/elastic/elasticsearch/issues/"
|
||||
es-pull: "https://github.com/elastic/elasticsearch/pull/"
|
||||
es-commit: "https://github.com/elastic/elasticsearch/commit/"
|
||||
kib-repo: "https://github.com/elastic/kibana/"
|
||||
kib-issue: "https://github.com/elastic/kibana/issues/"
|
||||
kibana-issue: "'{kib-repo}issues/'"
|
||||
kib-pull: "https://github.com/elastic/kibana/pull/"
|
||||
kibana-pull: "'{kib-repo}pull/'"
|
||||
kib-commit: "https://github.com/elastic/kibana/commit/"
|
||||
ml-repo: "https://github.com/elastic/ml-cpp/"
|
||||
ml-issue: "https://github.com/elastic/ml-cpp/issues/"
|
||||
ml-pull: "https://github.com/elastic/ml-cpp/pull/"
|
||||
ml-commit: "https://github.com/elastic/ml-cpp/commit/"
|
||||
apm-repo: "https://github.com/elastic/apm-server/"
|
||||
apm-issue: "https://github.com/elastic/apm-server/issues/"
|
||||
apm-pull: "https://github.com/elastic/apm-server/pull/"
|
||||
kibana-blob: "https://github.com/elastic/kibana/blob/current/"
|
||||
apm-get-started-ref: "https://www.elastic.co/guide/en/apm/get-started/current"
|
||||
apm-server-ref: "https://www.elastic.co/guide/en/apm/server/current"
|
||||
apm-server-ref-v: "https://www.elastic.co/guide/en/apm/server/current"
|
||||
apm-server-ref-m: "https://www.elastic.co/guide/en/apm/server/master"
|
||||
apm-server-ref-62: "https://www.elastic.co/guide/en/apm/server/6.2"
|
||||
apm-server-ref-64: "https://www.elastic.co/guide/en/apm/server/6.4"
|
||||
apm-server-ref-70: "https://www.elastic.co/guide/en/apm/server/7.0"
|
||||
apm-overview-ref-v: "https://www.elastic.co/guide/en/apm/get-started/current"
|
||||
apm-overview-ref-70: "https://www.elastic.co/guide/en/apm/get-started/7.0"
|
||||
apm-overview-ref-m: "https://www.elastic.co/guide/en/apm/get-started/master"
|
||||
infra-guide: "https://www.elastic.co/guide/en/infrastructure/guide/current"
|
||||
a-data-source: "a data view"
|
||||
icon-bug: "pass:[<span class=\"eui-icon icon-bug\"></span>]"
|
||||
icon-checkInCircleFilled: "pass:[<span class=\"eui-icon icon-checkInCircleFilled\"></span>]"
|
||||
icon-warningFilled: "pass:[<span class=\"eui-icon icon-warningFilled\"></span>]"
|
||||
636
docs/extend/codec-new-plugin.md
Normal file
636
docs/extend/codec-new-plugin.md
Normal file
|
|
@ -0,0 +1,636 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/codec-new-plugin.html
|
||||
---
|
||||
|
||||
# How to write a Logstash codec plugin [codec-new-plugin]
|
||||
|
||||
To develop a new codec for Logstash, build a self-contained Ruby gem whose source code lives in its own GitHub repository. The Ruby gem can then be hosted and shared on RubyGems.org. You can use the example codec implementation as a starting point. (If you’re unfamiliar with Ruby, you can find an excellent quickstart guide at [https://www.ruby-lang.org/en/documentation/quickstart/](https://www.ruby-lang.org/en/documentation/quickstart/).)
|
||||
|
||||
## Get started [_get_started_2]
|
||||
|
||||
Let’s step through creating a codec plugin using the [example codec plugin](https://github.com/logstash-plugins/logstash-codec-example/).
|
||||
|
||||
### Create a GitHub repo for your new plugin [_create_a_github_repo_for_your_new_plugin_2]
|
||||
|
||||
Each Logstash plugin lives in its own GitHub repository. To create a new repository for your plugin:
|
||||
|
||||
1. Log in to GitHub.
|
||||
2. Click the **Repositories** tab. You’ll see a list of other repositories you’ve forked or contributed to.
|
||||
3. Click the green **New** button in the upper right.
|
||||
4. Specify the following settings for your new repo:
|
||||
|
||||
* **Repository name** — a unique name of the form `logstash-codec-pluginname`.
|
||||
* **Public or Private** — your choice, but the repository must be Public if you want to submit it as an official plugin.
|
||||
* **Initialize this repository with a README** — enables you to immediately clone the repository to your computer.
|
||||
|
||||
5. Click **Create Repository**.
|
||||
|
||||
|
||||
### Use the plugin generator tool [_use_the_plugin_generator_tool_2]
|
||||
|
||||
You can create your own Logstash plugin in seconds! The `generate` subcommand of `bin/logstash-plugin` creates the foundation for a new Logstash plugin with templatized files. It creates the correct directory structure, gemspec files, and dependencies so you can start adding custom code to process data with Logstash.
|
||||
|
||||
For more information, see [Generating plugins](/reference/plugin-generator.md)
|
||||
|
||||
|
||||
### Copy the codec code [_copy_the_codec_code]
|
||||
|
||||
Alternatively, you can use the examples repo we host on github.com
|
||||
|
||||
1. **Clone your plugin.** Replace `GITUSERNAME` with your github username, and `MYPLUGINNAME` with your plugin name.
|
||||
|
||||
* `git clone https://github.com/GITUSERNAME/logstash-``codec-MYPLUGINNAME.git`
|
||||
|
||||
* alternately, via ssh: `git clone git@github.com:GITUSERNAME/logstash``-codec-MYPLUGINNAME.git`
|
||||
|
||||
* `cd logstash-codec-MYPLUGINNAME`
|
||||
|
||||
2. **Clone the codec plugin example and copy it to your plugin branch.**
|
||||
|
||||
You don’t want to include the example .git directory or its contents, so delete it before you copy the example.
|
||||
|
||||
* `cd /tmp`
|
||||
* `git clone https://github.com/logstash-plugins/logstash``-codec-example.git`
|
||||
* `cd logstash-codec-example`
|
||||
* `rm -rf .git`
|
||||
* `cp -R * /path/to/logstash-codec-mypluginname/`
|
||||
|
||||
3. **Rename the following files to match the name of your plugin.**
|
||||
|
||||
* `logstash-codec-example.gemspec`
|
||||
* `example.rb`
|
||||
* `example_spec.rb`
|
||||
|
||||
```txt
|
||||
cd /path/to/logstash-codec-mypluginname
|
||||
mv logstash-codec-example.gemspec logstash-codec-mypluginname.gemspec
|
||||
mv lib/logstash/codecs/example.rb lib/logstash/codecs/mypluginname.rb
|
||||
mv spec/codecs/example_spec.rb spec/codecs/mypluginname_spec.rb
|
||||
```
|
||||
|
||||
|
||||
Your file structure should look like this:
|
||||
|
||||
```txt
|
||||
$ tree logstash-codec-mypluginname
|
||||
├── Gemfile
|
||||
├── LICENSE
|
||||
├── README.md
|
||||
├── Rakefile
|
||||
├── lib
|
||||
│ └── logstash
|
||||
│ └── codecs
|
||||
│ └── mypluginname.rb
|
||||
├── logstash-codec-mypluginname.gemspec
|
||||
└── spec
|
||||
└── codecs
|
||||
└── mypluginname_spec.rb
|
||||
```
|
||||
|
||||
For more information about the Ruby gem file structure and an excellent walkthrough of the Ruby gem creation process, see [http://timelessrepo.com/making-ruby-gems](http://timelessrepo.com/making-ruby-gems)
|
||||
|
||||
|
||||
### See what your plugin looks like [_see_what_your_plugin_looks_like_2]
|
||||
|
||||
Before we dive into the details, open up the plugin file in your favorite text editor and take a look.
|
||||
|
||||
```ruby
|
||||
require "logstash/codecs/base"
|
||||
require "logstash/codecs/line"
|
||||
|
||||
# Add any asciidoc formatted documentation here
|
||||
class LogStash::Codecs::Example < LogStash::Codecs::Base
|
||||
|
||||
# This example codec will append a string to the message field
|
||||
# of an event, either in the decoding or encoding methods
|
||||
#
|
||||
# This is only intended to be used as an example.
|
||||
#
|
||||
# input {
|
||||
# stdin { codec => example }
|
||||
# }
|
||||
#
|
||||
# or
|
||||
#
|
||||
# output {
|
||||
# stdout { codec => example }
|
||||
# }
|
||||
config_name "example"
|
||||
|
||||
# Append a string to the message
|
||||
config :append, :validate => :string, :default => ', Hello World!'
|
||||
|
||||
public
|
||||
def register
|
||||
@lines = LogStash::Codecs::Line.new
|
||||
@lines.charset = "UTF-8"
|
||||
end
|
||||
|
||||
public
|
||||
def decode(data)
|
||||
@lines.decode(data) do |line|
|
||||
replace = { "message" => line["message"].to_s + @append }
|
||||
yield LogStash::Event.new(replace)
|
||||
end
|
||||
end # def decode
|
||||
|
||||
public
|
||||
def encode(event)
|
||||
@on_event.call(event, event.get("message").to_s + @append + NL)
|
||||
end # def encode
|
||||
|
||||
end # class LogStash::Codecs::Example
|
||||
```
|
||||
|
||||
|
||||
|
||||
## Coding codec plugins [_coding_codec_plugins]
|
||||
|
||||
Now let’s take a line-by-line look at the example plugin.
|
||||
|
||||
### `require` Statements [_require_statements_2]
|
||||
|
||||
Logstash codec plugins require parent classes defined in `logstash/codecs/base` and logstash/namespace:
|
||||
|
||||
```ruby
|
||||
require "logstash/codecs/base"
|
||||
require "logstash/namespace"
|
||||
```
|
||||
|
||||
Of course, the plugin you build may depend on other code, or even gems. Just put them here along with these Logstash dependencies.
|
||||
|
||||
|
||||
|
||||
## Plugin Body [_plugin_body_2]
|
||||
|
||||
Let’s go through the various elements of the plugin itself.
|
||||
|
||||
### `class` Declaration [_class_declaration_2]
|
||||
|
||||
The codec plugin class should be a subclass of `LogStash::Codecs::Base`:
|
||||
|
||||
```ruby
|
||||
class LogStash::Codecs::Example < LogStash::Codecs::Base
|
||||
```
|
||||
|
||||
The class name should closely mirror the plugin name, for example:
|
||||
|
||||
```ruby
|
||||
LogStash::Codecs::Example
|
||||
```
|
||||
|
||||
|
||||
### `config_name` [_config_name_2]
|
||||
|
||||
```ruby
|
||||
config_name "example"
|
||||
```
|
||||
|
||||
This is the name your plugin will call inside the codec configuration block.
|
||||
|
||||
If you set `config_name "example"` in your plugin code, the corresponding Logstash configuration block would need to look like this:
|
||||
|
||||
|
||||
|
||||
## Configuration Parameters [_configuration_parameters_2]
|
||||
|
||||
```ruby
|
||||
config :variable_name, :validate => :variable_type, :default => "Default value", :required => boolean, :deprecated => boolean, :obsolete => string
|
||||
```
|
||||
|
||||
The configuration, or `config` section allows you to define as many (or as few) parameters as are needed to enable Logstash to process events.
|
||||
|
||||
There are several configuration attributes:
|
||||
|
||||
* `:validate` - allows you to enforce passing a particular data type to Logstash for this configuration option, such as `:string`, `:password`, `:boolean`, `:number`, `:array`, `:hash`, `:path` (a file-system path), `uri`, `:codec` (since 1.2.0), `:bytes`. Note that this also works as a coercion in that if I specify "true" for boolean (even though technically a string), it will become a valid boolean in the config. This coercion works for the `:number` type as well where "1.2" becomes a float and "22" is an integer.
|
||||
* `:default` - lets you specify a default value for a parameter
|
||||
* `:required` - whether or not this parameter is mandatory (a Boolean `true` or
|
||||
* `:list` - whether or not this value should be a list of values. Will typecheck the list members, and convert scalars to one element lists. Note that this mostly obviates the array type, though if you need lists of complex objects that will be more suitable. `false`)
|
||||
* `:deprecated` - informational (also a Boolean `true` or `false`)
|
||||
* `:obsolete` - used to declare that a given setting has been removed and is no longer functioning. The idea is to provide an informed upgrade path to users who are still using a now-removed setting.
|
||||
|
||||
|
||||
## Plugin Methods [_plugin_methods_2]
|
||||
|
||||
Logstash codecs must implement the `register` method, and the `decode` method or the `encode` method (or both).
|
||||
|
||||
### `register` Method [_register_method_2]
|
||||
|
||||
```ruby
|
||||
public
|
||||
def register
|
||||
end # def register
|
||||
```
|
||||
|
||||
The Logstash `register` method is like an `initialize` method. It was originally created to enforce having `super` called, preventing headaches for newbies. (Note: It may go away in favor of `initialize`, in conjunction with some enforced testing to ensure `super` is called.)
|
||||
|
||||
`public` means the method can be called anywhere, not just within the class. This is the default behavior for methods in Ruby, but it is specified explicitly here anyway.
|
||||
|
||||
You can also assign instance variables here (variables prepended by `@`). Configuration variables are now in scope as instance variables, like `@message`
|
||||
|
||||
|
||||
### `decode` Method [_decode_method]
|
||||
|
||||
```ruby
|
||||
public
|
||||
def decode(data)
|
||||
@lines.decode(data) do |line|
|
||||
replace = { "message" => line["message"].to_s + @append }
|
||||
yield LogStash::Event.new(replace)
|
||||
end
|
||||
end # def decode
|
||||
```
|
||||
|
||||
The codec’s `decode` method is where data coming in from an input is transformed into an event. There are complex examples like the [collectd](https://github.com/logstash-plugins/logstash-codec-collectd/blob/main/lib/logstash/codecs/collectd.rb#L386-L484) codec, and simpler examples like the [spool](https://github.com/logstash-plugins/logstash-codec-spool/blob/main/lib/logstash/codecs/spool.rb#L11-L16) codec.
|
||||
|
||||
There must be a `yield` statement as part of the `decode` method which will return decoded events to the pipeline.
|
||||
|
||||
|
||||
### `encode` Method [_encode_method]
|
||||
|
||||
```ruby
|
||||
public
|
||||
def encode(event)
|
||||
@on_event.call(event, event.get("message").to_s + @append + NL)
|
||||
end # def encode
|
||||
```
|
||||
|
||||
The `encode` method takes an event and serializes it (*encodes*) into another format. Good examples of `encode` methods include the simple [plain](https://github.com/logstash-plugins/logstash-codec-plain/blob/main/lib/logstash/codecs/plain.rb#L39-L46) codec, the slightly more involved [msgpack](https://github.com/logstash-plugins/logstash-codec-msgpack/blob/main/lib/logstash/codecs/msgpack.rb#L38-L46) codec, and even an [avro](https://github.com/logstash-plugins/logstash-codec-avro/blob/main/lib/logstash/codecs/avro.rb#L38-L45) codec.
|
||||
|
||||
In most cases, your `encode` method should have an `@on_event.call()` statement. This call will output data per event in the described way.
|
||||
|
||||
|
||||
|
||||
## Building the Plugin [_building_the_plugin_2]
|
||||
|
||||
At this point in the process you have coded your plugin and are ready to build a Ruby Gem from it. The following information will help you complete the process.
|
||||
|
||||
### External dependencies [_external_dependencies_2]
|
||||
|
||||
A `require` statement in Ruby is used to include necessary code. In some cases your plugin may require additional files. For example, the collectd plugin [uses](https://github.com/logstash-plugins/logstash-codec-collectd/blob/main/lib/logstash/codecs/collectd.rb#L148) the `types.db` file provided by collectd. In the main directory of your plugin, a file called `vendor.json` is where these files are described.
|
||||
|
||||
The `vendor.json` file contains an array of JSON objects, each describing a file dependency. This example comes from the [collectd](https://github.com/logstash-plugins/logstash-codec-collectd/blob/main/vendor.json) codec plugin:
|
||||
|
||||
```txt
|
||||
[{
|
||||
"sha1": "a90fe6cc53b76b7bdd56dc57950d90787cb9c96e",
|
||||
"url": "http://collectd.org/files/collectd-5.4.0.tar.gz",
|
||||
"files": [ "/src/types.db" ]
|
||||
}]
|
||||
```
|
||||
|
||||
* `sha1` is the sha1 signature used to verify the integrity of the file referenced by `url`.
|
||||
* `url` is the address from where Logstash will download the file.
|
||||
* `files` is an optional array of files to extract from the downloaded file. Note that while tar archives can use absolute or relative paths, treat them as absolute in this array. If `files` is not present, all files will be uncompressed and extracted into the vendor directory.
|
||||
|
||||
Another example of the `vendor.json` file is the [`geoip` filter](https://github.com/logstash-plugins/logstash-filter-geoip/blob/main/vendor.json)
|
||||
|
||||
The process used to download these dependencies is to call `rake vendor`. This will be discussed further in the testing section of this document.
|
||||
|
||||
Another kind of external dependency is on jar files. This will be described in the "Add a `gemspec` file" section.
|
||||
|
||||
|
||||
### Deprecated features [_deprecated_features_2]
|
||||
|
||||
As a plugin evolves, an option or feature may no longer serve the intended purpose, and the developer may want to *deprecate* its usage. Deprecation warns users about the option’s status, so they aren’t caught by surprise when it is removed in a later release.
|
||||
|
||||
{{ls}} 7.6 introduced a *deprecation logger* to make handling those situations easier. You can use the [adapter](https://github.com/logstash-plugins/logstash-mixin-deprecation_logger_support) to ensure that your plugin can use the deprecation logger while still supporting older versions of {{ls}}. See the [readme](https://github.com/logstash-plugins/logstash-mixin-deprecation_logger_support/blob/main/README.md) for more information and for instructions on using the adapter.
|
||||
|
||||
Deprecations are noted in the `logstash-deprecation.log` file in the `log` directory.
|
||||
|
||||
|
||||
### Add a Gemfile [_add_a_gemfile_2]
|
||||
|
||||
Gemfiles allow Ruby’s Bundler to maintain the dependencies for your plugin. Currently, all we’ll need is the Logstash gem, for testing, but if you require other gems, you should add them in here.
|
||||
|
||||
::::{tip}
|
||||
See [Bundler’s Gemfile page](http://bundler.io/gemfile.html) for more details.
|
||||
::::
|
||||
|
||||
|
||||
```ruby
|
||||
source 'https://rubygems.org'
|
||||
gemspec
|
||||
gem "logstash", :github => "elastic/logstash", :branch => "master"
|
||||
```
|
||||
|
||||
|
||||
|
||||
## Add a `gemspec` file [_add_a_gemspec_file_2]
|
||||
|
||||
Gemspecs define the Ruby gem which will be built and contain your plugin.
|
||||
|
||||
::::{tip}
|
||||
More information can be found on the [Rubygems Specification page](http://guides.rubygems.org/specification-reference/).
|
||||
::::
|
||||
|
||||
|
||||
```ruby
|
||||
Gem::Specification.new do |s|
|
||||
s.name = 'logstash-codec-example'
|
||||
s.version = '0.1.0'
|
||||
s.licenses = ['Apache License (2.0)']
|
||||
s.summary = "This codec does x, y, z in Logstash"
|
||||
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
||||
s.authors = ["Elastic"]
|
||||
s.email = 'info@elastic.co'
|
||||
s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
|
||||
s.require_paths = ["lib"]
|
||||
|
||||
# Files
|
||||
s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
|
||||
# Tests
|
||||
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
||||
|
||||
# Special flag to let us know this is actually a logstash plugin
|
||||
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "codec" }
|
||||
|
||||
# Gem dependencies
|
||||
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
||||
s.add_development_dependency 'logstash-devutils'
|
||||
end
|
||||
```
|
||||
|
||||
It is appropriate to change these values to fit your plugin. In particular, `s.name` and `s.summary` should reflect your plugin’s name and behavior.
|
||||
|
||||
`s.licenses` and `s.version` are also important and will come into play when you are ready to publish your plugin.
|
||||
|
||||
Logstash and all its plugins are licensed under [Apache License, version 2 ("ALv2")](https://github.com/elastic/logstash/blob/main/LICENSE.txt). If you make your plugin publicly available via [RubyGems.org](http://rubygems.org), please make sure to have this line in your gemspec:
|
||||
|
||||
* `s.licenses = ['Apache License (2.0)']`
|
||||
|
||||
The gem version, designated by `s.version`, helps track changes to plugins over time. You should use [semver versioning](http://semver.org/) strategy for version numbers.
|
||||
|
||||
### Runtime and Development Dependencies [_runtime_and_development_dependencies_2]
|
||||
|
||||
At the bottom of the `gemspec` file is a section with a comment: `Gem dependencies`. This is where any other needed gems must be mentioned. If a gem is necessary for your plugin to function, it is a runtime dependency. If a gem are only used for testing, then it would be a development dependency.
|
||||
|
||||
::::{note}
|
||||
You can also have versioning requirements for your dependencies—including other Logstash plugins:
|
||||
|
||||
```ruby
|
||||
# Gem dependencies
|
||||
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
||||
s.add_development_dependency 'logstash-devutils'
|
||||
```
|
||||
|
||||
This gemspec has a runtime dependency on the logstash-core-plugin-api and requires that it have a version number greater than or equal to version 1.60 and less than or equal to version 2.99.
|
||||
|
||||
::::
|
||||
|
||||
|
||||
::::{important}
|
||||
All plugins have a runtime dependency on the `logstash-core-plugin-api` gem, and a development dependency on `logstash-devutils`.
|
||||
::::
|
||||
|
||||
|
||||
|
||||
### Jar dependencies [_jar_dependencies_2]
|
||||
|
||||
In some cases, such as the [Elasticsearch output plugin](https://github.com/logstash-plugins/logstash-output-elasticsearch/blob/main/logstash-output-elasticsearch.gemspec#L22-L23), your code may depend on a jar file. In cases such as this, the dependency is added in the gemspec file in this manner:
|
||||
|
||||
```ruby
|
||||
# Jar dependencies
|
||||
s.requirements << "jar 'org.elasticsearch:elasticsearch', '5.0.0'"
|
||||
s.add_runtime_dependency 'jar-dependencies'
|
||||
```
|
||||
|
||||
With these both defined, the install process will search for the required jar file at [http://mvnrepository.com](http://mvnrepository.com) and download the specified version.
|
||||
|
||||
|
||||
|
||||
## Document your plugin [_document_your_plugin_2]
|
||||
|
||||
Documentation is an important part of your plugin. All plugin documentation is rendered and placed in the [Logstash Reference](/reference/index.md) and the [Versioned plugin docs](logstash-docs://reference/integration-plugins.md).
|
||||
|
||||
See [Document your plugin](/extend/plugin-doc.md) for tips and guidelines.
|
||||
|
||||
|
||||
## Add Tests [_add_tests_2]
|
||||
|
||||
Logstash loves tests. Lots of tests. If you’re using your new codec plugin in a production environment, you’ll want to have some tests to ensure you are not breaking any existing functionality.
|
||||
|
||||
::::{note}
|
||||
A full exposition on RSpec is outside the scope of this document. Learn more about RSpec at [http://rspec.info](http://rspec.info)
|
||||
::::
|
||||
|
||||
|
||||
For help learning about tests and testing, look in the `spec/codecs/` directory of several other similar plugins.
|
||||
|
||||
|
||||
## Clone and test! [_clone_and_test_2]
|
||||
|
||||
Now let’s start with a fresh clone of the plugin, build it and run the tests.
|
||||
|
||||
* **Clone your plugin into a temporary location** Replace `GITUSERNAME` with your github username, and `MYPLUGINNAME` with your plugin name.
|
||||
|
||||
* `git clone https://github.com/GITUSERNAME/logstash-``codec-MYPLUGINNAME.git`
|
||||
|
||||
* alternately, via ssh: `git clone git@github.com:GITUSERNAME/logstash-``codec-MYPLUGINNAME.git`
|
||||
|
||||
* `cd logstash-codec-MYPLUGINNAME`
|
||||
|
||||
|
||||
Then, you’ll need to install your plugins dependencies with bundler:
|
||||
|
||||
```
|
||||
bundle install
|
||||
```
|
||||
|
||||
::::{important}
|
||||
If your plugin has an external file dependency described in `vendor.json`, you must download that dependency before running or testing. You can do this by running:
|
||||
|
||||
```
|
||||
rake vendor
|
||||
```
|
||||
|
||||
::::
|
||||
|
||||
|
||||
And finally, run the tests:
|
||||
|
||||
```
|
||||
bundle exec rspec
|
||||
```
|
||||
|
||||
You should see a success message, which looks something like this:
|
||||
|
||||
```
|
||||
Finished in 0.034 seconds
|
||||
1 example, 0 failures
|
||||
```
|
||||
|
||||
Hooray! You’re almost there! (Unless you saw failures… you should fix those first).
|
||||
|
||||
|
||||
## Building and Testing [_building_and_testing_2]
|
||||
|
||||
Now you’re ready to build your (well-tested) plugin into a Ruby gem.
|
||||
|
||||
### Build [_build_2]
|
||||
|
||||
You already have all the necessary ingredients, so let’s go ahead and run the build command:
|
||||
|
||||
```sh
|
||||
gem build logstash-codec-example.gemspec
|
||||
```
|
||||
|
||||
That’s it! Your gem should be built and be in the same path with the name
|
||||
|
||||
```sh
|
||||
logstash-codec-mypluginname-0.1.0.gem
|
||||
```
|
||||
|
||||
The `s.version` number from your gemspec file will provide the gem version, in this case, `0.1.0`.
|
||||
|
||||
|
||||
### Test installation [_test_installation_2]
|
||||
|
||||
You should test install your plugin into a clean installation of Logstash. Download the latest version from the [Logstash downloads page](https://www.elastic.co/downloads/logstash/).
|
||||
|
||||
1. Untar and cd in to the directory:
|
||||
|
||||
```sh
|
||||
curl -O https://download.elastic.co/logstash/logstash/logstash-9.0.0.tar.gz
|
||||
tar xzvf logstash-9.0.0.tar.gz
|
||||
cd logstash-9.0.0
|
||||
```
|
||||
|
||||
2. Using the plugin tool, we can install the gem we just built.
|
||||
|
||||
* Replace `/my/logstash/plugins` with the correct path to the gem for your environment, and `0.1.0` with the correct version number from the gemspec file.
|
||||
|
||||
```sh
|
||||
bin/logstash-plugin install /my/logstash/plugins/logstash-codec-example/logstash-codec-example-0.1.0.gem
|
||||
```
|
||||
|
||||
* After running this, you should see feedback from Logstash that it was successfully installed:
|
||||
|
||||
```sh
|
||||
validating /my/logstash/plugins/logstash-codec-example/logstash-codec-example-0.1.0.gem >= 0
|
||||
Valid logstash plugin. Continuing...
|
||||
Successfully installed 'logstash-codec-example' with version '0.1.0'
|
||||
```
|
||||
|
||||
::::{tip}
|
||||
You can also use the Logstash plugin tool to determine which plugins are currently available:
|
||||
|
||||
```sh
|
||||
bin/logstash-plugin list
|
||||
```
|
||||
|
||||
Depending on what you have installed, you might see a short or long list of plugins: inputs, codecs, filters and outputs.
|
||||
|
||||
::::
|
||||
|
||||
3. Now try running Logstash with a simple configuration passed in via the command-line, using the `-e` flag.
|
||||
|
||||
::::{note}
|
||||
Your results will depend on what your codec plugin is designed to do.
|
||||
::::
|
||||
|
||||
|
||||
```sh
|
||||
bin/logstash -e 'input { stdin{ codec => example{}} } output {stdout { codec => rubydebug }}'
|
||||
```
|
||||
|
||||
The example codec plugin will append the contents of `append` (which by default appends ", Hello World!")
|
||||
|
||||
After starting Logstash, type something, for example "Random output string". The resulting output message field contents should be, "Random output string, Hello World!":
|
||||
|
||||
```sh
|
||||
Random output string
|
||||
{
|
||||
"message" => "Random output string, Hello World!",
|
||||
"@version" => "1",
|
||||
"@timestamp" => "2015-01-27T19:17:18.932Z",
|
||||
"host" => "cadenza"
|
||||
}
|
||||
```
|
||||
|
||||
Feel free to experiment and test this by changing the `append` parameter:
|
||||
|
||||
```sh
|
||||
bin/logstash -e 'input { stdin{ codec => example{ append => ", I am appending this! }} } output {stdout { codec => rubydebug }}'
|
||||
```
|
||||
|
||||
Congratulations! You’ve built, deployed and successfully run a Logstash codec.
|
||||
|
||||
|
||||
|
||||
## Submitting your plugin to [RubyGems.org](http://rubygems.org) and [logstash-plugins](https://github.com/logstash-plugins) [_submitting_your_plugin_to_rubygems_orghttprubygems_org_and_logstash_pluginshttpsgithub_comlogstash_plugins_2]
|
||||
|
||||
Logstash uses [RubyGems.org](http://rubygems.org) as its repository for all plugin artifacts. Once you have developed your new plugin, you can make it available to Logstash users by simply publishing it to RubyGems.org.
|
||||
|
||||
### Licensing [_licensing_2]
|
||||
|
||||
Logstash and all its plugins are licensed under [Apache License, version 2 ("ALv2")](https://github.com/elasticsearch/logstash/blob/main/LICENSE). If you make your plugin publicly available via [RubyGems.org](http://rubygems.org), please make sure to have this line in your gemspec:
|
||||
|
||||
* `s.licenses = ['Apache License (2.0)']`
|
||||
|
||||
|
||||
### Publishing to [RubyGems.org](http://rubygems.org) [_publishing_to_rubygems_orghttprubygems_org_2]
|
||||
|
||||
To begin, you’ll need an account on RubyGems.org
|
||||
|
||||
* [Sign-up for a RubyGems account](https://rubygems.org/sign_up).
|
||||
|
||||
After creating an account, [obtain](http://guides.rubygems.org/rubygems-org-api/#api-authorization) an API key from RubyGems.org. By default, RubyGems uses the file `~/.gem/credentials` to store your API key. These credentials will be used to publish the gem. Replace `username` and `password` with the credentials you created at RubyGems.org:
|
||||
|
||||
```sh
|
||||
curl -u username:password https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials
|
||||
chmod 0600 ~/.gem/credentials
|
||||
```
|
||||
|
||||
Before proceeding, make sure you have the right version in your gemspec file and commit your changes.
|
||||
|
||||
* `s.version = '0.1.0'`
|
||||
|
||||
To publish version 0.1.0 of your new logstash gem:
|
||||
|
||||
```sh
|
||||
bundle install
|
||||
bundle exec rake vendor
|
||||
bundle exec rspec
|
||||
bundle exec rake publish_gem
|
||||
```
|
||||
|
||||
::::{note}
|
||||
Executing `rake publish_gem`:
|
||||
|
||||
1. Reads the version from the gemspec file (`s.version = '0.1.0'`)
|
||||
2. Checks in your local repository if a tag exists for that version. If the tag already exists, it aborts the process. Otherwise, it creates a new version tag in your local repository.
|
||||
3. Builds the gem
|
||||
4. Publishes the gem to RubyGems.org
|
||||
|
||||
::::
|
||||
|
||||
|
||||
That’s it! Your plugin is published! Logstash users can now install your plugin by running:
|
||||
|
||||
```sh
|
||||
bin/logstash-plugin install logstash-codec-mypluginname
|
||||
```
|
||||
|
||||
|
||||
|
||||
## Contributing your source code to [logstash-plugins](https://github.com/logstash-plugins) [_contributing_your_source_code_to_logstash_pluginshttpsgithub_comlogstash_plugins_2]
|
||||
|
||||
It is not required to contribute your source code to [logstash-plugins](https://github.com/logstash-plugins) github organization, but we always welcome new plugins!
|
||||
|
||||
### Benefits [_benefits_2]
|
||||
|
||||
Some of the many benefits of having your plugin in the logstash-plugins repository are:
|
||||
|
||||
* **Discovery.** Your plugin will appear in the [Logstash Reference](/reference/index.md), where Logstash users look first for plugins and documentation.
|
||||
* **Documentation.** Your plugin documentation will automatically be added to the [Logstash Reference](/reference/index.md).
|
||||
* **Testing.** With our testing infrastructure, your plugin will be continuously tested against current and future releases of Logstash. As a result, users will have the assurance that if incompatibilities arise, they will be quickly discovered and corrected.
|
||||
|
||||
|
||||
### Acceptance Guidelines [_acceptance_guidelines_2]
|
||||
|
||||
* **Code Review.** Your plugin must be reviewed by members of the community for coherence, quality, readability, stability and security.
|
||||
* **Tests.** Your plugin must contain tests to be accepted. These tests are also subject to code review for scope and completeness. It’s ok if you don’t know how to write tests — we will guide you. We are working on publishing a guide to creating tests for Logstash which will make it easier. In the meantime, you can refer to [http://betterspecs.org/](http://betterspecs.org/) for examples.
|
||||
|
||||
To begin migrating your plugin to logstash-plugins, simply create a new [issue](https://github.com/elasticsearch/logstash/issues) in the Logstash repository. When the acceptance guidelines are completed, we will facilitate the move to the logstash-plugins organization using the recommended [github process](https://help.github.com/articles/transferring-a-repository/#transferring-from-a-user-to-an-organization).
|
||||
193
docs/extend/community-maintainer.md
Normal file
193
docs/extend/community-maintainer.md
Normal file
|
|
@ -0,0 +1,193 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/community-maintainer.html
|
||||
---
|
||||
|
||||
# Logstash Plugins Community Maintainer Guide [community-maintainer]
|
||||
|
||||
This document, to be read by new Maintainers, should explain their responsibilities. It was inspired by the [C4](http://rfc.zeromq.org/spec:22) document from the ZeroMQ project. This document is subject to change and suggestions through Pull Requests and issues are strongly encouraged.
|
||||
|
||||
|
||||
## Contribution Guidelines [_contribution_guidelines]
|
||||
|
||||
For general guidance around contributing to Logstash Plugins, see the [*Contributing to Logstash*](/extend/index.md) section.
|
||||
|
||||
|
||||
## Document Goals [_document_goals]
|
||||
|
||||
To help make the Logstash plugins community participation easy with positive feedback.
|
||||
|
||||
To increase diversity.
|
||||
|
||||
To reduce code review, merge and release dependencies on the core team by providing support and tools to the Community and Maintainers.
|
||||
|
||||
To support the natural life cycle of a plugin.
|
||||
|
||||
To codify the roles and responsibilities of: Maintainers and Contributors with specific focus on patch testing, code review, merging and release.
|
||||
|
||||
|
||||
## Development Workflow [_development_workflow]
|
||||
|
||||
All Issues and Pull Requests must be tracked using the Github issue tracker.
|
||||
|
||||
The plugin uses the [Apache 2.0 license](http://www.apache.org/licenses/LICENSE-2.0). Maintainers should check whether a patch introduces code which has an incompatible license. Patch ownership and copyright is defined in the Elastic [Contributor License Agreement](https://www.elastic.co/contributor-agreement) (CLA).
|
||||
|
||||
|
||||
### Terminology [_terminology_2]
|
||||
|
||||
A "Contributor" is a role a person assumes when providing a patch. Contributors will not have commit access to the repository. They need to sign the Elastic [Contributor License Agreement](https://www.elastic.co/contributor-agreement) before a patch can be reviewed. Contributors can add themselves to the plugin Contributor list.
|
||||
|
||||
A "Maintainer" is a role a person assumes when maintaining a plugin and keeping it healthy, including triaging issues, and reviewing and merging patches.
|
||||
|
||||
|
||||
### Patch Requirements [_patch_requirements]
|
||||
|
||||
A patch is a minimal and accurate answer to exactly one identified and agreed upon problem. It must conform to the [code style guidelines](https://github.com/elastic/logstash/blob/main/STYLE.md) and must include RSpec tests that verify the fitness of the solution.
|
||||
|
||||
A patch will be automatically tested by a CI system that will report on the Pull Request status.
|
||||
|
||||
A patch CLA will be automatically verified and reported on the Pull Request status.
|
||||
|
||||
A patch commit message has a single short (less than 50 character) first line summarizing the change, a blank second line, and any additional lines as necessary for change explanation and rationale.
|
||||
|
||||
A patch is mergeable when it satisfies the above requirements and has been reviewed positively by at least one other person.
|
||||
|
||||
|
||||
### Development Process [_development_process]
|
||||
|
||||
A user will log an issue on the issue tracker describing the problem they face or observe with as much detail as possible.
|
||||
|
||||
To work on an issue, a Contributor forks the plugin repository and then works on their forked repository and submits a patch by creating a pull request back to the plugin.
|
||||
|
||||
Maintainers must not merge patches where the author has not signed the CLA.
|
||||
|
||||
Before a patch can be accepted it should be reviewed. Maintainers should merge accepted patches without delay.
|
||||
|
||||
Maintainers should not merge their own patches except in exceptional cases, such as non-responsiveness from other Maintainers or core team for an extended period (more than 2 weeks).
|
||||
|
||||
Reviewer’s comments should not be based on personal preferences.
|
||||
|
||||
The Maintainers should label Issues and Pull Requests.
|
||||
|
||||
Maintainers should involve the core team if help is needed to reach consensus.
|
||||
|
||||
Review non-source changes such as documentation in the same way as source code changes.
|
||||
|
||||
|
||||
### Branch Management [_branch_management]
|
||||
|
||||
The plugin has a main branch that always holds the latest in-progress version and should always build. Topic branches should kept to the minimum.
|
||||
|
||||
|
||||
### Changelog Management [_changelog_management]
|
||||
|
||||
Every plugin should have a changelog (https://www.elastic.co/guide/en/logstash/current/CHANGELOG.html). If not, please create one. When changes are made to a plugin, make sure to include a changelog entry under the respective version to document the change. The changelog should be easily understood from a user point of view. As we iterate and release plugins rapidly, users use the changelog as a mechanism for deciding whether to update.
|
||||
|
||||
Changes that are not user facing should be tagged as `internal:`. For example:
|
||||
|
||||
```markdown
|
||||
- internal: Refactored specs for better testing
|
||||
- config: Default timeout configuration changed from 10s to 5s
|
||||
```
|
||||
|
||||
|
||||
#### Detailed format of https://www.elastic.co/guide/en/logstash/current/CHANGELOG.html [_detailed_format_of_changelog_md]
|
||||
|
||||
Sharing a similar format of https://www.elastic.co/guide/en/logstash/current/CHANGELOG.html in plugins ease readability for users. Please see following annotated example and see a concrete example in [logstash-filter-date](https://raw.githubusercontent.com/logstash-plugins/logstash-filter-date/main/https://www.elastic.co/guide/en/logstash/current/CHANGELOG.html).
|
||||
|
||||
```markdown
|
||||
## 1.0.x <1>
|
||||
- change description <2>
|
||||
- tag: change description <3>
|
||||
- tag1,tag2: change description <4>
|
||||
- tag: Multi-line description <5>
|
||||
must be indented and can use
|
||||
additional markdown syntax
|
||||
<6>
|
||||
## 1.0.0 <7>
|
||||
[...]
|
||||
```
|
||||
|
||||
1. Latest version is the first line of https://www.elastic.co/guide/en/logstash/current/CHANGELOG.html. Each version identifier should be a level-2 header using `##`
|
||||
2. One change description is described as a list item using a dash `-` aligned under the version identifier
|
||||
3. One change can be tagged by a word and suffixed by `:`.<br> Common tags are `bugfix`, `feature`, `doc`, `test` or `internal`.
|
||||
4. One change can have multiple tags separated by a comma and suffixed by `:`
|
||||
5. A multi-line change description must be properly indented
|
||||
6. Please take care to **separate versions with an empty line**
|
||||
7. Previous version identifier
|
||||
|
||||
|
||||
|
||||
### Continuous Integration [_continuous_integration]
|
||||
|
||||
Plugins are setup with automated continuous integration (CI) environments and there should be a corresponding badge on each Github page. If it’s missing, please contact the Logstash core team.
|
||||
|
||||
Every Pull Request opened automatically triggers a CI run. To conduct a manual run, comment “Jenkins, please test this.” on the Pull Request.
|
||||
|
||||
|
||||
## Versioning Plugins [_versioning_plugins]
|
||||
|
||||
Logstash core and its plugins have separate product development lifecycles. Hence the versioning and release strategy for the core and plugins do not have to be aligned. In fact, this was one of our goals during the great separation of plugins work in Logstash 1.5.
|
||||
|
||||
At times, there will be changes in core API in Logstash, which will require mass update of plugins to reflect the changes in core. However, this does not happen frequently.
|
||||
|
||||
For plugins, we would like to adhere to a versioning and release strategy that can better inform our users, about any breaking changes to the Logstash configuration formats and functionality.
|
||||
|
||||
Plugin releases follows a three-placed numbering scheme X.Y.Z. where X denotes a major release version which may break compatibility with existing configuration or functionality. Y denotes releases which includes features which are backward compatible. Z denotes releases which includes bug fixes and patches.
|
||||
|
||||
|
||||
### Changing the version [_changing_the_version]
|
||||
|
||||
Version can be changed in the Gemspec, which needs to be associated with a changelog entry. Following this, we can publish the gem to RubyGem.org manually. At this point only the core developers can publish a gem.
|
||||
|
||||
|
||||
### Labeling [_labeling]
|
||||
|
||||
Labeling is a critical aspect of maintaining plugins. All issues in GitHub should be labeled correctly so it can:
|
||||
|
||||
* Provide good feedback to users/developers
|
||||
* Help prioritize changes
|
||||
* Be used in release notes
|
||||
|
||||
Most labels are self explanatory, but here’s a quick recap of few important labels:
|
||||
|
||||
* `bug`: Labels an issue as an unintentional defect
|
||||
* `needs details`: If a the issue reporter has incomplete details, please ask them for more info and label as needs details.
|
||||
* `missing cla`: Contributor License Agreement is missing and patch cannot be accepted without it
|
||||
* `adopt me`: Ask for help from the community to take over this issue
|
||||
* `enhancement`: New feature, not a bug fix
|
||||
* `needs tests`: Patch has no tests, and cannot be accepted without unit/integration tests
|
||||
* `docs`: Documentation related issue/PR
|
||||
|
||||
|
||||
## Logging [_logging]
|
||||
|
||||
Although it’s important not to bog down performance with excessive logging, debug level logs can be immensely helpful when diagnosing and troubleshooting issues with Logstash. Please remember to liberally add debug logs wherever it makes sense as users will be forever gracious.
|
||||
|
||||
```shell
|
||||
@logger.debug("Logstash loves debug logs!", :actions => actions)
|
||||
```
|
||||
|
||||
|
||||
## Contributor License Agreement (CLA) Guidance [_contributor_license_agreement_cla_guidance]
|
||||
|
||||
Why is a [CLA](https://www.elastic.co/contributor-agreement) required?
|
||||
: We ask this of all Contributors in order to assure our users of the origin and continuing existence of the code. We are not asking Contributors to assign copyright to us, but to give us the right to distribute a Contributor’s code without restriction.
|
||||
|
||||
Please make sure the CLA is signed by every Contributor prior to reviewing PRs and commits.
|
||||
: Contributors only need to sign the CLA once and should sign with the same email as used in Github. If a Contributor signs the CLA after a PR is submitted, they can refresh the automated CLA checker by pushing another comment on the PR after 5 minutes of signing.
|
||||
|
||||
|
||||
## Need Help? [_need_help]
|
||||
|
||||
Ping @logstash-core on Github to get the attention of the Logstash core team.
|
||||
|
||||
|
||||
## Community Administration [_community_administration]
|
||||
|
||||
The core team is there to support the plugin Maintainers and overall ecosystem.
|
||||
|
||||
Maintainers should propose Contributors to become a Maintainer.
|
||||
|
||||
Contributors and Maintainers should follow the Elastic Community [Code of Conduct](https://www.elastic.co/community/codeofconduct). The core team should block or ban "bad actors".
|
||||
|
||||
11
docs/extend/contribute-to-core.md
Normal file
11
docs/extend/contribute-to-core.md
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/contribute-to-core.html
|
||||
---
|
||||
|
||||
# Extending Logstash core [contribute-to-core]
|
||||
|
||||
We also welcome contributions and bug fixes to the Logstash core feature set.
|
||||
|
||||
Please read through our [contribution](https://github.com/elastic/logstash/blob/main/CONTRIBUTING.md) guide, and the Logstash [readme](https://github.com/elastic/logstash/blob/main/README.md) document.
|
||||
|
||||
386
docs/extend/contributing-patch-plugin.md
Normal file
386
docs/extend/contributing-patch-plugin.md
Normal file
|
|
@ -0,0 +1,386 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/contributing-patch-plugin.html
|
||||
---
|
||||
|
||||
# Contributing a patch to a Logstash plugin [contributing-patch-plugin]
|
||||
|
||||
This section discusses the information you need to know to successfully contribute a patch to a Logstash plugin.
|
||||
|
||||
Each plugin defines its own configuration options. These control the behavior of the plugin to some degree. Configuration option definitions commonly include:
|
||||
|
||||
* Data validation
|
||||
* Default value
|
||||
* Any required flags
|
||||
|
||||
Plugins are subclasses of a Logstash base class. A plugin’s base class defines common configuration and methods.
|
||||
|
||||
## Input plugins [contrib-patch-input]
|
||||
|
||||
Input plugins ingest data from an external source. Input plugins are always associated with a codec. An input plugin always has an associated codec plugin. Input and codec plugins operate in conjunction to create a Logstash event and add that event to the processing queue. An input codec is a subclass of the `LogStash::Inputs::Base` class.
|
||||
|
||||
### Input API [input-api]
|
||||
|
||||
`#register() -> nil`
|
||||
: Required. This API sets up resources for the plugin, typically the connection to the external source.
|
||||
|
||||
`#run(queue) -> nil`
|
||||
: Required. This API fetches or listens for source data, typically looping until stopped. Must handle errors inside the loop. Pushes any created events to the queue object specified in the method argument. Some inputs may receive batched data to minimize the external call overhead.
|
||||
|
||||
`#stop() -> nil`
|
||||
: Optional. Stops external connections and cleans up.
|
||||
|
||||
|
||||
|
||||
## Codec plugins [contrib-patch-codec]
|
||||
|
||||
Codec plugins decode input data that has a specific structure, such as JSON input data. A codec plugin is a subclass of `LogStash::Codecs::Base`.
|
||||
|
||||
### Codec API [codec-api]
|
||||
|
||||
`#register() -> nil`
|
||||
: Identical to the API of the same name for input plugins.
|
||||
|
||||
`#decode(data){|event| block} -> nil`
|
||||
: Must be implemented. Used to create an Event from the raw data given in the method argument. Must handle errors. The caller must provide a Ruby block. The block is called with the created Event.
|
||||
|
||||
`#encode(event) -> nil`
|
||||
: Required. Used to create a structured data object from the given Event. May handle errors. This method calls a block that was previously stored as @on_event with two arguments: the original event and the data object.
|
||||
|
||||
|
||||
|
||||
## Filter plugins [contrib-patch-filter]
|
||||
|
||||
A mechanism to change, mutate or merge one or more Events. A filter plugin is a subclass of the `LogStash::Filters::Base` class.
|
||||
|
||||
### Filter API [filter-api]
|
||||
|
||||
`#register() -> nil`
|
||||
: Identical to the API of the same name for input plugins.
|
||||
|
||||
`#filter(event) -> nil`
|
||||
: Required. May handle errors. Used to apply a mutation function to the given event.
|
||||
|
||||
|
||||
|
||||
## Output plugins [contrib-patch-output]
|
||||
|
||||
A mechanism to send an event to an external destination. This process may require serialization. An output plugin is a subclass of the `LogStash::Outputs::Base` class.
|
||||
|
||||
### Output API [output-api]
|
||||
|
||||
`#register() -> nil`
|
||||
: Identical to the API of the same name for input plugins.
|
||||
|
||||
`#receive(event) -> nil`
|
||||
: Required. Must handle errors. Used to prepare the given event for transmission to the external destination. Some outputs may buffer the prepared events to batch transmit to the destination.
|
||||
|
||||
|
||||
|
||||
## Process [patch-process]
|
||||
|
||||
A bug or feature is identified. An issue is created in the plugin repository. A patch is created and a pull request (PR) is submitted. After review and possible rework the PR is merged and the plugin is published.
|
||||
|
||||
The [Community Maintainer Guide](/extend/community-maintainer.md) explains, in more detail, the process of getting a patch accepted, merged and published. The Community Maintainer Guide also details the roles that contributors and maintainers are expected to perform.
|
||||
|
||||
|
||||
## Testing methodologies [test-methods]
|
||||
|
||||
### Test driven development [tdd]
|
||||
|
||||
Test driven development (TDD) describes a methodology for using tests to guide evolution of source code. For our purposes, we are use only a part of it. Before writing the fix, we create tests that illustrate the bug by failing. We stop when we have written enough code to make the tests pass and submit the fix and tests as a patch. It is not necessary to write the tests before the fix, but it is very easy to write a passing test afterwards that may not actually verify that the fault is really fixed especially if the fault can be triggered via multiple execution paths or varying input data.
|
||||
|
||||
|
||||
### RSpec framework [rspec]
|
||||
|
||||
Logstash uses Rspec, a Ruby testing framework, to define and run the test suite. What follows is a summary of various sources.
|
||||
|
||||
```ruby
|
||||
2 require "logstash/devutils/rspec/spec_helper"
|
||||
3 require "logstash/plugin"
|
||||
4
|
||||
5 describe "outputs/riemann" do
|
||||
6 describe "#register" do
|
||||
7 let(:output) do
|
||||
8 LogStash::Plugin.lookup("output", "riemann").new(configuration)
|
||||
9 end
|
||||
10
|
||||
11 context "when no protocol is specified" do
|
||||
12 let(:configuration) { Hash.new }
|
||||
13
|
||||
14 it "the method completes without error" do
|
||||
15 expect {output.register}.not_to raise_error
|
||||
16 end
|
||||
17 end
|
||||
18
|
||||
19 context "when a bad protocol is specified" do
|
||||
20 let(:configuration) { {"protocol" => "fake"} }
|
||||
21
|
||||
22 it "the method fails with error" do
|
||||
23 expect {output.register}.to raise_error
|
||||
24 end
|
||||
25 end
|
||||
26
|
||||
27 context "when the tcp protocol is specified" do
|
||||
28 let(:configuration) { {"protocol" => "tcp"} }
|
||||
29
|
||||
30 it "the method completes without error" do
|
||||
31 expect {output.register}.not_to raise_error
|
||||
32 end
|
||||
33 end
|
||||
34 end
|
||||
35
|
||||
36 describe "#receive" do
|
||||
37 let(:output) do
|
||||
38 LogStash::Plugin.lookup("output", "riemann").new(configuration)
|
||||
39 end
|
||||
40
|
||||
41 context "when operating normally" do
|
||||
42 let(:configuration) { Hash.new }
|
||||
43 let(:event) do
|
||||
44 data = {"message"=>"hello", "@version"=>"1",
|
||||
45 "@timestamp"=>"2015-06-03T23:34:54.076Z",
|
||||
46 "host"=>"vagrant-ubuntu-trusty-64"}
|
||||
47 LogStash::Event.new(data)
|
||||
48 end
|
||||
49
|
||||
50 before(:example) do
|
||||
51 output.register
|
||||
52 end
|
||||
53
|
||||
54 it "should accept the event" do
|
||||
55 expect { output.receive event }.not_to raise_error
|
||||
56 end
|
||||
57 end
|
||||
58 end
|
||||
59 end
|
||||
```
|
||||
|
||||
```ruby
|
||||
describe(string){block} -> nil
|
||||
describe(Class){block} -> nil
|
||||
```
|
||||
|
||||
With RSpec, we are always describing the plugin method behavior. The describe block is added in logical sections and can accept either an existing class name or a string. The string used in line 5 is the plugin name. Line 6 is the register method, line 36 is the receive method. It is a RSpec convention to prefix instance methods with one hash and class methods with one dot.
|
||||
|
||||
```ruby
|
||||
context(string){block} -> nil
|
||||
```
|
||||
|
||||
In RSpec, context blocks define sections that group tests by a variation. The string should start with the word `when` and then detail the variation. See line 11. The tests in the content block should should only be for that variation.
|
||||
|
||||
```ruby
|
||||
let(symbol){block} -> nil
|
||||
```
|
||||
|
||||
In RSpec, `let` blocks define resources for use in the test blocks. These resources are reinitialized for every test block. They are available as method calls inside the test block. Define `let` blocks in `describe` and `context` blocks, which scope the `let` block and any other nested blocks. You can use other `let` methods defined later within the `let` block body. See lines 7-9, which define the output resource and use the configuration method, defined with different variations in lines 12, 20 and 28.
|
||||
|
||||
```ruby
|
||||
before(symbol){block} -> nil - symbol is one of :suite, :context, :example, but :all and :each are synonyms for :suite and :example respectively.
|
||||
```
|
||||
|
||||
In RSpec, `before` blocks are used to further set up any resources that would have been initialized in a `let` block. You cannot define `let` blocks inside `before` blocks.
|
||||
|
||||
You can also define `after` blocks, which are typically used to clean up any setup activity performed by a `before` block.
|
||||
|
||||
```ruby
|
||||
it(string){block} -> nil
|
||||
```
|
||||
|
||||
In RSpec, `it` blocks set the expectations that verify the behavior of the tested code. The string should not start with *it* or *should*, but needs to express the outcome of the expectation. When put together the texts from the enclosing describe, `context` and `it` blocks should form a fairly readable sentence, as in lines 5, 6, 11 and 14:
|
||||
|
||||
```ruby
|
||||
outputs/riemann
|
||||
#register when no protocol is specified the method completes without error
|
||||
```
|
||||
|
||||
Readable code like this make the goals of tests easy to understand.
|
||||
|
||||
```ruby
|
||||
expect(object){block} -> nil
|
||||
```
|
||||
|
||||
In RSpec, the expect method verifies a statement that compares an actual result to an expected result. The `expect` method is usually paired with a call to the `to` or `not_to` methods. Use the block form when expecting errors or observing for changes. The `to` or `not_to` methods require a `matcher` object that encapsulates the expected value. The argument form of the `expect` method encapsulates the actual value. When put together the whole line tests the actual against the expected value.
|
||||
|
||||
```ruby
|
||||
raise_error(error class|nil) -> matcher instance
|
||||
be(object) -> matcher instance
|
||||
eq(object) -> matcher instance
|
||||
eql(object) -> matcher instance
|
||||
for more see http://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers
|
||||
```
|
||||
|
||||
In RSpec, a matcher is an object generated by the equivalent method call (be, eq) that will be used to evaluate the expected against the actual values.
|
||||
|
||||
|
||||
|
||||
## Putting it all together [all-together]
|
||||
|
||||
This example fixes an [issue](https://github.com/logstash-plugins/logstash-output-zeromq/issues/9) in the ZeroMQ output plugin. The issue does not require knowledge of ZeroMQ.
|
||||
|
||||
The activities in this example have the following prerequisites:
|
||||
|
||||
* A minimal knowledge of Git and Github. See the [Github boot camp](https://help.github.com/categories/bootcamp/).
|
||||
* A text editor.
|
||||
* A JRuby [runtime](https://www.ruby-lang.org/en/documentation/installation/#managers) [environment](https://howistart.org/posts/ruby/1). The `chruby` tool manages Ruby versions.
|
||||
* JRuby 1.7.22 or later.
|
||||
* The `bundler` and `rake` gems installed.
|
||||
* ZeroMQ [installed](http://zeromq.org/intro:get-the-software).
|
||||
|
||||
1. In Github, fork the ZeroMQ [output plugin repository](https://github.com/logstash-plugins/logstash-output-zeromq).
|
||||
2. On your local machine, [clone](https://help.github.com/articles/fork-a-repo/) the fork to a known folder such as `logstash/`.
|
||||
3. Open the following files in a text editor:
|
||||
|
||||
* `logstash-output-zeromq/lib/logstash/outputs/zeromq.rb`
|
||||
* `logstash-output-zeromq/lib/logstash/util/zeromq.rb`
|
||||
* `logstash-output-zeromq/spec/outputs/zeromq_spec.rb`
|
||||
|
||||
4. According to the issue, log output in server mode must indicate `bound`. Furthermore, the test file contains no tests.
|
||||
|
||||
::::{note}
|
||||
Line 21 of `util/zeromq.rb` reads `@logger.info("0mq: #{server? ? 'connected' : 'bound'}", :address => address)`
|
||||
::::
|
||||
|
||||
5. In the text editor, require `zeromq.rb` for the file `zeromq_spec.rb` by adding the following lines:
|
||||
|
||||
```ruby
|
||||
require "logstash/outputs/zeromq"
|
||||
require "logstash/devutils/rspec/spec_helper"
|
||||
```
|
||||
|
||||
6. The desired error message should read:
|
||||
|
||||
```ruby
|
||||
LogStash::Outputs::ZeroMQ when in server mode a 'bound' info line is logged
|
||||
```
|
||||
|
||||
To properly generate this message, add a `describe` block with the fully qualified class name as the argument, a context block, and an `it` block.
|
||||
|
||||
```ruby
|
||||
describe LogStash::Outputs::ZeroMQ do
|
||||
context "when in server mode" do
|
||||
it "a 'bound' info line is logged" do
|
||||
end
|
||||
end
|
||||
end
|
||||
```
|
||||
|
||||
7. To add the missing test, use an instance of the ZeroMQ output and a substitute logger. This example uses an RSpec feature called *test doubles* as the substitute logger.
|
||||
|
||||
Add the following lines to `zeromq_spec.rb`, after `describe LogStash::Outputs::ZeroMQ do` and before `context "when in server mode" do`:
|
||||
|
||||
```ruby
|
||||
let(:output) { described_class.new("mode" => "server", "topology" => "pushpull" }
|
||||
let(:tracer) { double("logger") }
|
||||
```
|
||||
|
||||
8. Add the body to the `it` block. Add the following five lines after the line `context "when in server mode" do`:
|
||||
|
||||
```ruby
|
||||
allow(tracer).to receive(:debug)<1>
|
||||
output.logger = logger<2>
|
||||
expect(tracer).to receive(:info).with("0mq: bound", {:address=>"tcp://127.0.0.1:2120"})<3>
|
||||
output.register<4>
|
||||
output.do_close<5>
|
||||
```
|
||||
|
||||
|
||||
1. Allow the double to receive `debug` method calls.
|
||||
2. Make the output use the test double.
|
||||
3. Set an expectation on the test to receive an `info` method call.
|
||||
4. Call `register` on the output.
|
||||
5. Call `do_close` on the output so the test does not hang.
|
||||
|
||||
|
||||
At the end of the modifications, the relevant code section reads:
|
||||
|
||||
```ruby
|
||||
require "logstash/outputs/zeromq"
|
||||
require "logstash/devutils/rspec/spec_helper"
|
||||
|
||||
describe LogStash::Outputs::ZeroMQ do
|
||||
let(:output) { described_class.new("mode" => "server", "topology" => "pushpull") }
|
||||
let(:tracer) { double("logger") }
|
||||
|
||||
context "when in server mode" do
|
||||
it "a ‘bound’ info line is logged" do
|
||||
allow(tracer).to receive(:debug)
|
||||
output.logger = tracer
|
||||
expect(tracer).to receive(:info).with("0mq: bound", {:address=>"tcp://127.0.0.1:2120"})
|
||||
output.register
|
||||
output.do_close
|
||||
end
|
||||
end
|
||||
end
|
||||
```
|
||||
|
||||
To run this test:
|
||||
|
||||
1. Open a terminal window
|
||||
2. Navigate to the cloned plugin folder
|
||||
3. The first time you run the test, run the command `bundle install`
|
||||
4. Run the command `bundle exec rspec`
|
||||
|
||||
Assuming all prerequisites were installed correctly, the test fails with output similar to:
|
||||
|
||||
```shell
|
||||
Using Accessor#strict_set for specs
|
||||
Run options: exclude {:redis=>true, :socket=>true, :performance=>true, :couchdb=>true, :elasticsearch=>true,
|
||||
:elasticsearch_secure=>true, :export_cypher=>true, :integration=>true, :windows=>true}
|
||||
|
||||
LogStash::Outputs::ZeroMQ
|
||||
when in server mode
|
||||
a ‘bound’ info line is logged (FAILED - 1)
|
||||
|
||||
Failures:
|
||||
|
||||
1) LogStash::Outputs::ZeroMQ when in server mode a ‘bound’ info line is logged
|
||||
Failure/Error: output.register
|
||||
Double "logger" received :info with unexpected arguments
|
||||
expected: ("0mq: bound", {:address=>"tcp://127.0.0.1:2120"})
|
||||
got: ("0mq: connected", {:address=>"tcp://127.0.0.1:2120"})
|
||||
# ./lib/logstash/util/zeromq.rb:21:in `setup'
|
||||
# ./lib/logstash/outputs/zeromq.rb:92:in `register'
|
||||
# ./lib/logstash/outputs/zeromq.rb:91:in `register'
|
||||
# ./spec/outputs/zeromq_spec.rb:13:in `(root)'
|
||||
# /Users/guy/.gem/jruby/1.9.3/gems/rspec-wait-0.0.7/lib/rspec/wait.rb:46:in `(root)'
|
||||
|
||||
Finished in 0.133 seconds (files took 1.28 seconds to load)
|
||||
1 example, 1 failure
|
||||
|
||||
Failed examples:
|
||||
|
||||
rspec ./spec/outputs/zeromq_spec.rb:10 # LogStash::Outputs::ZeroMQ when in server mode a ‘bound’ info line is logged
|
||||
|
||||
Randomized with seed 2568
|
||||
```
|
||||
|
||||
To correct the error, open the `util/zeromq.rb` file in your text editor and swap the positions of the words `connected` and `bound` on line 21. Line 21 now reads:
|
||||
|
||||
```ruby
|
||||
@logger.info("0mq: #{server? ? 'bound' : 'connected'}", :address => address)
|
||||
```
|
||||
|
||||
Run the test again with the `bundle exec rspec` command.
|
||||
|
||||
The test passes with output similar to:
|
||||
|
||||
```shell
|
||||
Using Accessor#strict_set for specs
|
||||
Run options: exclude {:redis=>true, :socket=>true, :performance=>true, :couchdb=>true, :elasticsearch=>true, :elasticsearch_secure=>true, :export_cypher=>true, :integration=>true, :windows=>true}
|
||||
|
||||
LogStash::Outputs::ZeroMQ
|
||||
when in server mode
|
||||
a ‘bound’ info line is logged
|
||||
|
||||
Finished in 0.114 seconds (files took 1.22 seconds to load)
|
||||
1 example, 0 failures
|
||||
|
||||
Randomized with seed 45887
|
||||
```
|
||||
|
||||
[Commit](https://help.github.com/articles/fork-a-repo/#next-steps) the changes to git and Github.
|
||||
|
||||
Your pull request is visible from the [Pull Requests](https://github.com/logstash-plugins/logstash-output-zeromq/pulls) section of the original Github repository. The plugin maintainers review your work, suggest changes if necessary, and merge and publish a new version of the plugin.
|
||||
|
||||
|
||||
47
docs/extend/create-logstash-plugins.md
Normal file
47
docs/extend/create-logstash-plugins.md
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/contributing-java-plugin.html
|
||||
---
|
||||
|
||||
# Create Logstash plugins [contributing-java-plugin]
|
||||
|
||||
Now you can write your own Java plugin for use with {{ls}}. We have provided instructions and GitHub examples to give you a head start.
|
||||
|
||||
Native support for Java plugins in {{ls}} consists of several components:
|
||||
|
||||
* Extensions to the Java execution engine to support running Java plugins in Logstash pipelines
|
||||
* APIs for developing Java plugins. The APIs are in the `co.elastic.logstash.api` package. A Java plugin might break if it references classes or specific concrete implementations of API interfaces outside that package. The implementation of classes outside of the API package may change at any time.
|
||||
* Tooling to automate the packaging and deployment of Java plugins in Logstash.
|
||||
|
||||
|
||||
## Process overview [_process_overview]
|
||||
|
||||
Here are the steps:
|
||||
|
||||
1. Choose the type of plugin you want to create: input, codec, filter, or output.
|
||||
2. Set up your environment.
|
||||
3. Code the plugin.
|
||||
4. Package and deploy the plugin.
|
||||
5. Run Logstash with your new plugin.
|
||||
|
||||
|
||||
### Let’s get started [_lets_get_started]
|
||||
|
||||
Here are the example repos:
|
||||
|
||||
* [Input plugin example](https://github.com/logstash-plugins/logstash-input-java_input_example)
|
||||
* [Codec plugin example](https://github.com/logstash-plugins/logstash-codec-java_codec_example)
|
||||
* [Filter plugin example](https://github.com/logstash-plugins/logstash-filter-java_filter_example)
|
||||
* [Output plugin example](https://github.com/logstash-plugins/logstash-output-java_output_example)
|
||||
|
||||
Here are the instructions:
|
||||
|
||||
* [How to write a Java input plugin](/extend/java-input-plugin.md)
|
||||
* [How to write a Java codec plugin](/extend/java-codec-plugin.md)
|
||||
* [How to write a Java filter plugin](/extend/java-filter-plugin.md)
|
||||
* [How to write a Java output plugin](/extend/java-output-plugin.md)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
637
docs/extend/filter-new-plugin.md
Normal file
637
docs/extend/filter-new-plugin.md
Normal file
|
|
@ -0,0 +1,637 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/filter-new-plugin.html
|
||||
---
|
||||
|
||||
# How to write a Logstash filter plugin [filter-new-plugin]
|
||||
|
||||
To develop a new filter for Logstash, build a self-contained Ruby gem whose source code lives in its own GitHub repository. The Ruby gem can then be hosted and shared on RubyGems.org. You can use the example filter implementation as a starting point. (If you’re unfamiliar with Ruby, you can find an excellent quickstart guide at [https://www.ruby-lang.org/en/documentation/quickstart/](https://www.ruby-lang.org/en/documentation/quickstart/).)
|
||||
|
||||
## Get started [_get_started_3]
|
||||
|
||||
Let’s step through creating a filter plugin using the [example filter plugin](https://github.com/logstash-plugins/logstash-filter-example/).
|
||||
|
||||
### Create a GitHub repo for your new plugin [_create_a_github_repo_for_your_new_plugin_3]
|
||||
|
||||
Each Logstash plugin lives in its own GitHub repository. To create a new repository for your plugin:
|
||||
|
||||
1. Log in to GitHub.
|
||||
2. Click the **Repositories** tab. You’ll see a list of other repositories you’ve forked or contributed to.
|
||||
3. Click the green **New** button in the upper right.
|
||||
4. Specify the following settings for your new repo:
|
||||
|
||||
* **Repository name** — a unique name of the form `logstash-filter-pluginname`.
|
||||
* **Public or Private** — your choice, but the repository must be Public if you want to submit it as an official plugin.
|
||||
* **Initialize this repository with a README** — enables you to immediately clone the repository to your computer.
|
||||
|
||||
5. Click **Create Repository**.
|
||||
|
||||
|
||||
### Use the plugin generator tool [_use_the_plugin_generator_tool_3]
|
||||
|
||||
You can create your own Logstash plugin in seconds! The `generate` subcommand of `bin/logstash-plugin` creates the foundation for a new Logstash plugin with templatized files. It creates the correct directory structure, gemspec files, and dependencies so you can start adding custom code to process data with Logstash.
|
||||
|
||||
For more information, see [Generating plugins](/reference/plugin-generator.md)
|
||||
|
||||
|
||||
### Copy the filter code [_copy_the_filter_code]
|
||||
|
||||
Alternatively, you can use the examples repo we host on github.com
|
||||
|
||||
1. **Clone your plugin.** Replace `GITUSERNAME` with your github username, and `MYPLUGINNAME` with your plugin name.
|
||||
|
||||
* `git clone https://github.com/GITUSERNAME/logstash-``filter-MYPLUGINNAME.git`
|
||||
|
||||
* alternately, via ssh: `git clone git@github.com:GITUSERNAME/logstash``-filter-MYPLUGINNAME.git`
|
||||
|
||||
* `cd logstash-filter-MYPLUGINNAME`
|
||||
|
||||
2. **Clone the filter plugin example and copy it to your plugin branch.**
|
||||
|
||||
You don’t want to include the example .git directory or its contents, so delete it before you copy the example.
|
||||
|
||||
* `cd /tmp`
|
||||
* `git clone https://github.com/logstash-plugins/logstash``-filter-example.git`
|
||||
* `cd logstash-filter-example`
|
||||
* `rm -rf .git`
|
||||
* `cp -R * /path/to/logstash-filter-mypluginname/`
|
||||
|
||||
3. **Rename the following files to match the name of your plugin.**
|
||||
|
||||
* `logstash-filter-example.gemspec`
|
||||
* `example.rb`
|
||||
* `example_spec.rb`
|
||||
|
||||
```txt
|
||||
cd /path/to/logstash-filter-mypluginname
|
||||
mv logstash-filter-example.gemspec logstash-filter-mypluginname.gemspec
|
||||
mv lib/logstash/filters/example.rb lib/logstash/filters/mypluginname.rb
|
||||
mv spec/filters/example_spec.rb spec/filters/mypluginname_spec.rb
|
||||
```
|
||||
|
||||
|
||||
Your file structure should look like this:
|
||||
|
||||
```txt
|
||||
$ tree logstash-filter-mypluginname
|
||||
├── Gemfile
|
||||
├── LICENSE
|
||||
├── README.md
|
||||
├── Rakefile
|
||||
├── lib
|
||||
│ └── logstash
|
||||
│ └── filters
|
||||
│ └── mypluginname.rb
|
||||
├── logstash-filter-mypluginname.gemspec
|
||||
└── spec
|
||||
└── filters
|
||||
└── mypluginname_spec.rb
|
||||
```
|
||||
|
||||
For more information about the Ruby gem file structure and an excellent walkthrough of the Ruby gem creation process, see [http://timelessrepo.com/making-ruby-gems](http://timelessrepo.com/making-ruby-gems)
|
||||
|
||||
|
||||
### See what your plugin looks like [_see_what_your_plugin_looks_like_3]
|
||||
|
||||
Before we dive into the details, open up the plugin file in your favorite text editor and take a look.
|
||||
|
||||
```ruby
|
||||
require "logstash/filters/base"
|
||||
require "logstash/namespace"
|
||||
|
||||
# Add any asciidoc formatted documentation here
|
||||
# This example filter will replace the contents of the default
|
||||
# message field with whatever you specify in the configuration.
|
||||
#
|
||||
# It is only intended to be used as an example.
|
||||
class LogStash::Filters::Example < LogStash::Filters::Base
|
||||
|
||||
# Setting the config_name here is required. This is how you
|
||||
# configure this filter from your Logstash config.
|
||||
#
|
||||
# filter {
|
||||
# example { message => "My message..." }
|
||||
# }
|
||||
config_name "example"
|
||||
|
||||
# Replace the message with this value.
|
||||
config :message, :validate => :string, :default => "Hello World!"
|
||||
|
||||
|
||||
public
|
||||
def register
|
||||
# Add instance variables
|
||||
end # def register
|
||||
|
||||
public
|
||||
def filter(event)
|
||||
|
||||
if @message
|
||||
# Replace the event message with our message as configured in the
|
||||
# config file.
|
||||
event.set("message", @message)
|
||||
end
|
||||
|
||||
# filter_matched should go in the last line of our successful code
|
||||
filter_matched(event)
|
||||
end # def filter
|
||||
|
||||
end # class LogStash::Filters::Example
|
||||
```
|
||||
|
||||
|
||||
|
||||
## Coding filter plugins [_coding_filter_plugins]
|
||||
|
||||
Now let’s take a line-by-line look at the example plugin.
|
||||
|
||||
### `require` Statements [_require_statements_3]
|
||||
|
||||
Logstash filter plugins require parent classes defined in `logstash/filters/base` and logstash/namespace:
|
||||
|
||||
```ruby
|
||||
require "logstash/filters/base"
|
||||
require "logstash/namespace"
|
||||
```
|
||||
|
||||
Of course, the plugin you build may depend on other code, or even gems. Just put them here along with these Logstash dependencies.
|
||||
|
||||
|
||||
|
||||
## Plugin Body [_plugin_body_3]
|
||||
|
||||
Let’s go through the various elements of the plugin itself.
|
||||
|
||||
### `class` Declaration [_class_declaration_3]
|
||||
|
||||
The filter plugin class should be a subclass of `LogStash::Filters::Base`:
|
||||
|
||||
```ruby
|
||||
class LogStash::Filters::Example < LogStash::Filters::Base
|
||||
```
|
||||
|
||||
The class name should closely mirror the plugin name, for example:
|
||||
|
||||
```ruby
|
||||
LogStash::Filters::Example
|
||||
```
|
||||
|
||||
|
||||
### `config_name` [_config_name_3]
|
||||
|
||||
```ruby
|
||||
config_name "example"
|
||||
```
|
||||
|
||||
This is the name your plugin will call inside the filter configuration block.
|
||||
|
||||
If you set `config_name "example"` in your plugin code, the corresponding Logstash configuration block would need to look like this:
|
||||
|
||||
|
||||
|
||||
## Configuration Parameters [_configuration_parameters_3]
|
||||
|
||||
```ruby
|
||||
config :variable_name, :validate => :variable_type, :default => "Default value", :required => boolean, :deprecated => boolean, :obsolete => string
|
||||
```
|
||||
|
||||
The configuration, or `config` section allows you to define as many (or as few) parameters as are needed to enable Logstash to process events.
|
||||
|
||||
There are several configuration attributes:
|
||||
|
||||
* `:validate` - allows you to enforce passing a particular data type to Logstash for this configuration option, such as `:string`, `:password`, `:boolean`, `:number`, `:array`, `:hash`, `:path` (a file-system path), `uri`, `:codec` (since 1.2.0), `:bytes`. Note that this also works as a coercion in that if I specify "true" for boolean (even though technically a string), it will become a valid boolean in the config. This coercion works for the `:number` type as well where "1.2" becomes a float and "22" is an integer.
|
||||
* `:default` - lets you specify a default value for a parameter
|
||||
* `:required` - whether or not this parameter is mandatory (a Boolean `true` or
|
||||
* `:list` - whether or not this value should be a list of values. Will typecheck the list members, and convert scalars to one element lists. Note that this mostly obviates the array type, though if you need lists of complex objects that will be more suitable. `false`)
|
||||
* `:deprecated` - informational (also a Boolean `true` or `false`)
|
||||
* `:obsolete` - used to declare that a given setting has been removed and is no longer functioning. The idea is to provide an informed upgrade path to users who are still using a now-removed setting.
|
||||
|
||||
|
||||
## Plugin Methods [_plugin_methods_3]
|
||||
|
||||
Logstash filters must implement the `register` and `filter` methods.
|
||||
|
||||
### `register` Method [_register_method_3]
|
||||
|
||||
```ruby
|
||||
public
|
||||
def register
|
||||
end # def register
|
||||
```
|
||||
|
||||
The Logstash `register` method is like an `initialize` method. It was originally created to enforce having `super` called, preventing headaches for newbies. (Note: It may go away in favor of `initialize`, in conjunction with some enforced testing to ensure `super` is called.)
|
||||
|
||||
`public` means the method can be called anywhere, not just within the class. This is the default behavior for methods in Ruby, but it is specified explicitly here anyway.
|
||||
|
||||
You can also assign instance variables here (variables prepended by `@`). Configuration variables are now in scope as instance variables, like `@message`
|
||||
|
||||
|
||||
### `filter` Method [_filter_method]
|
||||
|
||||
```ruby
|
||||
public
|
||||
def filter(event)
|
||||
|
||||
if @message
|
||||
# Replace the event message with our message as configured in the
|
||||
# config file.
|
||||
event.set("message", @message)
|
||||
end
|
||||
|
||||
# filter_matched should go in the last line of our successful code
|
||||
filter_matched(event)
|
||||
end # def filter
|
||||
```
|
||||
|
||||
The plugin’s `filter` method is where the actual filtering work takes place! Inside the `filter` method you can refer to the event data using the `Event` object. Event is the main object that encapsulates data flow internally in Logstash and provides an [API](/reference/event-api.md) for the plugin developers to interact with the event’s content.
|
||||
|
||||
The `filter` method should also handle any [event dependent configuration](/reference/event-dependent-configuration.md) by explicitly calling the `sprintf` method available in Event class. For example:
|
||||
|
||||
```ruby
|
||||
field_foo = event.sprintf(field)
|
||||
```
|
||||
|
||||
Note that configuration variables are now in scope as instance variables, like `@message`
|
||||
|
||||
```ruby
|
||||
filter_matched(event)
|
||||
```
|
||||
|
||||
Calling the `filter_matched` method upon successful execution of the plugin will ensure that any fields or tags added through the Logstash configuration for this filter will be handled correctly. For example, any `add_field`, `remove_field`, `add_tag` and/or `remove_tag` actions will be performed at this time.
|
||||
|
||||
Event methods such as `event.cancel` are now available to control the workflow of the event being processed.
|
||||
|
||||
|
||||
|
||||
## Building the Plugin [_building_the_plugin_3]
|
||||
|
||||
At this point in the process you have coded your plugin and are ready to build a Ruby Gem from it. The following information will help you complete the process.
|
||||
|
||||
### External dependencies [_external_dependencies_3]
|
||||
|
||||
A `require` statement in Ruby is used to include necessary code. In some cases your plugin may require additional files. For example, the collectd plugin [uses](https://github.com/logstash-plugins/logstash-codec-collectd/blob/main/lib/logstash/codecs/collectd.rb#L148) the `types.db` file provided by collectd. In the main directory of your plugin, a file called `vendor.json` is where these files are described.
|
||||
|
||||
The `vendor.json` file contains an array of JSON objects, each describing a file dependency. This example comes from the [collectd](https://github.com/logstash-plugins/logstash-codec-collectd/blob/main/vendor.json) codec plugin:
|
||||
|
||||
```txt
|
||||
[{
|
||||
"sha1": "a90fe6cc53b76b7bdd56dc57950d90787cb9c96e",
|
||||
"url": "http://collectd.org/files/collectd-5.4.0.tar.gz",
|
||||
"files": [ "/src/types.db" ]
|
||||
}]
|
||||
```
|
||||
|
||||
* `sha1` is the sha1 signature used to verify the integrity of the file referenced by `url`.
|
||||
* `url` is the address from where Logstash will download the file.
|
||||
* `files` is an optional array of files to extract from the downloaded file. Note that while tar archives can use absolute or relative paths, treat them as absolute in this array. If `files` is not present, all files will be uncompressed and extracted into the vendor directory.
|
||||
|
||||
Another example of the `vendor.json` file is the [`geoip` filter](https://github.com/logstash-plugins/logstash-filter-geoip/blob/main/vendor.json)
|
||||
|
||||
The process used to download these dependencies is to call `rake vendor`. This will be discussed further in the testing section of this document.
|
||||
|
||||
Another kind of external dependency is on jar files. This will be described in the "Add a `gemspec` file" section.
|
||||
|
||||
|
||||
### Deprecated features [_deprecated_features_3]
|
||||
|
||||
As a plugin evolves, an option or feature may no longer serve the intended purpose, and the developer may want to *deprecate* its usage. Deprecation warns users about the option’s status, so they aren’t caught by surprise when it is removed in a later release.
|
||||
|
||||
{{ls}} 7.6 introduced a *deprecation logger* to make handling those situations easier. You can use the [adapter](https://github.com/logstash-plugins/logstash-mixin-deprecation_logger_support) to ensure that your plugin can use the deprecation logger while still supporting older versions of {{ls}}. See the [readme](https://github.com/logstash-plugins/logstash-mixin-deprecation_logger_support/blob/main/README.md) for more information and for instructions on using the adapter.
|
||||
|
||||
Deprecations are noted in the `logstash-deprecation.log` file in the `log` directory.
|
||||
|
||||
|
||||
### Add a Gemfile [_add_a_gemfile_3]
|
||||
|
||||
Gemfiles allow Ruby’s Bundler to maintain the dependencies for your plugin. Currently, all we’ll need is the Logstash gem, for testing, but if you require other gems, you should add them in here.
|
||||
|
||||
::::{tip}
|
||||
See [Bundler’s Gemfile page](http://bundler.io/gemfile.html) for more details.
|
||||
::::
|
||||
|
||||
|
||||
```ruby
|
||||
source 'https://rubygems.org'
|
||||
gemspec
|
||||
gem "logstash", :github => "elastic/logstash", :branch => "master"
|
||||
```
|
||||
|
||||
|
||||
|
||||
## Add a `gemspec` file [_add_a_gemspec_file_3]
|
||||
|
||||
Gemspecs define the Ruby gem which will be built and contain your plugin.
|
||||
|
||||
::::{tip}
|
||||
More information can be found on the [Rubygems Specification page](http://guides.rubygems.org/specification-reference/).
|
||||
::::
|
||||
|
||||
|
||||
```ruby
|
||||
Gem::Specification.new do |s|
|
||||
s.name = 'logstash-filter-example'
|
||||
s.version = '0.1.0'
|
||||
s.licenses = ['Apache License (2.0)']
|
||||
s.summary = "This filter does x, y, z in Logstash"
|
||||
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
||||
s.authors = ["Elastic"]
|
||||
s.email = 'info@elastic.co'
|
||||
s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
|
||||
s.require_paths = ["lib"]
|
||||
|
||||
# Files
|
||||
s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
|
||||
# Tests
|
||||
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
||||
|
||||
# Special flag to let us know this is actually a logstash plugin
|
||||
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
|
||||
|
||||
# Gem dependencies
|
||||
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
||||
s.add_development_dependency 'logstash-devutils'
|
||||
end
|
||||
```
|
||||
|
||||
It is appropriate to change these values to fit your plugin. In particular, `s.name` and `s.summary` should reflect your plugin’s name and behavior.
|
||||
|
||||
`s.licenses` and `s.version` are also important and will come into play when you are ready to publish your plugin.
|
||||
|
||||
Logstash and all its plugins are licensed under [Apache License, version 2 ("ALv2")](https://github.com/elastic/logstash/blob/main/LICENSE.txt). If you make your plugin publicly available via [RubyGems.org](http://rubygems.org), please make sure to have this line in your gemspec:
|
||||
|
||||
* `s.licenses = ['Apache License (2.0)']`
|
||||
|
||||
The gem version, designated by `s.version`, helps track changes to plugins over time. You should use [semver versioning](http://semver.org/) strategy for version numbers.
|
||||
|
||||
### Runtime and Development Dependencies [_runtime_and_development_dependencies_3]
|
||||
|
||||
At the bottom of the `gemspec` file is a section with a comment: `Gem dependencies`. This is where any other needed gems must be mentioned. If a gem is necessary for your plugin to function, it is a runtime dependency. If a gem are only used for testing, then it would be a development dependency.
|
||||
|
||||
::::{note}
|
||||
You can also have versioning requirements for your dependencies—including other Logstash plugins:
|
||||
|
||||
```ruby
|
||||
# Gem dependencies
|
||||
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
||||
s.add_development_dependency 'logstash-devutils'
|
||||
```
|
||||
|
||||
This gemspec has a runtime dependency on the logstash-core-plugin-api and requires that it have a version number greater than or equal to version 1.60 and less than or equal to version 2.99.
|
||||
|
||||
::::
|
||||
|
||||
|
||||
::::{important}
|
||||
All plugins have a runtime dependency on the `logstash-core-plugin-api` gem, and a development dependency on `logstash-devutils`.
|
||||
::::
|
||||
|
||||
|
||||
|
||||
### Jar dependencies [_jar_dependencies_3]
|
||||
|
||||
In some cases, such as the [Elasticsearch output plugin](https://github.com/logstash-plugins/logstash-output-elasticsearch/blob/main/logstash-output-elasticsearch.gemspec#L22-L23), your code may depend on a jar file. In cases such as this, the dependency is added in the gemspec file in this manner:
|
||||
|
||||
```ruby
|
||||
# Jar dependencies
|
||||
s.requirements << "jar 'org.elasticsearch:elasticsearch', '5.0.0'"
|
||||
s.add_runtime_dependency 'jar-dependencies'
|
||||
```
|
||||
|
||||
With these both defined, the install process will search for the required jar file at [http://mvnrepository.com](http://mvnrepository.com) and download the specified version.
|
||||
|
||||
|
||||
|
||||
## Document your plugin [_document_your_plugin_3]
|
||||
|
||||
Documentation is an important part of your plugin. All plugin documentation is rendered and placed in the [Logstash Reference](/reference/index.md) and the [Versioned plugin docs](logstash-docs://reference/integration-plugins.md).
|
||||
|
||||
See [Document your plugin](/extend/plugin-doc.md) for tips and guidelines.
|
||||
|
||||
|
||||
## Add Tests [_add_tests_3]
|
||||
|
||||
Logstash loves tests. Lots of tests. If you’re using your new filter plugin in a production environment, you’ll want to have some tests to ensure you are not breaking any existing functionality.
|
||||
|
||||
::::{note}
|
||||
A full exposition on RSpec is outside the scope of this document. Learn more about RSpec at [http://rspec.info](http://rspec.info)
|
||||
::::
|
||||
|
||||
|
||||
For help learning about tests and testing, look in the `spec/filters/` directory of several other similar plugins.
|
||||
|
||||
|
||||
## Clone and test! [_clone_and_test_3]
|
||||
|
||||
Now let’s start with a fresh clone of the plugin, build it and run the tests.
|
||||
|
||||
* **Clone your plugin into a temporary location** Replace `GITUSERNAME` with your github username, and `MYPLUGINNAME` with your plugin name.
|
||||
|
||||
* `git clone https://github.com/GITUSERNAME/logstash-``filter-MYPLUGINNAME.git`
|
||||
|
||||
* alternately, via ssh: `git clone git@github.com:GITUSERNAME/logstash-``filter-MYPLUGINNAME.git`
|
||||
|
||||
* `cd logstash-filter-MYPLUGINNAME`
|
||||
|
||||
|
||||
Then, you’ll need to install your plugins dependencies with bundler:
|
||||
|
||||
```
|
||||
bundle install
|
||||
```
|
||||
|
||||
::::{important}
|
||||
If your plugin has an external file dependency described in `vendor.json`, you must download that dependency before running or testing. You can do this by running:
|
||||
|
||||
```
|
||||
rake vendor
|
||||
```
|
||||
|
||||
::::
|
||||
|
||||
|
||||
And finally, run the tests:
|
||||
|
||||
```
|
||||
bundle exec rspec
|
||||
```
|
||||
|
||||
You should see a success message, which looks something like this:
|
||||
|
||||
```
|
||||
Finished in 0.034 seconds
|
||||
1 example, 0 failures
|
||||
```
|
||||
|
||||
Hooray! You’re almost there! (Unless you saw failures… you should fix those first).
|
||||
|
||||
|
||||
## Building and Testing [_building_and_testing_3]
|
||||
|
||||
Now you’re ready to build your (well-tested) plugin into a Ruby gem.
|
||||
|
||||
### Build [_build_3]
|
||||
|
||||
You already have all the necessary ingredients, so let’s go ahead and run the build command:
|
||||
|
||||
```sh
|
||||
gem build logstash-filter-example.gemspec
|
||||
```
|
||||
|
||||
That’s it! Your gem should be built and be in the same path with the name
|
||||
|
||||
```sh
|
||||
logstash-filter-mypluginname-0.1.0.gem
|
||||
```
|
||||
|
||||
The `s.version` number from your gemspec file will provide the gem version, in this case, `0.1.0`.
|
||||
|
||||
|
||||
### Test installation [_test_installation_3]
|
||||
|
||||
You should test install your plugin into a clean installation of Logstash. Download the latest version from the [Logstash downloads page](https://www.elastic.co/downloads/logstash/).
|
||||
|
||||
1. Untar and cd in to the directory:
|
||||
|
||||
```sh
|
||||
curl -O https://download.elastic.co/logstash/logstash/logstash-9.0.0.tar.gz
|
||||
tar xzvf logstash-9.0.0.tar.gz
|
||||
cd logstash-9.0.0
|
||||
```
|
||||
|
||||
2. Using the plugin tool, we can install the gem we just built.
|
||||
|
||||
* Replace `/my/logstash/plugins` with the correct path to the gem for your environment, and `0.1.0` with the correct version number from the gemspec file.
|
||||
|
||||
```sh
|
||||
bin/logstash-plugin install /my/logstash/plugins/logstash-filter-example/logstash-filter-example-0.1.0.gem
|
||||
```
|
||||
|
||||
* After running this, you should see feedback from Logstash that it was successfully installed:
|
||||
|
||||
```sh
|
||||
validating /my/logstash/plugins/logstash-filter-example/logstash-filter-example-0.1.0.gem >= 0
|
||||
Valid logstash plugin. Continuing...
|
||||
Successfully installed 'logstash-filter-example' with version '0.1.0'
|
||||
```
|
||||
|
||||
::::{tip}
|
||||
You can also use the Logstash plugin tool to determine which plugins are currently available:
|
||||
|
||||
```sh
|
||||
bin/logstash-plugin list
|
||||
```
|
||||
|
||||
Depending on what you have installed, you might see a short or long list of plugins: inputs, codecs, filters and outputs.
|
||||
|
||||
::::
|
||||
|
||||
3. Now try running Logstash with a simple configuration passed in via the command-line, using the `-e` flag.
|
||||
|
||||
::::{note}
|
||||
Your results will depend on what your filter plugin is designed to do.
|
||||
::::
|
||||
|
||||
|
||||
```sh
|
||||
bin/logstash -e 'input { stdin{} } filter { example {} } output {stdout { codec => rubydebug }}'
|
||||
```
|
||||
|
||||
Test your filter by sending input through `stdin` and output (after filtering) through `stdout` with the `rubydebug` codec, which enhances readability.
|
||||
|
||||
In the case of the example filter plugin, any text you send will be replaced by the contents of the `message` configuration parameter, the default value being "Hello World!":
|
||||
|
||||
```sh
|
||||
Testing 1, 2, 3
|
||||
{
|
||||
"message" => "Hello World!",
|
||||
"@version" => "1",
|
||||
"@timestamp" => "2015-01-27T19:17:18.932Z",
|
||||
"host" => "cadenza"
|
||||
}
|
||||
```
|
||||
|
||||
Feel free to experiment and test this by changing the `message` parameter:
|
||||
|
||||
```sh
|
||||
bin/logstash -e 'input { stdin{} } filter { example { message => "This is a new message!"} } output {stdout { codec => rubydebug }}'
|
||||
```
|
||||
|
||||
Congratulations! You’ve built, deployed and successfully run a Logstash filter.
|
||||
|
||||
|
||||
|
||||
## Submitting your plugin to [RubyGems.org](http://rubygems.org) and [logstash-plugins](https://github.com/logstash-plugins) [_submitting_your_plugin_to_rubygems_orghttprubygems_org_and_logstash_pluginshttpsgithub_comlogstash_plugins_3]
|
||||
|
||||
Logstash uses [RubyGems.org](http://rubygems.org) as its repository for all plugin artifacts. Once you have developed your new plugin, you can make it available to Logstash users by simply publishing it to RubyGems.org.
|
||||
|
||||
### Licensing [_licensing_3]
|
||||
|
||||
Logstash and all its plugins are licensed under [Apache License, version 2 ("ALv2")](https://github.com/elasticsearch/logstash/blob/main/LICENSE). If you make your plugin publicly available via [RubyGems.org](http://rubygems.org), please make sure to have this line in your gemspec:
|
||||
|
||||
* `s.licenses = ['Apache License (2.0)']`
|
||||
|
||||
|
||||
### Publishing to [RubyGems.org](http://rubygems.org) [_publishing_to_rubygems_orghttprubygems_org_3]
|
||||
|
||||
To begin, you’ll need an account on RubyGems.org
|
||||
|
||||
* [Sign-up for a RubyGems account](https://rubygems.org/sign_up).
|
||||
|
||||
After creating an account, [obtain](http://guides.rubygems.org/rubygems-org-api/#api-authorization) an API key from RubyGems.org. By default, RubyGems uses the file `~/.gem/credentials` to store your API key. These credentials will be used to publish the gem. Replace `username` and `password` with the credentials you created at RubyGems.org:
|
||||
|
||||
```sh
|
||||
curl -u username:password https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials
|
||||
chmod 0600 ~/.gem/credentials
|
||||
```
|
||||
|
||||
Before proceeding, make sure you have the right version in your gemspec file and commit your changes.
|
||||
|
||||
* `s.version = '0.1.0'`
|
||||
|
||||
To publish version 0.1.0 of your new logstash gem:
|
||||
|
||||
```sh
|
||||
bundle install
|
||||
bundle exec rake vendor
|
||||
bundle exec rspec
|
||||
bundle exec rake publish_gem
|
||||
```
|
||||
|
||||
::::{note}
|
||||
Executing `rake publish_gem`:
|
||||
|
||||
1. Reads the version from the gemspec file (`s.version = '0.1.0'`)
|
||||
2. Checks in your local repository if a tag exists for that version. If the tag already exists, it aborts the process. Otherwise, it creates a new version tag in your local repository.
|
||||
3. Builds the gem
|
||||
4. Publishes the gem to RubyGems.org
|
||||
|
||||
::::
|
||||
|
||||
|
||||
That’s it! Your plugin is published! Logstash users can now install your plugin by running:
|
||||
|
||||
```sh
|
||||
bin/logstash-plugin install logstash-filter-mypluginname
|
||||
```
|
||||
|
||||
|
||||
|
||||
## Contributing your source code to [logstash-plugins](https://github.com/logstash-plugins) [_contributing_your_source_code_to_logstash_pluginshttpsgithub_comlogstash_plugins_3]
|
||||
|
||||
It is not required to contribute your source code to [logstash-plugins](https://github.com/logstash-plugins) github organization, but we always welcome new plugins!
|
||||
|
||||
### Benefits [_benefits_3]
|
||||
|
||||
Some of the many benefits of having your plugin in the logstash-plugins repository are:
|
||||
|
||||
* **Discovery.** Your plugin will appear in the [Logstash Reference](/reference/index.md), where Logstash users look first for plugins and documentation.
|
||||
* **Documentation.** Your plugin documentation will automatically be added to the [Logstash Reference](/reference/index.md).
|
||||
* **Testing.** With our testing infrastructure, your plugin will be continuously tested against current and future releases of Logstash. As a result, users will have the assurance that if incompatibilities arise, they will be quickly discovered and corrected.
|
||||
|
||||
|
||||
### Acceptance Guidelines [_acceptance_guidelines_3]
|
||||
|
||||
* **Code Review.** Your plugin must be reviewed by members of the community for coherence, quality, readability, stability and security.
|
||||
* **Tests.** Your plugin must contain tests to be accepted. These tests are also subject to code review for scope and completeness. It’s ok if you don’t know how to write tests — we will guide you. We are working on publishing a guide to creating tests for Logstash which will make it easier. In the meantime, you can refer to [http://betterspecs.org/](http://betterspecs.org/) for examples.
|
||||
|
||||
To begin migrating your plugin to logstash-plugins, simply create a new [issue](https://github.com/elasticsearch/logstash/issues) in the Logstash repository. When the acceptance guidelines are completed, we will facilitate the move to the logstash-plugins organization using the recommended [github process](https://help.github.com/articles/transferring-a-repository/#transferring-from-a-user-to-an-organization).
|
||||
58
docs/extend/index.md
Normal file
58
docs/extend/index.md
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/contributing-to-logstash.html
|
||||
---
|
||||
|
||||
# Contributing to Logstash [contributing-to-logstash]
|
||||
|
||||
You can add your own input, codec, filter, or output plugins to Logstash.
|
||||
|
||||
|
||||
### Acceptance guidelines [plugin-acceptance]
|
||||
|
||||
Start with the end in mind. These guidelines and best practices can help you build a better plugin, even if you choose not to share it with the world.
|
||||
|
||||
* **Consistency.** Your plugin must be consistent in quality and naming conventions used by other plugins. The plugin name must be unique and in this format: `logstash-plugintype-pluginname`. If the plugin name is more than one word, separate words after plugin type with underscores. Example: *logstash-output-elastic_app_search*
|
||||
* **Documentation.** Documentation is a required component of your plugin. If we list your plugin in the Logstash Reference, we point to your documentation—a readme.md, docs/index.asciidoc, or both—in your plugin repo.
|
||||
* **Code Review.** Your plugin must be reviewed by members of the community for coherence, quality, readability, stability and security.
|
||||
* **Tests.** Your plugin must contain tests to be accepted. You can refer to [http://betterspecs.org/](http://betterspecs.org/) for examples.
|
||||
|
||||
* Step 1. Enable travis on your account
|
||||
* Step 2. Import our standard travis.yml [https://github.com/logstash-plugins/.ci/blob/1.x/travis/travis.yml](https://github.com/logstash-plugins/.ci/blob/1.x/travis/travis.yml), as shown in the [fingerprint filter example](https://github.com/logstash-plugins/logstash-filter-fingerprint/blob/main/.travis.yml).
|
||||
* Step 3. Have specs in the spec folder.
|
||||
|
||||
|
||||
|
||||
## Add a plugin [add-plugin]
|
||||
|
||||
Plugins can be developed and deployed independently of the Logstash core. Here are some documents to guide you through the process of coding, deploying, and sharing your plugin:
|
||||
|
||||
* Write a new plugin
|
||||
|
||||
* [How to write a Logstash input plugin](/extend/input-new-plugin.md)
|
||||
* [How to write a Logstash codec plugin](/extend/codec-new-plugin.md)
|
||||
* [How to write a Logstash filter plugin](/extend/filter-new-plugin.md)
|
||||
* [How to write a Logstash output plugin](/extend/output-new-plugin.md)
|
||||
* [Community Maintainer’s Guide](/extend/community-maintainer.md)
|
||||
|
||||
* [Document your plugin](/extend/plugin-doc.md)
|
||||
* [Publish your plugin to RubyGems.org](/extend/publish-plugin.md)
|
||||
* [List your plugin](/extend/plugin-listing.md)
|
||||
* Contribute a patch
|
||||
|
||||
* [Contributing a patch to a Logstash plugin](/extend/contributing-patch-plugin.md)
|
||||
* [Extending Logstash core](/extend/contribute-to-core.md)
|
||||
|
||||
|
||||
|
||||
#### Plugin Shutdown APIs [shutdown-apis]
|
||||
|
||||
You have three options for shutting down a plugin: `stop`, `stop?`, and `close`.
|
||||
|
||||
* Call the `stop` method from outside the plugin thread. This method signals the plugin to stop.
|
||||
* The `stop?` method returns `true` when the `stop` method has already been called for that plugin.
|
||||
* The `close` method performs final bookkeeping and cleanup after the plugin’s `run` method and the plugin’s thread both exit. The `close` method is a a new name for the method known as `teardown` in previous versions of Logstash.
|
||||
|
||||
The `shutdown`, `finished`, `finished?`, `running?`, and `terminating?` methods are redundant and no longer present in the Plugin Base class.
|
||||
|
||||
Sample code for the plugin shutdown APIs is [available](https://github.com/logstash-plugins/logstash-input-example/blob/main/lib/logstash/inputs/example.rb).
|
||||
674
docs/extend/input-new-plugin.md
Normal file
674
docs/extend/input-new-plugin.md
Normal file
|
|
@ -0,0 +1,674 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/input-new-plugin.html
|
||||
---
|
||||
|
||||
# How to write a Logstash input plugin [input-new-plugin]
|
||||
|
||||
To develop a new input for Logstash, build a self-contained Ruby gem whose source code lives in its own GitHub repository. The Ruby gem can then be hosted and shared on RubyGems.org. You can use the example input implementation as a starting point. (If you’re unfamiliar with Ruby, you can find an excellent quickstart guide at [https://www.ruby-lang.org/en/documentation/quickstart/](https://www.ruby-lang.org/en/documentation/quickstart/).)
|
||||
|
||||
## Get started [_get_started]
|
||||
|
||||
Let’s step through creating an input plugin using the [example input plugin](https://github.com/logstash-plugins/logstash-input-example/).
|
||||
|
||||
### Create a GitHub repo for your new plugin [_create_a_github_repo_for_your_new_plugin]
|
||||
|
||||
Each Logstash plugin lives in its own GitHub repository. To create a new repository for your plugin:
|
||||
|
||||
1. Log in to GitHub.
|
||||
2. Click the **Repositories** tab. You’ll see a list of other repositories you’ve forked or contributed to.
|
||||
3. Click the green **New** button in the upper right.
|
||||
4. Specify the following settings for your new repo:
|
||||
|
||||
* **Repository name** — a unique name of the form `logstash-input-pluginname`.
|
||||
* **Public or Private** — your choice, but the repository must be Public if you want to submit it as an official plugin.
|
||||
* **Initialize this repository with a README** — enables you to immediately clone the repository to your computer.
|
||||
|
||||
5. Click **Create Repository**.
|
||||
|
||||
|
||||
### Use the plugin generator tool [_use_the_plugin_generator_tool]
|
||||
|
||||
You can create your own Logstash plugin in seconds! The `generate` subcommand of `bin/logstash-plugin` creates the foundation for a new Logstash plugin with templatized files. It creates the correct directory structure, gemspec files, and dependencies so you can start adding custom code to process data with Logstash.
|
||||
|
||||
For more information, see [Generating plugins](/reference/plugin-generator.md)
|
||||
|
||||
|
||||
### Copy the input code [_copy_the_input_code]
|
||||
|
||||
Alternatively, you can use the examples repo we host on github.com
|
||||
|
||||
1. **Clone your plugin.** Replace `GITUSERNAME` with your github username, and `MYPLUGINNAME` with your plugin name.
|
||||
|
||||
* `git clone https://github.com/GITUSERNAME/logstash-``input-MYPLUGINNAME.git`
|
||||
|
||||
* alternately, via ssh: `git clone git@github.com:GITUSERNAME/logstash``-input-MYPLUGINNAME.git`
|
||||
|
||||
* `cd logstash-input-MYPLUGINNAME`
|
||||
|
||||
2. **Clone the input plugin example and copy it to your plugin branch.**
|
||||
|
||||
You don’t want to include the example .git directory or its contents, so delete it before you copy the example.
|
||||
|
||||
* `cd /tmp`
|
||||
* `git clone https://github.com/logstash-plugins/logstash``-input-example.git`
|
||||
* `cd logstash-input-example`
|
||||
* `rm -rf .git`
|
||||
* `cp -R * /path/to/logstash-input-mypluginname/`
|
||||
|
||||
3. **Rename the following files to match the name of your plugin.**
|
||||
|
||||
* `logstash-input-example.gemspec`
|
||||
* `example.rb`
|
||||
* `example_spec.rb`
|
||||
|
||||
```txt
|
||||
cd /path/to/logstash-input-mypluginname
|
||||
mv logstash-input-example.gemspec logstash-input-mypluginname.gemspec
|
||||
mv lib/logstash/inputs/example.rb lib/logstash/inputs/mypluginname.rb
|
||||
mv spec/inputs/example_spec.rb spec/inputs/mypluginname_spec.rb
|
||||
```
|
||||
|
||||
|
||||
Your file structure should look like this:
|
||||
|
||||
```txt
|
||||
$ tree logstash-input-mypluginname
|
||||
├── Gemfile
|
||||
├── LICENSE
|
||||
├── README.md
|
||||
├── Rakefile
|
||||
├── lib
|
||||
│ └── logstash
|
||||
│ └── inputs
|
||||
│ └── mypluginname.rb
|
||||
├── logstash-input-mypluginname.gemspec
|
||||
└── spec
|
||||
└── inputs
|
||||
└── mypluginname_spec.rb
|
||||
```
|
||||
|
||||
For more information about the Ruby gem file structure and an excellent walkthrough of the Ruby gem creation process, see [http://timelessrepo.com/making-ruby-gems](http://timelessrepo.com/making-ruby-gems)
|
||||
|
||||
|
||||
### See what your plugin looks like [_see_what_your_plugin_looks_like]
|
||||
|
||||
Before we dive into the details, open up the plugin file in your favorite text editor and take a look.
|
||||
|
||||
```ruby
|
||||
require "logstash/inputs/base"
|
||||
require "logstash/namespace"
|
||||
require "stud/interval"
|
||||
require "socket" # for Socket.gethostname
|
||||
|
||||
# Add any asciidoc formatted documentation here
|
||||
# Generate a repeating message.
|
||||
#
|
||||
# This plugin is intended only as an example.
|
||||
|
||||
class LogStash::Inputs::Example < LogStash::Inputs::Base
|
||||
config_name "example"
|
||||
|
||||
# If undefined, Logstash will complain, even if codec is unused.
|
||||
default :codec, "plain"
|
||||
|
||||
# The message string to use in the event.
|
||||
config :message, :validate => :string, :default => "Hello World!"
|
||||
|
||||
# Set how frequently messages should be sent.
|
||||
#
|
||||
# The default, `1`, means send a message every second.
|
||||
config :interval, :validate => :number, :default => 1
|
||||
|
||||
public
|
||||
def register
|
||||
@host = Socket.gethostname
|
||||
end # def register
|
||||
|
||||
def run(queue)
|
||||
Stud.interval(@interval) do
|
||||
event = LogStash::Event.new("message" => @message, "host" => @host)
|
||||
decorate(event)
|
||||
queue << event
|
||||
end # loop
|
||||
end # def run
|
||||
|
||||
end # class LogStash::Inputs::Example
|
||||
```
|
||||
|
||||
|
||||
|
||||
## Coding input plugins [_coding_input_plugins]
|
||||
|
||||
Now let’s take a line-by-line look at the example plugin.
|
||||
|
||||
### `require` Statements [_require_statements]
|
||||
|
||||
Logstash input plugins require parent classes defined in `logstash/inputs/base` and logstash/namespace:
|
||||
|
||||
```ruby
|
||||
require "logstash/inputs/base"
|
||||
require "logstash/namespace"
|
||||
```
|
||||
|
||||
Of course, the plugin you build may depend on other code, or even gems. Just put them here along with these Logstash dependencies.
|
||||
|
||||
|
||||
|
||||
## Plugin Body [_plugin_body]
|
||||
|
||||
Let’s go through the various elements of the plugin itself.
|
||||
|
||||
### `class` Declaration [_class_declaration]
|
||||
|
||||
The input plugin class should be a subclass of `LogStash::Inputs::Base`:
|
||||
|
||||
```ruby
|
||||
class LogStash::Inputs::Example < LogStash::Inputs::Base
|
||||
```
|
||||
|
||||
The class name should closely mirror the plugin name, for example:
|
||||
|
||||
```ruby
|
||||
LogStash::Inputs::Example
|
||||
```
|
||||
|
||||
|
||||
### `config_name` [_config_name]
|
||||
|
||||
```ruby
|
||||
config_name "example"
|
||||
```
|
||||
|
||||
This is the name your plugin will call inside the input configuration block.
|
||||
|
||||
If you set `config_name "example"` in your plugin code, the corresponding Logstash configuration block would need to look like this:
|
||||
|
||||
```js
|
||||
input {
|
||||
example {...}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
|
||||
## Configuration Parameters [_configuration_parameters]
|
||||
|
||||
```ruby
|
||||
config :variable_name, :validate => :variable_type, :default => "Default value", :required => boolean, :deprecated => boolean, :obsolete => string
|
||||
```
|
||||
|
||||
The configuration, or `config` section allows you to define as many (or as few) parameters as are needed to enable Logstash to process events.
|
||||
|
||||
There are several configuration attributes:
|
||||
|
||||
* `:validate` - allows you to enforce passing a particular data type to Logstash for this configuration option, such as `:string`, `:password`, `:boolean`, `:number`, `:array`, `:hash`, `:path` (a file-system path), `uri`, `:codec` (since 1.2.0), `:bytes`. Note that this also works as a coercion in that if I specify "true" for boolean (even though technically a string), it will become a valid boolean in the config. This coercion works for the `:number` type as well where "1.2" becomes a float and "22" is an integer.
|
||||
* `:default` - lets you specify a default value for a parameter
|
||||
* `:required` - whether or not this parameter is mandatory (a Boolean `true` or
|
||||
* `:list` - whether or not this value should be a list of values. Will typecheck the list members, and convert scalars to one element lists. Note that this mostly obviates the array type, though if you need lists of complex objects that will be more suitable. `false`)
|
||||
* `:deprecated` - informational (also a Boolean `true` or `false`)
|
||||
* `:obsolete` - used to declare that a given setting has been removed and is no longer functioning. The idea is to provide an informed upgrade path to users who are still using a now-removed setting.
|
||||
|
||||
|
||||
## Plugin Methods [_plugin_methods]
|
||||
|
||||
Logstash inputs must implement two main methods: `register` and `run`.
|
||||
|
||||
### `register` Method [_register_method]
|
||||
|
||||
```ruby
|
||||
public
|
||||
def register
|
||||
end # def register
|
||||
```
|
||||
|
||||
The Logstash `register` method is like an `initialize` method. It was originally created to enforce having `super` called, preventing headaches for newbies. (Note: It may go away in favor of `initialize`, in conjunction with some enforced testing to ensure `super` is called.)
|
||||
|
||||
`public` means the method can be called anywhere, not just within the class. This is the default behavior for methods in Ruby, but it is specified explicitly here anyway.
|
||||
|
||||
You can also assign instance variables here (variables prepended by `@`). Configuration variables are now in scope as instance variables, like `@message`
|
||||
|
||||
|
||||
### `run` Method [_run_method]
|
||||
|
||||
The example input plugin has the following `run` Method:
|
||||
|
||||
```ruby
|
||||
def run(queue)
|
||||
Stud.interval(@interval) do
|
||||
event = LogStash::Event.new("message" => @message, "host" => @host)
|
||||
decorate(event)
|
||||
queue << event
|
||||
end # loop
|
||||
end # def run
|
||||
```
|
||||
|
||||
The `run` method is where a stream of data from an input becomes an event.
|
||||
|
||||
The stream can be plain or generated as with the [heartbeat](https://github.com/logstash-plugins/logstash-input-heartbeat/blob/main/lib/logstash/inputs/heartbeat.rb#L43-L61) input plugin. In these cases, though no codec is used, [a default codec](https://github.com/logstash-plugins/logstash-input-heartbeat/blob/main/lib/logstash/inputs/heartbeat.rb#L17) must be set in the code to avoid errors.
|
||||
|
||||
Here’s another example `run` method:
|
||||
|
||||
```ruby
|
||||
def run(queue)
|
||||
while true
|
||||
begin
|
||||
# Based on some testing, there is no way to interrupt an IO.sysread nor
|
||||
# IO.select call in JRuby.
|
||||
data = $stdin.sysread(16384)
|
||||
@codec.decode(data) do |event|
|
||||
decorate(event)
|
||||
event.set("host", @host) if !event.include?("host")
|
||||
queue << event
|
||||
end
|
||||
rescue IOError, EOFError, LogStash::ShutdownSignal
|
||||
# stdin closed or a requested shutdown
|
||||
break
|
||||
end
|
||||
end # while true
|
||||
finished
|
||||
end # def run
|
||||
```
|
||||
|
||||
In this example, the `data` is being sent to the codec defined in the configuration block to `decode` the data stream and return an event.
|
||||
|
||||
In both examples, the resulting `event` is passed to the `decorate` method:
|
||||
|
||||
```ruby
|
||||
decorate(event)
|
||||
```
|
||||
|
||||
This applies any tags you might have set in the input configuration block. For example, `tags => ["tag1", "tag2"]`.
|
||||
|
||||
Also in both examples, the `event`, after being "decorated," is appended to the queue:
|
||||
|
||||
```ruby
|
||||
queue << event
|
||||
```
|
||||
|
||||
This inserts the event into the pipeline.
|
||||
|
||||
::::{tip}
|
||||
Because input plugins can range from simple to complex, it is helpful to see more examples of how they have been created:
|
||||
|
||||
* [syslog](https://github.com/logstash-plugins/logstash-input-syslog/blob/main/lib/logstash/inputs/syslog.rb)
|
||||
* [zeromq](https://github.com/logstash-plugins/logstash-input-zeromq/blob/main/lib/logstash/inputs/zeromq.rb)
|
||||
* [stdin](https://github.com/logstash-plugins/logstash-input-stdin/blob/main/lib/logstash/inputs/stdin.rb)
|
||||
* [tcp](https://github.com/logstash-plugins/logstash-input-tcp/blob/main/lib/logstash/inputs/tcp.rb)
|
||||
|
||||
There are many more more examples in the [logstash-plugin github repository](https://github.com/logstash-plugins?query=logstash-input).
|
||||
|
||||
::::
|
||||
|
||||
|
||||
|
||||
|
||||
## Building the Plugin [_building_the_plugin]
|
||||
|
||||
At this point in the process you have coded your plugin and are ready to build a Ruby Gem from it. The following information will help you complete the process.
|
||||
|
||||
### External dependencies [_external_dependencies]
|
||||
|
||||
A `require` statement in Ruby is used to include necessary code. In some cases your plugin may require additional files. For example, the collectd plugin [uses](https://github.com/logstash-plugins/logstash-codec-collectd/blob/main/lib/logstash/codecs/collectd.rb#L148) the `types.db` file provided by collectd. In the main directory of your plugin, a file called `vendor.json` is where these files are described.
|
||||
|
||||
The `vendor.json` file contains an array of JSON objects, each describing a file dependency. This example comes from the [collectd](https://github.com/logstash-plugins/logstash-codec-collectd/blob/main/vendor.json) codec plugin:
|
||||
|
||||
```txt
|
||||
[{
|
||||
"sha1": "a90fe6cc53b76b7bdd56dc57950d90787cb9c96e",
|
||||
"url": "http://collectd.org/files/collectd-5.4.0.tar.gz",
|
||||
"files": [ "/src/types.db" ]
|
||||
}]
|
||||
```
|
||||
|
||||
* `sha1` is the sha1 signature used to verify the integrity of the file referenced by `url`.
|
||||
* `url` is the address from where Logstash will download the file.
|
||||
* `files` is an optional array of files to extract from the downloaded file. Note that while tar archives can use absolute or relative paths, treat them as absolute in this array. If `files` is not present, all files will be uncompressed and extracted into the vendor directory.
|
||||
|
||||
Another example of the `vendor.json` file is the [`geoip` filter](https://github.com/logstash-plugins/logstash-filter-geoip/blob/main/vendor.json)
|
||||
|
||||
The process used to download these dependencies is to call `rake vendor`. This will be discussed further in the testing section of this document.
|
||||
|
||||
Another kind of external dependency is on jar files. This will be described in the "Add a `gemspec` file" section.
|
||||
|
||||
|
||||
### Deprecated features [_deprecated_features]
|
||||
|
||||
As a plugin evolves, an option or feature may no longer serve the intended purpose, and the developer may want to *deprecate* its usage. Deprecation warns users about the option’s status, so they aren’t caught by surprise when it is removed in a later release.
|
||||
|
||||
{{ls}} 7.6 introduced a *deprecation logger* to make handling those situations easier. You can use the [adapter](https://github.com/logstash-plugins/logstash-mixin-deprecation_logger_support) to ensure that your plugin can use the deprecation logger while still supporting older versions of {{ls}}. See the [readme](https://github.com/logstash-plugins/logstash-mixin-deprecation_logger_support/blob/main/README.md) for more information and for instructions on using the adapter.
|
||||
|
||||
Deprecations are noted in the `logstash-deprecation.log` file in the `log` directory.
|
||||
|
||||
|
||||
### Add a Gemfile [_add_a_gemfile]
|
||||
|
||||
Gemfiles allow Ruby’s Bundler to maintain the dependencies for your plugin. Currently, all we’ll need is the Logstash gem, for testing, but if you require other gems, you should add them in here.
|
||||
|
||||
::::{tip}
|
||||
See [Bundler’s Gemfile page](http://bundler.io/gemfile.html) for more details.
|
||||
::::
|
||||
|
||||
|
||||
```ruby
|
||||
source 'https://rubygems.org'
|
||||
gemspec
|
||||
gem "logstash", :github => "elastic/logstash", :branch => "master"
|
||||
```
|
||||
|
||||
|
||||
|
||||
## Add a `gemspec` file [_add_a_gemspec_file]
|
||||
|
||||
Gemspecs define the Ruby gem which will be built and contain your plugin.
|
||||
|
||||
::::{tip}
|
||||
More information can be found on the [Rubygems Specification page](http://guides.rubygems.org/specification-reference/).
|
||||
::::
|
||||
|
||||
|
||||
```ruby
|
||||
Gem::Specification.new do |s|
|
||||
s.name = 'logstash-input-example'
|
||||
s.version = '0.1.0'
|
||||
s.licenses = ['Apache License (2.0)']
|
||||
s.summary = "This input does x, y, z in Logstash"
|
||||
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
||||
s.authors = ["Elastic"]
|
||||
s.email = 'info@elastic.co'
|
||||
s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
|
||||
s.require_paths = ["lib"]
|
||||
|
||||
# Files
|
||||
s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
|
||||
# Tests
|
||||
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
||||
|
||||
# Special flag to let us know this is actually a logstash plugin
|
||||
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
|
||||
|
||||
# Gem dependencies
|
||||
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
||||
s.add_development_dependency 'logstash-devutils'
|
||||
end
|
||||
```
|
||||
|
||||
It is appropriate to change these values to fit your plugin. In particular, `s.name` and `s.summary` should reflect your plugin’s name and behavior.
|
||||
|
||||
`s.licenses` and `s.version` are also important and will come into play when you are ready to publish your plugin.
|
||||
|
||||
Logstash and all its plugins are licensed under [Apache License, version 2 ("ALv2")](https://github.com/elastic/logstash/blob/main/LICENSE.txt). If you make your plugin publicly available via [RubyGems.org](http://rubygems.org), please make sure to have this line in your gemspec:
|
||||
|
||||
* `s.licenses = ['Apache License (2.0)']`
|
||||
|
||||
The gem version, designated by `s.version`, helps track changes to plugins over time. You should use [semver versioning](http://semver.org/) strategy for version numbers.
|
||||
|
||||
### Runtime and Development Dependencies [_runtime_and_development_dependencies]
|
||||
|
||||
At the bottom of the `gemspec` file is a section with a comment: `Gem dependencies`. This is where any other needed gems must be mentioned. If a gem is necessary for your plugin to function, it is a runtime dependency. If a gem are only used for testing, then it would be a development dependency.
|
||||
|
||||
::::{note}
|
||||
You can also have versioning requirements for your dependencies—including other Logstash plugins:
|
||||
|
||||
```ruby
|
||||
# Gem dependencies
|
||||
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
||||
s.add_development_dependency 'logstash-devutils'
|
||||
```
|
||||
|
||||
This gemspec has a runtime dependency on the logstash-core-plugin-api and requires that it have a version number greater than or equal to version 1.60 and less than or equal to version 2.99.
|
||||
|
||||
::::
|
||||
|
||||
|
||||
::::{important}
|
||||
All plugins have a runtime dependency on the `logstash-core-plugin-api` gem, and a development dependency on `logstash-devutils`.
|
||||
::::
|
||||
|
||||
|
||||
|
||||
### Jar dependencies [_jar_dependencies]
|
||||
|
||||
In some cases, such as the [Elasticsearch output plugin](https://github.com/logstash-plugins/logstash-output-elasticsearch/blob/main/logstash-output-elasticsearch.gemspec#L22-L23), your code may depend on a jar file. In cases such as this, the dependency is added in the gemspec file in this manner:
|
||||
|
||||
```ruby
|
||||
# Jar dependencies
|
||||
s.requirements << "jar 'org.elasticsearch:elasticsearch', '5.0.0'"
|
||||
s.add_runtime_dependency 'jar-dependencies'
|
||||
```
|
||||
|
||||
With these both defined, the install process will search for the required jar file at [http://mvnrepository.com](http://mvnrepository.com) and download the specified version.
|
||||
|
||||
|
||||
|
||||
## Document your plugin [_document_your_plugin]
|
||||
|
||||
Documentation is an important part of your plugin. All plugin documentation is rendered and placed in the [Logstash Reference](/reference/index.md) and the [Versioned plugin docs](logstash-docs://reference/integration-plugins.md).
|
||||
|
||||
See [Document your plugin](/extend/plugin-doc.md) for tips and guidelines.
|
||||
|
||||
|
||||
## Add Tests [_add_tests]
|
||||
|
||||
Logstash loves tests. Lots of tests. If you’re using your new input plugin in a production environment, you’ll want to have some tests to ensure you are not breaking any existing functionality.
|
||||
|
||||
::::{note}
|
||||
A full exposition on RSpec is outside the scope of this document. Learn more about RSpec at [http://rspec.info](http://rspec.info)
|
||||
::::
|
||||
|
||||
|
||||
For help learning about tests and testing, look in the `spec/inputs/` directory of several other similar plugins.
|
||||
|
||||
|
||||
## Clone and test! [_clone_and_test]
|
||||
|
||||
Now let’s start with a fresh clone of the plugin, build it and run the tests.
|
||||
|
||||
* **Clone your plugin into a temporary location** Replace `GITUSERNAME` with your github username, and `MYPLUGINNAME` with your plugin name.
|
||||
|
||||
* `git clone https://github.com/GITUSERNAME/logstash-``input-MYPLUGINNAME.git`
|
||||
|
||||
* alternately, via ssh: `git clone git@github.com:GITUSERNAME/logstash-``input-MYPLUGINNAME.git`
|
||||
|
||||
* `cd logstash-input-MYPLUGINNAME`
|
||||
|
||||
|
||||
Then, you’ll need to install your plugins dependencies with bundler:
|
||||
|
||||
```
|
||||
bundle install
|
||||
```
|
||||
|
||||
::::{important}
|
||||
If your plugin has an external file dependency described in `vendor.json`, you must download that dependency before running or testing. You can do this by running:
|
||||
|
||||
```
|
||||
rake vendor
|
||||
```
|
||||
|
||||
::::
|
||||
|
||||
|
||||
And finally, run the tests:
|
||||
|
||||
```
|
||||
bundle exec rspec
|
||||
```
|
||||
|
||||
You should see a success message, which looks something like this:
|
||||
|
||||
```
|
||||
Finished in 0.034 seconds
|
||||
1 example, 0 failures
|
||||
```
|
||||
|
||||
Hooray! You’re almost there! (Unless you saw failures… you should fix those first).
|
||||
|
||||
|
||||
## Building and Testing [_building_and_testing]
|
||||
|
||||
Now you’re ready to build your (well-tested) plugin into a Ruby gem.
|
||||
|
||||
### Build [_build]
|
||||
|
||||
You already have all the necessary ingredients, so let’s go ahead and run the build command:
|
||||
|
||||
```sh
|
||||
gem build logstash-input-example.gemspec
|
||||
```
|
||||
|
||||
That’s it! Your gem should be built and be in the same path with the name
|
||||
|
||||
```sh
|
||||
logstash-input-mypluginname-0.1.0.gem
|
||||
```
|
||||
|
||||
The `s.version` number from your gemspec file will provide the gem version, in this case, `0.1.0`.
|
||||
|
||||
|
||||
### Test installation [_test_installation]
|
||||
|
||||
You should test install your plugin into a clean installation of Logstash. Download the latest version from the [Logstash downloads page](https://www.elastic.co/downloads/logstash/).
|
||||
|
||||
1. Untar and cd in to the directory:
|
||||
|
||||
```sh
|
||||
curl -O https://download.elastic.co/logstash/logstash/logstash-9.0.0.tar.gz
|
||||
tar xzvf logstash-9.0.0.tar.gz
|
||||
cd logstash-9.0.0
|
||||
```
|
||||
|
||||
2. Using the plugin tool, we can install the gem we just built.
|
||||
|
||||
* Replace `/my/logstash/plugins` with the correct path to the gem for your environment, and `0.1.0` with the correct version number from the gemspec file.
|
||||
|
||||
```sh
|
||||
bin/logstash-plugin install /my/logstash/plugins/logstash-input-example/logstash-input-example-0.1.0.gem
|
||||
```
|
||||
|
||||
* After running this, you should see feedback from Logstash that it was successfully installed:
|
||||
|
||||
```sh
|
||||
validating /my/logstash/plugins/logstash-input-example/logstash-input-example-0.1.0.gem >= 0
|
||||
Valid logstash plugin. Continuing...
|
||||
Successfully installed 'logstash-input-example' with version '0.1.0'
|
||||
```
|
||||
|
||||
::::{tip}
|
||||
You can also use the Logstash plugin tool to determine which plugins are currently available:
|
||||
|
||||
```sh
|
||||
bin/logstash-plugin list
|
||||
```
|
||||
|
||||
Depending on what you have installed, you might see a short or long list of plugins: inputs, codecs, filters and outputs.
|
||||
|
||||
::::
|
||||
|
||||
3. Now try running Logstash with a simple configuration passed in via the command-line, using the `-e` flag.
|
||||
|
||||
::::{note}
|
||||
Your results will depend on what your input plugin is designed to do.
|
||||
::::
|
||||
|
||||
|
||||
```sh
|
||||
bin/logstash -e 'input { example{} } output {stdout { codec => rubydebug }}'
|
||||
```
|
||||
|
||||
The example input plugin will send the contents of `message` (with a default message of "Hello World!") every second.
|
||||
|
||||
```sh
|
||||
{
|
||||
"message" => "Hello World!",
|
||||
"@version" => "1",
|
||||
"@timestamp" => "2015-01-27T19:17:18.932Z",
|
||||
"host" => "cadenza"
|
||||
}
|
||||
```
|
||||
|
||||
Feel free to experiment and test this by changing the `message` and `interval` parameters:
|
||||
|
||||
```sh
|
||||
bin/logstash -e 'input { example{ message => "A different message" interval => 5 } } output {stdout { codec => rubydebug }}'
|
||||
```
|
||||
|
||||
Congratulations! You’ve built, deployed and successfully run a Logstash input.
|
||||
|
||||
|
||||
|
||||
## Submitting your plugin to [RubyGems.org](http://rubygems.org) and [logstash-plugins](https://github.com/logstash-plugins) [_submitting_your_plugin_to_rubygems_orghttprubygems_org_and_logstash_pluginshttpsgithub_comlogstash_plugins]
|
||||
|
||||
Logstash uses [RubyGems.org](http://rubygems.org) as its repository for all plugin artifacts. Once you have developed your new plugin, you can make it available to Logstash users by simply publishing it to RubyGems.org.
|
||||
|
||||
### Licensing [_licensing]
|
||||
|
||||
Logstash and all its plugins are licensed under [Apache License, version 2 ("ALv2")](https://github.com/elasticsearch/logstash/blob/main/LICENSE). If you make your plugin publicly available via [RubyGems.org](http://rubygems.org), please make sure to have this line in your gemspec:
|
||||
|
||||
* `s.licenses = ['Apache License (2.0)']`
|
||||
|
||||
|
||||
### Publishing to [RubyGems.org](http://rubygems.org) [_publishing_to_rubygems_orghttprubygems_org]
|
||||
|
||||
To begin, you’ll need an account on RubyGems.org
|
||||
|
||||
* [Sign-up for a RubyGems account](https://rubygems.org/sign_up).
|
||||
|
||||
After creating an account, [obtain](http://guides.rubygems.org/rubygems-org-api/#api-authorization) an API key from RubyGems.org. By default, RubyGems uses the file `~/.gem/credentials` to store your API key. These credentials will be used to publish the gem. Replace `username` and `password` with the credentials you created at RubyGems.org:
|
||||
|
||||
```sh
|
||||
curl -u username:password https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials
|
||||
chmod 0600 ~/.gem/credentials
|
||||
```
|
||||
|
||||
Before proceeding, make sure you have the right version in your gemspec file and commit your changes.
|
||||
|
||||
* `s.version = '0.1.0'`
|
||||
|
||||
To publish version 0.1.0 of your new logstash gem:
|
||||
|
||||
```sh
|
||||
bundle install
|
||||
bundle exec rake vendor
|
||||
bundle exec rspec
|
||||
bundle exec rake publish_gem
|
||||
```
|
||||
|
||||
::::{note}
|
||||
Executing `rake publish_gem`:
|
||||
|
||||
1. Reads the version from the gemspec file (`s.version = '0.1.0'`)
|
||||
2. Checks in your local repository if a tag exists for that version. If the tag already exists, it aborts the process. Otherwise, it creates a new version tag in your local repository.
|
||||
3. Builds the gem
|
||||
4. Publishes the gem to RubyGems.org
|
||||
|
||||
::::
|
||||
|
||||
|
||||
That’s it! Your plugin is published! Logstash users can now install your plugin by running:
|
||||
|
||||
```sh
|
||||
bin/logstash-plugin install logstash-input-mypluginname
|
||||
```
|
||||
|
||||
|
||||
|
||||
## Contributing your source code to [logstash-plugins](https://github.com/logstash-plugins) [_contributing_your_source_code_to_logstash_pluginshttpsgithub_comlogstash_plugins]
|
||||
|
||||
It is not required to contribute your source code to [logstash-plugins](https://github.com/logstash-plugins) github organization, but we always welcome new plugins!
|
||||
|
||||
### Benefits [_benefits]
|
||||
|
||||
Some of the many benefits of having your plugin in the logstash-plugins repository are:
|
||||
|
||||
* **Discovery.** Your plugin will appear in the [Logstash Reference](/reference/index.md), where Logstash users look first for plugins and documentation.
|
||||
* **Documentation.** Your plugin documentation will automatically be added to the [Logstash Reference](/reference/index.md).
|
||||
* **Testing.** With our testing infrastructure, your plugin will be continuously tested against current and future releases of Logstash. As a result, users will have the assurance that if incompatibilities arise, they will be quickly discovered and corrected.
|
||||
|
||||
|
||||
### Acceptance Guidelines [_acceptance_guidelines]
|
||||
|
||||
* **Code Review.** Your plugin must be reviewed by members of the community for coherence, quality, readability, stability and security.
|
||||
* **Tests.** Your plugin must contain tests to be accepted. These tests are also subject to code review for scope and completeness. It’s ok if you don’t know how to write tests — we will guide you. We are working on publishing a guide to creating tests for Logstash which will make it easier. In the meantime, you can refer to [http://betterspecs.org/](http://betterspecs.org/) for examples.
|
||||
|
||||
To begin migrating your plugin to logstash-plugins, simply create a new [issue](https://github.com/elasticsearch/logstash/issues) in the Logstash repository. When the acceptance guidelines are completed, we will facilitate the move to the logstash-plugins organization using the recommended [github process](https://help.github.com/articles/transferring-a-repository/#transferring-from-a-user-to-an-organization).
|
||||
348
docs/extend/java-codec-plugin.md
Normal file
348
docs/extend/java-codec-plugin.md
Normal file
|
|
@ -0,0 +1,348 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/java-codec-plugin.html
|
||||
---
|
||||
|
||||
# How to write a Java codec plugin [java-codec-plugin]
|
||||
|
||||
::::{note}
|
||||
Java codecs are currently supported only for Java input and output plugins. They will not work with Ruby input or output plugins.
|
||||
::::
|
||||
|
||||
|
||||
To develop a new Java codec for Logstash, you write a new Java class that conforms to the Logstash Java Codecs API, package it, and install it with the logstash-plugin utility. We’ll go through each of those steps.
|
||||
|
||||
|
||||
## Set up your environment [_set_up_your_environment_2]
|
||||
|
||||
|
||||
### Copy the example repo [_copy_the_example_repo_2]
|
||||
|
||||
Start by copying the [example codec plugin](https://github.com/logstash-plugins/logstash-codec-java_codec_example). The plugin API is currently part of the Logstash codebase so you must have a local copy of that available. You can obtain a copy of the Logstash codebase with the following `git` command:
|
||||
|
||||
```shell
|
||||
git clone --branch <branch_name> --single-branch https://github.com/elastic/logstash.git <target_folder>
|
||||
```
|
||||
|
||||
The `branch_name` should correspond to the version of Logstash containing the preferred revision of the Java plugin API.
|
||||
|
||||
::::{note}
|
||||
The GA version of the Java plugin API is available in the `7.2` and later branches of the Logstash codebase.
|
||||
::::
|
||||
|
||||
|
||||
Specify the `target_folder` for your local copy of the Logstash codebase. If you do not specify `target_folder`, it defaults to a new folder called `logstash` under your current folder.
|
||||
|
||||
|
||||
### Generate the .jar file [_generate_the_jar_file_2]
|
||||
|
||||
After you have obtained a copy of the appropriate revision of the Logstash codebase, you need to compile it to generate the .jar file containing the Java plugin API. From the root directory of your Logstash codebase ($LS_HOME), you can compile it with `./gradlew assemble` (or `gradlew.bat assemble` if you’re running on Windows). This should produce the `$LS_HOME/logstash-core/build/libs/logstash-core-x.y.z.jar` where `x`, `y`, and `z` refer to the version of Logstash.
|
||||
|
||||
After you have successfully compiled Logstash, you need to tell your Java plugin where to find the `logstash-core-x.y.z.jar` file. Create a new file named `gradle.properties` in the root folder of your plugin project. That file should have a single line:
|
||||
|
||||
```txt
|
||||
LOGSTASH_CORE_PATH=<target_folder>/logstash-core
|
||||
```
|
||||
|
||||
where `target_folder` is the root folder of your local copy of the Logstash codebase.
|
||||
|
||||
|
||||
## Code the plugin [_code_the_plugin_2]
|
||||
|
||||
The example codec plugin decodes messages separated by a configurable delimiter and encodes messages by writing their string representation separated by a delimiter. For example, if the codec were configured with `/` as the delimiter, the input text `event1/event2/` would be decoded into two separate events with `message` fields of `event1` and `event2`, respectively. Note that this is only an example codec and does not cover all the edge cases that a production-grade codec should cover.
|
||||
|
||||
Let’s look at the main class in that codec filter:
|
||||
|
||||
```java
|
||||
@LogstashPlugin(name="java_codec_example")
|
||||
public class JavaCodecExample implements Codec {
|
||||
|
||||
public static final PluginConfigSpec<String> DELIMITER_CONFIG =
|
||||
PluginConfigSpec.stringSetting("delimiter", ",");
|
||||
|
||||
private final String id;
|
||||
private final String delimiter;
|
||||
|
||||
public JavaCodecExample(final Configuration config, final Context context) {
|
||||
this(config.get(DELIMITER_CONFIG));
|
||||
}
|
||||
|
||||
private JavaCodecExample(String delimiter) {
|
||||
this.id = UUID.randomUUID().toString();
|
||||
this.delimiter = delimiter;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void decode(ByteBuffer byteBuffer, Consumer<Map<String, Object>> consumer) {
|
||||
// a not-production-grade delimiter decoder
|
||||
byte[] byteInput = new byte[byteBuffer.remaining()];
|
||||
byteBuffer.get(byteInput);
|
||||
if (byteInput.length > 0) {
|
||||
String input = new String(byteInput);
|
||||
String[] split = input.split(delimiter);
|
||||
for (String s : split) {
|
||||
Map<String, Object> map = new HashMap<>();
|
||||
map.put("message", s);
|
||||
consumer.accept(map);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void flush(ByteBuffer byteBuffer, Consumer<Map<String, Object>> consumer) {
|
||||
// if the codec maintains any internal state such as partially-decoded input, this
|
||||
// method should flush that state along with any additional input supplied in
|
||||
// the ByteBuffer
|
||||
|
||||
decode(byteBuffer, consumer); // this is a simplistic implementation
|
||||
}
|
||||
|
||||
@Override
|
||||
public void encode(Event event, OutputStream outputStream) throws IOException {
|
||||
outputStream.write((event.toString() + delimiter).getBytes(Charset.defaultCharset()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Collection<PluginConfigSpec<?>> configSchema() {
|
||||
// should return a list of all configuration options for this plugin
|
||||
return Collections.singletonList(DELIMITER_CONFIG);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Codec cloneCodec() {
|
||||
return new JavaCodecExample(this.delimiter);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getId() {
|
||||
return this.id;
|
||||
}
|
||||
|
||||
}
|
||||
```
|
||||
|
||||
Let’s step through and examine each part of that class.
|
||||
|
||||
|
||||
### Class declaration [_class_declaration_6]
|
||||
|
||||
```java
|
||||
@LogstashPlugin(name="java_codec_example")
|
||||
public class JavaCodecExample implements Codec {
|
||||
```
|
||||
|
||||
Notes about the class declaration:
|
||||
|
||||
* All Java plugins must be annotated with the `@LogstashPlugin` annotation. Additionally:
|
||||
|
||||
* The `name` property of the annotation must be supplied and defines the name of the plugin as it will be used in the Logstash pipeline definition. For example, this codec would be referenced in the codec section of the an appropriate input or output in the Logstash pipeline defintion as `codec => java_codec_example { }`
|
||||
* The value of the `name` property must match the name of the class excluding casing and underscores.
|
||||
|
||||
* The class must implement the `co.elastic.logstash.api.Codec` interface.
|
||||
* Java plugins may not be created in the `org.logstash` or `co.elastic.logstash` packages to prevent potential clashes with classes in Logstash itself.
|
||||
|
||||
|
||||
#### Plugin settings [_plugin_settings_2]
|
||||
|
||||
The snippet below contains both the setting definition and the method referencing it:
|
||||
|
||||
```java
|
||||
public static final PluginConfigSpec<String> DELIMITER_CONFIG =
|
||||
PluginConfigSpec.stringSetting("delimiter", ",");
|
||||
|
||||
@Override
|
||||
public Collection<PluginConfigSpec<?>> configSchema() {
|
||||
return Collections.singletonList(DELIMITER_CONFIG);
|
||||
}
|
||||
```
|
||||
|
||||
The `PluginConfigSpec` class allows developers to specify the settings that a plugin supports complete with setting name, data type, deprecation status, required status, and default value. In this example, the `delimiter` setting defines the delimiter on which the codec will split events. It is not a required setting and if it is not explicitly set, its default value will be `,`.
|
||||
|
||||
The `configSchema` method must return a list of all settings that the plugin supports. The Logstash execution engine will validate that all required settings are present and that no unsupported settings are present.
|
||||
|
||||
|
||||
#### Constructor and initialization [_constructor_and_initialization_2]
|
||||
|
||||
```java
|
||||
private final String id;
|
||||
private final String delimiter;
|
||||
|
||||
public JavaCodecExample(final Configuration config, final Context context) {
|
||||
this(config.get(DELIMITER_CONFIG));
|
||||
}
|
||||
|
||||
private JavaCodecExample(String delimiter) {
|
||||
this.id = UUID.randomUUID().toString();
|
||||
this.delimiter = delimiter;
|
||||
}
|
||||
```
|
||||
|
||||
All Java codec plugins must have a constructor taking a `Configuration` and `Context` argument. This is the constructor that will be used to instantiate them at runtime. The retrieval and validation of all plugin settings should occur in this constructor. In this example, the delimiter to be used for delimiting events is retrieved from its setting and stored in a local variable so that it can be used later in the `decode` and `encode` methods. The codec’s ID is initialized to a random UUID (as should be done for most codecs), and a local `encoder` variable is initialized to encode and decode with a specified character set.
|
||||
|
||||
Any additional initialization may occur in the constructor as well. If there are any unrecoverable errors encountered in the configuration or initialization of the codec plugin, a descriptive exception should be thrown. The exception will be logged and will prevent Logstash from starting.
|
||||
|
||||
|
||||
### Codec methods [_codec_methods]
|
||||
|
||||
```java
|
||||
@Override
|
||||
public void decode(ByteBuffer byteBuffer, Consumer<Map<String, Object>> consumer) {
|
||||
// a not-production-grade delimiter decoder
|
||||
byte[] byteInput = new byte[byteBuffer.remaining()];
|
||||
byteBuffer.get(byteInput);
|
||||
if (byteInput.length > 0) {
|
||||
String input = new String(byteInput);
|
||||
String[] split = input.split(delimiter);
|
||||
for (String s : split) {
|
||||
Map<String, Object> map = new HashMap<>();
|
||||
map.put("message", s);
|
||||
consumer.accept(map);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void flush(ByteBuffer byteBuffer, Consumer<Map<String, Object>> consumer) {
|
||||
// if the codec maintains any internal state such as partially-decoded input, this
|
||||
// method should flush that state along with any additional input supplied in
|
||||
// the ByteBuffer
|
||||
|
||||
decode(byteBuffer, consumer); // this is a simplistic implementation
|
||||
}
|
||||
|
||||
@Override
|
||||
public void encode(Event event, OutputStream outputStream) throws IOException {
|
||||
outputStream.write((event.toString() + delimiter).getBytes(Charset.defaultCharset()));
|
||||
}
|
||||
```
|
||||
|
||||
The `decode`, `flush`, and `encode` methods provide the core functionality of the codec. Codecs may be used by inputs to decode a sequence or stream of bytes into events or by outputs to encode events into a sequence of bytes.
|
||||
|
||||
The `decode` method decodes events from the specified `ByteBuffer` and passes them to the provided `Consumer`. The input must provide a `ByteBuffer` that is ready for reading with `byteBuffer.position()` indicating the next position to read and `byteBuffer.limit()` indicating the first byte in the buffer that is not safe to read. Codecs must ensure that `byteBuffer.position()` reflects the last-read position before returning control to the input. The input is then responsible for returning the buffer to write mode via either `byteBuffer.clear()` or `byteBuffer.compact()` before resuming writes. In the example above, the `decode` method simply splits the incoming byte stream on the specified delimiter. A production-grade codec such as [`java-line`](https://github.com/elastic/logstash/blob/main/logstash-core/src/main/java/org/logstash/plugins/codecs/Line.java) would not make the simplifying assumption that the end of the supplied byte stream corresponded with the end of an event.
|
||||
|
||||
Events should be constructed as instances of `Map<String, Object>` and pushed into the event pipeline via the `Consumer<Map<String, Object>>.accept()` method. To reduce allocations and GC pressure, codecs may reuse the same map instance by modifying its fields between calls to `Consumer<Map<String, Object>>.accept()` because the event pipeline will create events based on a copy of the map’s data.
|
||||
|
||||
The `flush` method works in coordination with the `decode` method to decode all remaining events from the specified `ByteBuffer` along with any internal state that may remain after previous calls to the `decode` method. As an example of internal state that a codec might maintain, consider an input stream of bytes `event1/event2/event3` with a delimiter of `/`. Due to buffering or other reasons, the input might supply a partial stream of bytes such as `event1/eve` to the codec’s `decode` method. In this case, the codec could save the beginning three characters `eve` of the second event rather than assuming that the supplied byte stream ends on an event boundary. If the next call to `decode` supplied the `nt2/ev` bytes, the codec would prepend the saved `eve` bytes to produce the full `event2` event and then save the remaining `ev` bytes for decoding when the remainder of the bytes for that event were supplied. A call to `flush` signals the codec that the supplied bytes represent the end of an event stream and all remaining bytes should be decoded to events. The `flush` example above is a simplistic implementation that does not maintain any state about partially-supplied byte streams across calls to `decode`.
|
||||
|
||||
The `encode` method encodes an event into a sequence of bytes and writes it into the specified `OutputStream`. Because a single codec instance is shared across all pipeline workers in the output stage of the Logstash pipeline, codecs should *not* retain state across calls to their `encode` methods.
|
||||
|
||||
|
||||
### cloneCodec method [_clonecodec_method]
|
||||
|
||||
```java
|
||||
@Override
|
||||
public Codec cloneCodec() {
|
||||
return new JavaCodecExample(this.delimiter);
|
||||
}
|
||||
```
|
||||
|
||||
The `cloneCodec` method should return an identical instance of the codec with the exception of its ID. Because codecs may be stateful across calls to their `decode` methods, input plugins that are multi-threaded should use a separate instance of each codec via the `cloneCodec` method for each of their threads. Because a single codec instance is shared across all pipeline workers in the output stage of the Logstash pipeline, codecs should *not* retain state across calls to their `encode` methods. In the example above, the codec is cloned with the same delimiter but a different ID.
|
||||
|
||||
|
||||
### getId method [_getid_method_2]
|
||||
|
||||
```java
|
||||
@Override
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
```
|
||||
|
||||
For codec plugins, the `getId` method should always return the id that was set at instantiation time. This is typically an UUID.
|
||||
|
||||
|
||||
### Unit tests [_unit_tests_2]
|
||||
|
||||
Lastly, but certainly not least importantly, unit tests are strongly encouraged. The example codec plugin includes an [example unit test](https://github.com/logstash-plugins/logstash-codec-java_codec_example/blob/main/src/test/java/org/logstashplugins/JavaCodecExampleTest.java) that you can use as a template for your own.
|
||||
|
||||
|
||||
## Package and deploy [_package_and_deploy_2]
|
||||
|
||||
Java plugins are packaged as Ruby gems for dependency management and interoperability with Ruby plugins. Once they are packaged as gems, they may be installed with the `logstash-plugin` utility just as Ruby plugins are. Because no knowledge of Ruby or its toolchain should be required for Java plugin development, the procedure for packaging Java plugins as Ruby gems has been automated through a custom task in the Gradle build file provided with the example Java plugins. The following sections describe how to configure and execute that packaging task as well as how to install the packaged Java plugin in Logstash.
|
||||
|
||||
|
||||
### Configuring the Gradle packaging task [_configuring_the_gradle_packaging_task_2]
|
||||
|
||||
The following section appears near the top of the `build.gradle` file supplied with the example Java plugins:
|
||||
|
||||
```java
|
||||
// ===========================================================================
|
||||
// plugin info
|
||||
// ===========================================================================
|
||||
group 'org.logstashplugins' // must match the package of the main plugin class
|
||||
version "${file("VERSION").text.trim()}" // read from required VERSION file
|
||||
description = "Example Java filter implementation"
|
||||
pluginInfo.licenses = ['Apache-2.0'] // list of SPDX license IDs
|
||||
pluginInfo.longDescription = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using \$LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
||||
pluginInfo.authors = ['Elasticsearch']
|
||||
pluginInfo.email = ['info@elastic.co']
|
||||
pluginInfo.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
|
||||
pluginInfo.pluginType = "filter"
|
||||
pluginInfo.pluginClass = "JavaFilterExample"
|
||||
pluginInfo.pluginName = "java_filter_example"
|
||||
// ===========================================================================
|
||||
```
|
||||
|
||||
You should configure the values above for your plugin.
|
||||
|
||||
* The `version` value will be automatically read from the `VERSION` file in the root of your plugin’s codebase.
|
||||
* `pluginInfo.pluginType` should be set to one of `input`, `filter`, `codec`, or `output`.
|
||||
* `pluginInfo.pluginName` must match the name specified on the `@LogstashPlugin` annotation on the main plugin class. The Gradle packaging task will validate that and return an error if they do not match.
|
||||
|
||||
|
||||
### Running the Gradle packaging task [_running_the_gradle_packaging_task_2]
|
||||
|
||||
Several Ruby source files along with a `gemspec` file and a `Gemfile` are required to package the plugin as a Ruby gem. These Ruby files are used only for defining the Ruby gem structure or at Logstash startup time to register the Java plugin. They are not used during runtime event processing. The Gradle packaging task automatically generates all of these files based on the values configured in the section above.
|
||||
|
||||
You run the Gradle packaging task with the following command:
|
||||
|
||||
```shell
|
||||
./gradlew gem
|
||||
```
|
||||
|
||||
For Windows platforms: Substitute `gradlew.bat` for `./gradlew` as appropriate in the command.
|
||||
|
||||
That task will produce a gem file in the root directory of your plugin’s codebase with the name `logstash-{{plugintype}}-<pluginName>-<version>.gem`
|
||||
|
||||
|
||||
### Installing the Java plugin in Logstash [_installing_the_java_plugin_in_logstash_2]
|
||||
|
||||
After you have packaged your Java plugin as a Ruby gem, you can install it in Logstash with this command:
|
||||
|
||||
```shell
|
||||
bin/logstash-plugin install --no-verify --local /path/to/javaPlugin.gem
|
||||
```
|
||||
|
||||
For Windows platforms: Substitute backslashes for forward slashes as appropriate in the command.
|
||||
|
||||
|
||||
## Run Logstash with the Java codec plugin [_run_logstash_with_the_java_codec_plugin]
|
||||
|
||||
To test the plugin, start Logstash with:
|
||||
|
||||
```java
|
||||
echo "foo,bar" | bin/logstash -e 'input { java_stdin { codec => java_codec_example } }'
|
||||
```
|
||||
|
||||
The expected Logstash output (excluding initialization) with the configuration above is:
|
||||
|
||||
```txt
|
||||
{
|
||||
"@version" => "1",
|
||||
"message" => "foo",
|
||||
"@timestamp" => yyyy-MM-ddThh:mm:ss.SSSZ,
|
||||
"host" => "<yourHostName>"
|
||||
}
|
||||
{
|
||||
"@version" => "1",
|
||||
"message" => "bar\n",
|
||||
"@timestamp" => yyyy-MM-ddThh:mm:ss.SSSZ,
|
||||
"host" => "<yourHostName>"
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
## Feedback [_feedback_2]
|
||||
|
||||
If you have any feedback on Java plugin support in Logstash, please comment on our [main Github issue](https://github.com/elastic/logstash/issues/9215) or post in the [Logstash forum](https://discuss.elastic.co/c/logstash).
|
||||
|
||||
307
docs/extend/java-filter-plugin.md
Normal file
307
docs/extend/java-filter-plugin.md
Normal file
|
|
@ -0,0 +1,307 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/java-filter-plugin.html
|
||||
---
|
||||
|
||||
# How to write a Java filter plugin [java-filter-plugin]
|
||||
|
||||
To develop a new Java filter for Logstash, you write a new Java class that conforms to the Logstash Java Filters API, package it, and install it with the logstash-plugin utility. We’ll go through each of those steps.
|
||||
|
||||
|
||||
## Set up your environment [_set_up_your_environment_3]
|
||||
|
||||
|
||||
### Copy the example repo [_copy_the_example_repo_3]
|
||||
|
||||
Start by copying the [example filter plugin](https://github.com/logstash-plugins/logstash-filter-java_filter_example). The plugin API is currently part of the Logstash codebase so you must have a local copy of that available. You can obtain a copy of the Logstash codebase with the following `git` command:
|
||||
|
||||
```shell
|
||||
git clone --branch <branch_name> --single-branch https://github.com/elastic/logstash.git <target_folder>
|
||||
```
|
||||
|
||||
The `branch_name` should correspond to the version of Logstash containing the preferred revision of the Java plugin API.
|
||||
|
||||
::::{note}
|
||||
The GA version of the Java plugin API is available in the `7.2` and later branches of the Logstash codebase.
|
||||
::::
|
||||
|
||||
|
||||
Specify the `target_folder` for your local copy of the Logstash codebase. If you do not specify `target_folder`, it defaults to a new folder called `logstash` under your current folder.
|
||||
|
||||
|
||||
### Generate the .jar file [_generate_the_jar_file_3]
|
||||
|
||||
After you have obtained a copy of the appropriate revision of the Logstash codebase, you need to compile it to generate the .jar file containing the Java plugin API. From the root directory of your Logstash codebase ($LS_HOME), you can compile it with `./gradlew assemble` (or `gradlew.bat assemble` if you’re running on Windows). This should produce the `$LS_HOME/logstash-core/build/libs/logstash-core-x.y.z.jar` where `x`, `y`, and `z` refer to the version of Logstash.
|
||||
|
||||
After you have successfully compiled Logstash, you need to tell your Java plugin where to find the `logstash-core-x.y.z.jar` file. Create a new file named `gradle.properties` in the root folder of your plugin project. That file should have a single line:
|
||||
|
||||
```txt
|
||||
LOGSTASH_CORE_PATH=<target_folder>/logstash-core
|
||||
```
|
||||
|
||||
where `target_folder` is the root folder of your local copy of the Logstash codebase.
|
||||
|
||||
|
||||
## Code the plugin [_code_the_plugin_3]
|
||||
|
||||
The example filter plugin allows one to configure a field in each event that will be reversed. For example, if the filter were configured to reverse the `day_of_week` field, an event with `day_of_week: "Monday"` would be transformed to `day_of_week: "yadnoM"`. Let’s look at the main class in that example filter:
|
||||
|
||||
```java
|
||||
@LogstashPlugin(name = "java_filter_example")
|
||||
public class JavaFilterExample implements Filter {
|
||||
|
||||
public static final PluginConfigSpec<String> SOURCE_CONFIG =
|
||||
PluginConfigSpec.stringSetting("source", "message");
|
||||
|
||||
private String id;
|
||||
private String sourceField;
|
||||
|
||||
public JavaFilterExample(String id, Configuration config, Context context) {
|
||||
this.id = id;
|
||||
this.sourceField = config.get(SOURCE_CONFIG);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Collection<Event> filter(Collection<Event> events, FilterMatchListener matchListener) {
|
||||
for (Event e : events) {
|
||||
Object f = e.getField(sourceField);
|
||||
if (f instanceof String) {
|
||||
e.setField(sourceField, StringUtils.reverse((String)f));
|
||||
matchListener.filterMatched(e);
|
||||
}
|
||||
}
|
||||
return events;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Collection<PluginConfigSpec<?>> configSchema() {
|
||||
return Collections.singletonList(SOURCE_CONFIG);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getId() {
|
||||
return this.id;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
this.sourceField = null;
|
||||
return;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Let’s step through and examine each part of that class.
|
||||
|
||||
|
||||
### Class declaration [_class_declaration_7]
|
||||
|
||||
```java
|
||||
@LogstashPlugin(name = "java_filter_example")
|
||||
public class JavaFilterExample implements Filter {
|
||||
```
|
||||
|
||||
Notes about the class declaration:
|
||||
|
||||
* All Java plugins must be annotated with the `@LogstashPlugin` annotation. Additionally:
|
||||
|
||||
* The `name` property of the annotation must be supplied and defines the name of the plugin as it will be used in the Logstash pipeline definition. For example, this filter would be referenced in the filter section of the Logstash pipeline defintion as `filter { java_filter_example => { .... } }`
|
||||
* The value of the `name` property must match the name of the class excluding casing and underscores.
|
||||
|
||||
* The class must implement the `co.elastic.logstash.api.Filter` interface.
|
||||
* Java plugins may not be created in the `org.logstash` or `co.elastic.logstash` packages to prevent potential clashes with classes in Logstash itself.
|
||||
|
||||
|
||||
### Plugin settings [_plugin_settings_3]
|
||||
|
||||
The snippet below contains both the setting definition and the method referencing it:
|
||||
|
||||
```java
|
||||
public static final PluginConfigSpec<String> SOURCE_CONFIG =
|
||||
PluginConfigSpec.stringSetting("source", "message");
|
||||
|
||||
@Override
|
||||
public Collection<PluginConfigSpec<?>> configSchema() {
|
||||
return Collections.singletonList(SOURCE_CONFIG);
|
||||
}
|
||||
```
|
||||
|
||||
The `PluginConfigSpec` class allows developers to specify the settings that a plugin supports complete with setting name, data type, deprecation status, required status, and default value. In this example, the `source` setting defines the name of the field in each event that will be reversed. It is not a required setting and if it is not explicitly set, its default value will be `message`.
|
||||
|
||||
The `configSchema` method must return a list of all settings that the plugin supports. In a future phase of the Java plugin project, the Logstash execution engine will validate that all required settings are present and that no unsupported settings are present.
|
||||
|
||||
|
||||
### Constructor and initialization [_constructor_and_initialization_3]
|
||||
|
||||
```java
|
||||
private String id;
|
||||
private String sourceField;
|
||||
|
||||
public JavaFilterExample(String id, Configuration config, Context context) {
|
||||
this.id = id;
|
||||
this.sourceField = config.get(SOURCE_CONFIG);
|
||||
}
|
||||
```
|
||||
|
||||
All Java filter plugins must have a constructor taking a `String` id and a `Configuration` and `Context` argument. This is the constructor that will be used to instantiate them at runtime. The retrieval and validation of all plugin settings should occur in this constructor. In this example, the name of the field to be reversed in each event is retrieved from its setting and stored in a local variable so that it can be used later in the `filter` method.
|
||||
|
||||
Any additional initialization may occur in the constructor as well. If there are any unrecoverable errors encountered in the configuration or initialization of the filter plugin, a descriptive exception should be thrown. The exception will be logged and will prevent Logstash from starting.
|
||||
|
||||
|
||||
### Filter method [_filter_method_2]
|
||||
|
||||
```java
|
||||
@Override
|
||||
public Collection<Event> filter(Collection<Event> events, FilterMatchListener matchListener) {
|
||||
for (Event e : events) {
|
||||
Object f = e.getField(sourceField);
|
||||
if (f instanceof String) {
|
||||
e.setField(sourceField, StringUtils.reverse((String)f));
|
||||
matchListener.filterMatched(e);
|
||||
}
|
||||
}
|
||||
return events;
|
||||
```
|
||||
|
||||
Finally, we come to the `filter` method that is invoked by the Logstash execution engine on batches of events as they flow through the event processing pipeline. The events to be filtered are supplied in the `events` argument and the method should return a collection of filtered events. Filters may perform a variety of actions on events as they flow through the pipeline including:
|
||||
|
||||
* Mutation — Fields in events may be added, removed, or changed by a filter. This is the most common scenario for filters that perform various kinds of enrichment on events. In this scenario, the incoming `events` collection may be returned unmodified since the events in the collection are mutated in place.
|
||||
* Deletion — Events may be removed from the event pipeline by a filter so that subsequent filters and outputs do not receive them. In this scenario, the events to be deleted must be removed from the collection of filtered events before it is returned.
|
||||
* Creation — A filter may insert new events into the event pipeline that will be seen only by subsequent filters and outputs. In this scenario, the new events must be added to the collection of filtered events before it is returned.
|
||||
* Observation — Events may pass unchanged by a filter through the event pipeline. This may be useful in scenarios where a filter performs external actions (e.g., updating an external cache) based on the events observed in the event pipeline. In this scenario, the incoming `events` collection may be returned unmodified since no changes were made.
|
||||
|
||||
In the example above, the value of the `source` field is retrieved from each event and reversed if it is a string value. Because each event is mutated in place, the incoming `events` collection can be returned.
|
||||
|
||||
The `matchListener` is the mechanism by which filters indicate which events "match". The common actions for filters such as `add_field` and `add_tag` are applied only to events that are designated as "matching". Some filters such as the [grok filter](/reference/plugins-filters-grok.md) have a clear definition for what constitutes a matching event and will notify the listener only for matching events. Other filters such as the [UUID filter](/reference/plugins-filters-uuid.md) have no specific match criteria and should notify the listener for every event filtered. In this example, the filter notifies the match listener for any event that had a `String` value in its `source` field and was therefore able to be reversed.
|
||||
|
||||
|
||||
### getId method [_getid_method_3]
|
||||
|
||||
```java
|
||||
@Override
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
```
|
||||
|
||||
For filter plugins, the `getId` method should always return the id that was provided to the plugin through its constructor at instantiation time.
|
||||
|
||||
|
||||
### close method [_close_method]
|
||||
|
||||
```java
|
||||
@Override
|
||||
public void close() {
|
||||
// shutdown a resource that was instantiated during the filter initialization phase.
|
||||
this.sourceField = null;
|
||||
return;
|
||||
}
|
||||
```
|
||||
|
||||
Filter plugins can use additional resources to perform operations, such as creating new database connections. Implementing the `close` method will allow the plugins to free up those resources when shutting down the pipeline.
|
||||
|
||||
|
||||
### Unit tests [_unit_tests_3]
|
||||
|
||||
Lastly, but certainly not least importantly, unit tests are strongly encouraged. The example filter plugin includes an [example unit test](https://github.com/logstash-plugins/logstash-filter-java_filter_example/blob/main/src/test/java/org/logstashplugins/JavaFilterExampleTest.java) that you can use as a template for your own.
|
||||
|
||||
|
||||
## Package and deploy [_package_and_deploy_3]
|
||||
|
||||
Java plugins are packaged as Ruby gems for dependency management and interoperability with Ruby plugins. Once they are packaged as gems, they may be installed with the `logstash-plugin` utility just as Ruby plugins are. Because no knowledge of Ruby or its toolchain should be required for Java plugin development, the procedure for packaging Java plugins as Ruby gems has been automated through a custom task in the Gradle build file provided with the example Java plugins. The following sections describe how to configure and execute that packaging task as well as how to install the packaged Java plugin in Logstash.
|
||||
|
||||
|
||||
### Configuring the Gradle packaging task [_configuring_the_gradle_packaging_task_3]
|
||||
|
||||
The following section appears near the top of the `build.gradle` file supplied with the example Java plugins:
|
||||
|
||||
```java
|
||||
// ===========================================================================
|
||||
// plugin info
|
||||
// ===========================================================================
|
||||
group 'org.logstashplugins' // must match the package of the main plugin class
|
||||
version "${file("VERSION").text.trim()}" // read from required VERSION file
|
||||
description = "Example Java filter implementation"
|
||||
pluginInfo.licenses = ['Apache-2.0'] // list of SPDX license IDs
|
||||
pluginInfo.longDescription = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using \$LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
||||
pluginInfo.authors = ['Elasticsearch']
|
||||
pluginInfo.email = ['info@elastic.co']
|
||||
pluginInfo.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
|
||||
pluginInfo.pluginType = "filter"
|
||||
pluginInfo.pluginClass = "JavaFilterExample"
|
||||
pluginInfo.pluginName = "java_filter_example"
|
||||
// ===========================================================================
|
||||
```
|
||||
|
||||
You should configure the values above for your plugin.
|
||||
|
||||
* The `version` value will be automatically read from the `VERSION` file in the root of your plugin’s codebase.
|
||||
* `pluginInfo.pluginType` should be set to one of `input`, `filter`, `codec`, or `output`.
|
||||
* `pluginInfo.pluginName` must match the name specified on the `@LogstashPlugin` annotation on the main plugin class. The Gradle packaging task will validate that and return an error if they do not match.
|
||||
|
||||
|
||||
### Running the Gradle packaging task [_running_the_gradle_packaging_task_3]
|
||||
|
||||
Several Ruby source files along with a `gemspec` file and a `Gemfile` are required to package the plugin as a Ruby gem. These Ruby files are used only for defining the Ruby gem structure or at Logstash startup time to register the Java plugin. They are not used during runtime event processing. The Gradle packaging task automatically generates all of these files based on the values configured in the section above.
|
||||
|
||||
You run the Gradle packaging task with the following command:
|
||||
|
||||
```shell
|
||||
./gradlew gem
|
||||
```
|
||||
|
||||
For Windows platforms: Substitute `gradlew.bat` for `./gradlew` as appropriate in the command.
|
||||
|
||||
That task will produce a gem file in the root directory of your plugin’s codebase with the name `logstash-{{plugintype}}-<pluginName>-<version>.gem`
|
||||
|
||||
|
||||
### Installing the Java plugin in Logstash [_installing_the_java_plugin_in_logstash_3]
|
||||
|
||||
After you have packaged your Java plugin as a Ruby gem, you can install it in Logstash with this command:
|
||||
|
||||
```shell
|
||||
bin/logstash-plugin install --no-verify --local /path/to/javaPlugin.gem
|
||||
```
|
||||
|
||||
For Windows platforms: Substitute backslashes for forward slashes as appropriate in the command.
|
||||
|
||||
|
||||
## Run Logstash with the Java filter plugin [_run_logstash_with_the_java_filter_plugin]
|
||||
|
||||
The following is a minimal Logstash configuration that can be used to test that the Java filter plugin is correctly installed and functioning.
|
||||
|
||||
```java
|
||||
input {
|
||||
generator { message => "Hello world!" count => 1 }
|
||||
}
|
||||
filter {
|
||||
java_filter_example {}
|
||||
}
|
||||
output {
|
||||
stdout { codec => rubydebug }
|
||||
}
|
||||
```
|
||||
|
||||
Copy the above Logstash configuration to a file such as `java_filter.conf`. Start Logstash with:
|
||||
|
||||
```shell
|
||||
bin/logstash -f /path/to/java_filter.conf
|
||||
```
|
||||
|
||||
The expected Logstash output (excluding initialization) with the configuration above is:
|
||||
|
||||
```txt
|
||||
{
|
||||
"sequence" => 0,
|
||||
"@version" => "1",
|
||||
"message" => "!dlrow olleH",
|
||||
"@timestamp" => yyyy-MM-ddThh:mm:ss.SSSZ,
|
||||
"host" => "<yourHostName>"
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
## Feedback [_feedback_3]
|
||||
|
||||
If you have any feedback on Java plugin support in Logstash, please comment on our [main Github issue](https://github.com/elastic/logstash/issues/9215) or post in the [Logstash forum](https://discuss.elastic.co/c/logstash).
|
||||
|
||||
341
docs/extend/java-input-plugin.md
Normal file
341
docs/extend/java-input-plugin.md
Normal file
|
|
@ -0,0 +1,341 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/java-input-plugin.html
|
||||
---
|
||||
|
||||
# How to write a Java input plugin [java-input-plugin]
|
||||
|
||||
To develop a new Java input for Logstash, you write a new Java class that conforms to the Logstash Java Inputs API, package it, and install it with the logstash-plugin utility. We’ll go through each of those steps.
|
||||
|
||||
|
||||
## Set up your environment [_set_up_your_environment]
|
||||
|
||||
|
||||
### Copy the example repo [_copy_the_example_repo]
|
||||
|
||||
Start by copying the [example input plugin](https://github.com/logstash-plugins/logstash-input-java_input_example). The plugin API is currently part of the Logstash codebase so you must have a local copy of that available. You can obtain a copy of the Logstash codebase with the following `git` command:
|
||||
|
||||
```shell
|
||||
git clone --branch <branch_name> --single-branch https://github.com/elastic/logstash.git <target_folder>
|
||||
```
|
||||
|
||||
The `branch_name` should correspond to the version of Logstash containing the preferred revision of the Java plugin API.
|
||||
|
||||
::::{note}
|
||||
The GA version of the Java plugin API is available in the `7.2` and later branches of the Logstash codebase.
|
||||
::::
|
||||
|
||||
|
||||
Specify the `target_folder` for your local copy of the Logstash codebase. If you do not specify `target_folder`, it defaults to a new folder called `logstash` under your current folder.
|
||||
|
||||
|
||||
### Generate the .jar file [_generate_the_jar_file]
|
||||
|
||||
After you have obtained a copy of the appropriate revision of the Logstash codebase, you need to compile it to generate the .jar file containing the Java plugin API. From the root directory of your Logstash codebase ($LS_HOME), you can compile it with `./gradlew assemble` (or `gradlew.bat assemble` if you’re running on Windows). This should produce the `$LS_HOME/logstash-core/build/libs/logstash-core-x.y.z.jar` where `x`, `y`, and `z` refer to the version of Logstash.
|
||||
|
||||
After you have successfully compiled Logstash, you need to tell your Java plugin where to find the `logstash-core-x.y.z.jar` file. Create a new file named `gradle.properties` in the root folder of your plugin project. That file should have a single line:
|
||||
|
||||
```txt
|
||||
LOGSTASH_CORE_PATH=<target_folder>/logstash-core
|
||||
```
|
||||
|
||||
where `target_folder` is the root folder of your local copy of the Logstash codebase.
|
||||
|
||||
|
||||
## Code the plugin [_code_the_plugin]
|
||||
|
||||
The example input plugin generates a configurable number of simple events before terminating. Let’s look at the main class in the example input.
|
||||
|
||||
```java
|
||||
@LogstashPlugin(name="java_input_example")
|
||||
public class JavaInputExample implements Input {
|
||||
|
||||
public static final PluginConfigSpec<Long> EVENT_COUNT_CONFIG =
|
||||
PluginConfigSpec.numSetting("count", 3);
|
||||
|
||||
public static final PluginConfigSpec<String> PREFIX_CONFIG =
|
||||
PluginConfigSpec.stringSetting("prefix", "message");
|
||||
|
||||
private String id;
|
||||
private long count;
|
||||
private String prefix;
|
||||
private final CountDownLatch done = new CountDownLatch(1);
|
||||
private volatile boolean stopped;
|
||||
|
||||
|
||||
public JavaInputExample(String id, Configuration config, Context context) {
|
||||
this.id = id;
|
||||
count = config.get(EVENT_COUNT_CONFIG);
|
||||
prefix = config.get(PREFIX_CONFIG);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void start(Consumer<Map<String, Object>> consumer) {
|
||||
int eventCount = 0;
|
||||
try {
|
||||
while (!stopped && eventCount < count) {
|
||||
eventCount++;
|
||||
consumer.accept.push(Collections.singletonMap("message",
|
||||
prefix + " " + StringUtils.center(eventCount + " of " + count, 20)));
|
||||
}
|
||||
} finally {
|
||||
stopped = true;
|
||||
done.countDown();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void stop() {
|
||||
stopped = true; // set flag to request cooperative stop of input
|
||||
}
|
||||
|
||||
@Override
|
||||
public void awaitStop() throws InterruptedException {
|
||||
done.await(); // blocks until input has stopped
|
||||
}
|
||||
|
||||
@Override
|
||||
public Collection<PluginConfigSpec<?>> configSchema() {
|
||||
return Arrays.asList(EVENT_COUNT_CONFIG, PREFIX_CONFIG);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getId() {
|
||||
return this.id;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Let’s step through and examine each part of that class.
|
||||
|
||||
|
||||
### Class declaration [_class_declaration_5]
|
||||
|
||||
```java
|
||||
@LogstashPlugin(name="java_input_example")
|
||||
public class JavaInputExample implements Input {
|
||||
```
|
||||
|
||||
Notes about the class declaration:
|
||||
|
||||
* All Java plugins must be annotated with the `@LogstashPlugin` annotation. Additionally:
|
||||
|
||||
* The `name` property of the annotation must be supplied and defines the name of the plugin as it will be used in the Logstash pipeline definition. For example, this input would be referenced in the input section of the Logstash pipeline defintion as `input { java_input_example => { .... } }`
|
||||
* The value of the `name` property must match the name of the class excluding casing and underscores.
|
||||
|
||||
* The class must implement the `co.elastic.logstash.api.Input` interface.
|
||||
* Java plugins may not be created in the `org.logstash` or `co.elastic.logstash` packages to prevent potential clashes with classes in Logstash itself.
|
||||
|
||||
|
||||
### Plugin settings [_plugin_settings]
|
||||
|
||||
The snippet below contains both the setting definition and the method referencing it.
|
||||
|
||||
```java
|
||||
public static final PluginConfigSpec<Long> EVENT_COUNT_CONFIG =
|
||||
PluginConfigSpec.numSetting("count", 3);
|
||||
|
||||
public static final PluginConfigSpec<String> PREFIX_CONFIG =
|
||||
PluginConfigSpec.stringSetting("prefix", "message");
|
||||
|
||||
@Override
|
||||
public Collection<PluginConfigSpec<?>> configSchema() {
|
||||
return Arrays.asList(EVENT_COUNT_CONFIG, PREFIX_CONFIG);
|
||||
}
|
||||
```
|
||||
|
||||
The `PluginConfigSpec` class allows developers to specify the settings that a plugin supports complete with setting name, data type, deprecation status, required status, and default value. In this example, the `count` setting defines the number of events that will be generated and the `prefix` setting defines an optional prefix to include in the event field. Neither setting is required and if it is not explicitly set, the settings default to `3` and `message`, respectively.
|
||||
|
||||
The `configSchema` method must return a list of all settings that the plugin supports. In a future phase of the Java plugin project, the Logstash execution engine will validate that all required settings are present and that no unsupported settings are present.
|
||||
|
||||
|
||||
### Constructor and initialization [_constructor_and_initialization]
|
||||
|
||||
```java
|
||||
private String id;
|
||||
private long count;
|
||||
private String prefix;
|
||||
|
||||
public JavaInputExample(String id, Configuration config, Context context) {
|
||||
this.id = id;
|
||||
count = config.get(EVENT_COUNT_CONFIG);
|
||||
prefix = config.get(PREFIX_CONFIG);
|
||||
}
|
||||
```
|
||||
|
||||
All Java input plugins must have a constructor taking a `String` id and `Configuration` and `Context` argument. This is the constructor that will be used to instantiate them at runtime. The retrieval and validation of all plugin settings should occur in this constructor. In this example, the values of the two plugin settings are retrieved and stored in local variables for later use in the `start` method.
|
||||
|
||||
Any additional initialization may occur in the constructor as well. If there are any unrecoverable errors encountered in the configuration or initialization of the input plugin, a descriptive exception should be thrown. The exception will be logged and will prevent Logstash from starting.
|
||||
|
||||
|
||||
### Start method [_start_method]
|
||||
|
||||
```java
|
||||
@Override
|
||||
public void start(Consumer<Map<String, Object>> consumer) {
|
||||
int eventCount = 0;
|
||||
try {
|
||||
while (!stopped && eventCount < count) {
|
||||
eventCount++;
|
||||
consumer.accept.push(Collections.singletonMap("message",
|
||||
prefix + " " + StringUtils.center(eventCount + " of " + count, 20)));
|
||||
}
|
||||
} finally {
|
||||
stopped = true;
|
||||
done.countDown();
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The `start` method begins the event-producing loop in an input. Inputs are flexible and may produce events through many different mechanisms including:
|
||||
|
||||
* a pull mechanism such as periodic queries of external database
|
||||
* a push mechanism such as events sent from clients to a local network port
|
||||
* a timed computation such as a heartbeat
|
||||
* any other mechanism that produces a useful stream of events. Event streams may be either finite or infinite. If the input produces an infinite stream of events, this method should loop until a stop request is made through the `stop` method. If the input produces a finite stream of events, this method should terminate when the last event in the stream is produced or a stop request is made, whichever comes first.
|
||||
|
||||
Events should be constructed as instances of `Map<String, Object>` and pushed into the event pipeline via the `Consumer<Map<String, Object>>.accept()` method. To reduce allocations and GC pressure, inputs may reuse the same map instance by modifying its fields between calls to `Consumer<Map<String, Object>>.accept()` because the event pipeline will create events based on a copy of the map’s data.
|
||||
|
||||
|
||||
### Stop and awaitStop methods [_stop_and_awaitstop_methods]
|
||||
|
||||
```java
|
||||
private final CountDownLatch done = new CountDownLatch(1);
|
||||
private volatile boolean stopped;
|
||||
|
||||
@Override
|
||||
public void stop() {
|
||||
stopped = true; // set flag to request cooperative stop of input
|
||||
}
|
||||
|
||||
@Override
|
||||
public void awaitStop() throws InterruptedException {
|
||||
done.await(); // blocks until input has stopped
|
||||
}
|
||||
```
|
||||
|
||||
The `stop` method notifies the input to stop producing events. The stop mechanism may be implemented in any way that honors the API contract though a `volatile boolean` flag works well for many use cases.
|
||||
|
||||
Inputs stop both asynchronously and cooperatively. Use the `awaitStop` method to block until the input has completed the stop process. Note that this method should **not** signal the input to stop as the `stop` method does. The awaitStop mechanism may be implemented in any way that honors the API contract though a `CountDownLatch` works well for many use cases.
|
||||
|
||||
|
||||
### getId method [_getid_method]
|
||||
|
||||
```java
|
||||
@Override
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
```
|
||||
|
||||
For input plugins, the `getId` method should always return the id that was provided to the plugin through its constructor at instantiation time.
|
||||
|
||||
|
||||
### Unit tests [_unit_tests]
|
||||
|
||||
Lastly, but certainly not least importantly, unit tests are strongly encouraged. The example input plugin includes an [example unit test](https://github.com/logstash-plugins/logstash-input-java_input_example/blob/main/src/test/java/org/logstashplugins/JavaInputExampleTest.java) that you can use as a template for your own.
|
||||
|
||||
|
||||
## Package and deploy [_package_and_deploy]
|
||||
|
||||
Java plugins are packaged as Ruby gems for dependency management and interoperability with Ruby plugins. Once they are packaged as gems, they may be installed with the `logstash-plugin` utility just as Ruby plugins are. Because no knowledge of Ruby or its toolchain should be required for Java plugin development, the procedure for packaging Java plugins as Ruby gems has been automated through a custom task in the Gradle build file provided with the example Java plugins. The following sections describe how to configure and execute that packaging task as well as how to install the packaged Java plugin in Logstash.
|
||||
|
||||
|
||||
### Configuring the Gradle packaging task [_configuring_the_gradle_packaging_task]
|
||||
|
||||
The following section appears near the top of the `build.gradle` file supplied with the example Java plugins:
|
||||
|
||||
```java
|
||||
// ===========================================================================
|
||||
// plugin info
|
||||
// ===========================================================================
|
||||
group 'org.logstashplugins' // must match the package of the main plugin class
|
||||
version "${file("VERSION").text.trim()}" // read from required VERSION file
|
||||
description = "Example Java filter implementation"
|
||||
pluginInfo.licenses = ['Apache-2.0'] // list of SPDX license IDs
|
||||
pluginInfo.longDescription = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using \$LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
||||
pluginInfo.authors = ['Elasticsearch']
|
||||
pluginInfo.email = ['info@elastic.co']
|
||||
pluginInfo.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
|
||||
pluginInfo.pluginType = "filter"
|
||||
pluginInfo.pluginClass = "JavaFilterExample"
|
||||
pluginInfo.pluginName = "java_filter_example"
|
||||
// ===========================================================================
|
||||
```
|
||||
|
||||
You should configure the values above for your plugin.
|
||||
|
||||
* The `version` value will be automatically read from the `VERSION` file in the root of your plugin’s codebase.
|
||||
* `pluginInfo.pluginType` should be set to one of `input`, `filter`, `codec`, or `output`.
|
||||
* `pluginInfo.pluginName` must match the name specified on the `@LogstashPlugin` annotation on the main plugin class. The Gradle packaging task will validate that and return an error if they do not match.
|
||||
|
||||
|
||||
### Running the Gradle packaging task [_running_the_gradle_packaging_task]
|
||||
|
||||
Several Ruby source files along with a `gemspec` file and a `Gemfile` are required to package the plugin as a Ruby gem. These Ruby files are used only for defining the Ruby gem structure or at Logstash startup time to register the Java plugin. They are not used during runtime event processing. The Gradle packaging task automatically generates all of these files based on the values configured in the section above.
|
||||
|
||||
You run the Gradle packaging task with the following command:
|
||||
|
||||
```shell
|
||||
./gradlew gem
|
||||
```
|
||||
|
||||
For Windows platforms: Substitute `gradlew.bat` for `./gradlew` as appropriate in the command.
|
||||
|
||||
That task will produce a gem file in the root directory of your plugin’s codebase with the name `logstash-{{plugintype}}-<pluginName>-<version>.gem`
|
||||
|
||||
|
||||
### Installing the Java plugin in Logstash [_installing_the_java_plugin_in_logstash]
|
||||
|
||||
After you have packaged your Java plugin as a Ruby gem, you can install it in Logstash with this command:
|
||||
|
||||
```shell
|
||||
bin/logstash-plugin install --no-verify --local /path/to/javaPlugin.gem
|
||||
```
|
||||
|
||||
For Windows platforms: Substitute backslashes for forward slashes as appropriate in the command.
|
||||
|
||||
|
||||
## Running Logstash with the Java input plugin [_running_logstash_with_the_java_input_plugin]
|
||||
|
||||
The following is a minimal Logstash configuration that can be used to test that the Java input plugin is correctly installed and functioning.
|
||||
|
||||
```java
|
||||
input {
|
||||
java_input_example {}
|
||||
}
|
||||
output {
|
||||
stdout { codec => rubydebug }
|
||||
}
|
||||
```
|
||||
|
||||
Copy the above Logstash configuration to a file such as `java_input.conf`. Start {{ls}} with:
|
||||
|
||||
```shell
|
||||
bin/logstash -f /path/to/java_input.conf
|
||||
```
|
||||
|
||||
The expected Logstash output (excluding initialization) with the configuration above is:
|
||||
|
||||
```txt
|
||||
{
|
||||
"@version" => "1",
|
||||
"message" => "message 1 of 3 ",
|
||||
"@timestamp" => yyyy-MM-ddThh:mm:ss.SSSZ
|
||||
}
|
||||
{
|
||||
"@version" => "1",
|
||||
"message" => "message 2 of 3 ",
|
||||
"@timestamp" => yyyy-MM-ddThh:mm:ss.SSSZ
|
||||
}
|
||||
{
|
||||
"@version" => "1",
|
||||
"message" => "message 3 of 3 ",
|
||||
"@timestamp" => yyyy-MM-ddThh:mm:ss.SSSZ
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
## Feedback [_feedback]
|
||||
|
||||
If you have any feedback on Java plugin support in Logstash, please comment on our [main Github issue](https://github.com/elastic/logstash/issues/9215) or post in the [Logstash forum](https://discuss.elastic.co/c/logstash).
|
||||
311
docs/extend/java-output-plugin.md
Normal file
311
docs/extend/java-output-plugin.md
Normal file
|
|
@ -0,0 +1,311 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/java-output-plugin.html
|
||||
---
|
||||
|
||||
# How to write a Java output plugin [java-output-plugin]
|
||||
|
||||
To develop a new Java output for Logstash, you write a new Java class that conforms to the Logstash Java Outputs API, package it, and install it with the logstash-plugin utility. We’ll go through each of those steps.
|
||||
|
||||
|
||||
## Set up your environment [_set_up_your_environment_4]
|
||||
|
||||
|
||||
### Copy the example repo [_copy_the_example_repo_4]
|
||||
|
||||
Start by copying the [example output plugin](https://github.com/logstash-plugins/logstash-output-java_output_example). The plugin API is currently part of the Logstash codebase so you must have a local copy of that available. You can obtain a copy of the Logstash codebase with the following `git` command:
|
||||
|
||||
```shell
|
||||
git clone --branch <branch_name> --single-branch https://github.com/elastic/logstash.git <target_folder>
|
||||
```
|
||||
|
||||
The `branch_name` should correspond to the version of Logstash containing the preferred revision of the Java plugin API.
|
||||
|
||||
::::{note}
|
||||
The GA version of the Java plugin API is available in the `7.2` and later branches of the Logstash codebase.
|
||||
::::
|
||||
|
||||
|
||||
Specify the `target_folder` for your local copy of the Logstash codebase. If you do not specify `target_folder`, it defaults to a new folder called `logstash` under your current folder.
|
||||
|
||||
|
||||
### Generate the .jar file [_generate_the_jar_file_4]
|
||||
|
||||
After you have obtained a copy of the appropriate revision of the Logstash codebase, you need to compile it to generate the .jar file containing the Java plugin API. From the root directory of your Logstash codebase ($LS_HOME), you can compile it with `./gradlew assemble` (or `gradlew.bat assemble` if you’re running on Windows). This should produce the `$LS_HOME/logstash-core/build/libs/logstash-core-x.y.z.jar` where `x`, `y`, and `z` refer to the version of Logstash.
|
||||
|
||||
After you have successfully compiled Logstash, you need to tell your Java plugin where to find the `logstash-core-x.y.z.jar` file. Create a new file named `gradle.properties` in the root folder of your plugin project. That file should have a single line:
|
||||
|
||||
```txt
|
||||
LOGSTASH_CORE_PATH=<target_folder>/logstash-core
|
||||
```
|
||||
|
||||
where `target_folder` is the root folder of your local copy of the Logstash codebase.
|
||||
|
||||
|
||||
## Code the plugin [_code_the_plugin_4]
|
||||
|
||||
The example output plugin prints events to the console using the event’s `toString` method. Let’s look at the main class in the example output:
|
||||
|
||||
```java
|
||||
@LogstashPlugin(name = "java_output_example")
|
||||
public class JavaOutputExample implements Output {
|
||||
|
||||
public static final PluginConfigSpec<String> PREFIX_CONFIG =
|
||||
PluginConfigSpec.stringSetting("prefix", "");
|
||||
|
||||
private final String id;
|
||||
private String prefix;
|
||||
private PrintStream printer;
|
||||
private final CountDownLatch done = new CountDownLatch(1);
|
||||
private volatile boolean stopped = false;
|
||||
|
||||
public JavaOutputExample(final String id, final Configuration configuration, final Context context) {
|
||||
this(id, configuration, context, System.out);
|
||||
}
|
||||
|
||||
JavaOutputExample(final String id, final Configuration config, final Context context, OutputStream targetStream) {
|
||||
this.id = id;
|
||||
prefix = config.get(PREFIX_CONFIG);
|
||||
printer = new PrintStream(targetStream);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void output(final Collection<Event> events) {
|
||||
Iterator<Event> z = events.iterator();
|
||||
while (z.hasNext() && !stopped) {
|
||||
String s = prefix + z.next();
|
||||
printer.println(s);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void stop() {
|
||||
stopped = true;
|
||||
done.countDown();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void awaitStop() throws InterruptedException {
|
||||
done.await();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Collection<PluginConfigSpec<?>> configSchema() {
|
||||
return Collections.singletonList(PREFIX_CONFIG);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Let’s step through and examine each part of that class.
|
||||
|
||||
|
||||
### Class declaration [_class_declaration_8]
|
||||
|
||||
```java
|
||||
@LogstashPlugin(name="java_output_example")
|
||||
public class JavaOutputExample implements Output {
|
||||
```
|
||||
|
||||
Notes about the class declaration:
|
||||
|
||||
* All Java plugins must be annotated with the `@LogstashPlugin` annotation. Additionally:
|
||||
|
||||
* The `name` property of the annotation must be supplied and defines the name of the plugin as it will be used in the Logstash pipeline definition. For example, this output would be referenced in the output section of the Logstash pipeline definition as `output { java_output_example => { .... } }`
|
||||
* The value of the `name` property must match the name of the class excluding casing and underscores.
|
||||
|
||||
* The class must implement the `co.elastic.logstash.api.Output` interface.
|
||||
* Java plugins may not be created in the `org.logstash` or `co.elastic.logstash` packages to prevent potential clashes with classes in Logstash itself.
|
||||
|
||||
|
||||
### Plugin settings [_plugin_settings_4]
|
||||
|
||||
The snippet below contains both the setting definition and the method referencing it:
|
||||
|
||||
```java
|
||||
public static final PluginConfigSpec<String> PREFIX_CONFIG =
|
||||
PluginConfigSpec.stringSetting("prefix", "");
|
||||
|
||||
@Override
|
||||
public Collection<PluginConfigSpec<?>> configSchema() {
|
||||
return Collections.singletonList(PREFIX_CONFIG);
|
||||
}
|
||||
```
|
||||
|
||||
The `PluginConfigSpec` class allows developers to specify the settings that a plugin supports complete with setting name, data type, deprecation status, required status, and default value. In this example, the `prefix` setting defines an optional prefix to include in the output of the event. The setting is not required and if it is not explicitly set, it defaults to the empty string.
|
||||
|
||||
The `configSchema` method must return a list of all settings that the plugin supports. In a future phase of the Java plugin project, the Logstash execution engine will validate that all required settings are present and that no unsupported settings are present.
|
||||
|
||||
|
||||
### Constructor and initialization [_constructor_and_initialization_4]
|
||||
|
||||
```java
|
||||
private final String id;
|
||||
private String prefix;
|
||||
private PrintStream printer;
|
||||
|
||||
public JavaOutputExample(final String id, final Configuration configuration, final Context context) {
|
||||
this(configuration, context, System.out);
|
||||
}
|
||||
|
||||
JavaOutputExample(final String id, final Configuration config, final Context context, OutputStream targetStream) {
|
||||
this.id = id;
|
||||
prefix = config.get(PREFIX_CONFIG);
|
||||
printer = new PrintStream(targetStream);
|
||||
}
|
||||
```
|
||||
|
||||
All Java output plugins must have a constructor taking a `String` id and a `Configuration` and `Context` argument. This is the constructor that will be used to instantiate them at runtime. The retrieval and validation of all plugin settings should occur in this constructor. In this example, the values of the `prefix` setting is retrieved and stored in a local variable for later use in the `output` method. In this example, a second, pacakge private constructor is defined that is useful for unit testing with a `Stream` other than `System.out`.
|
||||
|
||||
Any additional initialization may occur in the constructor as well. If there are any unrecoverable errors encountered in the configuration or initialization of the output plugin, a descriptive exception should be thrown. The exception will be logged and will prevent Logstash from starting.
|
||||
|
||||
|
||||
### Output method [_output_method]
|
||||
|
||||
```java
|
||||
@Override
|
||||
public void output(final Collection<Event> events) {
|
||||
Iterator<Event> z = events.iterator();
|
||||
while (z.hasNext() && !stopped) {
|
||||
String s = prefix + z.next();
|
||||
printer.println(s);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Outputs may send events to local sinks such as the console or a file or to remote systems such as Elasticsearch or other external systems. In this example, the events are printed to the local console.
|
||||
|
||||
|
||||
### Stop and awaitStop methods [_stop_and_awaitstop_methods_2]
|
||||
|
||||
```java
|
||||
private final CountDownLatch done = new CountDownLatch(1);
|
||||
private volatile boolean stopped;
|
||||
|
||||
@Override
|
||||
public void stop() {
|
||||
stopped = true;
|
||||
done.countDown();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void awaitStop() throws InterruptedException {
|
||||
done.await();
|
||||
}
|
||||
```
|
||||
|
||||
The `stop` method notifies the output to stop sending events. The stop mechanism may be implemented in any way that honors the API contract though a `volatile boolean` flag works well for many use cases. Because this output example is so simple, its `output` method does not check for the stop flag.
|
||||
|
||||
Outputs stop both asynchronously and cooperatively. Use the `awaitStop` method to block until the output has completed the stop process. Note that this method should **not** signal the output to stop as the `stop` method does. The awaitStop mechanism may be implemented in any way that honors the API contract though a `CountDownLatch` works well for many use cases.
|
||||
|
||||
|
||||
### getId method [_getid_method_4]
|
||||
|
||||
```java
|
||||
@Override
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
```
|
||||
|
||||
For output plugins, the `getId` method should always return the id that was provided to the plugin through its constructor at instantiation time.
|
||||
|
||||
|
||||
### Unit tests [_unit_tests_4]
|
||||
|
||||
Lastly, but certainly not least importantly, unit tests are strongly encouraged. The example output plugin includes an [example unit test](https://github.com/logstash-plugins/logstash-output-java_output_example/blob/main/src/test/java/org/logstashplugins/JavaOutputExampleTest.java) that you can use as a template for your own.
|
||||
|
||||
|
||||
## Package and deploy [_package_and_deploy_4]
|
||||
|
||||
Java plugins are packaged as Ruby gems for dependency management and interoperability with Ruby plugins. Once they are packaged as gems, they may be installed with the `logstash-plugin` utility just as Ruby plugins are. Because no knowledge of Ruby or its toolchain should be required for Java plugin development, the procedure for packaging Java plugins as Ruby gems has been automated through a custom task in the Gradle build file provided with the example Java plugins. The following sections describe how to configure and execute that packaging task as well as how to install the packaged Java plugin in Logstash.
|
||||
|
||||
|
||||
### Configuring the Gradle packaging task [_configuring_the_gradle_packaging_task_4]
|
||||
|
||||
The following section appears near the top of the `build.gradle` file supplied with the example Java plugins:
|
||||
|
||||
```java
|
||||
// ===========================================================================
|
||||
// plugin info
|
||||
// ===========================================================================
|
||||
group 'org.logstashplugins' // must match the package of the main plugin class
|
||||
version "${file("VERSION").text.trim()}" // read from required VERSION file
|
||||
description = "Example Java filter implementation"
|
||||
pluginInfo.licenses = ['Apache-2.0'] // list of SPDX license IDs
|
||||
pluginInfo.longDescription = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using \$LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
||||
pluginInfo.authors = ['Elasticsearch']
|
||||
pluginInfo.email = ['info@elastic.co']
|
||||
pluginInfo.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
|
||||
pluginInfo.pluginType = "filter"
|
||||
pluginInfo.pluginClass = "JavaFilterExample"
|
||||
pluginInfo.pluginName = "java_filter_example"
|
||||
// ===========================================================================
|
||||
```
|
||||
|
||||
You should configure the values above for your plugin.
|
||||
|
||||
* The `version` value will be automatically read from the `VERSION` file in the root of your plugin’s codebase.
|
||||
* `pluginInfo.pluginType` should be set to one of `input`, `filter`, `codec`, or `output`.
|
||||
* `pluginInfo.pluginName` must match the name specified on the `@LogstashPlugin` annotation on the main plugin class. The Gradle packaging task will validate that and return an error if they do not match.
|
||||
|
||||
|
||||
### Running the Gradle packaging task [_running_the_gradle_packaging_task_4]
|
||||
|
||||
Several Ruby source files along with a `gemspec` file and a `Gemfile` are required to package the plugin as a Ruby gem. These Ruby files are used only for defining the Ruby gem structure or at Logstash startup time to register the Java plugin. They are not used during runtime event processing. The Gradle packaging task automatically generates all of these files based on the values configured in the section above.
|
||||
|
||||
You run the Gradle packaging task with the following command:
|
||||
|
||||
```shell
|
||||
./gradlew gem
|
||||
```
|
||||
|
||||
For Windows platforms: Substitute `gradlew.bat` for `./gradlew` as appropriate in the command.
|
||||
|
||||
That task will produce a gem file in the root directory of your plugin’s codebase with the name `logstash-{{plugintype}}-<pluginName>-<version>.gem`
|
||||
|
||||
|
||||
### Installing the Java plugin in Logstash [_installing_the_java_plugin_in_logstash_4]
|
||||
|
||||
After you have packaged your Java plugin as a Ruby gem, you can install it in Logstash with this command:
|
||||
|
||||
```shell
|
||||
bin/logstash-plugin install --no-verify --local /path/to/javaPlugin.gem
|
||||
```
|
||||
|
||||
For Windows platforms: Substitute backslashes for forward slashes as appropriate in the command.
|
||||
|
||||
|
||||
## Running Logstash with the Java output plugin [_running_logstash_with_the_java_output_plugin]
|
||||
|
||||
The following is a minimal Logstash configuration that can be used to test that the Java output plugin is correctly installed and functioning.
|
||||
|
||||
```java
|
||||
input {
|
||||
generator { message => "Hello world!" count => 1 }
|
||||
}
|
||||
output {
|
||||
java_output_example {}
|
||||
}
|
||||
```
|
||||
|
||||
Copy the above Logstash configuration to a file such as `java_output.conf`. Logstash should then be started with:
|
||||
|
||||
```txt
|
||||
bin/logstash -f /path/to/java_output.conf
|
||||
```
|
||||
|
||||
The expected Logstash output (excluding initialization) with the configuration above is:
|
||||
|
||||
```txt
|
||||
{"@timestamp":"yyyy-MM-ddTHH:mm:ss.SSSZ","message":"Hello world!","@version":"1","host":"<yourHostname>","sequence":0}
|
||||
```
|
||||
|
||||
|
||||
## Feedback [_feedback_4]
|
||||
|
||||
If you have any feedback on Java plugin support in Logstash, please comment on our [main Github issue](https://github.com/elastic/logstash/issues/9215) or post in the [Logstash forum](https://discuss.elastic.co/c/logstash).
|
||||
570
docs/extend/output-new-plugin.md
Normal file
570
docs/extend/output-new-plugin.md
Normal file
|
|
@ -0,0 +1,570 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/output-new-plugin.html
|
||||
---
|
||||
|
||||
# How to write a Logstash output plugin [output-new-plugin]
|
||||
|
||||
To develop a new output for Logstash, build a self-contained Ruby gem whose source code lives in its own GitHub repository. The Ruby gem can then be hosted and shared on RubyGems.org. You can use the example output implementation as a starting point. (If you’re unfamiliar with Ruby, you can find an excellent quickstart guide at [https://www.ruby-lang.org/en/documentation/quickstart/](https://www.ruby-lang.org/en/documentation/quickstart/).)
|
||||
|
||||
## Get started [_get_started_4]
|
||||
|
||||
Let’s step through creating an output plugin using the [example output plugin](https://github.com/logstash-plugins/logstash-output-example/).
|
||||
|
||||
### Create a GitHub repo for your new plugin [_create_a_github_repo_for_your_new_plugin_4]
|
||||
|
||||
Each Logstash plugin lives in its own GitHub repository. To create a new repository for your plugin:
|
||||
|
||||
1. Log in to GitHub.
|
||||
2. Click the **Repositories** tab. You’ll see a list of other repositories you’ve forked or contributed to.
|
||||
3. Click the green **New** button in the upper right.
|
||||
4. Specify the following settings for your new repo:
|
||||
|
||||
* **Repository name** — a unique name of the form `logstash-output-pluginname`.
|
||||
* **Public or Private** — your choice, but the repository must be Public if you want to submit it as an official plugin.
|
||||
* **Initialize this repository with a README** — enables you to immediately clone the repository to your computer.
|
||||
|
||||
5. Click **Create Repository**.
|
||||
|
||||
|
||||
### Use the plugin generator tool [_use_the_plugin_generator_tool_4]
|
||||
|
||||
You can create your own Logstash plugin in seconds! The `generate` subcommand of `bin/logstash-plugin` creates the foundation for a new Logstash plugin with templatized files. It creates the correct directory structure, gemspec files, and dependencies so you can start adding custom code to process data with Logstash.
|
||||
|
||||
For more information, see [Generating plugins](/reference/plugin-generator.md)
|
||||
|
||||
|
||||
### Copy the output code [_copy_the_output_code]
|
||||
|
||||
Alternatively, you can use the examples repo we host on github.com
|
||||
|
||||
1. **Clone your plugin.** Replace `GITUSERNAME` with your github username, and `MYPLUGINNAME` with your plugin name.
|
||||
|
||||
* `git clone https://github.com/GITUSERNAME/logstash-``output-MYPLUGINNAME.git`
|
||||
|
||||
* alternately, via ssh: `git clone git@github.com:GITUSERNAME/logstash``-output-MYPLUGINNAME.git`
|
||||
|
||||
* `cd logstash-output-MYPLUGINNAME`
|
||||
|
||||
2. **Clone the output plugin example and copy it to your plugin branch.**
|
||||
|
||||
You don’t want to include the example .git directory or its contents, so delete it before you copy the example.
|
||||
|
||||
* `cd /tmp`
|
||||
* `git clone https://github.com/logstash-plugins/logstash``-output-example.git`
|
||||
* `cd logstash-output-example`
|
||||
* `rm -rf .git`
|
||||
* `cp -R * /path/to/logstash-output-mypluginname/`
|
||||
|
||||
3. **Rename the following files to match the name of your plugin.**
|
||||
|
||||
* `logstash-output-example.gemspec`
|
||||
* `example.rb`
|
||||
* `example_spec.rb`
|
||||
|
||||
```txt
|
||||
cd /path/to/logstash-output-mypluginname
|
||||
mv logstash-output-example.gemspec logstash-output-mypluginname.gemspec
|
||||
mv lib/logstash/outputs/example.rb lib/logstash/outputs/mypluginname.rb
|
||||
mv spec/outputs/example_spec.rb spec/outputs/mypluginname_spec.rb
|
||||
```
|
||||
|
||||
|
||||
Your file structure should look like this:
|
||||
|
||||
```txt
|
||||
$ tree logstash-output-mypluginname
|
||||
├── Gemfile
|
||||
├── LICENSE
|
||||
├── README.md
|
||||
├── Rakefile
|
||||
├── lib
|
||||
│ └── logstash
|
||||
│ └── outputs
|
||||
│ └── mypluginname.rb
|
||||
├── logstash-output-mypluginname.gemspec
|
||||
└── spec
|
||||
└── outputs
|
||||
└── mypluginname_spec.rb
|
||||
```
|
||||
|
||||
For more information about the Ruby gem file structure and an excellent walkthrough of the Ruby gem creation process, see [http://timelessrepo.com/making-ruby-gems](http://timelessrepo.com/making-ruby-gems)
|
||||
|
||||
|
||||
### See what your plugin looks like [_see_what_your_plugin_looks_like_4]
|
||||
|
||||
Before we dive into the details, open up the plugin file in your favorite text editor and take a look.
|
||||
|
||||
```ruby
|
||||
require "logstash/outputs/base"
|
||||
require "logstash/namespace"
|
||||
|
||||
# Add any asciidoc formatted documentation here
|
||||
# An example output that does nothing.
|
||||
class LogStash::Outputs::Example < LogStash::Outputs::Base
|
||||
config_name "example"
|
||||
|
||||
# This sets the concurrency behavior of this plugin. By default it is :legacy, which was the standard
|
||||
# way concurrency worked before Logstash 2.4
|
||||
#
|
||||
# You should explicitly set it to either :single or :shared as :legacy will be removed in Logstash 6.0
|
||||
#
|
||||
# When configured as :single a single instance of the Output will be shared among the
|
||||
# pipeline worker threads. Access to the `#multi_receive/#multi_receive_encoded/#receive` method will be synchronized
|
||||
# i.e. only one thread will be active at a time making threadsafety much simpler.
|
||||
#
|
||||
# You can set this to :shared if your output is threadsafe. This will maximize
|
||||
# concurrency but you will need to make appropriate uses of mutexes in `#multi_receive/#receive`.
|
||||
#
|
||||
# Only the `#multi_receive/#multi_receive_encoded` methods need to actually be threadsafe, the other methods
|
||||
# will only be executed in a single thread
|
||||
concurrency :single
|
||||
|
||||
public
|
||||
def register
|
||||
end # def register
|
||||
|
||||
public
|
||||
# Takes an array of events
|
||||
# Must be threadsafe if `concurrency :shared` is set
|
||||
def multi_receive(events)
|
||||
end # def multi_receive
|
||||
end # class LogStash::Outputs::Example
|
||||
```
|
||||
|
||||
|
||||
|
||||
## Coding output plugins [_coding_output_plugins]
|
||||
|
||||
Now let’s take a line-by-line look at the example plugin.
|
||||
|
||||
### `require` Statements [_require_statements_4]
|
||||
|
||||
Logstash output plugins require parent classes defined in `logstash/outputs/base` and logstash/namespace:
|
||||
|
||||
```ruby
|
||||
require "logstash/outputs/base"
|
||||
require "logstash/namespace"
|
||||
```
|
||||
|
||||
Of course, the plugin you build may depend on other code, or even gems. Just put them here along with these Logstash dependencies.
|
||||
|
||||
|
||||
|
||||
## Plugin Body [_plugin_body_4]
|
||||
|
||||
Let’s go through the various elements of the plugin itself.
|
||||
|
||||
### `class` Declaration [_class_declaration_4]
|
||||
|
||||
The output plugin class should be a subclass of `LogStash::Outputs::Base`:
|
||||
|
||||
```ruby
|
||||
class LogStash::Outputs::Example < LogStash::Outputs::Base
|
||||
```
|
||||
|
||||
The class name should closely mirror the plugin name, for example:
|
||||
|
||||
```ruby
|
||||
LogStash::Outputs::Example
|
||||
```
|
||||
|
||||
|
||||
### `config_name` [_config_name_4]
|
||||
|
||||
```ruby
|
||||
config_name "example"
|
||||
```
|
||||
|
||||
This is the name your plugin will call inside the output configuration block.
|
||||
|
||||
If you set `config_name "example"` in your plugin code, the corresponding Logstash configuration block would need to look like this:
|
||||
|
||||
|
||||
|
||||
## Configuration Parameters [_configuration_parameters_4]
|
||||
|
||||
```ruby
|
||||
config :variable_name, :validate => :variable_type, :default => "Default value", :required => boolean, :deprecated => boolean, :obsolete => string
|
||||
```
|
||||
|
||||
The configuration, or `config` section allows you to define as many (or as few) parameters as are needed to enable Logstash to process events.
|
||||
|
||||
There are several configuration attributes:
|
||||
|
||||
* `:validate` - allows you to enforce passing a particular data type to Logstash for this configuration option, such as `:string`, `:password`, `:boolean`, `:number`, `:array`, `:hash`, `:path` (a file-system path), `uri`, `:codec` (since 1.2.0), `:bytes`. Note that this also works as a coercion in that if I specify "true" for boolean (even though technically a string), it will become a valid boolean in the config. This coercion works for the `:number` type as well where "1.2" becomes a float and "22" is an integer.
|
||||
* `:default` - lets you specify a default value for a parameter
|
||||
* `:required` - whether or not this parameter is mandatory (a Boolean `true` or
|
||||
* `:list` - whether or not this value should be a list of values. Will typecheck the list members, and convert scalars to one element lists. Note that this mostly obviates the array type, though if you need lists of complex objects that will be more suitable. `false`)
|
||||
* `:deprecated` - informational (also a Boolean `true` or `false`)
|
||||
* `:obsolete` - used to declare that a given setting has been removed and is no longer functioning. The idea is to provide an informed upgrade path to users who are still using a now-removed setting.
|
||||
|
||||
|
||||
## Plugin Methods [_plugin_methods_4]
|
||||
|
||||
Logstash outputs must implement the `register` and `multi_receive` methods.
|
||||
|
||||
### `register` Method [_register_method_4]
|
||||
|
||||
```ruby
|
||||
public
|
||||
def register
|
||||
end # def register
|
||||
```
|
||||
|
||||
The Logstash `register` method is like an `initialize` method. It was originally created to enforce having `super` called, preventing headaches for newbies. (Note: It may go away in favor of `initialize`, in conjunction with some enforced testing to ensure `super` is called.)
|
||||
|
||||
`public` means the method can be called anywhere, not just within the class. This is the default behavior for methods in Ruby, but it is specified explicitly here anyway.
|
||||
|
||||
You can also assign instance variables here (variables prepended by `@`). Configuration variables are now in scope as instance variables, like `@message`
|
||||
|
||||
|
||||
|
||||
## Building the Plugin [_building_the_plugin_4]
|
||||
|
||||
At this point in the process you have coded your plugin and are ready to build a Ruby Gem from it. The following information will help you complete the process.
|
||||
|
||||
### External dependencies [_external_dependencies_4]
|
||||
|
||||
A `require` statement in Ruby is used to include necessary code. In some cases your plugin may require additional files. For example, the collectd plugin [uses](https://github.com/logstash-plugins/logstash-codec-collectd/blob/main/lib/logstash/codecs/collectd.rb#L148) the `types.db` file provided by collectd. In the main directory of your plugin, a file called `vendor.json` is where these files are described.
|
||||
|
||||
The `vendor.json` file contains an array of JSON objects, each describing a file dependency. This example comes from the [collectd](https://github.com/logstash-plugins/logstash-codec-collectd/blob/main/vendor.json) codec plugin:
|
||||
|
||||
```txt
|
||||
[{
|
||||
"sha1": "a90fe6cc53b76b7bdd56dc57950d90787cb9c96e",
|
||||
"url": "http://collectd.org/files/collectd-5.4.0.tar.gz",
|
||||
"files": [ "/src/types.db" ]
|
||||
}]
|
||||
```
|
||||
|
||||
* `sha1` is the sha1 signature used to verify the integrity of the file referenced by `url`.
|
||||
* `url` is the address from where Logstash will download the file.
|
||||
* `files` is an optional array of files to extract from the downloaded file. Note that while tar archives can use absolute or relative paths, treat them as absolute in this array. If `files` is not present, all files will be uncompressed and extracted into the vendor directory.
|
||||
|
||||
Another example of the `vendor.json` file is the [`geoip` filter](https://github.com/logstash-plugins/logstash-filter-geoip/blob/main/vendor.json)
|
||||
|
||||
The process used to download these dependencies is to call `rake vendor`. This will be discussed further in the testing section of this document.
|
||||
|
||||
Another kind of external dependency is on jar files. This will be described in the "Add a `gemspec` file" section.
|
||||
|
||||
|
||||
### Deprecated features [_deprecated_features_4]
|
||||
|
||||
As a plugin evolves, an option or feature may no longer serve the intended purpose, and the developer may want to *deprecate* its usage. Deprecation warns users about the option’s status, so they aren’t caught by surprise when it is removed in a later release.
|
||||
|
||||
{{ls}} 7.6 introduced a *deprecation logger* to make handling those situations easier. You can use the [adapter](https://github.com/logstash-plugins/logstash-mixin-deprecation_logger_support) to ensure that your plugin can use the deprecation logger while still supporting older versions of {{ls}}. See the [readme](https://github.com/logstash-plugins/logstash-mixin-deprecation_logger_support/blob/main/README.md) for more information and for instructions on using the adapter.
|
||||
|
||||
Deprecations are noted in the `logstash-deprecation.log` file in the `log` directory.
|
||||
|
||||
|
||||
### Add a Gemfile [_add_a_gemfile_4]
|
||||
|
||||
Gemfiles allow Ruby’s Bundler to maintain the dependencies for your plugin. Currently, all we’ll need is the Logstash gem, for testing, but if you require other gems, you should add them in here.
|
||||
|
||||
::::{tip}
|
||||
See [Bundler’s Gemfile page](http://bundler.io/gemfile.html) for more details.
|
||||
::::
|
||||
|
||||
|
||||
```ruby
|
||||
source 'https://rubygems.org'
|
||||
gemspec
|
||||
gem "logstash", :github => "elastic/logstash", :branch => "master"
|
||||
```
|
||||
|
||||
|
||||
|
||||
## Add a `gemspec` file [_add_a_gemspec_file_4]
|
||||
|
||||
Gemspecs define the Ruby gem which will be built and contain your plugin.
|
||||
|
||||
::::{tip}
|
||||
More information can be found on the [Rubygems Specification page](http://guides.rubygems.org/specification-reference/).
|
||||
::::
|
||||
|
||||
|
||||
```ruby
|
||||
Gem::Specification.new do |s|
|
||||
s.name = 'logstash-output-example'
|
||||
s.version = '0.1.0'
|
||||
s.licenses = ['Apache License (2.0)']
|
||||
s.summary = "This output does x, y, z in Logstash"
|
||||
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
||||
s.authors = ["Elastic"]
|
||||
s.email = 'info@elastic.co'
|
||||
s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
|
||||
s.require_paths = ["lib"]
|
||||
|
||||
# Files
|
||||
s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
|
||||
# Tests
|
||||
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
||||
|
||||
# Special flag to let us know this is actually a logstash plugin
|
||||
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
|
||||
|
||||
# Gem dependencies
|
||||
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
||||
s.add_development_dependency 'logstash-devutils'
|
||||
end
|
||||
```
|
||||
|
||||
It is appropriate to change these values to fit your plugin. In particular, `s.name` and `s.summary` should reflect your plugin’s name and behavior.
|
||||
|
||||
`s.licenses` and `s.version` are also important and will come into play when you are ready to publish your plugin.
|
||||
|
||||
Logstash and all its plugins are licensed under [Apache License, version 2 ("ALv2")](https://github.com/elastic/logstash/blob/main/LICENSE.txt). If you make your plugin publicly available via [RubyGems.org](http://rubygems.org), please make sure to have this line in your gemspec:
|
||||
|
||||
* `s.licenses = ['Apache License (2.0)']`
|
||||
|
||||
The gem version, designated by `s.version`, helps track changes to plugins over time. You should use [semver versioning](http://semver.org/) strategy for version numbers.
|
||||
|
||||
### Runtime and Development Dependencies [_runtime_and_development_dependencies_4]
|
||||
|
||||
At the bottom of the `gemspec` file is a section with a comment: `Gem dependencies`. This is where any other needed gems must be mentioned. If a gem is necessary for your plugin to function, it is a runtime dependency. If a gem are only used for testing, then it would be a development dependency.
|
||||
|
||||
::::{note}
|
||||
You can also have versioning requirements for your dependencies—including other Logstash plugins:
|
||||
|
||||
```ruby
|
||||
# Gem dependencies
|
||||
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
||||
s.add_development_dependency 'logstash-devutils'
|
||||
```
|
||||
|
||||
This gemspec has a runtime dependency on the logstash-core-plugin-api and requires that it have a version number greater than or equal to version 1.60 and less than or equal to version 2.99.
|
||||
|
||||
::::
|
||||
|
||||
|
||||
::::{important}
|
||||
All plugins have a runtime dependency on the `logstash-core-plugin-api` gem, and a development dependency on `logstash-devutils`.
|
||||
::::
|
||||
|
||||
|
||||
|
||||
### Jar dependencies [_jar_dependencies_4]
|
||||
|
||||
In some cases, such as the [Elasticsearch output plugin](https://github.com/logstash-plugins/logstash-output-elasticsearch/blob/main/logstash-output-elasticsearch.gemspec#L22-L23), your code may depend on a jar file. In cases such as this, the dependency is added in the gemspec file in this manner:
|
||||
|
||||
```ruby
|
||||
# Jar dependencies
|
||||
s.requirements << "jar 'org.elasticsearch:elasticsearch', '5.0.0'"
|
||||
s.add_runtime_dependency 'jar-dependencies'
|
||||
```
|
||||
|
||||
With these both defined, the install process will search for the required jar file at [http://mvnrepository.com](http://mvnrepository.com) and download the specified version.
|
||||
|
||||
|
||||
|
||||
## Document your plugin [_document_your_plugin_4]
|
||||
|
||||
Documentation is an important part of your plugin. All plugin documentation is rendered and placed in the [Logstash Reference](/reference/index.md) and the [Versioned plugin docs](logstash-docs://reference/integration-plugins.md).
|
||||
|
||||
See [Document your plugin](/extend/plugin-doc.md) for tips and guidelines.
|
||||
|
||||
|
||||
## Add Tests [_add_tests_4]
|
||||
|
||||
Logstash loves tests. Lots of tests. If you’re using your new output plugin in a production environment, you’ll want to have some tests to ensure you are not breaking any existing functionality.
|
||||
|
||||
::::{note}
|
||||
A full exposition on RSpec is outside the scope of this document. Learn more about RSpec at [http://rspec.info](http://rspec.info)
|
||||
::::
|
||||
|
||||
|
||||
For help learning about tests and testing, look in the `spec/outputs/` directory of several other similar plugins.
|
||||
|
||||
|
||||
## Clone and test! [_clone_and_test_4]
|
||||
|
||||
Now let’s start with a fresh clone of the plugin, build it and run the tests.
|
||||
|
||||
* **Clone your plugin into a temporary location** Replace `GITUSERNAME` with your github username, and `MYPLUGINNAME` with your plugin name.
|
||||
|
||||
* `git clone https://github.com/GITUSERNAME/logstash-``output-MYPLUGINNAME.git`
|
||||
|
||||
* alternately, via ssh: `git clone git@github.com:GITUSERNAME/logstash-``output-MYPLUGINNAME.git`
|
||||
|
||||
* `cd logstash-output-MYPLUGINNAME`
|
||||
|
||||
|
||||
Then, you’ll need to install your plugins dependencies with bundler:
|
||||
|
||||
```
|
||||
bundle install
|
||||
```
|
||||
|
||||
::::{important}
|
||||
If your plugin has an external file dependency described in `vendor.json`, you must download that dependency before running or testing. You can do this by running:
|
||||
|
||||
```
|
||||
rake vendor
|
||||
```
|
||||
|
||||
::::
|
||||
|
||||
|
||||
And finally, run the tests:
|
||||
|
||||
```
|
||||
bundle exec rspec
|
||||
```
|
||||
|
||||
You should see a success message, which looks something like this:
|
||||
|
||||
```
|
||||
Finished in 0.034 seconds
|
||||
1 example, 0 failures
|
||||
```
|
||||
|
||||
Hooray! You’re almost there! (Unless you saw failures… you should fix those first).
|
||||
|
||||
|
||||
## Building and Testing [_building_and_testing_4]
|
||||
|
||||
Now you’re ready to build your (well-tested) plugin into a Ruby gem.
|
||||
|
||||
### Build [_build_4]
|
||||
|
||||
You already have all the necessary ingredients, so let’s go ahead and run the build command:
|
||||
|
||||
```sh
|
||||
gem build logstash-output-example.gemspec
|
||||
```
|
||||
|
||||
That’s it! Your gem should be built and be in the same path with the name
|
||||
|
||||
```sh
|
||||
logstash-output-mypluginname-0.1.0.gem
|
||||
```
|
||||
|
||||
The `s.version` number from your gemspec file will provide the gem version, in this case, `0.1.0`.
|
||||
|
||||
|
||||
### Test installation [_test_installation_4]
|
||||
|
||||
You should test install your plugin into a clean installation of Logstash. Download the latest version from the [Logstash downloads page](https://www.elastic.co/downloads/logstash/).
|
||||
|
||||
1. Untar and cd in to the directory:
|
||||
|
||||
```sh
|
||||
curl -O https://download.elastic.co/logstash/logstash/logstash-9.0.0.tar.gz
|
||||
tar xzvf logstash-9.0.0.tar.gz
|
||||
cd logstash-9.0.0
|
||||
```
|
||||
|
||||
2. Using the plugin tool, we can install the gem we just built.
|
||||
|
||||
* Replace `/my/logstash/plugins` with the correct path to the gem for your environment, and `0.1.0` with the correct version number from the gemspec file.
|
||||
|
||||
```sh
|
||||
bin/logstash-plugin install /my/logstash/plugins/logstash-output-example/logstash-output-example-0.1.0.gem
|
||||
```
|
||||
|
||||
* After running this, you should see feedback from Logstash that it was successfully installed:
|
||||
|
||||
```sh
|
||||
validating /my/logstash/plugins/logstash-output-example/logstash-output-example-0.1.0.gem >= 0
|
||||
Valid logstash plugin. Continuing...
|
||||
Successfully installed 'logstash-output-example' with version '0.1.0'
|
||||
```
|
||||
|
||||
::::{tip}
|
||||
You can also use the Logstash plugin tool to determine which plugins are currently available:
|
||||
|
||||
```sh
|
||||
bin/logstash-plugin list
|
||||
```
|
||||
|
||||
Depending on what you have installed, you might see a short or long list of plugins: inputs, codecs, filters and outputs.
|
||||
|
||||
::::
|
||||
|
||||
3. Now try running Logstash with a simple configuration passed in via the command-line, using the `-e` flag.
|
||||
|
||||
::::{note}
|
||||
Your results will depend on what your output plugin is designed to do.
|
||||
::::
|
||||
|
||||
|
||||
Congratulations! You’ve built, deployed and successfully run a Logstash output.
|
||||
|
||||
|
||||
|
||||
## Submitting your plugin to [RubyGems.org](http://rubygems.org) and [logstash-plugins](https://github.com/logstash-plugins) [_submitting_your_plugin_to_rubygems_orghttprubygems_org_and_logstash_pluginshttpsgithub_comlogstash_plugins_4]
|
||||
|
||||
Logstash uses [RubyGems.org](http://rubygems.org) as its repository for all plugin artifacts. Once you have developed your new plugin, you can make it available to Logstash users by simply publishing it to RubyGems.org.
|
||||
|
||||
### Licensing [_licensing_4]
|
||||
|
||||
Logstash and all its plugins are licensed under [Apache License, version 2 ("ALv2")](https://github.com/elasticsearch/logstash/blob/main/LICENSE). If you make your plugin publicly available via [RubyGems.org](http://rubygems.org), please make sure to have this line in your gemspec:
|
||||
|
||||
* `s.licenses = ['Apache License (2.0)']`
|
||||
|
||||
|
||||
### Publishing to [RubyGems.org](http://rubygems.org) [_publishing_to_rubygems_orghttprubygems_org_4]
|
||||
|
||||
To begin, you’ll need an account on RubyGems.org
|
||||
|
||||
* [Sign-up for a RubyGems account](https://rubygems.org/sign_up).
|
||||
|
||||
After creating an account, [obtain](http://guides.rubygems.org/rubygems-org-api/#api-authorization) an API key from RubyGems.org. By default, RubyGems uses the file `~/.gem/credentials` to store your API key. These credentials will be used to publish the gem. Replace `username` and `password` with the credentials you created at RubyGems.org:
|
||||
|
||||
```sh
|
||||
curl -u username:password https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials
|
||||
chmod 0600 ~/.gem/credentials
|
||||
```
|
||||
|
||||
Before proceeding, make sure you have the right version in your gemspec file and commit your changes.
|
||||
|
||||
* `s.version = '0.1.0'`
|
||||
|
||||
To publish version 0.1.0 of your new logstash gem:
|
||||
|
||||
```sh
|
||||
bundle install
|
||||
bundle exec rake vendor
|
||||
bundle exec rspec
|
||||
bundle exec rake publish_gem
|
||||
```
|
||||
|
||||
::::{note}
|
||||
Executing `rake publish_gem`:
|
||||
|
||||
1. Reads the version from the gemspec file (`s.version = '0.1.0'`)
|
||||
2. Checks in your local repository if a tag exists for that version. If the tag already exists, it aborts the process. Otherwise, it creates a new version tag in your local repository.
|
||||
3. Builds the gem
|
||||
4. Publishes the gem to RubyGems.org
|
||||
|
||||
::::
|
||||
|
||||
|
||||
That’s it! Your plugin is published! Logstash users can now install your plugin by running:
|
||||
|
||||
```sh
|
||||
bin/logstash-plugin install logstash-output-mypluginname
|
||||
```
|
||||
|
||||
|
||||
|
||||
## Contributing your source code to [logstash-plugins](https://github.com/logstash-plugins) [_contributing_your_source_code_to_logstash_pluginshttpsgithub_comlogstash_plugins_4]
|
||||
|
||||
It is not required to contribute your source code to [logstash-plugins](https://github.com/logstash-plugins) github organization, but we always welcome new plugins!
|
||||
|
||||
### Benefits [_benefits_4]
|
||||
|
||||
Some of the many benefits of having your plugin in the logstash-plugins repository are:
|
||||
|
||||
* **Discovery.** Your plugin will appear in the [Logstash Reference](/reference/index.md), where Logstash users look first for plugins and documentation.
|
||||
* **Documentation.** Your plugin documentation will automatically be added to the [Logstash Reference](/reference/index.md).
|
||||
* **Testing.** With our testing infrastructure, your plugin will be continuously tested against current and future releases of Logstash. As a result, users will have the assurance that if incompatibilities arise, they will be quickly discovered and corrected.
|
||||
|
||||
|
||||
### Acceptance Guidelines [_acceptance_guidelines_4]
|
||||
|
||||
* **Code Review.** Your plugin must be reviewed by members of the community for coherence, quality, readability, stability and security.
|
||||
* **Tests.** Your plugin must contain tests to be accepted. These tests are also subject to code review for scope and completeness. It’s ok if you don’t know how to write tests — we will guide you. We are working on publishing a guide to creating tests for Logstash which will make it easier. In the meantime, you can refer to [http://betterspecs.org/](http://betterspecs.org/) for examples.
|
||||
|
||||
To begin migrating your plugin to logstash-plugins, simply create a new [issue](https://github.com/elasticsearch/logstash/issues) in the Logstash repository. When the acceptance guidelines are completed, we will facilitate the move to the logstash-plugins organization using the recommended [github process](https://help.github.com/articles/transferring-a-repository/#transferring-from-a-user-to-an-organization).
|
||||
172
docs/extend/plugin-doc.md
Normal file
172
docs/extend/plugin-doc.md
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/plugin-doc.html
|
||||
---
|
||||
|
||||
# Document your plugin [plugin-doc]
|
||||
|
||||
Documentation is a required component of your plugin. Quality documentation with good examples contributes to the adoption of your plugin.
|
||||
|
||||
The documentation that you write for your plugin will be generated and published in the [Logstash Reference](/reference/index.md) and the [Logstash Versioned Plugin Reference](logstash-docs://reference/integration-plugins.md).
|
||||
|
||||
::::{admonition} Plugin listing in {{ls}} Reference
|
||||
:class: note
|
||||
|
||||
We may list your plugin in the [Logstash Reference](/reference/index.md) if it meets our [requirements and quality standards](/extend/index.md#plugin-acceptance). When we list your plugin, we point to *your* documentation—a readme.md, docs/index.asciidoc, or both—in your plugin repo. For more info on this option, see [List your plugin](/extend/plugin-listing.md).
|
||||
|
||||
::::
|
||||
|
||||
|
||||
The following sections contain guidelines for documenting plugins hosted in the Github [logstash-plugins](https://github.com/logstash-plugins/) organization.
|
||||
|
||||
## Documentation file [plugin-doc-file]
|
||||
|
||||
Documentation belongs in a single file called *docs/index.asciidoc*. It belongs in a single file called *docs/index.asciidoc*. The [plugin generation utility](/reference/plugin-generator.md) creates a starter file for you.
|
||||
|
||||
|
||||
## Heading IDs [heading-ids]
|
||||
|
||||
Format heading anchors with variables that can support generated IDs. This approach creates unique IDs when the [Logstash Versioned Plugin Reference](logstash-docs://reference/integration-plugins.md) is built. Unique heading IDs are required to avoid duplication over multiple versions of a plugin.
|
||||
|
||||
**Example**
|
||||
|
||||
Don’t hardcode a plugin heading ID like this: `[[config_models]]`
|
||||
|
||||
Instead, use variables to define it:
|
||||
|
||||
```txt
|
||||
[id="plugins-{type}s-{plugin}-config_models"]
|
||||
==== Configuration models
|
||||
```
|
||||
|
||||
If you hardcode an ID, the [Logstash Versioned Plugin Reference](logstash-docs://reference/integration-plugins.md) builds correctly the first time. The second time the doc build runs, the ID is flagged as a duplicate, and the build fails.
|
||||
|
||||
|
||||
## Link formats [link-format]
|
||||
|
||||
Correct link formatting is essential for directing users to the content you want them to see. Incorrect link formatting or duplicate links can break the documentation build. Let’s not do that.
|
||||
|
||||
### Link to content in the same file [_link_to_content_in_the_same_file]
|
||||
|
||||
Use angle brackets to format links to content in the same asciidoc file.
|
||||
|
||||
**Example**
|
||||
|
||||
This link:
|
||||
|
||||
```txt
|
||||
<<plugins-{type}s-{plugin}-config_models>>
|
||||
```
|
||||
|
||||
Points to this heading in the same file:
|
||||
|
||||
```txt
|
||||
[id="plugins-{type}s-{plugin}-config_models"]
|
||||
==== Configuration models
|
||||
```
|
||||
|
||||
|
||||
### Link to content in the Logstash Reference Guide [_link_to_content_in_the_logstash_reference_guide]
|
||||
|
||||
Use external link syntax for links that point to documentation for other plugins or content in the Logstash Reference Guide.
|
||||
|
||||
**Examples**
|
||||
|
||||
```txt
|
||||
{logstash-ref}/plugins-codecs-multiline.html[Multiline codec plugin]
|
||||
```
|
||||
|
||||
```txt
|
||||
{logstash-ref}/getting-started-with-logstash.html
|
||||
```
|
||||
|
||||
|
||||
### Link text [_link_text]
|
||||
|
||||
If you don’t specify link text, the URL is used as the link text.
|
||||
|
||||
**Examples**
|
||||
|
||||
If you want your link to display as {{logstash-ref}}/getting-started-with-logstash.html, use this format:
|
||||
|
||||
```txt
|
||||
{logstash-ref}/getting-started-with-logstash.html
|
||||
```
|
||||
|
||||
If you want your link to display as [Getting Started with Logstash](/reference/getting-started-with-logstash.md), use this format:
|
||||
|
||||
```txt
|
||||
{logstash-ref}/getting-started-with-logstash.html[Getting Started with Logstash]
|
||||
```
|
||||
|
||||
|
||||
### Link to data type descriptions [_link_to_data_type_descriptions]
|
||||
|
||||
We make an exception for links that point to data type descriptions, such as `<<boolean,boolean>>`, because they are used so frequently. We have a cleanup step in the conversion script that converts the links to the correct syntax.
|
||||
|
||||
|
||||
|
||||
## Code samples [format-code]
|
||||
|
||||
We all love code samples. Asciidoc supports code blocks and config examples. To include Ruby code, use the asciidoc `[source,ruby]` directive.
|
||||
|
||||
Note that the hashmarks (#) are present to make the example render correctly. Don’t include the hashmarks in your asciidoc file.
|
||||
|
||||
```txt
|
||||
# [source,ruby]
|
||||
# -----
|
||||
# match => {
|
||||
# "field1" => "value1"
|
||||
# "field2" => "value2"
|
||||
# ...
|
||||
# }
|
||||
# -----
|
||||
```
|
||||
|
||||
The sample above (with hashmarks removed) renders in the documentation like this:
|
||||
|
||||
```ruby
|
||||
match => {
|
||||
"field1" => "value1"
|
||||
"field2" => "value2"
|
||||
...
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
## Where’s my doc? [_wheres_my_doc]
|
||||
|
||||
Plugin documentation goes through several steps before it gets published in the [Logstash Versioned Plugin Reference](logstash-docs://reference/integration-plugins.md) and the [Logstash Reference](/reference/index.md).
|
||||
|
||||
Here’s an overview of the workflow:
|
||||
|
||||
* Be sure that you have signed the contributor license agreement (CLA) and have all necessary approvals and sign offs.
|
||||
* Merge the pull request for your plugin (including the `index.asciidoc` file, the `changelog.md` file, and the gemspec).
|
||||
* Wait for the continuous integration build to complete successfully.
|
||||
* Publish the plugin to [https://rubygems.org](https://rubygems.org).
|
||||
* A script detects the new or changed version, and picks up the `index.asciidoc` file for inclusion in the doc build.
|
||||
* The documentation for your new plugin is published in the [Logstash Versioned Plugin Reference](logstash-docs://reference/integration-plugins.md).
|
||||
|
||||
We’re not done yet.
|
||||
|
||||
* For each release, we package the new and changed documentation files into a pull request to add or update content. (We sometimes package plugin docs between releases if we make significant changes to plugin documentation or add a new plugin.)
|
||||
* The script detects the new or changed version, and picks up the `index.asciidoc` file for inclusion in the doc build.
|
||||
* We create a pull request, and merge the new and changed content into the appropriate version branches.
|
||||
* For a new plugin, we add a link to the list of plugins in the [Logstash Reference](/reference/index.md).
|
||||
* The documentation for your new (or changed) plugin is published in the [Logstash Reference](/reference/index.md).
|
||||
|
||||
### Documentation or plugin updates [_documentation_or_plugin_updates]
|
||||
|
||||
When you make updates to your plugin or the documentation, consider bumping the version number in the changelog and gemspec (or version file). The version change triggers the doc build to pick up your changes for publishing.
|
||||
|
||||
|
||||
|
||||
## Resources [_resources]
|
||||
|
||||
For more asciidoc formatting tips, see the excellent reference at [https://github.com/elastic/docs#asciidoc-guide](https://github.com/elastic/docs#asciidoc-guide).
|
||||
|
||||
For tips on contributing and changelog guidelines, see [CONTRIBUTING.md](https://github.com/elastic/logstash/blob/main/CONTRIBUTING.md#logstash-plugin-changelog-guidelines).
|
||||
|
||||
For general information about contributing, see [Contributing to Logstash](/extend/index.md).
|
||||
|
||||
|
||||
23
docs/extend/plugin-listing.md
Normal file
23
docs/extend/plugin-listing.md
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/plugin-listing.html
|
||||
---
|
||||
|
||||
# List your plugin [plugin-listing]
|
||||
|
||||
The [Logstash Reference](/reference/index.md) is the first place {{ls}} users look for plugins and documentation. If your plugin meets the [quality and acceptance guidelines](/extend/index.md#plugin-acceptance), we may be able to list it in the guide.
|
||||
|
||||
The plugin source and documentation will continue to live in your repo, and we will direct users there.
|
||||
|
||||
If you would like to have your plugin included in the [Logstash Reference](/reference/index.md), create a new [issue](https://github.com/elasticsearch/logstash/issues) in the Logstash repository with the following information:
|
||||
|
||||
* Title: `PluginListing: <your-plugin-name>`
|
||||
* Body:
|
||||
|
||||
* Brief description of the plugin (what it is and what it does).
|
||||
* Link to the plugin repository.
|
||||
* Link to the README.md or docs/index.asciidoc.
|
||||
* Describe how your plugin meets our [quality and acceptance guidelines](/extend/index.md#plugin-acceptance).
|
||||
|
||||
* Labels: `docs`, `new-plugin`
|
||||
|
||||
62
docs/extend/publish-plugin.md
Normal file
62
docs/extend/publish-plugin.md
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/publish-plugin.html
|
||||
---
|
||||
|
||||
# Publish your plugin to RubyGems.org [publish-plugin]
|
||||
|
||||
Logstash uses [RubyGems.org](http://rubygems.org) as its repository for all plugin artifacts. After you have developed your new plugin, you can make it available to Logstash users by publishing it to RubyGems.org.
|
||||
|
||||
## Licensing [_licensing_5]
|
||||
|
||||
Logstash and all its plugins are licensed under [Apache License, version 2 ("ALv2")](https://github.com/elasticsearch/logstash/blob/main/LICENSE). If you make your plugin publicly available via [RubyGems.org](http://rubygems.org), please make sure to have this line in your gemspec:
|
||||
|
||||
* `s.licenses = ['Apache License (2.0)']`
|
||||
|
||||
|
||||
## Publish to [RubyGems.org](http://rubygems.org) [_publish_to_rubygems_orghttprubygems_org]
|
||||
|
||||
You’ll need an account on RubyGems.org
|
||||
|
||||
* [Sign-up for a RubyGems account](https://rubygems.org/sign_up).
|
||||
|
||||
After creating an account, [obtain](http://guides.rubygems.org/rubygems-org-api/#api-authorization) an API key from RubyGems.org. By default, RubyGems uses the file `~/.gem/credentials` to store your API key. These credentials will be used to publish the gem. Replace `username` and `password` with the credentials you created at RubyGems.org:
|
||||
|
||||
```sh
|
||||
curl -u username:password https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials
|
||||
chmod 0600 ~/.gem/credentials
|
||||
```
|
||||
|
||||
Before proceeding, make sure you have the right version in your gemspec file and commit your changes.
|
||||
|
||||
* `s.version = '0.1.0'`
|
||||
|
||||
To publish version 0.1.0 of your new logstash gem:
|
||||
|
||||
```sh
|
||||
bundle install
|
||||
bundle exec rake vendor
|
||||
bundle exec rspec
|
||||
bundle exec rake publish_gem
|
||||
```
|
||||
|
||||
::::{note}
|
||||
Execute `rake publish_gem`:
|
||||
|
||||
1. Reads the version from the gemspec file (`s.version = '0.1.0'`)
|
||||
2. Checks in your local repository if a tag exists for that version. If the tag already exists, it aborts the process. Otherwise, it creates a new version tag in your local repository.
|
||||
3. Builds the gem
|
||||
4. Publishes the gem to RubyGems.org
|
||||
|
||||
::::
|
||||
|
||||
|
||||
That’s it! Your plugin is published! Logstash users can now install your plugin by running:
|
||||
|
||||
```sh
|
||||
bin/plugin install logstash-output-mypluginname
|
||||
```
|
||||
|
||||
Where <plugintype> is `input`, `output`, `filter`, or `codec`, and <mypluginname> is the name of your new plugin.
|
||||
|
||||
|
||||
18
docs/extend/toc.yml
Normal file
18
docs/extend/toc.yml
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
toc:
|
||||
- file: index.md
|
||||
- file: input-new-plugin.md
|
||||
- file: codec-new-plugin.md
|
||||
- file: filter-new-plugin.md
|
||||
- file: output-new-plugin.md
|
||||
- file: community-maintainer.md
|
||||
- file: plugin-doc.md
|
||||
- file: publish-plugin.md
|
||||
- file: plugin-listing.md
|
||||
- file: contributing-patch-plugin.md
|
||||
- file: contribute-to-core.md
|
||||
- file: create-logstash-plugins.md
|
||||
children:
|
||||
- file: java-input-plugin.md
|
||||
- file: java-codec-plugin.md
|
||||
- file: java-filter-plugin.md
|
||||
- file: java-output-plugin.md
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
[[logstash-reference]]
|
||||
= Logstash Reference
|
||||
|
||||
:branch: 5.4
|
||||
:major-version: 5.4
|
||||
:logstash_version: 5.4.0
|
||||
:elasticsearch_version: 5.4.0
|
||||
:docker-image: docker.elastic.co/logstash/logstash:{logstash_version}
|
||||
|
||||
//////////
|
||||
release-state can be: released | prerelease | unreleased
|
||||
//////////
|
||||
:release-state: released
|
||||
|
||||
:jdk: 1.8.0
|
||||
:guide: https://www.elastic.co/guide/en/elasticsearch/guide/current/
|
||||
:ref: https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/
|
||||
:xpack: https://www.elastic.co/guide/en/x-pack/{branch}/
|
||||
:logstash: https://www.elastic.co/guide/en/logstash/{branch}/
|
||||
:filebeat: https://www.elastic.co/guide/en/beats/filebeat/{branch}/
|
||||
:lsissue: https://github.com/elastic/logstash/issues/
|
||||
:security: X-Pack Security
|
||||
|
||||
[[introduction]]
|
||||
== Logstash Introduction
|
||||
|
||||
Logstash is an open source data collection engine with real-time pipelining capabilities. Logstash can dynamically
|
||||
unify data from disparate sources and normalize the data into destinations of your choice. Cleanse and democratize all
|
||||
your data for diverse advanced downstream analytics and visualization use cases.
|
||||
|
||||
While Logstash originally drove innovation in log collection, its capabilities extend well beyond that use case. Any
|
||||
type of event can be enriched and transformed with a broad array of input, filter, and output plugins, with many
|
||||
native codecs further simplifying the ingestion process. Logstash accelerates your insights by harnessing a greater
|
||||
volume and variety of data.
|
||||
|
||||
include::static/introduction.asciidoc[]
|
||||
|
||||
include::static/getting-started-with-logstash.asciidoc[]
|
||||
|
Before Width: | Height: | Size: 39 KiB After Width: | Height: | Size: 39 KiB |
|
Before Width: | Height: | Size: 143 KiB After Width: | Height: | Size: 143 KiB |
|
Before Width: | Height: | Size: 46 KiB After Width: | Height: | Size: 46 KiB |
|
Before Width: | Height: | Size: 79 KiB After Width: | Height: | Size: 79 KiB |
|
Before Width: | Height: | Size: 134 KiB After Width: | Height: | Size: 134 KiB |
|
Before Width: | Height: | Size: 186 KiB After Width: | Height: | Size: 186 KiB |
|
Before Width: | Height: | Size: 196 KiB After Width: | Height: | Size: 196 KiB |
|
Before Width: | Height: | Size: 197 KiB After Width: | Height: | Size: 197 KiB |
|
Before Width: | Height: | Size: 595 KiB After Width: | Height: | Size: 595 KiB |
|
Before Width: | Height: | Size: 168 KiB After Width: | Height: | Size: 168 KiB |
|
Before Width: | Height: | Size: 348 KiB After Width: | Height: | Size: 348 KiB |
|
Before Width: | Height: | Size: 388 KiB After Width: | Height: | Size: 388 KiB |
|
Before Width: | Height: | Size: 178 KiB After Width: | Height: | Size: 178 KiB |
|
Before Width: | Height: | Size: 126 KiB After Width: | Height: | Size: 126 KiB |
|
Before Width: | Height: | Size: 125 KiB After Width: | Height: | Size: 125 KiB |
|
Before Width: | Height: | Size: 105 KiB After Width: | Height: | Size: 105 KiB |
|
Before Width: | Height: | Size: 118 KiB After Width: | Height: | Size: 118 KiB |
|
Before Width: | Height: | Size: 113 KiB After Width: | Height: | Size: 113 KiB |
|
|
@ -1,10 +0,0 @@
|
|||
/////
|
||||
These settings control attributes for Logstash core content
|
||||
in the Logstash Reference (LSR) only.
|
||||
|
||||
Shared attributes for the plugin docs (in the LSR and VPR) should
|
||||
go in /docs/include/attributes-lsplugins.asciidoc instead
|
||||
with a corresponding change to the VPR settings in
|
||||
logstash-docs/docs/versioned-plugins/include/attributes-ls-vpr.asciidoc
|
||||
/////
|
||||
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
/////
|
||||
These settings control attributes in the LSR only.
|
||||
They correspond to the VPR settings in logstash-docs/docs/versioned-plugins/include/attributes-ls-vpr.asciidoc
|
||||
When we update one, we must update settings in the other location,
|
||||
|
||||
Attribute text formatted without hard wrap is deliberate.
|
||||
Otherwise, text breaks at return and content after the return is dropped.
|
||||
|
||||
Text is written to accommodate multiple versions because plugins are not stack versioned.
|
||||
/////
|
||||
|
||||
|
||||
:ecs-default: When the `ecs_compatibility` option for this plugin is not explicitly set, its effective value depends on the `pipeline.ecs_compatibility` setting for the pipeline in `pipelines.yml`, or globally in {logstash-ref}/logstash-settings-file.html[`logstash.yml`], allowing you to specify your preferred behavior at the plugin, pipeline, or system level. If no preference is specified, the default value is `v8` for Logstash 8 or `disabled` for all earlier releases of Logstash. For more information about ECS compatibility settings in Logstash and plugins, see {logstash-ref}/ecs-ls.html[ECS in Logstash].
|
||||
|
|
@ -1,234 +0,0 @@
|
|||
==== Common options
|
||||
|
||||
// Contributors: You must conditionally code all internal links and IDs in this
|
||||
// file to make the common files work in both the LS Reference and the versioned
|
||||
// plugin docs
|
||||
|
||||
These configuration options are supported by all filter plugins:
|
||||
|
||||
ifeval::["{versioned_docs}"!="true"]
|
||||
[cols="<,<,<",options="header",]
|
||||
|=======================================================================
|
||||
|Setting |Input type|Required
|
||||
| <<plugins-{type}s-{plugin}-add_field>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No
|
||||
| <<plugins-{type}s-{plugin}-add_tag>> |{logstash-ref}/configuration-file-structure.html#array[array]|No
|
||||
| <<plugins-{type}s-{plugin}-enable_metric>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No
|
||||
| <<plugins-{type}s-{plugin}-id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No
|
||||
| <<plugins-{type}s-{plugin}-periodic_flush>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No
|
||||
| <<plugins-{type}s-{plugin}-remove_field>> |{logstash-ref}/configuration-file-structure.html#array[array]|No
|
||||
| <<plugins-{type}s-{plugin}-remove_tag>> |{logstash-ref}/configuration-file-structure.html#array[array]|No
|
||||
|=======================================================================
|
||||
endif::[]
|
||||
ifeval::["{versioned_docs}"=="true"]
|
||||
[cols="<,<,<",options="header",]
|
||||
|=======================================================================
|
||||
|Setting |Input type|Required
|
||||
| <<{version}-plugins-{type}s-{plugin}-add_field>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No
|
||||
| <<{version}-plugins-{type}s-{plugin}-add_tag>> |{logstash-ref}/configuration-file-structure.html#array[array]|No
|
||||
| <<{version}-plugins-{type}s-{plugin}-enable_metric>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No
|
||||
| <<{version}-plugins-{type}s-{plugin}-id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No
|
||||
| <<{version}-plugins-{type}s-{plugin}-periodic_flush>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No
|
||||
| <<{version}-plugins-{type}s-{plugin}-remove_field>> |{logstash-ref}/configuration-file-structure.html#array[array]|No
|
||||
| <<{version}-plugins-{type}s-{plugin}-remove_tag>> |{logstash-ref}/configuration-file-structure.html#array[array]|No
|
||||
|=======================================================================
|
||||
endif::[]
|
||||
|
||||
ifeval::["{versioned_docs}"!="true"]
|
||||
[id="plugins-{type}s-{plugin}-add_field"]
|
||||
endif::[]
|
||||
ifeval::["{versioned_docs}"=="true"]
|
||||
[id="{version}-plugins-{type}s-{plugin}-add_field"]
|
||||
endif::[]
|
||||
===== `add_field`
|
||||
|
||||
* Value type is {logstash-ref}/configuration-file-structure.html#hash[hash]
|
||||
* Default value is `{}`
|
||||
|
||||
If this filter is successful, add any arbitrary fields to this event.
|
||||
Field names can be dynamic and include parts of the event using the `%{field}`.
|
||||
|
||||
Example:
|
||||
|
||||
["source","json",subs="attributes"]
|
||||
filter {
|
||||
{plugin} {
|
||||
add_field => { "foo_%\{somefield\}" => "Hello world, from %\{host\}" }
|
||||
}
|
||||
}
|
||||
|
||||
["source","json",subs="attributes"]
|
||||
# You can also add multiple fields at once:
|
||||
filter {
|
||||
{plugin} {
|
||||
add_field => {
|
||||
"foo_%\{somefield\}" => "Hello world, from %\{host\}"
|
||||
"new_field" => "new_static_value"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
If the event has field `"somefield" == "hello"` this filter, on success,
|
||||
would add field `foo_hello` if it is present, with the
|
||||
value above and the `%{host}` piece replaced with that value from the
|
||||
event. The second example would also add a hardcoded field.
|
||||
|
||||
ifeval::["{versioned_docs}"!="true"]
|
||||
[id="plugins-{type}s-{plugin}-add_tag"]
|
||||
endif::[]
|
||||
ifeval::["{versioned_docs}"=="true"]
|
||||
[id="{version}-plugins-{type}s-{plugin}-add_tag"]
|
||||
endif::[]
|
||||
===== `add_tag`
|
||||
|
||||
* Value type is {logstash-ref}/configuration-file-structure.html#array[array]
|
||||
* Default value is `[]`
|
||||
|
||||
If this filter is successful, add arbitrary tags to the event.
|
||||
Tags can be dynamic and include parts of the event using the `%{field}`
|
||||
syntax.
|
||||
|
||||
Example:
|
||||
|
||||
["source","json",subs="attributes"]
|
||||
filter {
|
||||
{plugin} {
|
||||
add_tag => [ "foo_%\{somefield\}" ]
|
||||
}
|
||||
}
|
||||
|
||||
["source","json",subs="attributes"]
|
||||
# You can also add multiple tags at once:
|
||||
filter {
|
||||
{plugin} {
|
||||
add_tag => [ "foo_%\{somefield\}", "taggedy_tag"]
|
||||
}
|
||||
}
|
||||
|
||||
If the event has field `"somefield" == "hello"` this filter, on success,
|
||||
would add a tag `foo_hello` (and the second example would of course add a `taggedy_tag` tag).
|
||||
|
||||
ifeval::["{versioned_docs}"!="true"]
|
||||
[id="plugins-{type}s-{plugin}-enable_metric"]
|
||||
endif::[]
|
||||
ifeval::["{versioned_docs}"=="true"]
|
||||
[id="{version}-plugins-{type}s-{plugin}-enable_metric"]
|
||||
endif::[]
|
||||
===== `enable_metric`
|
||||
|
||||
* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean]
|
||||
* Default value is `true`
|
||||
|
||||
Disable or enable metric logging for this specific plugin instance.
|
||||
By default we record all the metrics we can, but you can disable metrics collection
|
||||
for a specific plugin.
|
||||
|
||||
ifeval::["{versioned_docs}"!="true"]
|
||||
[id="plugins-{type}s-{plugin}-id"]
|
||||
endif::[]
|
||||
ifeval::["{versioned_docs}"=="true"]
|
||||
[id="{version}-plugins-{type}s-{plugin}-id"]
|
||||
endif::[]
|
||||
===== `id`
|
||||
|
||||
* Value type is {logstash-ref}/configuration-file-structure.html#string[string]
|
||||
* There is no default value for this setting.
|
||||
|
||||
Add a unique `ID` to the plugin configuration. If no ID is specified, Logstash will generate one.
|
||||
It is strongly recommended to set this ID in your configuration. This is particularly useful
|
||||
when you have two or more plugins of the same type, for example, if you have 2 {plugin} filters.
|
||||
Adding a named ID in this case will help in monitoring Logstash when using the monitoring APIs.
|
||||
|
||||
|
||||
["source","json",subs="attributes"]
|
||||
filter {
|
||||
{plugin} {
|
||||
id => "ABC"
|
||||
}
|
||||
}
|
||||
|
||||
NOTE: Variable substitution in the `id` field only supports environment variables
|
||||
and does not support the use of values from the secret store.
|
||||
|
||||
ifeval::["{versioned_docs}"!="true"]
|
||||
[id="plugins-{type}s-{plugin}-periodic_flush"]
|
||||
endif::[]
|
||||
ifeval::["{versioned_docs}"=="true"]
|
||||
[id="{version}-plugins-{type}s-{plugin}-periodic_flush"]
|
||||
endif::[]
|
||||
===== `periodic_flush`
|
||||
|
||||
* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean]
|
||||
* Default value is `false`
|
||||
|
||||
Call the filter flush method at regular interval.
|
||||
Optional.
|
||||
|
||||
ifeval::["{versioned_docs}"!="true"]
|
||||
[id="plugins-{type}s-{plugin}-remove_field"]
|
||||
endif::[]
|
||||
ifeval::["{versioned_docs}"=="true"]
|
||||
[id="{version}-plugins-{type}s-{plugin}-remove_field"]
|
||||
endif::[]
|
||||
===== `remove_field`
|
||||
|
||||
* Value type is {logstash-ref}/configuration-file-structure.html#array[array]
|
||||
* Default value is `[]`
|
||||
|
||||
If this filter is successful, remove arbitrary fields from this event.
|
||||
Fields names can be dynamic and include parts of the event using the %{field}
|
||||
Example:
|
||||
|
||||
["source","json",subs="attributes"]
|
||||
filter {
|
||||
{plugin} {
|
||||
remove_field => [ "foo_%\{somefield\}" ]
|
||||
}
|
||||
}
|
||||
|
||||
["source","json",subs="attributes"]
|
||||
# You can also remove multiple fields at once:
|
||||
filter {
|
||||
{plugin} {
|
||||
remove_field => [ "foo_%\{somefield\}", "my_extraneous_field" ]
|
||||
}
|
||||
}
|
||||
|
||||
If the event has field `"somefield" == "hello"` this filter, on success,
|
||||
would remove the field with name `foo_hello` if it is present. The second
|
||||
example would remove an additional, non-dynamic field.
|
||||
|
||||
ifeval::["{versioned_docs}"!="true"]
|
||||
[id="plugins-{type}s-{plugin}-remove_tag"]
|
||||
endif::[]
|
||||
ifeval::["{versioned_docs}"=="true"]
|
||||
[id="{version}-plugins-{type}s-{plugin}-remove_tag"]
|
||||
endif::[]
|
||||
===== `remove_tag`
|
||||
|
||||
* Value type is {logstash-ref}/configuration-file-structure.html#array[array]
|
||||
* Default value is `[]`
|
||||
|
||||
If this filter is successful, remove arbitrary tags from the event.
|
||||
Tags can be dynamic and include parts of the event using the `%{field}`
|
||||
syntax.
|
||||
|
||||
Example:
|
||||
|
||||
["source","json",subs="attributes"]
|
||||
filter {
|
||||
{plugin} {
|
||||
remove_tag => [ "foo_%\{somefield\}" ]
|
||||
}
|
||||
}
|
||||
|
||||
["source","json",subs="attributes"]
|
||||
# You can also remove multiple tags at once:
|
||||
filter {
|
||||
{plugin} {
|
||||
remove_tag => [ "foo_%\{somefield\}", "sad_unwanted_tag"]
|
||||
}
|
||||
}
|
||||
|
||||
If the event has field `"somefield" == "hello"` this filter, on success,
|
||||
would remove the tag `foo_hello` if it is present. The second example
|
||||
would remove a sad, unwanted tag as well.
|
||||
|
|
@ -1,172 +0,0 @@
|
|||
==== Common options
|
||||
|
||||
// Contributors: You must conditionally code all internal links and IDs in this
|
||||
// file to make the common files work in both the LS Reference and the versioned
|
||||
// plugin docs
|
||||
|
||||
These configuration options are supported by all input plugins:
|
||||
|
||||
[cols="<,<,<",options="header",]
|
||||
ifeval::["{versioned_docs}"!="true"]
|
||||
|=======================================================================
|
||||
|Setting |Input type|Required
|
||||
| <<plugins-{type}s-{plugin}-add_field>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No
|
||||
ifndef::no_codec[]
|
||||
| <<plugins-{type}s-{plugin}-codec>> |{logstash-ref}/configuration-file-structure.html#codec[codec]|No
|
||||
endif::no_codec[]
|
||||
| <<plugins-{type}s-{plugin}-enable_metric>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No
|
||||
| <<plugins-{type}s-{plugin}-id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No
|
||||
| <<plugins-{type}s-{plugin}-tags>> |{logstash-ref}/configuration-file-structure.html#array[array]|No
|
||||
| <<plugins-{type}s-{plugin}-type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No
|
||||
|=======================================================================
|
||||
endif::[]
|
||||
ifeval::["{versioned_docs}"=="true"]
|
||||
|=======================================================================
|
||||
|Setting |Input type|Required
|
||||
| <<{version}-plugins-{type}s-{plugin}-add_field>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No
|
||||
ifndef::no_codec[]
|
||||
| <<{version}-plugins-{type}s-{plugin}-codec>> |{logstash-ref}/configuration-file-structure.html#codec[codec]|No
|
||||
endif::no_codec[]
|
||||
| <<{version}-plugins-{type}s-{plugin}-enable_metric>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No
|
||||
| <<{version}-plugins-{type}s-{plugin}-id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No
|
||||
| <<{version}-plugins-{type}s-{plugin}-tags>> |{logstash-ref}/configuration-file-structure.html#array[array]|No
|
||||
| <<{version}-plugins-{type}s-{plugin}-type>> |{logstash-ref}/configuration-file-structure.html#string[string]|No
|
||||
|=======================================================================
|
||||
endif::[]
|
||||
|
||||
|
||||
ifeval::["{versioned_docs}"!="true"]
|
||||
[id="plugins-{type}s-{plugin}-add_field"]
|
||||
endif::[]
|
||||
ifeval::["{versioned_docs}"=="true"]
|
||||
[id="{version}-plugins-{type}s-{plugin}-add_field"]
|
||||
endif::[]
|
||||
===== `add_field`
|
||||
|
||||
* Value type is {logstash-ref}/configuration-file-structure.html#hash[hash]
|
||||
* Default value is `{}`
|
||||
|
||||
Add a field to an event
|
||||
|
||||
ifndef::no_codec[]
|
||||
ifeval::["{versioned_docs}"!="true"]
|
||||
[id="plugins-{type}s-{plugin}-codec"]
|
||||
endif::[]
|
||||
ifeval::["{versioned_docs}"=="true"]
|
||||
[id="{version}-plugins-{type}s-{plugin}-codec"]
|
||||
endif::[]
|
||||
===== `codec`
|
||||
|
||||
* Value type is {logstash-ref}/configuration-file-structure.html#codec[codec]
|
||||
ifdef::default_codec[]
|
||||
* Default value is +"{default_codec}"+
|
||||
endif::[]
|
||||
ifndef::default_codec[]
|
||||
* Default value is `"plain"`
|
||||
endif::[]
|
||||
|
||||
The codec used for input data. Input codecs are a convenient method for decoding your data before it enters the input, without needing a separate filter in your Logstash pipeline.
|
||||
endif::no_codec[]
|
||||
|
||||
|
||||
ifeval::["{versioned_docs}"!="true"]
|
||||
[id="plugins-{type}s-{plugin}-enable_metric"]
|
||||
endif::[]
|
||||
ifeval::["{versioned_docs}"=="true"]
|
||||
[id="{version}-plugins-{type}s-{plugin}-enable_metric"]
|
||||
endif::[]
|
||||
===== `enable_metric`
|
||||
|
||||
* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean]
|
||||
* Default value is `true`
|
||||
|
||||
Disable or enable metric logging for this specific plugin instance
|
||||
by default we record all the metrics we can, but you can disable metrics collection
|
||||
for a specific plugin.
|
||||
|
||||
ifeval::["{versioned_docs}"!="true"]
|
||||
[id="plugins-{type}s-{plugin}-id"]
|
||||
endif::[]
|
||||
ifeval::["{versioned_docs}"=="true"]
|
||||
[id="{version}-plugins-{type}s-{plugin}-id"]
|
||||
endif::[]
|
||||
===== `id`
|
||||
|
||||
* Value type is {logstash-ref}/configuration-file-structure.html#string[string]
|
||||
* There is no default value for this setting.
|
||||
|
||||
Add a unique `ID` to the plugin configuration. If no ID is specified, Logstash will generate one.
|
||||
It is strongly recommended to set this ID in your configuration. This is particularly useful
|
||||
when you have two or more plugins of the same type, for example, if you have 2 {plugin} inputs.
|
||||
Adding a named ID in this case will help in monitoring Logstash when using the monitoring APIs.
|
||||
|
||||
["source","json",subs="attributes"]
|
||||
---------------------------------------------------------------------------------------------------
|
||||
input {
|
||||
{plugin} {
|
||||
id => "my_plugin_id"
|
||||
}
|
||||
}
|
||||
---------------------------------------------------------------------------------------------------
|
||||
|
||||
NOTE: Variable substitution in the `id` field only supports environment variables
|
||||
and does not support the use of values from the secret store.
|
||||
|
||||
ifeval::["{versioned_docs}"!="true"]
|
||||
[id="plugins-{type}s-{plugin}-tags"]
|
||||
endif::[]
|
||||
ifeval::["{versioned_docs}"=="true"]
|
||||
[id="{version}-plugins-{type}s-{plugin}-tags"]
|
||||
endif::[]
|
||||
===== `tags`
|
||||
|
||||
* Value type is {logstash-ref}/configuration-file-structure.html#array[array]
|
||||
* There is no default value for this setting.
|
||||
|
||||
Add any number of arbitrary tags to your event.
|
||||
|
||||
This can help with processing later.
|
||||
|
||||
ifeval::["{versioned_docs}"!="true"]
|
||||
[id="plugins-{type}s-{plugin}-type"]
|
||||
endif::[]
|
||||
ifeval::["{versioned_docs}"=="true"]
|
||||
[id="{version}-plugins-{type}s-{plugin}-type"]
|
||||
endif::[]
|
||||
===== `type`
|
||||
|
||||
* Value type is {logstash-ref}/configuration-file-structure.html#string[string]
|
||||
* There is no default value for this setting.
|
||||
|
||||
Add a `type` field to all events handled by this input.
|
||||
|
||||
Types are used mainly for filter activation.
|
||||
|
||||
The type is stored as part of the event itself, so you can
|
||||
also use the type to search for it in Kibana.
|
||||
|
||||
If you try to set a type on an event that already has one (for
|
||||
example when you send an event from a shipper to an indexer) then
|
||||
a new input will not override the existing type. A type set at
|
||||
the shipper stays with that event for its life even
|
||||
when sent to another Logstash server.
|
||||
|
||||
ifeval::["{type}"=="input"]
|
||||
ifeval::["{plugin}"=="beats"]
|
||||
|
||||
ifeval::["{versioned_docs}"!="true"]
|
||||
NOTE: The Beats shipper automatically sets the `type` field on the event.
|
||||
You cannot override this setting in the Logstash config. If you specify
|
||||
a setting for the <<plugins-inputs-beats-type,`type`>> config option in
|
||||
Logstash, it is ignored.
|
||||
endif::[]
|
||||
ifeval::["{versioned_docs}"=="true"]
|
||||
NOTE: The Beats shipper automatically sets the `type` field on the event.
|
||||
You cannot override this setting in the Logstash config. If you specify
|
||||
a setting for the <<{version}-plugins-inputs-beats-type,`type`>> config option in
|
||||
Logstash, it is ignored.
|
||||
endif::[]
|
||||
|
||||
endif::[]
|
||||
endif::[]
|
||||
|
||||
|
|
@ -1,94 +0,0 @@
|
|||
==== Common options
|
||||
|
||||
// Contributors: You must conditionally code all internal links and IDs in this
|
||||
// file to make the common files work in both the LS Reference and the versioned
|
||||
// plugin docs
|
||||
|
||||
These configuration options are supported by all output plugins:
|
||||
|
||||
ifeval::["{versioned_docs}"!="true"]
|
||||
[cols="<,<,<",options="header",]
|
||||
|=======================================================================
|
||||
|Setting |Input type|Required
|
||||
ifndef::no_codec[]
|
||||
| <<plugins-{type}s-{plugin}-codec>> |{logstash-ref}/configuration-file-structure.html#codec[codec]|No
|
||||
endif::no_codec[]
|
||||
| <<plugins-{type}s-{plugin}-enable_metric>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No
|
||||
| <<plugins-{type}s-{plugin}-id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No
|
||||
|=======================================================================
|
||||
endif::[]
|
||||
ifeval::["{versioned_docs}"=="true"]
|
||||
[cols="<,<,<",options="header",]
|
||||
|=======================================================================
|
||||
|Setting |Input type|Required
|
||||
ifndef::no_codec[]
|
||||
| <<{version}-plugins-{type}s-{plugin}-codec>> |{logstash-ref}/configuration-file-structure.html#codec[codec]|No
|
||||
endif::no_codec[]
|
||||
| <<{version}-plugins-{type}s-{plugin}-enable_metric>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No
|
||||
| <<{version}-plugins-{type}s-{plugin}-id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No
|
||||
|=======================================================================
|
||||
endif::[]
|
||||
|
||||
ifndef::no_codec[]
|
||||
ifeval::["{versioned_docs}"!="true"]
|
||||
[id="plugins-{type}s-{plugin}-codec"]
|
||||
endif::[]
|
||||
ifeval::["{versioned_docs}"=="true"]
|
||||
[id="{version}-plugins-{type}s-{plugin}-codec"]
|
||||
endif::[]
|
||||
===== `codec`
|
||||
|
||||
* Value type is {logstash-ref}/configuration-file-structure.html#codec[codec]
|
||||
ifdef::default_codec[]
|
||||
* Default value is +"{default_codec}"+
|
||||
endif::[]
|
||||
ifndef::default_codec[]
|
||||
* Default value is `"plain"`
|
||||
endif::[]
|
||||
|
||||
The codec used for output data. Output codecs are a convenient method for encoding your data before it leaves the output without needing a separate filter in your Logstash pipeline.
|
||||
endif::no_codec[]
|
||||
|
||||
ifeval::["{versioned_docs}"!="true"]
|
||||
[id="plugins-{type}s-{plugin}-enable_metric"]
|
||||
endif::[]
|
||||
ifeval::["{versioned_docs}"=="true"]
|
||||
[id="{version}-plugins-{type}s-{plugin}-enable_metric"]
|
||||
endif::[]
|
||||
===== `enable_metric`
|
||||
|
||||
* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean]
|
||||
* Default value is `true`
|
||||
|
||||
Disable or enable metric logging for this specific plugin instance.
|
||||
By default we record all the metrics we can, but you can disable metrics collection
|
||||
for a specific plugin.
|
||||
|
||||
ifeval::["{versioned_docs}"!="true"]
|
||||
[id="plugins-{type}s-{plugin}-id"]
|
||||
endif::[]
|
||||
ifeval::["{versioned_docs}"=="true"]
|
||||
[id="{version}-plugins-{type}s-{plugin}-id"]
|
||||
endif::[]
|
||||
===== `id`
|
||||
|
||||
* Value type is {logstash-ref}/configuration-file-structure.html#string[string]
|
||||
* There is no default value for this setting.
|
||||
|
||||
Add a unique `ID` to the plugin configuration. If no ID is specified, Logstash will generate one.
|
||||
It is strongly recommended to set this ID in your configuration. This is particularly useful
|
||||
when you have two or more plugins of the same type. For example, if you have 2 {plugin} outputs.
|
||||
Adding a named ID in this case will help in monitoring Logstash when using the monitoring APIs.
|
||||
|
||||
["source","json",subs="attributes"]
|
||||
---------------------------------------------------------------------------------------------------
|
||||
output {
|
||||
{plugin} {
|
||||
id => "my_plugin_id"
|
||||
}
|
||||
}
|
||||
---------------------------------------------------------------------------------------------------
|
||||
|
||||
NOTE: Variable substitution in the `id` field only supports environment variables
|
||||
and does not support the use of values from the secret store.
|
||||
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
[subs="attributes"]
|
||||
++++
|
||||
<titleabbrev>{plugin}</titleabbrev>
|
||||
++++
|
||||
|
||||
*{ls} Core Plugin.* The {plugin} {type} plugin cannot be
|
||||
installed or uninstalled independently of {ls}.
|
||||
|
||||
==== Getting help
|
||||
|
||||
For questions about the plugin, open a topic in the
|
||||
http://discuss.elastic.co[Discuss] forums. For bugs or feature requests, open an
|
||||
issue in https://github.com/logstash[Github].
|
||||
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
[subs="attributes"]
|
||||
++++
|
||||
<titleabbrev>{plugin}</titleabbrev>
|
||||
++++
|
||||
|
||||
* A component of the <<plugins-integrations-{integration},{integration} integration plugin>>
|
||||
* Integration version: {version}
|
||||
* Released on: {release_date}
|
||||
* {changelog_url}[Changelog]
|
||||
|
||||
For other versions, see the
|
||||
{lsplugindocs}/{type}-{plugin}-index.html[Versioned plugin docs].
|
||||
|
||||
==== Getting help
|
||||
|
||||
For questions about the plugin, open a topic in the http://discuss.elastic.co[Discuss] forums.
|
||||
For bugs or feature requests, open an issue in https://github.com/logstash-plugins/logstash-integration-{integration}[Github].
|
||||
For the list of Elastic supported plugins, please consult the https://www.elastic.co/support/matrix#logstash_plugins[Elastic Support Matrix].
|
||||
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
[subs="attributes"]
|
||||
++++
|
||||
<titleabbrev>{plugin}</titleabbrev>
|
||||
++++
|
||||
|
||||
* Plugin version: {version}
|
||||
* Released on: {release_date}
|
||||
* {changelog_url}[Changelog]
|
||||
|
||||
For other versions, see the
|
||||
{lsplugindocs}/{type}-{plugin}-index.html[Versioned plugin docs].
|
||||
|
||||
ifeval::["{default_plugin}"=="0"]
|
||||
|
||||
==== Installation
|
||||
|
||||
For plugins not bundled by default, it is easy to install by running +bin/logstash-plugin install logstash-{type}-{plugin}+. See {logstash-ref}/working-with-plugins.html[Working with plugins] for more details.
|
||||
|
||||
endif::[]
|
||||
|
||||
==== Getting help
|
||||
|
||||
For questions about the plugin, open a topic in the http://discuss.elastic.co[Discuss] forums. For bugs or feature requests, open an issue in https://github.com/logstash-plugins/logstash-{type}-{plugin}[Github].
|
||||
For the list of Elastic supported plugins, please consult the https://www.elastic.co/support/matrix#logstash_plugins[Elastic Support Matrix].
|
||||
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
[id="{type}-{plugin}-index"]
|
||||
|
||||
== Versioned {plugin} {type} plugin docs
|
||||
[subs="attributes"]
|
||||
++++
|
||||
<titleabbrev>{plugin}</titleabbrev>
|
||||
++++
|
||||
|
||||
This page lists all available versions of the documentation for this plugin.
|
||||
To see which version of the plugin you have installed, run `bin/logstash-plugin
|
||||
list --verbose`.
|
||||
|
||||
NOTE: Versioned plugin documentation is not available for plugins released prior
|
||||
to Logstash 6.0.
|
||||
|
|
@ -1,238 +0,0 @@
|
|||
[[logstash-reference]]
|
||||
= Logstash Reference
|
||||
|
||||
include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
|
||||
include::{docs-root}/shared/attributes.asciidoc[]
|
||||
include::./include/attributes-ls.asciidoc[]
|
||||
include::./include/attributes-lsplugins.asciidoc[]
|
||||
|
||||
:include-xpack: true
|
||||
:lang: en
|
||||
:xls-repo-dir: {docdir}/../x-pack/docs/{lang}
|
||||
:log-repo-dir: {docdir}
|
||||
:plugins-repo-dir: {docdir}/../../logstash-docs/docs
|
||||
:docker-repo: docker.elastic.co/logstash/logstash
|
||||
:docker-image: {docker-repo}:{logstash_version}
|
||||
|
||||
:versioned_docs: false
|
||||
|
||||
:jdk: 1.8.0
|
||||
:lsissue: https://github.com/elastic/logstash/issues
|
||||
:lsplugindocs: https://www.elastic.co/guide/en/logstash-versioned-plugins/current
|
||||
:tab-widget-dir: {docdir}/static/tab-widgets
|
||||
|
||||
|
||||
[[introduction]]
|
||||
== Logstash Introduction
|
||||
|
||||
Logstash is an open source data collection engine with real-time pipelining capabilities. Logstash can dynamically
|
||||
unify data from disparate sources and normalize the data into destinations of your choice. Cleanse and democratize all
|
||||
your data for diverse advanced downstream analytics and visualization use cases.
|
||||
|
||||
While Logstash originally drove innovation in log collection, its capabilities extend well beyond that use case. Any
|
||||
type of event can be enriched and transformed with a broad array of input, filter, and output plugins, with many
|
||||
native codecs further simplifying the ingestion process. Logstash accelerates your insights by harnessing a greater
|
||||
volume and variety of data.
|
||||
|
||||
|
||||
[serverless]
|
||||
.Logstash to {serverless-full}
|
||||
****
|
||||
You'll use the {ls} <<plugins-outputs-elasticsearch,{es} output plugin>> to send data to {serverless-full}.
|
||||
Note these differences between {es-serverless} and both {ess} and self-managed {es}:
|
||||
|
||||
* Use *API keys* to access {serverless-full} from {ls}.
|
||||
Any user-based security settings in your in your <<plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors.
|
||||
* {serverless-full} uses *data streams* and {ref}/data-stream-lifecycle.html[{dlm} ({dlm-init})] instead of {ilm} ({ilm-init}).
|
||||
Any {ilm-init} settings in your <<plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors.
|
||||
* *{ls} monitoring* is available through the https://github.com/elastic/integrations/blob/main/packages/logstash/_dev/build/docs/README.md[{ls} Integration] in {serverless-docs}/observability/what-is-observability-serverless[Elastic Observability] on {serverless-full}.
|
||||
|
||||
|
||||
.Known issue for {ls} to {es-serverless}.
|
||||
|
||||
The logstash-output-elasticsearch `hosts` setting defaults to port :9200. Set the value to port :443 instead.
|
||||
****
|
||||
|
||||
|
||||
// The pass blocks here point to the correct repository for the edit links in the guide.
|
||||
|
||||
// Introduction
|
||||
|
||||
// Getting Started with Logstash
|
||||
include::static/getting-started-with-logstash.asciidoc[]
|
||||
|
||||
// Advanced LS Pipelines
|
||||
include::static/advanced-pipeline.asciidoc[]
|
||||
|
||||
// Processing Pipeline
|
||||
include::static/life-of-an-event.asciidoc[]
|
||||
|
||||
// Elastic Common Schema (ECS)
|
||||
include::static/ecs-compatibility.asciidoc[]
|
||||
|
||||
// Processing details
|
||||
include::static/processing-info.asciidoc[]
|
||||
|
||||
// Logstash setup
|
||||
include::static/setting-up-logstash.asciidoc[]
|
||||
|
||||
include::static/settings-file.asciidoc[]
|
||||
|
||||
include::static/keystore.asciidoc[]
|
||||
|
||||
include::static/running-logstash-command-line.asciidoc[]
|
||||
|
||||
include::static/running-logstash.asciidoc[]
|
||||
|
||||
include::static/docker.asciidoc[]
|
||||
|
||||
include::static/running-logstash-kubernetes.asciidoc[]
|
||||
|
||||
include::static/running-logstash-windows.asciidoc[]
|
||||
|
||||
include::static/logging.asciidoc[]
|
||||
|
||||
include::static/shutdown.asciidoc[]
|
||||
|
||||
// Upgrading Logstash
|
||||
include::static/upgrading.asciidoc[]
|
||||
|
||||
// Configuring pipelines
|
||||
include::static/pipeline-configuration.asciidoc[]
|
||||
|
||||
// Security
|
||||
include::static/security/logstash.asciidoc[]
|
||||
|
||||
// Advanced Logstash Configuration
|
||||
include::static/configuration-advanced.asciidoc[]
|
||||
|
||||
include::static/multiple-pipelines.asciidoc[]
|
||||
|
||||
include::static/pipeline-pipeline-config.asciidoc[]
|
||||
|
||||
include::static/reloading-config.asciidoc[]
|
||||
|
||||
include::static/managing-multiline-events.asciidoc[]
|
||||
|
||||
include::static/glob-support.asciidoc[]
|
||||
|
||||
include::static/field-reference.asciidoc[]
|
||||
|
||||
//The `field-reference.asciidoc` file (included above) contains a
|
||||
//`role="exclude"` attribute to pull in the topic and make it linkable in the LS
|
||||
//Ref, but not appear in the main TOC. The `exclude`attribute was carrying
|
||||
//forward for all subsequent topics under the `configuration.asciidoc` heading.
|
||||
//This include should remain after includes for all other topics under the
|
||||
//`Advanced Logstash Configuration` heading.
|
||||
|
||||
// Logstash-to-Logstash
|
||||
include::static/ls-ls-config.asciidoc[]
|
||||
|
||||
// Centralized configuration managements
|
||||
include::static/config-management.asciidoc[]
|
||||
|
||||
include::static/management/configuring-centralized-pipelines.asciidoc[]
|
||||
|
||||
// EA Integrations to Logstash
|
||||
// (Planting near module content for now. Will likely move it up in info architecture.)
|
||||
include::static/ea-integrations.asciidoc[]
|
||||
|
||||
|
||||
// Working with Filebeat Modules
|
||||
include::static/filebeat-modules.asciidoc[]
|
||||
|
||||
// Working with Winlogbeat Modules
|
||||
include::static/winlogbeat-modules.asciidoc[]
|
||||
|
||||
// Data resiliency
|
||||
include::static/resiliency.asciidoc[]
|
||||
|
||||
include::static/mem-queue.asciidoc[]
|
||||
|
||||
include::static/persistent-queues.asciidoc[]
|
||||
|
||||
include::static/dead-letter-queues.asciidoc[]
|
||||
|
||||
// Transforming Data
|
||||
include::static/transforming-data.asciidoc[]
|
||||
|
||||
// Deploying & Scaling
|
||||
include::static/deploying.asciidoc[]
|
||||
|
||||
// GeoIP Database Management
|
||||
include::static/geoip-database-management.asciidoc[]
|
||||
|
||||
// Troubleshooting performance
|
||||
include::static/performance-checklist.asciidoc[]
|
||||
|
||||
// Monitoring
|
||||
include::static/monitoring/monitoring-ea-intro.asciidoc[]
|
||||
|
||||
include::static/monitoring/monitoring-overview.asciidoc[]
|
||||
|
||||
include::static/monitoring/monitoring.asciidoc[]
|
||||
|
||||
// Working with Plugins
|
||||
include::static/plugin-manager.asciidoc[]
|
||||
|
||||
// These files do their own pass blocks
|
||||
|
||||
include::{plugins-repo-dir}/plugins/integrations.asciidoc[]
|
||||
|
||||
include::{plugins-repo-dir}/plugins/inputs.asciidoc[]
|
||||
|
||||
include::{plugins-repo-dir}/plugins/outputs.asciidoc[]
|
||||
|
||||
include::{plugins-repo-dir}/plugins/filters.asciidoc[]
|
||||
|
||||
include::{plugins-repo-dir}/plugins/codecs.asciidoc[]
|
||||
|
||||
// FAQ and Troubleshooting
|
||||
:edit_url!:
|
||||
include::static/best-practice.asciidoc[]
|
||||
|
||||
include::static/config-details.asciidoc[]
|
||||
|
||||
include::static/troubleshoot/troubleshooting.asciidoc[]
|
||||
|
||||
// Contributing to Logstash
|
||||
:edit_url:
|
||||
include::static/contributing-to-logstash.asciidoc[]
|
||||
|
||||
include::static/input.asciidoc[]
|
||||
|
||||
include::static/codec.asciidoc[]
|
||||
|
||||
include::static/filter.asciidoc[]
|
||||
|
||||
include::static/output.asciidoc[]
|
||||
|
||||
// Logstash Community Maintainer Guide
|
||||
include::static/maintainer-guide.asciidoc[]
|
||||
|
||||
// Plugin doc guidelines
|
||||
include::static/doc-for-plugin.asciidoc[]
|
||||
|
||||
// Submitting a Plugin
|
||||
include::static/submitting-a-plugin.asciidoc[]
|
||||
|
||||
include::static/listing-a-plugin.asciidoc[]
|
||||
|
||||
include::static/contributing-patch.asciidoc[]
|
||||
|
||||
include::static/contribute-core.asciidoc[]
|
||||
|
||||
// Contributing to Logstash - JAVA EDITION
|
||||
:edit_url:
|
||||
include::static/contributing-java-plugin.asciidoc[]
|
||||
|
||||
// Breaking Changes
|
||||
include::static/breaking-changes.asciidoc[]
|
||||
|
||||
// Release Notes
|
||||
include::static/releasenotes.asciidoc[]
|
||||
|
||||
:edit_url:
|
||||
include::static/redirects.asciidoc[]
|
||||
|
||||
:edit_url!:
|
||||
|
|
@ -1 +0,0 @@
|
|||
include::index.asciidoc[]
|
||||
14
docs/reference/advanced-logstash-configurations.md
Normal file
14
docs/reference/advanced-logstash-configurations.md
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/configuration-advanced.html
|
||||
---
|
||||
|
||||
# Advanced Logstash configurations [configuration-advanced]
|
||||
|
||||
You can take {{ls}} beyond basic configuration to handle more advanced requirements, such as multiple pipelines, communication between {{ls}} pipelines, and multiple line events.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
612
docs/reference/advanced-pipeline.md
Normal file
612
docs/reference/advanced-pipeline.md
Normal file
|
|
@ -0,0 +1,612 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/advanced-pipeline.html
|
||||
---
|
||||
|
||||
# Parsing Logs with Logstash [advanced-pipeline]
|
||||
|
||||
In [Stashing Your First Event](/reference/first-event.md), you created a basic Logstash pipeline to test your Logstash setup. In the real world, a Logstash pipeline is a bit more complex: it typically has one or more input, filter, and output plugins.
|
||||
|
||||
In this section, you create a Logstash pipeline that uses Filebeat to take Apache web logs as input, parses those logs to create specific, named fields from the logs, and writes the parsed data to an Elasticsearch cluster. Rather than defining the pipeline configuration at the command line, you’ll define the pipeline in a config file.
|
||||
|
||||
To get started, go [here](https://download.elastic.co/demos/logstash/gettingstarted/logstash-tutorial.log.gz) to download the sample data set used in this example. Unpack the file.
|
||||
|
||||
## Configuring Filebeat to Send Log Lines to Logstash [configuring-filebeat]
|
||||
|
||||
Before you create the Logstash pipeline, you’ll configure Filebeat to send log lines to Logstash. The [Filebeat](https://github.com/elastic/beats/tree/main/filebeat) client is a lightweight, resource-friendly tool that collects logs from files on the server and forwards these logs to your Logstash instance for processing. Filebeat is designed for reliability and low latency. Filebeat has a light resource footprint on the host machine, and the [`Beats input`](/reference/plugins-inputs-beats.md) plugin minimizes the resource demands on the Logstash instance.
|
||||
|
||||
::::{note}
|
||||
In a typical use case, Filebeat runs on a separate machine from the machine running your Logstash instance. For the purposes of this tutorial, Logstash and Filebeat are running on the same machine.
|
||||
::::
|
||||
|
||||
|
||||
The default Logstash installation includes the [`Beats input`](/reference/plugins-inputs-beats.md) plugin. The Beats input plugin enables Logstash to receive events from the Elastic Beats framework, which means that any Beat written to work with the Beats framework, such as Packetbeat and Metricbeat, can also send event data to Logstash.
|
||||
|
||||
To install Filebeat on your data source machine, download the appropriate package from the Filebeat [product page](https://www.elastic.co/downloads/beats/filebeat). You can also refer to [Filebeat quick start](beats://reference/filebeat/filebeat-installation-configuration.md) for additional installation instructions.
|
||||
|
||||
After installing Filebeat, you need to configure it. Open the `filebeat.yml` file located in your Filebeat installation directory, and replace the contents with the following lines. Make sure `paths` points to the example Apache log file, `logstash-tutorial.log`, that you downloaded earlier:
|
||||
|
||||
```yaml
|
||||
filebeat.inputs:
|
||||
- type: log
|
||||
paths:
|
||||
- /path/to/file/logstash-tutorial.log <1>
|
||||
output.logstash:
|
||||
hosts: ["localhost:5044"]
|
||||
```
|
||||
|
||||
1. Absolute path to the file or files that Filebeat processes.
|
||||
|
||||
|
||||
Save your changes.
|
||||
|
||||
To keep the configuration simple, you won’t specify TLS/SSL settings as you would in a real world scenario.
|
||||
|
||||
At the data source machine, run Filebeat with the following command:
|
||||
|
||||
```shell
|
||||
sudo ./filebeat -e -c filebeat.yml -d "publish"
|
||||
```
|
||||
|
||||
::::{note}
|
||||
If you run Filebeat as root, you need to change ownership of the configuration file (see [Config File Ownership and Permissions](beats://reference/libbeat/config-file-permissions.md) in the *Beats Platform Reference*).
|
||||
::::
|
||||
|
||||
|
||||
Filebeat will attempt to connect on port 5044. Until Logstash starts with an active Beats plugin, there won’t be any answer on that port, so any messages you see regarding failure to connect on that port are normal for now.
|
||||
|
||||
|
||||
## Configuring Logstash for Filebeat Input [_configuring_logstash_for_filebeat_input]
|
||||
|
||||
Next, you create a Logstash configuration pipeline that uses the Beats input plugin to receive events from Beats.
|
||||
|
||||
The following text represents the skeleton of a configuration pipeline:
|
||||
|
||||
```json
|
||||
# The # character at the beginning of a line indicates a comment. Use
|
||||
# comments to describe your configuration.
|
||||
input {
|
||||
}
|
||||
# The filter part of this file is commented out to indicate that it is
|
||||
# optional.
|
||||
# filter {
|
||||
#
|
||||
# }
|
||||
output {
|
||||
}
|
||||
```
|
||||
|
||||
This skeleton is non-functional, because the input and output sections don’t have any valid options defined.
|
||||
|
||||
To get started, copy and paste the skeleton configuration pipeline into a file named `first-pipeline.conf` in your home Logstash directory.
|
||||
|
||||
Next, configure your Logstash instance to use the Beats input plugin by adding the following lines to the `input` section of the `first-pipeline.conf` file:
|
||||
|
||||
```json
|
||||
beats {
|
||||
port => "5044"
|
||||
}
|
||||
```
|
||||
|
||||
You’ll configure Logstash to write to Elasticsearch later. For now, you can add the following line to the `output` section so that the output is printed to stdout when you run Logstash:
|
||||
|
||||
```json
|
||||
stdout { codec => rubydebug }
|
||||
```
|
||||
|
||||
When you’re done, the contents of `first-pipeline.conf` should look like this:
|
||||
|
||||
```json
|
||||
input {
|
||||
beats {
|
||||
port => "5044"
|
||||
}
|
||||
}
|
||||
# The filter part of this file is commented out to indicate that it is
|
||||
# optional.
|
||||
# filter {
|
||||
#
|
||||
# }
|
||||
output {
|
||||
stdout { codec => rubydebug }
|
||||
}
|
||||
```
|
||||
|
||||
To verify your configuration, run the following command:
|
||||
|
||||
```shell
|
||||
bin/logstash -f first-pipeline.conf --config.test_and_exit
|
||||
```
|
||||
|
||||
The `--config.test_and_exit` option parses your configuration file and reports any errors.
|
||||
|
||||
If the configuration file passes the configuration test, start Logstash with the following command:
|
||||
|
||||
```shell
|
||||
bin/logstash -f first-pipeline.conf --config.reload.automatic
|
||||
```
|
||||
|
||||
The `--config.reload.automatic` option enables automatic config reloading so that you don’t have to stop and restart Logstash every time you modify the configuration file.
|
||||
|
||||
As Logstash starts up, you might see one or more warning messages about Logstash ignoring the `pipelines.yml` file. You can safely ignore this warning. The `pipelines.yml` file is used for running [multiple pipelines](/reference/multiple-pipelines.md) in a single Logstash instance. For the examples shown here, you are running a single pipeline.
|
||||
|
||||
If your pipeline is working correctly, you should see a series of events like the following written to the console:
|
||||
|
||||
```json
|
||||
{
|
||||
"@timestamp" => 2017-11-09T01:44:20.071Z,
|
||||
"offset" => 325,
|
||||
"@version" => "1",
|
||||
"beat" => {
|
||||
"name" => "My-MacBook-Pro.local",
|
||||
"hostname" => "My-MacBook-Pro.local",
|
||||
"version" => "6.0.0"
|
||||
},
|
||||
"host" => "My-MacBook-Pro.local",
|
||||
"prospector" => {
|
||||
"type" => "log"
|
||||
},
|
||||
"input" => {
|
||||
"type" => "log"
|
||||
},
|
||||
"source" => "/path/to/file/logstash-tutorial.log",
|
||||
"message" => "83.149.9.216 - - [04/Jan/2015:05:13:42 +0000] \"GET /presentations/logstash-monitorama-2013/images/kibana-search.png HTTP/1.1\" 200 203023 \"http://semicomplete.com/presentations/logstash-monitorama-2013/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\"",
|
||||
"tags" => [
|
||||
[0] "beats_input_codec_plain_applied"
|
||||
]
|
||||
}
|
||||
...
|
||||
```
|
||||
|
||||
|
||||
### Parsing Web Logs with the Grok Filter Plugin [configuring-grok-filter]
|
||||
|
||||
Now you have a working pipeline that reads log lines from Filebeat. However you’ll notice that the format of the log messages is not ideal. You want to parse the log messages to create specific, named fields from the logs. To do this, you’ll use the `grok` filter plugin.
|
||||
|
||||
The [`grok`](/reference/plugins-filters-grok.md) filter plugin is one of several plugins that are available by default in Logstash. For details on how to manage Logstash plugins, see the [reference documentation](/reference/working-with-plugins.md) for the plugin manager.
|
||||
|
||||
The `grok` filter plugin enables you to parse the unstructured log data into something structured and queryable.
|
||||
|
||||
Because the `grok` filter plugin looks for patterns in the incoming log data, configuring the plugin requires you to make decisions about how to identify the patterns that are of interest to your use case. A representative line from the web server log sample looks like this:
|
||||
|
||||
```shell
|
||||
83.149.9.216 - - [04/Jan/2015:05:13:42 +0000] "GET /presentations/logstash-monitorama-2013/images/kibana-search.png
|
||||
HTTP/1.1" 200 203023 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel
|
||||
Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"
|
||||
```
|
||||
|
||||
The IP address at the beginning of the line is easy to identify, as is the timestamp in brackets. To parse the data, you can use the `%{{COMBINEDAPACHELOG}}` grok pattern, which structures lines from the Apache log using the following schema:
|
||||
|
||||
**Information**
|
||||
: **Field Name**
|
||||
|
||||
IP Address
|
||||
: `clientip`
|
||||
|
||||
User ID
|
||||
: `ident`
|
||||
|
||||
User Authentication
|
||||
: `auth`
|
||||
|
||||
timestamp
|
||||
: `timestamp`
|
||||
|
||||
HTTP Verb
|
||||
: `verb`
|
||||
|
||||
Request body
|
||||
: `request`
|
||||
|
||||
HTTP Version
|
||||
: `httpversion`
|
||||
|
||||
HTTP Status Code
|
||||
: `response`
|
||||
|
||||
Bytes served
|
||||
: `bytes`
|
||||
|
||||
Referrer URL
|
||||
: `referrer`
|
||||
|
||||
User agent
|
||||
: `agent`
|
||||
|
||||
::::{tip}
|
||||
If you need help building grok patterns, try out the [Grok Debugger](docs-content://explore-analyze/query-filter/tools/grok-debugger.md). The Grok Debugger is an {{xpack}} feature under the Basic License and is therefore **free to use**.
|
||||
::::
|
||||
|
||||
|
||||
Edit the `first-pipeline.conf` file and replace the entire `filter` section with the following text:
|
||||
|
||||
```json
|
||||
filter {
|
||||
grok {
|
||||
match => { "message" => "%{COMBINEDAPACHELOG}"}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
When you’re done, the contents of `first-pipeline.conf` should look like this:
|
||||
|
||||
```json
|
||||
input {
|
||||
beats {
|
||||
port => "5044"
|
||||
}
|
||||
}
|
||||
filter {
|
||||
grok {
|
||||
match => { "message" => "%{COMBINEDAPACHELOG}"}
|
||||
}
|
||||
}
|
||||
output {
|
||||
stdout { codec => rubydebug }
|
||||
}
|
||||
```
|
||||
|
||||
Save your changes. Because you’ve enabled automatic config reloading, you don’t have to restart Logstash to pick up your changes. However, you do need to force Filebeat to read the log file from scratch. To do this, go to the terminal window where Filebeat is running and press Ctrl+C to shut down Filebeat. Then delete the Filebeat registry file. For example, run:
|
||||
|
||||
```shell
|
||||
sudo rm data/registry
|
||||
```
|
||||
|
||||
Since Filebeat stores the state of each file it harvests in the registry, deleting the registry file forces Filebeat to read all the files it’s harvesting from scratch.
|
||||
|
||||
Next, restart Filebeat with the following command:
|
||||
|
||||
```shell
|
||||
sudo ./filebeat -e -c filebeat.yml -d "publish"
|
||||
```
|
||||
|
||||
There might be a slight delay before Filebeat begins processing events if it needs to wait for Logstash to reload the config file.
|
||||
|
||||
After Logstash applies the grok pattern, the events will have the following JSON representation:
|
||||
|
||||
```json
|
||||
{
|
||||
"request" => "/presentations/logstash-monitorama-2013/images/kibana-search.png",
|
||||
"agent" => "\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\"",
|
||||
"offset" => 325,
|
||||
"auth" => "-",
|
||||
"ident" => "-",
|
||||
"verb" => "GET",
|
||||
"prospector" => {
|
||||
"type" => "log"
|
||||
},
|
||||
"input" => {
|
||||
"type" => "log"
|
||||
},
|
||||
"source" => "/path/to/file/logstash-tutorial.log",
|
||||
"message" => "83.149.9.216 - - [04/Jan/2015:05:13:42 +0000] \"GET /presentations/logstash-monitorama-2013/images/kibana-search.png HTTP/1.1\" 200 203023 \"http://semicomplete.com/presentations/logstash-monitorama-2013/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\"",
|
||||
"tags" => [
|
||||
[0] "beats_input_codec_plain_applied"
|
||||
],
|
||||
"referrer" => "\"http://semicomplete.com/presentations/logstash-monitorama-2013/\"",
|
||||
"@timestamp" => 2017-11-09T02:51:12.416Z,
|
||||
"response" => "200",
|
||||
"bytes" => "203023",
|
||||
"clientip" => "83.149.9.216",
|
||||
"@version" => "1",
|
||||
"beat" => {
|
||||
"name" => "My-MacBook-Pro.local",
|
||||
"hostname" => "My-MacBook-Pro.local",
|
||||
"version" => "6.0.0"
|
||||
},
|
||||
"host" => "My-MacBook-Pro.local",
|
||||
"httpversion" => "1.1",
|
||||
"timestamp" => "04/Jan/2015:05:13:42 +0000"
|
||||
}
|
||||
```
|
||||
|
||||
Notice that the event includes the original message, but the log message is also broken down into specific fields.
|
||||
|
||||
|
||||
### Enhancing Your Data with the Geoip Filter Plugin [configuring-geoip-plugin]
|
||||
|
||||
In addition to parsing log data for better searches, filter plugins can derive supplementary information from existing data. As an example, the [`geoip`](/reference/plugins-filters-geoip.md) plugin looks up IP addresses, derives geographic location information from the addresses, and adds that location information to the logs.
|
||||
|
||||
Configure your Logstash instance to use the `geoip` filter plugin by adding the following lines to the `filter` section of the `first-pipeline.conf` file:
|
||||
|
||||
```json
|
||||
geoip {
|
||||
source => "clientip"
|
||||
}
|
||||
```
|
||||
|
||||
The `geoip` plugin configuration requires you to specify the name of the source field that contains the IP address to look up. In this example, the `clientip` field contains the IP address.
|
||||
|
||||
Since filters are evaluated in sequence, make sure that the `geoip` section is after the `grok` section of the configuration file and that both the `grok` and `geoip` sections are nested within the `filter` section.
|
||||
|
||||
When you’re done, the contents of `first-pipeline.conf` should look like this:
|
||||
|
||||
```json
|
||||
input {
|
||||
beats {
|
||||
port => "5044"
|
||||
}
|
||||
}
|
||||
filter {
|
||||
grok {
|
||||
match => { "message" => "%{COMBINEDAPACHELOG}"}
|
||||
}
|
||||
geoip {
|
||||
source => "clientip"
|
||||
}
|
||||
}
|
||||
output {
|
||||
stdout { codec => rubydebug }
|
||||
}
|
||||
```
|
||||
|
||||
Save your changes. To force Filebeat to read the log file from scratch, as you did earlier, shut down Filebeat (press Ctrl+C), delete the registry file, and then restart Filebeat with the following command:
|
||||
|
||||
```shell
|
||||
sudo ./filebeat -e -c filebeat.yml -d "publish"
|
||||
```
|
||||
|
||||
Notice that the event now contains geographic location information:
|
||||
|
||||
```json
|
||||
{
|
||||
"request" => "/presentations/logstash-monitorama-2013/images/kibana-search.png",
|
||||
"agent" => "\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\"",
|
||||
"geoip" => {
|
||||
"timezone" => "Europe/Moscow",
|
||||
"ip" => "83.149.9.216",
|
||||
"latitude" => 55.7485,
|
||||
"continent_code" => "EU",
|
||||
"city_name" => "Moscow",
|
||||
"country_name" => "Russia",
|
||||
"country_code2" => "RU",
|
||||
"country_code3" => "RU",
|
||||
"region_name" => "Moscow",
|
||||
"location" => {
|
||||
"lon" => 37.6184,
|
||||
"lat" => 55.7485
|
||||
},
|
||||
"postal_code" => "101194",
|
||||
"region_code" => "MOW",
|
||||
"longitude" => 37.6184
|
||||
},
|
||||
...
|
||||
```
|
||||
|
||||
|
||||
### Indexing Your Data into Elasticsearch [indexing-parsed-data-into-elasticsearch]
|
||||
|
||||
Now that the web logs are broken down into specific fields, you’re ready to get your data into Elasticsearch.
|
||||
|
||||
::::{tip}
|
||||
{ess-leadin}
|
||||
::::
|
||||
|
||||
|
||||
The Logstash pipeline can index the data into an Elasticsearch cluster. Edit the `first-pipeline.conf` file and replace the entire `output` section with the following text:
|
||||
|
||||
```json
|
||||
output {
|
||||
elasticsearch {
|
||||
hosts => [ "localhost:9200" ]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
With this configuration, Logstash uses http protocol to connect to Elasticsearch. The above example assumes that Logstash and Elasticsearch are running on the same instance. You can specify a remote Elasticsearch instance by using the `hosts` configuration to specify something like `hosts => [ "es-machine:9092" ]`.
|
||||
|
||||
At this point, your `first-pipeline.conf` file has input, filter, and output sections properly configured, and looks something like this:
|
||||
|
||||
```json
|
||||
input {
|
||||
beats {
|
||||
port => "5044"
|
||||
}
|
||||
}
|
||||
filter {
|
||||
grok {
|
||||
match => { "message" => "%{COMBINEDAPACHELOG}"}
|
||||
}
|
||||
geoip {
|
||||
source => "clientip"
|
||||
}
|
||||
}
|
||||
output {
|
||||
elasticsearch {
|
||||
hosts => [ "localhost:9200" ]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Save your changes. To force Filebeat to read the log file from scratch, as you did earlier, shut down Filebeat (press Ctrl+C), delete the registry file, and then restart Filebeat with the following command:
|
||||
|
||||
```shell
|
||||
sudo ./filebeat -e -c filebeat.yml -d "publish"
|
||||
```
|
||||
|
||||
|
||||
#### Testing Your Pipeline [testing-initial-pipeline]
|
||||
|
||||
Now that the Logstash pipeline is configured to index the data into an Elasticsearch cluster, you can query Elasticsearch.
|
||||
|
||||
Try a test query to Elasticsearch based on the fields created by the `grok` filter plugin. Replace $DATE with the current date, in YYYY.MM.DD format:
|
||||
|
||||
```shell
|
||||
curl -XGET 'localhost:9200/logstash-$DATE/_search?pretty&q=response=200'
|
||||
```
|
||||
|
||||
::::{note}
|
||||
The date used in the index name is based on UTC, not the timezone where Logstash is running. If the query returns `index_not_found_exception`, make sure that `logstash-$DATE` reflects the actual name of the index. To see a list of available indexes, use this query: `curl 'localhost:9200/_cat/indices?v'`.
|
||||
::::
|
||||
|
||||
|
||||
You should get multiple hits back. For example:
|
||||
|
||||
```json
|
||||
{
|
||||
"took": 50,
|
||||
"timed_out": false,
|
||||
"_shards": {
|
||||
"total": 5,
|
||||
"successful": 5,
|
||||
"skipped": 0,
|
||||
"failed": 0
|
||||
},
|
||||
"hits": {
|
||||
"total": 98,
|
||||
"max_score": 2.793642,
|
||||
"hits": [
|
||||
{
|
||||
"_index": "logstash-2017.11.09",
|
||||
"_type": "doc",
|
||||
"_id": "3IzDnl8BW52sR0fx5wdV",
|
||||
"_score": 2.793642,
|
||||
"_source": {
|
||||
"request": "/presentations/logstash-monitorama-2013/images/frontend-response-codes.png",
|
||||
"agent": """"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"""",
|
||||
"geoip": {
|
||||
"timezone": "Europe/Moscow",
|
||||
"ip": "83.149.9.216",
|
||||
"latitude": 55.7485,
|
||||
"continent_code": "EU",
|
||||
"city_name": "Moscow",
|
||||
"country_name": "Russia",
|
||||
"country_code2": "RU",
|
||||
"country_code3": "RU",
|
||||
"region_name": "Moscow",
|
||||
"location": {
|
||||
"lon": 37.6184,
|
||||
"lat": 55.7485
|
||||
},
|
||||
"postal_code": "101194",
|
||||
"region_code": "MOW",
|
||||
"longitude": 37.6184
|
||||
},
|
||||
"offset": 2932,
|
||||
"auth": "-",
|
||||
"ident": "-",
|
||||
"verb": "GET",
|
||||
"prospector": {
|
||||
"type": "log"
|
||||
},
|
||||
"input": {
|
||||
"type": "log"
|
||||
},
|
||||
"source": "/path/to/file/logstash-tutorial.log",
|
||||
"message": """83.149.9.216 - - [04/Jan/2015:05:13:45 +0000] "GET /presentations/logstash-monitorama-2013/images/frontend-response-codes.png HTTP/1.1" 200 52878 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"""",
|
||||
"tags": [
|
||||
"beats_input_codec_plain_applied"
|
||||
],
|
||||
"referrer": """"http://semicomplete.com/presentations/logstash-monitorama-2013/"""",
|
||||
"@timestamp": "2017-11-09T03:11:35.304Z",
|
||||
"response": "200",
|
||||
"bytes": "52878",
|
||||
"clientip": "83.149.9.216",
|
||||
"@version": "1",
|
||||
"beat": {
|
||||
"name": "My-MacBook-Pro.local",
|
||||
"hostname": "My-MacBook-Pro.local",
|
||||
"version": "6.0.0"
|
||||
},
|
||||
"host": "My-MacBook-Pro.local",
|
||||
"httpversion": "1.1",
|
||||
"timestamp": "04/Jan/2015:05:13:45 +0000"
|
||||
}
|
||||
},
|
||||
...
|
||||
```
|
||||
|
||||
Try another search for the geographic information derived from the IP address. Replace $DATE with the current date, in YYYY.MM.DD format:
|
||||
|
||||
```shell
|
||||
curl -XGET 'localhost:9200/logstash-$DATE/_search?pretty&q=geoip.city_name=Buffalo'
|
||||
```
|
||||
|
||||
A few log entries come from Buffalo, so the query produces the following response:
|
||||
|
||||
```json
|
||||
{
|
||||
"took": 9,
|
||||
"timed_out": false,
|
||||
"_shards": {
|
||||
"total": 5,
|
||||
"successful": 5,
|
||||
"skipped": 0,
|
||||
"failed": 0
|
||||
},
|
||||
"hits": {
|
||||
"total": 2,
|
||||
"max_score": 2.6390574,
|
||||
"hits": [
|
||||
{
|
||||
"_index": "logstash-2017.11.09",
|
||||
"_type": "doc",
|
||||
"_id": "L4zDnl8BW52sR0fx5whY",
|
||||
"_score": 2.6390574,
|
||||
"_source": {
|
||||
"request": "/blog/geekery/disabling-battery-in-ubuntu-vms.html?utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+semicomplete%2Fmain+%28semicomplete.com+-+Jordan+Sissel%29",
|
||||
"agent": """"Tiny Tiny RSS/1.11 (http://tt-rss.org/)"""",
|
||||
"geoip": {
|
||||
"timezone": "America/New_York",
|
||||
"ip": "198.46.149.143",
|
||||
"latitude": 42.8864,
|
||||
"continent_code": "NA",
|
||||
"city_name": "Buffalo",
|
||||
"country_name": "United States",
|
||||
"country_code2": "US",
|
||||
"dma_code": 514,
|
||||
"country_code3": "US",
|
||||
"region_name": "New York",
|
||||
"location": {
|
||||
"lon": -78.8781,
|
||||
"lat": 42.8864
|
||||
},
|
||||
"postal_code": "14202",
|
||||
"region_code": "NY",
|
||||
"longitude": -78.8781
|
||||
},
|
||||
"offset": 22795,
|
||||
"auth": "-",
|
||||
"ident": "-",
|
||||
"verb": "GET",
|
||||
"prospector": {
|
||||
"type": "log"
|
||||
},
|
||||
"input": {
|
||||
"type": "log"
|
||||
},
|
||||
"source": "/path/to/file/logstash-tutorial.log",
|
||||
"message": """198.46.149.143 - - [04/Jan/2015:05:29:13 +0000] "GET /blog/geekery/disabling-battery-in-ubuntu-vms.html?utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+semicomplete%2Fmain+%28semicomplete.com+-+Jordan+Sissel%29 HTTP/1.1" 200 9316 "-" "Tiny Tiny RSS/1.11 (http://tt-rss.org/)"""",
|
||||
"tags": [
|
||||
"beats_input_codec_plain_applied"
|
||||
],
|
||||
"referrer": """"-"""",
|
||||
"@timestamp": "2017-11-09T03:11:35.321Z",
|
||||
"response": "200",
|
||||
"bytes": "9316",
|
||||
"clientip": "198.46.149.143",
|
||||
"@version": "1",
|
||||
"beat": {
|
||||
"name": "My-MacBook-Pro.local",
|
||||
"hostname": "My-MacBook-Pro.local",
|
||||
"version": "6.0.0"
|
||||
},
|
||||
"host": "My-MacBook-Pro.local",
|
||||
"httpversion": "1.1",
|
||||
"timestamp": "04/Jan/2015:05:29:13 +0000"
|
||||
}
|
||||
},
|
||||
...
|
||||
```
|
||||
|
||||
If you are using Kibana to visualize your data, you can also explore the Filebeat data in Kibana:
|
||||
|
||||
:::{image} ../images/kibana-filebeat-data.png
|
||||
:alt: Discovering Filebeat data in Kibana
|
||||
:::
|
||||
|
||||
See the [Filebeat quick start docs](beats://reference/filebeat/filebeat-installation-configuration.md) for info about loading the Kibana index pattern for Filebeat.
|
||||
|
||||
You’ve successfully created a pipeline that uses Filebeat to take Apache web logs as input, parses those logs to create specific, named fields from the logs, and writes the parsed data to an Elasticsearch cluster. Next, you learn how to create a pipeline that uses multiple input and output plugins.
|
||||
|
||||
|
||||
67
docs/reference/codec-plugins.md
Normal file
67
docs/reference/codec-plugins.md
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/codec-plugins.html
|
||||
---
|
||||
|
||||
# Codec plugins [codec-plugins]
|
||||
|
||||
A codec plugin changes the data representation of an event. Codecs are essentially stream filters that can operate as part of an input or output.
|
||||
|
||||
The following codec plugins are available below. For a list of Elastic supported plugins, please consult the [Support Matrix](https://www.elastic.co/support/matrix#show_logstash_plugins).
|
||||
|
||||
| | | |
|
||||
| --- | --- | --- |
|
||||
| Plugin | Description | Github repository |
|
||||
| [avro](/reference/plugins-codecs-avro.md) | Reads serialized Avro records as Logstash events | [logstash-codec-avro](https://github.com/logstash-plugins/logstash-codec-avro) |
|
||||
| [cef](/reference/plugins-codecs-cef.md) | Reads the ArcSight Common Event Format (CEF). | [logstash-codec-cef](https://github.com/logstash-plugins/logstash-codec-cef) |
|
||||
| [cloudfront](/reference/plugins-codecs-cloudfront.md) | Reads AWS CloudFront reports | [logstash-codec-cloudfront](https://github.com/logstash-plugins/logstash-codec-cloudfront) |
|
||||
| [cloudtrail](/reference/plugins-codecs-cloudtrail.md) | Reads AWS CloudTrail log files | [logstash-codec-cloudtrail](https://github.com/logstash-plugins/logstash-codec-cloudtrail) |
|
||||
| [collectd](/reference/plugins-codecs-collectd.md) | Reads events from the `collectd` binary protocol using UDP. | [logstash-codec-collectd](https://github.com/logstash-plugins/logstash-codec-collectd) |
|
||||
| [csv](/reference/plugins-codecs-csv.md) | Takes CSV data, parses it, and passes it along. | [logstash-codec-csv](https://github.com/logstash-plugins/logstash-codec-csv) |
|
||||
| [dots](/reference/plugins-codecs-dots.md) | Sends 1 dot per event to `stdout` for performance tracking | [logstash-codec-dots](https://github.com/logstash-plugins/logstash-codec-dots) |
|
||||
| [edn](/reference/plugins-codecs-edn.md) | Reads EDN format data | [logstash-codec-edn](https://github.com/logstash-plugins/logstash-codec-edn) |
|
||||
| [edn_lines](/reference/plugins-codecs-edn_lines.md) | Reads newline-delimited EDN format data | [logstash-codec-edn_lines](https://github.com/logstash-plugins/logstash-codec-edn_lines) |
|
||||
| [es_bulk](/reference/plugins-codecs-es_bulk.md) | Reads the Elasticsearch bulk format into separate events, along with metadata | [logstash-codec-es_bulk](https://github.com/logstash-plugins/logstash-codec-es_bulk) |
|
||||
| [fluent](/reference/plugins-codecs-fluent.md) | Reads the `fluentd` `msgpack` schema | [logstash-codec-fluent](https://github.com/logstash-plugins/logstash-codec-fluent) |
|
||||
| [graphite](/reference/plugins-codecs-graphite.md) | Reads `graphite` formatted lines | [logstash-codec-graphite](https://github.com/logstash-plugins/logstash-codec-graphite) |
|
||||
| [gzip_lines](/reference/plugins-codecs-gzip_lines.md) | Reads `gzip` encoded content | [logstash-codec-gzip_lines](https://github.com/logstash-plugins/logstash-codec-gzip_lines) |
|
||||
| [jdots](/reference/plugins-codecs-jdots.md) | Renders each processed event as a dot | [core plugin](https://github.com/elastic/logstash/blob/master/logstash-core/src/main/java/org/logstash/plugins/codecs/Dots.java) |
|
||||
| [java_line](/reference/plugins-codecs-java_line.md) | Encodes and decodes line-oriented text data | [core plugin](https://github.com/elastic/logstash/blob/master/logstash-core/src/main/java/org/logstash/plugins/codecs/Line.java) |
|
||||
| [java_plain](/reference/plugins-codecs-java_plain.md) | Processes text data with no delimiters between events | [core plugin](https://github.com/elastic/logstash/blob/master/logstash-core/src/main/java/org/logstash/plugins/codecs/Plain.java) |
|
||||
| [json](/reference/plugins-codecs-json.md) | Reads JSON formatted content, creating one event per element in a JSON array | [logstash-codec-json](https://github.com/logstash-plugins/logstash-codec-json) |
|
||||
| [json_lines](/reference/plugins-codecs-json_lines.md) | Reads newline-delimited JSON | [logstash-codec-json_lines](https://github.com/logstash-plugins/logstash-codec-json_lines) |
|
||||
| [line](/reference/plugins-codecs-line.md) | Reads line-oriented text data | [logstash-codec-line](https://github.com/logstash-plugins/logstash-codec-line) |
|
||||
| [msgpack](/reference/plugins-codecs-msgpack.md) | Reads MessagePack encoded content | [logstash-codec-msgpack](https://github.com/logstash-plugins/logstash-codec-msgpack) |
|
||||
| [multiline](/reference/plugins-codecs-multiline.md) | Merges multiline messages into a single event | [logstash-codec-multiline](https://github.com/logstash-plugins/logstash-codec-multiline) |
|
||||
| [netflow](/reference/plugins-codecs-netflow.md) | Reads Netflow v5 and Netflow v9 data | [logstash-codec-netflow](https://github.com/logstash-plugins/logstash-codec-netflow) |
|
||||
| [nmap](/reference/plugins-codecs-nmap.md) | Reads Nmap data in XML format | [logstash-codec-nmap](https://github.com/logstash-plugins/logstash-codec-nmap) |
|
||||
| [plain](/reference/plugins-codecs-plain.md) | Reads plaintext with no delimiting between events | [logstash-codec-plain](https://github.com/logstash-plugins/logstash-codec-plain) |
|
||||
| [protobuf](/reference/plugins-codecs-protobuf.md) | Reads protobuf messages and converts to Logstash Events | [logstash-codec-protobuf](https://github.com/logstash-plugins/logstash-codec-protobuf) |
|
||||
| [rubydebug](/reference/plugins-codecs-rubydebug.md) | Applies the Ruby Awesome Print library to Logstash events | [logstash-codec-rubydebug](https://github.com/logstash-plugins/logstash-codec-rubydebug) |
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -1,18 +1,23 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/config-examples.html
|
||||
---
|
||||
|
||||
# Logstash configuration examples [config-examples]
|
||||
|
||||
[[config-examples]]
|
||||
=== Logstash configuration examples
|
||||
These examples illustrate how you can configure Logstash to filter events, process Apache logs and syslog messages, and use conditionals to control what events are processed by a filter or output.
|
||||
|
||||
TIP: If you need help building grok patterns, try out the
|
||||
{kibana-ref}/xpack-grokdebugger.html[Grok Debugger].
|
||||
::::{tip}
|
||||
If you need help building grok patterns, try out the [Grok Debugger](docs-content://explore-analyze/query-filter/tools/grok-debugger.md).
|
||||
::::
|
||||
|
||||
[discrete]
|
||||
[[filter-example]]
|
||||
==== Configuring filters
|
||||
Filters are an in-line processing mechanism that provide the flexibility to slice and dice your data to fit your needs. Let's take a look at some filters in action. The following configuration file sets up the `grok` and `date` filters.
|
||||
|
||||
[source,ruby]
|
||||
----------------------------------
|
||||
|
||||
## Configuring filters [filter-example]
|
||||
|
||||
Filters are an in-line processing mechanism that provide the flexibility to slice and dice your data to fit your needs. Let’s take a look at some filters in action. The following configuration file sets up the `grok` and `date` filters.
|
||||
|
||||
```ruby
|
||||
input { stdin { } }
|
||||
|
||||
filter {
|
||||
|
|
@ -28,27 +33,23 @@ output {
|
|||
elasticsearch { hosts => ["localhost:9200"] }
|
||||
stdout { codec => rubydebug }
|
||||
}
|
||||
----------------------------------
|
||||
```
|
||||
|
||||
Run Logstash with this configuration:
|
||||
|
||||
[source,ruby]
|
||||
----------------------------------
|
||||
```ruby
|
||||
bin/logstash -f logstash-filter.conf
|
||||
----------------------------------
|
||||
```
|
||||
|
||||
Now, paste the following line into your terminal and press Enter so it will be
|
||||
processed by the stdin input:
|
||||
Now, paste the following line into your terminal and press Enter so it will be processed by the stdin input:
|
||||
|
||||
[source,ruby]
|
||||
----------------------------------
|
||||
```ruby
|
||||
127.0.0.1 - - [11/Dec/2013:00:01:45 -0800] "GET /xampp/status.php HTTP/1.1" 200 3891 "http://cadenza/xampp/navi.php" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:25.0) Gecko/20100101 Firefox/25.0"
|
||||
----------------------------------
|
||||
```
|
||||
|
||||
You should see something returned to stdout that looks like this:
|
||||
|
||||
[source,ruby]
|
||||
----------------------------------
|
||||
```ruby
|
||||
{
|
||||
"message" => "127.0.0.1 - - [11/Dec/2013:00:01:45 -0800] \"GET /xampp/status.php HTTP/1.1\" 200 3891 \"http://cadenza/xampp/navi.php\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:25.0) Gecko/20100101 Firefox/25.0\"",
|
||||
"@timestamp" => "2013-12-11T08:01:45.000Z",
|
||||
|
|
@ -66,18 +67,18 @@ You should see something returned to stdout that looks like this:
|
|||
"referrer" => "\"http://cadenza/xampp/navi.php\"",
|
||||
"agent" => "\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:25.0) Gecko/20100101 Firefox/25.0\""
|
||||
}
|
||||
----------------------------------
|
||||
```
|
||||
|
||||
As you can see, Logstash (with help from the `grok` filter) was able to parse the log line (which happens to be in Apache "combined log" format) and break it up into many different discrete bits of information. This is extremely useful once you start querying and analyzing our log data. For example, you'll be able to easily run reports on HTTP response codes, IP addresses, referrers, and so on. There are quite a few grok patterns included with Logstash out-of-the-box, so it's quite likely if you need to parse a common log format, someone has already done the work for you. For more information, see the list of https://github.com/logstash-plugins/logstash-patterns-core/tree/main/patterns[Logstash grok patterns] on GitHub.
|
||||
As you can see, Logstash (with help from the `grok` filter) was able to parse the log line (which happens to be in Apache "combined log" format) and break it up into many different discrete bits of information. This is extremely useful once you start querying and analyzing our log data. For example, you’ll be able to easily run reports on HTTP response codes, IP addresses, referrers, and so on. There are quite a few grok patterns included with Logstash out-of-the-box, so it’s quite likely if you need to parse a common log format, someone has already done the work for you. For more information, see the list of [Logstash grok patterns](https://github.com/logstash-plugins/logstash-patterns-core/tree/main/patterns) on GitHub.
|
||||
|
||||
The other filter used in this example is the `date` filter. This filter parses out a timestamp and uses it as the timestamp for the event (regardless of when you're ingesting the log data). You'll notice that the `@timestamp` field in this example is set to December 11, 2013, even though Logstash is ingesting the event at some point afterwards. This is handy when backfilling logs. It gives you the ability to tell Logstash "use this value as the timestamp for this event".
|
||||
The other filter used in this example is the `date` filter. This filter parses out a timestamp and uses it as the timestamp for the event (regardless of when you’re ingesting the log data). You’ll notice that the `@timestamp` field in this example is set to December 11, 2013, even though Logstash is ingesting the event at some point afterwards. This is handy when backfilling logs. It gives you the ability to tell Logstash "use this value as the timestamp for this event".
|
||||
|
||||
[discrete]
|
||||
==== Processing Apache logs
|
||||
Let's do something that's actually *useful*: process apache2 access log files! We are going to read the input from a file on the localhost, and use a <<conditionals,conditional>> to process the event according to our needs. First, create a file called something like 'logstash-apache.conf' with the following contents (you can change the log's file path to suit your needs):
|
||||
|
||||
[source,js]
|
||||
----------------------------------
|
||||
## Processing Apache logs [_processing_apache_logs]
|
||||
|
||||
Let’s do something that’s actually **useful**: process apache2 access log files! We are going to read the input from a file on the localhost, and use a [conditional](/reference/event-dependent-configuration.md#conditionals) to process the event according to our needs. First, create a file called something like *logstash-apache.conf* with the following contents (you can change the log’s file path to suit your needs):
|
||||
|
||||
```js
|
||||
input {
|
||||
file {
|
||||
path => "/tmp/access_log"
|
||||
|
|
@ -103,48 +104,43 @@ output {
|
|||
}
|
||||
stdout { codec => rubydebug }
|
||||
}
|
||||
|
||||
----------------------------------
|
||||
```
|
||||
|
||||
Then, create the input file you configured above (in this example, "/tmp/access_log") with the following log entries (or use some from your own webserver):
|
||||
|
||||
[source,js]
|
||||
----------------------------------
|
||||
```js
|
||||
71.141.244.242 - kurt [18/May/2011:01:48:10 -0700] "GET /admin HTTP/1.1" 301 566 "-" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3"
|
||||
134.39.72.245 - - [18/May/2011:12:40:18 -0700] "GET /favicon.ico HTTP/1.1" 200 1189 "-" "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; InfoPath.2; .NET4.0C; .NET4.0E)"
|
||||
98.83.179.51 - - [18/May/2011:19:35:08 -0700] "GET /css/main.css HTTP/1.1" 200 1837 "http://www.safesand.com/information.htm" "Mozilla/5.0 (Windows NT 6.0; WOW64; rv:2.0.1) Gecko/20100101 Firefox/4.0.1"
|
||||
----------------------------------
|
||||
```
|
||||
|
||||
Now, run Logstash with the -f flag to pass in the configuration file:
|
||||
|
||||
[source,js]
|
||||
----------------------------------
|
||||
```js
|
||||
bin/logstash -f logstash-apache.conf
|
||||
----------------------------------
|
||||
```
|
||||
|
||||
Now you should see your apache log data in Elasticsearch! Logstash opened and read the specified input file, processing each event it encountered. Any additional lines logged to this file will also be captured, processed by Logstash as events, and stored in Elasticsearch. As an added bonus, they are stashed with the field "type" set to "apache_access" (this is done by the type => "apache_access" line in the input configuration).
|
||||
Now you should see your apache log data in Elasticsearch! Logstash opened and read the specified input file, processing each event it encountered. Any additional lines logged to this file will also be captured, processed by Logstash as events, and stored in Elasticsearch. As an added bonus, they are stashed with the field "type" set to "apache_access" (this is done by the type ⇒ "apache_access" line in the input configuration).
|
||||
|
||||
In this configuration, Logstash is only watching the apache access_log, but it's easy enough to watch both the access_log and the error_log (actually, any file matching `*log`), by changing one line in the above configuration:
|
||||
In this configuration, Logstash is only watching the apache access_log, but it’s easy enough to watch both the access_log and the error_log (actually, any file matching `*log`), by changing one line in the above configuration:
|
||||
|
||||
[source,js]
|
||||
----------------------------------
|
||||
```js
|
||||
input {
|
||||
file {
|
||||
path => "/tmp/*_log"
|
||||
...
|
||||
----------------------------------
|
||||
```
|
||||
|
||||
When you restart Logstash, it will process both the error and access logs. However, if you inspect your data (using elasticsearch-kopf, perhaps), you'll see that the access_log is broken up into discrete fields, but the error_log isn't. That's because we used a `grok` filter to match the standard combined apache log format and automatically split the data into separate fields. Wouldn't it be nice *if* we could control how a line was parsed, based on its format? Well, we can...
|
||||
When you restart Logstash, it will process both the error and access logs. However, if you inspect your data (using elasticsearch-kopf, perhaps), you’ll see that the access_log is broken up into discrete fields, but the error_log isn’t. That’s because we used a `grok` filter to match the standard combined apache log format and automatically split the data into separate fields. Wouldn’t it be nice **if** we could control how a line was parsed, based on its format? Well, we can…
|
||||
|
||||
Note that Logstash did not reprocess the events that were already seen in the access_log file. When reading from a file, Logstash saves its position and only processes new lines as they are added. Neat!
|
||||
|
||||
[discrete]
|
||||
[[using-conditionals]]
|
||||
==== Using conditionals
|
||||
|
||||
## Using conditionals [using-conditionals]
|
||||
|
||||
You use conditionals to control what events are processed by a filter or output. For example, you could label each event according to which file it appeared in (access_log, error_log, and other random files that end with "log").
|
||||
|
||||
[source,ruby]
|
||||
----------------------------------
|
||||
```ruby
|
||||
input {
|
||||
file {
|
||||
path => "/tmp/*_log"
|
||||
|
|
@ -171,9 +167,9 @@ output {
|
|||
elasticsearch { hosts => ["localhost:9200"] }
|
||||
stdout { codec => rubydebug }
|
||||
}
|
||||
----------------------------------
|
||||
```
|
||||
|
||||
This example labels all events using the `type` field, but doesn't actually parse the `error` or `random` files. There are so many types of error logs that how they should be labeled really depends on what logs you're working with.
|
||||
This example labels all events using the `type` field, but doesn’t actually parse the `error` or `random` files. There are so many types of error logs that how they should be labeled really depends on what logs you’re working with.
|
||||
|
||||
Similarly, you can use conditionals to direct events to particular outputs. For example, you could:
|
||||
|
||||
|
|
@ -181,14 +177,9 @@ Similarly, you can use conditionals to direct events to particular outputs. For
|
|||
* record any 4xx status to Elasticsearch
|
||||
* record all status code hits via statsd
|
||||
|
||||
To tell nagios about any http event that has a 5xx status code, you
|
||||
first need to check the value of the `type` field. If it's apache, then you can
|
||||
check to see if the `status` field contains a 5xx error. If it is, send it to nagios. If it isn't
|
||||
a 5xx error, check to see if the `status` field contains a 4xx error. If so, send it to Elasticsearch.
|
||||
Finally, send all apache status codes to statsd no matter what the `status` field contains:
|
||||
To tell nagios about any http event that has a 5xx status code, you first need to check the value of the `type` field. If it’s apache, then you can check to see if the `status` field contains a 5xx error. If it is, send it to nagios. If it isn’t a 5xx error, check to see if the `status` field contains a 4xx error. If so, send it to Elasticsearch. Finally, send all apache status codes to statsd no matter what the `status` field contains:
|
||||
|
||||
[source,js]
|
||||
----------------------------------
|
||||
```js
|
||||
output {
|
||||
if [type] == "apache" {
|
||||
if [status] =~ /^5\d\d/ {
|
||||
|
|
@ -199,16 +190,16 @@ output {
|
|||
statsd { increment => "apache.%{status}" }
|
||||
}
|
||||
}
|
||||
----------------------------------
|
||||
```
|
||||
|
||||
[discrete]
|
||||
==== Processing Syslog messages
|
||||
Syslog is one of the most common use cases for Logstash, and one it handles exceedingly well (as long as the log lines conform roughly to RFC3164). Syslog is the de facto UNIX networked logging standard, sending messages from client machines to a local file, or to a centralized log server via rsyslog. For this example, you won't need a functioning syslog instance; we'll fake it from the command line so you can get a feel for what happens.
|
||||
|
||||
First, let's make a simple configuration file for Logstash + syslog, called 'logstash-syslog.conf'.
|
||||
## Processing Syslog messages [_processing_syslog_messages]
|
||||
|
||||
[source,ruby]
|
||||
----------------------------------
|
||||
Syslog is one of the most common use cases for Logstash, and one it handles exceedingly well (as long as the log lines conform roughly to RFC3164). Syslog is the de facto UNIX networked logging standard, sending messages from client machines to a local file, or to a centralized log server via rsyslog. For this example, you won’t need a functioning syslog instance; we’ll fake it from the command line so you can get a feel for what happens.
|
||||
|
||||
First, let’s make a simple configuration file for Logstash + syslog, called *logstash-syslog.conf*.
|
||||
|
||||
```ruby
|
||||
input {
|
||||
tcp {
|
||||
port => 5000
|
||||
|
|
@ -237,36 +228,32 @@ output {
|
|||
elasticsearch { hosts => ["localhost:9200"] }
|
||||
stdout { codec => rubydebug }
|
||||
}
|
||||
----------------------------------
|
||||
```
|
||||
|
||||
Run Logstash with this new configuration:
|
||||
|
||||
[source,ruby]
|
||||
----------------------------------
|
||||
```ruby
|
||||
bin/logstash -f logstash-syslog.conf
|
||||
----------------------------------
|
||||
```
|
||||
|
||||
Normally, a client machine would connect to the Logstash instance on port 5000 and send its message. For this example, we'll just telnet to Logstash and enter a log line (similar to how we entered log lines into STDIN earlier). Open another shell window to interact with the Logstash syslog input and enter the following command:
|
||||
Normally, a client machine would connect to the Logstash instance on port 5000 and send its message. For this example, we’ll just telnet to Logstash and enter a log line (similar to how we entered log lines into STDIN earlier). Open another shell window to interact with the Logstash syslog input and enter the following command:
|
||||
|
||||
[source,ruby]
|
||||
----------------------------------
|
||||
```ruby
|
||||
telnet localhost 5000
|
||||
----------------------------------
|
||||
```
|
||||
|
||||
Copy and paste the following lines as samples. (Feel free to try some of your own, but keep in mind they might not parse if the `grok` filter is not correct for your data).
|
||||
|
||||
[source,ruby]
|
||||
----------------------------------
|
||||
```ruby
|
||||
Dec 23 12:11:43 louis postfix/smtpd[31499]: connect from unknown[95.75.93.154]
|
||||
Dec 23 14:42:56 louis named[16000]: client 199.48.164.7#64817: query (cache) 'amsterdamboothuren.com/MX/IN' denied
|
||||
Dec 23 14:30:01 louis CRON[619]: (www-data) CMD (php /usr/share/cacti/site/poller.php >/dev/null 2>/var/log/cacti/poller-error.log)
|
||||
Dec 22 18:28:06 louis rsyslogd: [origin software="rsyslogd" swVersion="4.2.0" x-pid="2253" x-info="http://www.rsyslog.com"] rsyslogd was HUPed, type 'lightweight'.
|
||||
----------------------------------
|
||||
```
|
||||
|
||||
Now you should see the output of Logstash in your original shell as it processes and parses messages!
|
||||
|
||||
[source,ruby]
|
||||
----------------------------------
|
||||
```ruby
|
||||
{
|
||||
"message" => "Dec 23 14:30:01 louis CRON[619]: (www-data) CMD (php /usr/share/cacti/site/poller.php >/dev/null 2>/var/log/cacti/poller-error.log)",
|
||||
"@timestamp" => "2013-12-23T22:30:01.000Z",
|
||||
|
|
@ -285,7 +272,5 @@ Now you should see the output of Logstash in your original shell as it processes
|
|||
"syslog_facility" => "user-level",
|
||||
"syslog_severity" => "notice"
|
||||
}
|
||||
----------------------------------
|
||||
|
||||
|
||||
```
|
||||
|
||||
33
docs/reference/config-setting-files.md
Normal file
33
docs/reference/config-setting-files.md
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/config-setting-files.html
|
||||
---
|
||||
|
||||
# Logstash Configuration Files [config-setting-files]
|
||||
|
||||
Logstash has two types of configuration files: *pipeline configuration files*, which define the Logstash processing pipeline, and *settings files*, which specify options that control Logstash startup and execution.
|
||||
|
||||
## Pipeline Configuration Files [pipeline-config-files]
|
||||
|
||||
You create pipeline configuration files when you define the stages of your Logstash processing pipeline. On deb and rpm, you place the pipeline configuration files in the `/etc/logstash/conf.d` directory. Logstash tries to load only files with `.conf` extension in the `/etc/logstash/conf.d directory` and ignores all other files.
|
||||
|
||||
See [*Creating a {{ls}} pipeline*](/reference/creating-logstash-pipeline.md) for more info.
|
||||
|
||||
|
||||
## Settings Files [settings-files]
|
||||
|
||||
The settings files are already defined in the Logstash installation. Logstash includes the following settings files:
|
||||
|
||||
**`logstash.yml`**
|
||||
: Contains Logstash configuration flags. You can set flags in this file instead of passing the flags at the command line. Any flags that you set at the command line override the corresponding settings in the `logstash.yml` file. See [logstash.yml](/reference/logstash-settings-file.md) for more info.
|
||||
|
||||
**`pipelines.yml`**
|
||||
: Contains the framework and instructions for running multiple pipelines in a single Logstash instance. See [Multiple Pipelines](/reference/multiple-pipelines.md) for more info.
|
||||
|
||||
**`jvm.options`**
|
||||
: Contains JVM configuration flags. Use this file to set initial and maximum values for total heap space. You can also use this file to set the locale for Logstash. Specify each flag on a separate line. All other settings in this file are considered expert settings.
|
||||
|
||||
**`log4j2.properties`**
|
||||
: Contains default settings for `log4j 2` library. See [Log4j2 configuration](/reference/logging.md#log4j2) for more info.
|
||||
|
||||
|
||||
235
docs/reference/configuration-file-structure.md
Normal file
235
docs/reference/configuration-file-structure.md
Normal file
|
|
@ -0,0 +1,235 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/configuration-file-structure.html
|
||||
---
|
||||
|
||||
# Structure of a pipeline [configuration-file-structure]
|
||||
|
||||
A {{ls}} pipeline config file has a separate section for each type of plugin you want to add to the event processing pipeline. For example:
|
||||
|
||||
```js
|
||||
# This is a comment. You should use comments to describe
|
||||
# parts of your configuration.
|
||||
input {
|
||||
...
|
||||
}
|
||||
|
||||
filter {
|
||||
...
|
||||
}
|
||||
|
||||
output {
|
||||
...
|
||||
}
|
||||
```
|
||||
|
||||
Each section contains configuration options for one or more plugins. If you specify multiple filters, they are applied in the order they appear in the configuration file. If you specify multiple outputs, events are sent to each destination sequentially, in the order they appear in the configuration file.
|
||||
|
||||
::::{tip}
|
||||
When you are ready to deploy a pipeline beyond your local machine, add the pipeline config file to [`logstash.yml`](/reference/logstash-settings-file.md) using the `pipeline.id` setting. When you are ready to deploy [multiple pipelines](/reference/multiple-pipelines.md), set up and configure your pipelines in the `pipelines.yml` file.
|
||||
::::
|
||||
|
||||
|
||||
|
||||
## Plugin configuration [plugin_configuration]
|
||||
|
||||
A plugin configuration consists of the plugin name followed by a block of settings for that plugin. For example, this input section configures two file inputs:
|
||||
|
||||
```js
|
||||
input {
|
||||
http {
|
||||
port => 3333
|
||||
tags => gateway
|
||||
}
|
||||
http {
|
||||
port => 4444
|
||||
tags => billing
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
In this example, two settings are configured for each of the file inputs: *port* and *tags*.
|
||||
|
||||
The settings you can configure vary according to the plugin type. For information about each plugin, see [Input Plugins](/reference/input-plugins.md), [Output Plugins](/reference/output-plugins.md), [Filter Plugins](/reference/filter-plugins.md), and [Codec Plugins](/reference/codec-plugins.md).
|
||||
|
||||
|
||||
## Value types [plugin-value-types]
|
||||
|
||||
A plugin can require that the value for a setting be a certain type, such as boolean, list, or hash. The following value types are supported.
|
||||
|
||||
## Array [array]
|
||||
|
||||
This type is now mostly deprecated in favor of using a standard type like `string` with the plugin defining the `:list => true` property for better type checking. It is still needed to handle lists of hashes or mixed types where type checking is not desired.
|
||||
|
||||
Example:
|
||||
|
||||
```js
|
||||
users => [ {id => 1, name => bob}, {id => 2, name => jane} ]
|
||||
```
|
||||
|
||||
|
||||
### Lists [list]
|
||||
|
||||
Not a type in and of itself, but a property types can have. This makes it possible to type check multiple values. Plugin authors can enable list checking by specifying `:list => true` when declaring an argument.
|
||||
|
||||
Example:
|
||||
|
||||
```js
|
||||
path => [ "/var/log/messages", "/var/log/*.log" ]
|
||||
uris => [ "http://elastic.co", "http://example.net" ]
|
||||
```
|
||||
|
||||
This example configures `path`, which is a `string` to be a list that contains an element for each of the three strings. It also will configure the `uris` parameter to be a list of URIs, failing if any of the URIs provided are not valid.
|
||||
|
||||
|
||||
### Boolean [boolean]
|
||||
|
||||
A boolean must be either `true` or `false`. Note that the `true` and `false` keywords are not enclosed in quotes.
|
||||
|
||||
Example:
|
||||
|
||||
```js
|
||||
ssl_enable => true
|
||||
```
|
||||
|
||||
|
||||
### Bytes [bytes]
|
||||
|
||||
A bytes field is a string field that represents a valid unit of bytes. It is a convenient way to declare specific sizes in your plugin options. Both SI (k M G T P E Z Y) and Binary (Ki Mi Gi Ti Pi Ei Zi Yi) units are supported. Binary units are in base-1024 and SI units are in base-1000. This field is case-insensitive and accepts space between the value and the unit. If no unit is specified, the integer string represents the number of bytes.
|
||||
|
||||
Examples:
|
||||
|
||||
```js
|
||||
my_bytes => "1113" # 1113 bytes
|
||||
my_bytes => "10MiB" # 10485760 bytes
|
||||
my_bytes => "100kib" # 102400 bytes
|
||||
my_bytes => "180 mb" # 180000000 bytes
|
||||
```
|
||||
|
||||
|
||||
### Codec [codec]
|
||||
|
||||
A codec is the name of Logstash codec used to represent the data. Codecs can be used in both inputs and outputs.
|
||||
|
||||
Input codecs provide a convenient way to decode your data before it enters the input. Output codecs provide a convenient way to encode your data before it leaves the output. Using an input or output codec eliminates the need for a separate filter in your Logstash pipeline.
|
||||
|
||||
A list of available codecs can be found at the [Codec Plugins](/reference/codec-plugins.md) page.
|
||||
|
||||
Example:
|
||||
|
||||
```js
|
||||
codec => "json"
|
||||
```
|
||||
|
||||
|
||||
### Hash [hash]
|
||||
|
||||
A hash is a collection of key value pairs specified in the format `"field1" => "value1"`. Note that multiple key value entries are separated by spaces rather than commas.
|
||||
|
||||
Example:
|
||||
|
||||
```js
|
||||
match => {
|
||||
"field1" => "value1"
|
||||
"field2" => "value2"
|
||||
...
|
||||
}
|
||||
# or as a single line. No commas between entries:
|
||||
match => { "field1" => "value1" "field2" => "value2" }
|
||||
```
|
||||
|
||||
|
||||
### Number [number]
|
||||
|
||||
Numbers must be valid numeric values (floating point or integer).
|
||||
|
||||
Example:
|
||||
|
||||
```js
|
||||
port => 33
|
||||
```
|
||||
|
||||
|
||||
### Password [password]
|
||||
|
||||
A password is a string with a single value that is not logged or printed.
|
||||
|
||||
Example:
|
||||
|
||||
```js
|
||||
my_password => "password"
|
||||
```
|
||||
|
||||
|
||||
### URI [uri]
|
||||
|
||||
A URI can be anything from a full URL like *http://elastic.co/* to a simple identifier like *foobar*. If the URI contains a password such as *http://user:pass@example.net* the password portion of the URI will not be logged or printed.
|
||||
|
||||
Example:
|
||||
|
||||
```js
|
||||
my_uri => "http://foo:bar@example.net"
|
||||
```
|
||||
|
||||
|
||||
### Path [path]
|
||||
|
||||
A path is a string that represents a valid operating system path.
|
||||
|
||||
Example:
|
||||
|
||||
```js
|
||||
my_path => "/tmp/logstash"
|
||||
```
|
||||
|
||||
|
||||
### String [string]
|
||||
|
||||
A string must be a single character sequence. Note that string values are enclosed in quotes, either double or single.
|
||||
|
||||
### Escape sequences [_escape_sequences]
|
||||
|
||||
By default, escape sequences are not enabled. If you wish to use escape sequences in quoted strings, you will need to set `config.support_escapes: true` in your `logstash.yml`. When `true`, quoted strings (double and single) will have this transformation:
|
||||
|
||||
| | |
|
||||
| --- | --- |
|
||||
| Text | Result |
|
||||
| \r | carriage return (ASCII 13) |
|
||||
| \n | new line (ASCII 10) |
|
||||
| \t | tab (ASCII 9) |
|
||||
| \\ | backslash (ASCII 92) |
|
||||
| \" | double quote (ASCII 34) |
|
||||
| \' | single quote (ASCII 39) |
|
||||
|
||||
Example:
|
||||
|
||||
```js
|
||||
name => "Hello world"
|
||||
name => 'It\'s a beautiful day'
|
||||
```
|
||||
|
||||
|
||||
### Field reference [field-reference]
|
||||
|
||||
A Field Reference is a special [String](#string) value representing the path to a field in an event, such as `@timestamp` or `[@timestamp]` to reference a top-level field, or `[client][ip]` to access a nested field. The [*Field References Deep Dive*](https://www.elastic.co/guide/en/logstash/current/field-references-deepdive.html) provides detailed information about the structure of Field References. When provided as a configuration option, Field References need to be quoted and special characters must be escaped following the same rules as [String](#string).
|
||||
|
||||
|
||||
## Comments [comments]
|
||||
|
||||
Comments are the same as in perl, ruby, and python. A comment starts with a *#* character, and does not need to be at the beginning of a line. For example:
|
||||
|
||||
```js
|
||||
# this is a comment
|
||||
|
||||
input { # comments can appear at the end of a line, too
|
||||
# ...
|
||||
}
|
||||
```
|
||||
|
||||
::::{note}
|
||||
Comments containing environment variable `${var}` references in `config.string` are still evaluated. Remove the `$` sign to avoid pipeline loading failures.
|
||||
::::
|
||||
|
||||
|
||||
|
||||
|
||||
157
docs/reference/configuring-centralized-pipelines.md
Normal file
157
docs/reference/configuring-centralized-pipelines.md
Normal file
|
|
@ -0,0 +1,157 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/configuring-centralized-pipelines.html
|
||||
---
|
||||
|
||||
# Configure Centralized Pipeline Management [configuring-centralized-pipelines]
|
||||
|
||||
To configure [centralized pipeline management](/reference/logstash-centralized-pipeline-management.md):
|
||||
|
||||
1. Verify that you are using a license that includes the pipeline management feature.
|
||||
|
||||
For more information, see [https://www.elastic.co/subscriptions](https://www.elastic.co/subscriptions) and [License management](docs-content://deploy-manage/license/manage-your-license-in-self-managed-cluster.md).
|
||||
|
||||
2. Specify [configuration management settings](#configuration-management-settings) in the `logstash.yml` file. At a minimum, set:
|
||||
|
||||
* `xpack.management.enabled: true` to enable centralized configuration management.
|
||||
* `xpack.management.elasticsearch.hosts` to specify the Elasticsearch instance that will store the Logstash pipeline configurations and metadata.
|
||||
* `xpack.management.pipeline.id` to register the pipelines that you want to centrally manage.
|
||||
|
||||
3. Restart Logstash.
|
||||
4. If your Elasticsearch cluster is protected with basic authentication, assign the built-in `logstash_admin` role as well as the `logstash_writer` role to any users who will use centralized pipeline management. See [Secure your connection](/reference/secure-connection.md) for more information.
|
||||
|
||||
::::{note}
|
||||
Centralized management is disabled until you configure and enable {{security-features}}.
|
||||
::::
|
||||
|
||||
|
||||
::::{important}
|
||||
After you’ve configured Logstash to use centralized pipeline management, you can no longer specify local pipeline configurations. This means that the `pipelines.yml` file and settings like `path.config` and `config.string` are inactive when this feature is enabled.
|
||||
::::
|
||||
|
||||
|
||||
## Configuration Management Settings in Logstash [configuration-management-settings]
|
||||
|
||||
|
||||
You can set the following `xpack.management` settings in `logstash.yml` to enable [centralized pipeline management](/reference/logstash-centralized-pipeline-management.md). For more information about configuring Logstash, see [logstash.yml](/reference/logstash-settings-file.md).
|
||||
|
||||
The following example shows basic settings that assume {{es}} and {{kib}} are installed on the localhost with basic AUTH enabled, but no SSL. If you’re using SSL, you need to specify additional SSL settings.
|
||||
|
||||
```shell
|
||||
xpack.management.enabled: true
|
||||
xpack.management.elasticsearch.hosts: "http://localhost:9200/"
|
||||
xpack.management.elasticsearch.username: logstash_admin_user
|
||||
xpack.management.elasticsearch.password: t0p.s3cr3t
|
||||
xpack.management.logstash.poll_interval: 5s
|
||||
xpack.management.pipeline.id: ["apache", "cloudwatch_logs"]
|
||||
```
|
||||
|
||||
`xpack.management.enabled`
|
||||
: Set to `true` to enable {{xpack}} centralized configuration management for Logstash.
|
||||
|
||||
`xpack.management.logstash.poll_interval`
|
||||
: How often the Logstash instance polls for pipeline changes from Elasticsearch. The default is 5s.
|
||||
|
||||
`xpack.management.pipeline.id`
|
||||
: Specify a comma-separated list of pipeline IDs to register for centralized pipeline management. After changing this setting, you need to restart Logstash to pick up changes. Pipeline IDs support `*` as a [wildcard](#wildcard-in-pipeline-id) for matching multiple IDs
|
||||
|
||||
`xpack.management.elasticsearch.hosts`
|
||||
: The {{es}} instance that will store the Logstash pipeline configurations and metadata. This might be the same {{es}} instance specified in the `outputs` section in your Logstash configuration, or a different one. Defaults to `http://localhost:9200`.
|
||||
|
||||
`xpack.management.elasticsearch.username` and `xpack.management.elasticsearch.password`
|
||||
: If your {{es}} cluster is protected with basic authentication, these settings provide the username and password that the Logstash instance uses to authenticate for accessing the configuration data. The username you specify here should have the built-in `logstash_admin` and `logstash_system` roles. These roles provide access to system indices for managing configurations.
|
||||
|
||||
::::{note}
|
||||
Starting with Elasticsearch version 7.10.0, the `logstash_admin` role inherits the `manage_logstash_pipelines` cluster privilege for centralized pipeline management. If a user has created their own roles and granted them access to the .logstash index, those roles will continue to work in 7.x but will need to be updated for 8.0.
|
||||
::::
|
||||
|
||||
|
||||
`xpack.management.elasticsearch.proxy`
|
||||
: Optional setting that allows you to specify a proxy URL if Logstash needs to use a proxy to reach your Elasticsearch cluster.
|
||||
|
||||
`xpack.management.elasticsearch.ssl.ca_trusted_fingerprint`
|
||||
: Optional setting that enables you to specify the hex-encoded SHA-256 fingerprint of the certificate authority for your {{es}} instance.
|
||||
|
||||
::::{note}
|
||||
A self-secured Elasticsearch cluster will provide the fingerprint of its CA to the console during setup.
|
||||
|
||||
You can also get the SHA256 fingerprint of an Elasticsearch’s CA using the `openssl` command-line utility on the Elasticsearch host:
|
||||
|
||||
```shell
|
||||
openssl x509 -fingerprint -sha256 -in $ES_HOME/config/certs/http_ca.crt
|
||||
```
|
||||
|
||||
::::
|
||||
|
||||
|
||||
`xpack.management.elasticsearch.ssl.certificate_authority`
|
||||
: Optional setting that enables you to specify a path to the `.pem` file for the certificate authority for your {{es}} instance.
|
||||
|
||||
`xpack.management.elasticsearch.ssl.truststore.path`
|
||||
: Optional setting that provides the path to the Java keystore (JKS) to validate the server’s certificate.
|
||||
|
||||
::::{note}
|
||||
You cannot use this setting and `xpack.management.elasticsearch.ssl.certificate_authority` at the same time.
|
||||
::::
|
||||
|
||||
|
||||
`xpack.management.elasticsearch.ssl.truststore.password`
|
||||
: Optional setting that provides the password to the truststore.
|
||||
|
||||
`xpack.management.elasticsearch.ssl.keystore.path`
|
||||
: Optional setting that provides the path to the Java keystore (JKS) to validate the client’s certificate.
|
||||
|
||||
::::{note}
|
||||
You cannot use this setting and `xpack.management.elasticsearch.ssl.keystore.certificate` at the same time.
|
||||
::::
|
||||
|
||||
|
||||
`xpack.management.elasticsearch.ssl.keystore.password`
|
||||
: Optional setting that provides the password to the keystore.
|
||||
|
||||
`xpack.management.elasticsearch.ssl.certificate`
|
||||
: Optional setting that provides the path to an SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file.
|
||||
|
||||
::::{note}
|
||||
This setting can be used only if `xpack.management.elasticsearch.ssl.key` is set.
|
||||
::::
|
||||
|
||||
|
||||
`xpack.management.elasticsearch.ssl.key`
|
||||
: Optional setting that provides the path to an OpenSSL-style RSA private key that corresponds to the `xpack.management.elasticsearch.ssl.certificate`.
|
||||
|
||||
::::{note}
|
||||
This setting can be used only if `xpack.management.elasticsearch.ssl.certificate` is set.
|
||||
::::
|
||||
|
||||
|
||||
`xpack.management.elasticsearch.ssl.verification_mode`
|
||||
: Option to validate the server’s certificate. Defaults to `full`. To disable, set to `none`. Disabling this severely compromises security.
|
||||
|
||||
`xpack.management.elasticsearch.ssl.cipher_suites`
|
||||
: Optional setting that provides the list of cipher suites to use, listed by priorities. Supported cipher suites vary depending on the Java and protocol versions.
|
||||
|
||||
`xpack.management.elasticsearch.cloud_id`
|
||||
: If you’re using {{es}} in {{ecloud}}, you should specify the identifier here. This setting is an alternative to `xpack.management.elasticsearch.hosts`. If `cloud_id` is configured, `xpack.management.elasticsearch.hosts` should not be used. This {{es}} instance will store the Logstash pipeline configurations and metadata.
|
||||
|
||||
`xpack.management.elasticsearch.cloud_auth`
|
||||
: If you’re using {{es}} in {{ecloud}}, you can set your auth credentials here. This setting is an alternative to both `xpack.management.elasticsearch.username` and `xpack.management.elasticsearch.password`. If `cloud_auth` is configured, those settings should not be used. The credentials you specify here should be for a user with the `logstash_admin` and `logstash_system` roles, which provide access to system indices for managing configurations.
|
||||
|
||||
`xpack.management.elasticsearch.api_key`
|
||||
: Authenticate using an Elasticsearch API key. Note that this option also requires using SSL. The API key Format is `id:api_key` where `id` and `api_key` are as returned by the Elasticsearch [Create API key API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-security-create-api-key).
|
||||
|
||||
|
||||
## Wildcard support in pipeline ID [wildcard-in-pipeline-id]
|
||||
|
||||
|
||||
Pipeline IDs must begin with a letter or underscore and contain only letters, underscores, dashes, and numbers. You can use `*` in `xpack.management.pipeline.id` to match any number of letters, underscores, dashes, and numbers.
|
||||
|
||||
```shell
|
||||
xpack.management.pipeline.id: ["*logs", "*apache*", "tomcat_log"]
|
||||
```
|
||||
|
||||
In this example, `"*logs"` matches all IDs ending in `logs`. `"*apache*"` matches any IDs with `apache` in the name.
|
||||
|
||||
Wildcard in pipeline IDs is available starting with Elasticsearch 7.10. Logstash can pick up new pipeline without a restart if the new pipeline ID matches the wildcard pattern.
|
||||
|
||||
|
||||
68
docs/reference/configuring-geoip-database-management.md
Normal file
68
docs/reference/configuring-geoip-database-management.md
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/configuring-geoip-database-management.html
|
||||
---
|
||||
|
||||
# Configure GeoIP Database Management [configuring-geoip-database-management]
|
||||
|
||||
To configure [GeoIP Database Management](/reference/logstash-geoip-database-management.md):
|
||||
|
||||
1. Verify that you are using a license that includes the geoip database management feature.
|
||||
|
||||
For more information, see [https://www.elastic.co/subscriptions](https://www.elastic.co/subscriptions) and [License management](docs-content://deploy-manage/license/manage-your-license-in-self-managed-cluster.md).
|
||||
|
||||
2. Specify [geoip database management settings](#geoip-database-management-settings) in the `logstash.yml` file to tune the configuration as-needed.
|
||||
|
||||
## GeoIP database Management settings in {{ls}} [geoip-database-management-settings]
|
||||
|
||||
|
||||
You can set the following `xpack.geoip` settings in `logstash.yml` to configure the [geoip database manager](/reference/logstash-geoip-database-management.md). For more information about configuring Logstash, see [logstash.yml](/reference/logstash-settings-file.md).
|
||||
|
||||
`xpack.geoip.downloader.enabled`
|
||||
: (Boolean) If `true`, Logstash automatically downloads and manages updates for GeoIP2 databases from the `xpack.geoip.downloader.endpoint`. If `false`, Logstash does not manage GeoIP2 databases and plugins that need a GeoIP2 database must be configured to provide their own.
|
||||
|
||||
`xpack.geoip.downloader.endpoint`
|
||||
: (String) Endpoint URL used to download updates for GeoIP2 databases. For example, `https://mydomain.com/overview.json`. Defaults to `https://geoip.elastic.co/v1/database`. Note that Logstash will periodically make a GET request to `${xpack.geoip.downloader.endpoint}?elastic_geoip_service_tos=agree`, expecting the list of metadata about databases typically found in `overview.json`.
|
||||
|
||||
`xpack.geoip.downloader.poll.interval`
|
||||
: (Time Value) How often Logstash checks for GeoIP2 database updates at the `xpack.geoip.downloader.endpoint`. For example, `6h` to check every six hours. Defaults to `24h` (24 hours).
|
||||
|
||||
|
||||
## Offline and air-gapped environments [configuring-geoip-database-management-offline]
|
||||
|
||||
If Logstash does not have access to the internet, or if you want to disable the database manager, set the `xpack.geoip.downloader.enabled` value to `false` in `logstash.yml`. When the database manager is disabled, plugins that require GeoIP lookups must be configured with their own source of GeoIP databases.
|
||||
|
||||
### Using an HTTP proxy [_using_an_http_proxy]
|
||||
|
||||
If you can’t connect directly to the Elastic GeoIP endpoint, consider setting up an HTTP proxy server. You can then specify the proxy with `http_proxy` environment variable.
|
||||
|
||||
```sh
|
||||
export http_proxy="http://PROXY_IP:PROXY_PORT"
|
||||
```
|
||||
|
||||
|
||||
### Using a custom endpoint [_using_a_custom_endpoint]
|
||||
|
||||
If you work in an air-gapped environment and can’t update your databases from the Elastic endpoint, You can then download databases from MaxMind and bootstrap the service.
|
||||
|
||||
1. Download both `GeoLite2-ASN.mmdb` and `GeoLite2-City.mmdb` database files from the [MaxMind site](http://dev.maxmind.com/geoip/geoip2/geolite2).
|
||||
2. Copy both database files to a single directory.
|
||||
3. [Download {{es}}](https://www.elastic.co/downloads/elasticsearch).
|
||||
4. From your {{es}} directory, run:
|
||||
|
||||
```sh
|
||||
./bin/elasticsearch-geoip -s my/database/dir
|
||||
```
|
||||
|
||||
5. Serve the static database files from your directory. For example, you can use Docker to serve the files from nginx server:
|
||||
|
||||
```sh
|
||||
docker run -p 8080:80 -v my/database/dir:/usr/share/nginx/html:ro nginx
|
||||
```
|
||||
|
||||
6. Specify the service’s endpoint URL in Logstash using the `xpack.geoip.download.endpoint=http://localhost:8080/overview.json` setting in `logstash.yml`.
|
||||
|
||||
Logstash gets automatic updates from this service.
|
||||
|
||||
|
||||
|
||||
47
docs/reference/connecting-to-cloud.md
Normal file
47
docs/reference/connecting-to-cloud.md
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html
|
||||
---
|
||||
|
||||
# Sending data to Elastic Cloud (hosted Elasticsearch Service) [connecting-to-cloud]
|
||||
|
||||
Our hosted {{ess}} on [Elastic Cloud](https://cloud.elastic.co/) simplifies safe, secure communication between {{ls}} and {{es}}. When you configure the Elasticsearch output plugin to use [`cloud_id`](/reference/plugins-outputs-elasticsearch.md#plugins-outputs-elasticsearch-cloud_id) with either the [`cloud_auth` option](/reference/plugins-outputs-elasticsearch.md#plugins-outputs-elasticsearch-cloud_auth) or the [`api_key` option](/reference/plugins-outputs-elasticsearch.md#plugins-outputs-elasticsearch-api_key), no additional SSL configuration is needed.
|
||||
|
||||
Examples:
|
||||
|
||||
* `output {elasticsearch { cloud_id => "<cloud id>" cloud_auth => "<cloud auth>" } }`
|
||||
* `output {elasticsearch { cloud_id => "<cloud id>" api_key => "<api key>" } }`
|
||||
|
||||
{ess-leadin-short}
|
||||
|
||||
## Cloud ID [cloud-id]
|
||||
|
||||
{{ls}} uses the Cloud ID, found in the Elastic Cloud web console, to build the Elasticsearch and Kibana hosts settings. It is a base64 encoded text value of about 120 characters made up of upper and lower case letters and numbers. If you have several Cloud IDs, you can add a label, which is ignored internally, to help you tell them apart. To add a label you should prefix your Cloud ID with a label and a `:` separator in this format "<label>:<cloud-id>"
|
||||
|
||||
|
||||
## Cloud Auth [cloud-auth]
|
||||
|
||||
Cloud Auth is optional. Construct this value by following this format "<username>:<password>". Use your Cloud username for the first part. Use your Cloud password for the second part, which is given once in the Cloud UI when you create a cluster. If you change your Cloud password in the Cloud UI, remember to change it here, too.
|
||||
|
||||
|
||||
## Using Cloud ID and Cloud Auth with plugins [cloud-id-plugins]
|
||||
|
||||
The Elasticsearch input, output, and filter plugins support cloud_id and cloud_auth in their configurations.
|
||||
|
||||
* [Elasticsearch input plugin](/reference/plugins-inputs-elasticsearch.md#plugins-inputs-elasticsearch-cloud_id)
|
||||
* [Elasticsearch filter plugin](/reference/plugins-filters-elasticsearch.md#plugins-filters-elasticsearch-cloud_id)
|
||||
* [Elasticsearch output plugin](/reference/plugins-outputs-elasticsearch.md#plugins-outputs-elasticsearch-cloud_id)
|
||||
|
||||
|
||||
## Sending {{ls}} management data to {{es}} Services [cloud-id-mgmt]
|
||||
|
||||
These settings in the `logstash.yml` config file can help you get set up to send management data to Elastic Cloud:
|
||||
|
||||
* `xpack.management.elasticsearch.cloud_id`
|
||||
* `xpack.management.elasticsearch.cloud_auth`
|
||||
|
||||
You can use the `xpack.management.elasticsearch.cloud_id` setting as an alternative to `xpack.management.elasticsearch.hosts`.
|
||||
|
||||
You can use the `xpack.management.elasticsearch.cloud_auth` setting as an alternative to both `xpack.management.elasticsearch.username` and `xpack.management.elasticsearch.password`. The credentials you specify here should be for a user with the logstash_admin role, which provides access to .logstash-* indices for managing configurations.
|
||||
|
||||
|
||||
92
docs/reference/core-operations.md
Normal file
92
docs/reference/core-operations.md
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/core-operations.html
|
||||
---
|
||||
|
||||
# Performing Core Operations [core-operations]
|
||||
|
||||
The plugins described in this section are useful for core operations, such as mutating and dropping events.
|
||||
|
||||
[date filter](/reference/plugins-filters-date.md)
|
||||
: Parses dates from fields to use as Logstash timestamps for events.
|
||||
|
||||
The following config parses a field called `logdate` to set the Logstash timestamp:
|
||||
|
||||
```json
|
||||
filter {
|
||||
date {
|
||||
match => [ "logdate", "MMM dd yyyy HH:mm:ss" ]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
[drop filter](/reference/plugins-filters-drop.md)
|
||||
: Drops events. This filter is typically used in combination with conditionals.
|
||||
|
||||
The following config drops `debug` level log messages:
|
||||
|
||||
```json
|
||||
filter {
|
||||
if [loglevel] == "debug" {
|
||||
drop { }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
[fingerprint filter](/reference/plugins-filters-fingerprint.md)
|
||||
: Fingerprints fields by applying a consistent hash.
|
||||
|
||||
The following config fingerprints the `IP`, `@timestamp`, and `message` fields and adds the hash to a metadata field called `generated_id`:
|
||||
|
||||
```json
|
||||
filter {
|
||||
fingerprint {
|
||||
source => ["IP", "@timestamp", "message"]
|
||||
method => "SHA1"
|
||||
key => "0123"
|
||||
target => "[@metadata][generated_id]"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
[mutate filter](/reference/plugins-filters-mutate.md)
|
||||
: Performs general mutations on fields. You can rename, remove, replace, and modify fields in your events.
|
||||
|
||||
The following config renames the `HOSTORIP` field to `client_ip`:
|
||||
|
||||
```json
|
||||
filter {
|
||||
mutate {
|
||||
rename => { "HOSTORIP" => "client_ip" }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The following config strips leading and trailing whitespace from the specified fields:
|
||||
|
||||
```json
|
||||
filter {
|
||||
mutate {
|
||||
strip => ["field1", "field2"]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
[ruby filter](/reference/plugins-filters-ruby.md)
|
||||
: Executes Ruby code.
|
||||
|
||||
The following config executes Ruby code that cancels 90% of the events:
|
||||
|
||||
```json
|
||||
filter {
|
||||
ruby {
|
||||
code => "event.cancel if rand <= 0.90"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
34
docs/reference/creating-logstash-pipeline.md
Normal file
34
docs/reference/creating-logstash-pipeline.md
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/configuration.html
|
||||
---
|
||||
|
||||
# Creating a Logstash Pipeline [configuration]
|
||||
|
||||
You can create a pipeline by stringing together plugins--[inputs](/reference/input-plugins.md), [outputs](/reference/output-plugins.md), [filters](/reference/filter-plugins.md), and sometimes [codecs](/reference/codec-plugins.md)--in order to process data. To build a Logstash pipeline, create a config file to specify which plugins you want to use and the settings for each plugin.
|
||||
|
||||
A very basic pipeline might contain only an input and an output. Most pipelines include at least one filter plugin because that’s where the "transform" part of the ETL (extract, transform, load) magic happens. You can reference event fields in a pipeline and use conditionals to process events when they meet certain criteria.
|
||||
|
||||
Let’s step through creating a simple pipeline config on your local machine and then using it to run Logstash. Create a file named "logstash-simple.conf" and save it in the same directory as Logstash.
|
||||
|
||||
```ruby
|
||||
input { stdin { } }
|
||||
output {
|
||||
elasticsearch { cloud_id => "<cloud id>" api_key => "<api key>" }
|
||||
stdout { codec => rubydebug }
|
||||
}
|
||||
```
|
||||
|
||||
Then, run {{ls}} and specify the configuration file with the `-f` flag.
|
||||
|
||||
```ruby
|
||||
bin/logstash -f logstash-simple.conf
|
||||
```
|
||||
|
||||
Et voilà! Logstash reads the specified configuration file and outputs to both Elasticsearch and stdout. Before we move on to [more complex examples](/reference/config-examples.md), let’s take a look at what’s in a pipeline config file.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
145
docs/reference/dashboard-monitoring-with-elastic-agent.md
Normal file
145
docs/reference/dashboard-monitoring-with-elastic-agent.md
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
---
|
||||
navigation_title: "Collect monitoring data for dashboards"
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/dashboard-monitoring-with-elastic-agent.html
|
||||
---
|
||||
|
||||
# Collect {{ls}} monitoring data for dashboards [dashboard-monitoring-with-elastic-agent]
|
||||
|
||||
|
||||
{{agent}} collects monitoring data from your {{ls}} instance, sends it directly to your monitoring cluster, and shows the data in {{ls}} dashboards.
|
||||
|
||||
You can enroll {{agent}} in [{{fleet}}](docs-content://reference/ingestion-tools/fleet/install-fleet-managed-elastic-agent.md) for management from a central location, or you can run [{{agent}} standalone](docs-content://reference/ingestion-tools/fleet/install-standalone-elastic-agent.md).
|
||||
|
||||
**Prerequisites**
|
||||
|
||||
Complete these steps as you prepare to collect and ship monitoring data for dashboards:
|
||||
|
||||
::::{dropdown} Disable default collection of {{ls}} monitoring metrics
|
||||
:name: disable-default-db
|
||||
|
||||
Set `monitoring.enabled` to `false` in logstash.yml to disable default collection:
|
||||
|
||||
```yaml
|
||||
monitoring.enabled: false
|
||||
```
|
||||
|
||||
::::
|
||||
|
||||
|
||||
::::{dropdown} Specify the target cluster_uuid (optional)
|
||||
:name: define-cluster__uuid-db
|
||||
|
||||
To bind the metrics of {{ls}} to a specific cluster, optionally define the `monitoring.cluster_uuid` in the configuration file (logstash.yml):
|
||||
|
||||
```yaml
|
||||
monitoring.cluster_uuid: PRODUCTION_ES_CLUSTER_UUID
|
||||
```
|
||||
|
||||
::::
|
||||
|
||||
|
||||
::::{dropdown} Create a monitoring user (standalone agent only)
|
||||
:name: create-user-db
|
||||
|
||||
Create a user on the production cluster that has the `remote_monitoring_collector` [built-in role](elasticsearch://reference/elasticsearch/roles.md).
|
||||
|
||||
::::
|
||||
|
||||
|
||||
|
||||
## Install and configure {{agent}} [install-and-configure-db]
|
||||
|
||||
Install and configure {{agent}} to collect {{ls}} monitoring data for dashboards. We’ll walk you through the process in these steps:
|
||||
|
||||
* [Add the {{agent}} {{ls}} integration to monitor host logs and metrics](#add-logstash-integration-ead)
|
||||
* [Install and run an {{agent}} on your machine](#add-agent-to-fleet-ead)
|
||||
* [View assets](#view-assets-ead)
|
||||
* [Monitor {{ls}} logs and metrics](#view-data-dashboard)
|
||||
|
||||
Check out [Installing {{agent}}](docs-content://reference/ingestion-tools/fleet/install-elastic-agents.md) in the *Fleet and Elastic Agent Guide* for more info.
|
||||
|
||||
|
||||
### Add the {{agent}} {{ls}} integration to monitor host logs and metrics [add-logstash-integration-ead]
|
||||
|
||||
1. Go to the {{kib}} home page, and click **Add integrations**.
|
||||
|
||||
:::{image} ../images/kibana-home.png
|
||||
:alt: {{kib}} home page
|
||||
:class: screenshot
|
||||
:::
|
||||
|
||||
2. In the query bar, search for **{{ls}}** and select the integration to see more details.
|
||||
3. Click **Add {{ls}}**.
|
||||
4. Configure the integration name and add a description (optional).
|
||||
5. Configure the integration to collect logs.
|
||||
|
||||
* Make sure that **Logs** is turned on if you want to collect logs from your {{ls}} instance. Be sure that the required settings are correctly configured.
|
||||
* Under **Logs**, modify the log paths to match your {{ls}} environment.
|
||||
|
||||
6. Configure the integration to collect metrics.
|
||||
|
||||
* Make sure that **Metrics (Technical Preview)** is turned on, and **Metrics (Stack Monitoring)** is turned off.
|
||||
* Under **Metrics (Technical Preview)**, make sure the {{ls}} URL setting points to your {{ls}} instance URLs.<br> By default, the integration collects {{ls}} monitoring metrics from `https://localhost:9600`. If that host and port number are not correct, update the `Logstash URL` setting. If you configured {{ls}} to use encrypted communications and/or a username and password, you must access it via HTTPS, and expand the **Advanced Settings** options, and fill in with the appropriate values for your {{ls}} instance.
|
||||
|
||||
7. Click **Save and continue**.<br> This step takes a minute or two to complete. When it’s done, you’ll have an agent policy that contains a system integration policy for the configuration you just specified.
|
||||
8. In the popup, click **Add {{agent}} to your hosts** to open the **Add agent** flyout.
|
||||
|
||||
::::{tip}
|
||||
If you accidentally close the popup, go to **{{fleet}} > Agents** and click **Add agent**.
|
||||
::::
|
||||
|
||||
|
||||
|
||||
## Install and run an {{agent}} on your machine [add-agent-to-fleet-ead]
|
||||
|
||||
The **Add agent** flyout has two options: **Enroll in {{fleet}}** and **Run standalone**. Enrolling agents in {{fleet}} (default) provides a centralized management tool in {{kib}}, reducing management overhead.
|
||||
|
||||
:::::::{tab-set}
|
||||
|
||||
::::::{tab-item} Fleet-managed
|
||||
1. When the **Add Agent flyout** appears, stay on the **Enroll in fleet** tab.
|
||||
2. Skip the **Select enrollment token** step. The enrollment token you need is already selected.
|
||||
|
||||
::::{note}
|
||||
The enrollment token is specific to the {{agent}} policy that you just created. When you run the command to enroll the agent in {{fleet}}, you will pass in the enrollment token.
|
||||
::::
|
||||
|
||||
3. Download, install, and enroll the {{agent}} on your host by selecting your host operating system and following the **Install {{agent}} on your host** step.
|
||||
|
||||
It takes about a minute for {{agent}} to enroll in {{fleet}}, download the configuration specified in the policy you just created, and start collecting data.
|
||||
::::::
|
||||
|
||||
::::::{tab-item} Run standalone
|
||||
1. When the **Add Agent flyout** appears, navigate to the **Run standalone** tab.
|
||||
2. Configure the agent. Follow the instructions in **Install Elastic Agent on your host**.
|
||||
3. After unpacking the binary, replace the `elastic-agent.yml` file with that supplied in the Add Agent flyout on the "Run standalone" tab, replacing the values of `ES_USERNAME` and `ES_PASSWORD` appropriately.
|
||||
4. Run `sudo ./elastic-agent install`
|
||||
::::::
|
||||
|
||||
:::::::
|
||||
|
||||
## View assets [view-assets-ead]
|
||||
|
||||
After you have confirmed enrollment and data is coming in, click **View assets** to access dashboards related to the {{ls}} integration.
|
||||
|
||||
For traditional Stack Monitoring UI, the dashboards marked **[Logs {{ls}}]** are used to visualize the logs produced by your {{ls}} instances, with those marked **[Metrics {{ls}}]** for the technical preview metrics dashboards. These are populated with data only if you selected the **Metrics (Technical Preview)** checkbox.
|
||||
|
||||
:::{image} ../images/integration-assets-dashboards.png
|
||||
:alt: Integration assets
|
||||
:class: screenshot
|
||||
:::
|
||||
|
||||
A number of dashboards are included to view {{ls}} as a whole, and dashboards that allow you to drill-down into how {{ls}} is performing on a node, pipeline and plugin basis.
|
||||
|
||||
|
||||
## Monitor {{ls}} logs and metrics [view-data-dashboard]
|
||||
|
||||
From the list of assets, open the **[Metrics {{ls}}] {{ls}} overview** dashboard to view overall performance. Then follow the navigation panel to further drill down into {{ls}} performance.
|
||||
|
||||
:::{image} ../images/integration-dashboard-overview.png
|
||||
:alt: The {{ls}} Overview dashboard in {{kib}} with various metrics from your monitored {ls}
|
||||
:class: screenshot
|
||||
:::
|
||||
|
||||
You can hover over any visualization to adjust its settings, or click the **Edit** button to make changes to the dashboard. To learn more, refer to [Dashboard and visualizations](docs-content://explore-analyze/dashboards.md).
|
||||
105
docs/reference/data-deserialization.md
Normal file
105
docs/reference/data-deserialization.md
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/data-deserialization.html
|
||||
---
|
||||
|
||||
# Deserializing Data [data-deserialization]
|
||||
|
||||
The plugins described in this section are useful for deserializing data into Logstash events.
|
||||
|
||||
[avro codec](/reference/plugins-codecs-avro.md)
|
||||
: Reads serialized Avro records as Logstash events. This plugin deserializes individual Avro records. It is not for reading Avro files. Avro files have a unique format that must be handled upon input.
|
||||
|
||||
The following config deserializes input from Kafka:
|
||||
|
||||
```json
|
||||
input {
|
||||
kafka {
|
||||
codec => {
|
||||
avro => {
|
||||
schema_uri => "/tmp/schema.avsc"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
...
|
||||
```
|
||||
|
||||
|
||||
[csv filter](/reference/plugins-filters-csv.md)
|
||||
: Parses comma-separated value data into individual fields. By default, the filter autogenerates field names (column1, column2, and so on), or you can specify a list of names. You can also change the column separator.
|
||||
|
||||
The following config parses CSV data into the field names specified in the `columns` field:
|
||||
|
||||
```json
|
||||
filter {
|
||||
csv {
|
||||
separator => ","
|
||||
columns => [ "Transaction Number", "Date", "Description", "Amount Debit", "Amount Credit", "Balance" ]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
[fluent codec](/reference/plugins-codecs-fluent.md)
|
||||
: Reads the Fluentd `msgpack` schema.
|
||||
|
||||
The following config decodes logs received from `fluent-logger-ruby`:
|
||||
|
||||
```json
|
||||
input {
|
||||
tcp {
|
||||
codec => fluent
|
||||
port => 4000
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
[json codec](/reference/plugins-codecs-json.md)
|
||||
: Decodes (via inputs) and encodes (via outputs) JSON formatted content, creating one event per element in a JSON array.
|
||||
|
||||
The following config decodes the JSON formatted content in a file:
|
||||
|
||||
```json
|
||||
input {
|
||||
file {
|
||||
path => "/path/to/myfile.json"
|
||||
codec =>"json"
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
[protobuf codec](/reference/plugins-codecs-protobuf.md)
|
||||
: Reads protobuf encoded messages and converts them to Logstash events. Requires the protobuf definitions to be compiled as Ruby files. You can compile them by using the [ruby-protoc compiler](https://github.com/codekitchen/ruby-protocol-buffers).
|
||||
|
||||
The following config decodes events from a Kafka stream:
|
||||
|
||||
```json
|
||||
input
|
||||
kafka {
|
||||
zk_connect => "127.0.0.1"
|
||||
topic_id => "your_topic_goes_here"
|
||||
codec => protobuf {
|
||||
class_name => "Animal::Unicorn"
|
||||
include_path => ['/path/to/protobuf/definitions/UnicornProtobuf.pb.rb']
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
[xml filter](/reference/plugins-filters-xml.md)
|
||||
: Parses XML into fields.
|
||||
|
||||
The following config parses the whole XML document stored in the `message` field:
|
||||
|
||||
```json
|
||||
filter {
|
||||
xml {
|
||||
source => "message"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
257
docs/reference/dead-letter-queues.md
Normal file
257
docs/reference/dead-letter-queues.md
Normal file
|
|
@ -0,0 +1,257 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/dead-letter-queues.html
|
||||
---
|
||||
|
||||
# Dead letter queues (DLQ) [dead-letter-queues]
|
||||
|
||||
The dead letter queue (DLQ) is designed as a place to temporarily write events that cannot be processed. The DLQ gives you flexibility to investigate problematic events without blocking the pipeline or losing the events. Your pipeline keeps flowing, and the immediate problem is averted. But those events still need to be addressed.
|
||||
|
||||
You can [process events from the DLQ](#es-proc-dlq) with the [`dead_letter_queue` input plugin](/reference/plugins-inputs-dead_letter_queue.md) .
|
||||
|
||||
Processing events does not delete items from the queue, and the DLQ sometimes needs attention. See [Track dead letter queue size](#dlq-size) and [Clear the dead letter queue](#dlq-clear) for more info.
|
||||
|
||||
## How the dead letter queue works [dead-letter-how]
|
||||
|
||||
By default, when Logstash encounters an event that it cannot process because the data contains a mapping error or some other issue, the Logstash pipeline either hangs or drops the unsuccessful event. In order to protect against data loss in this situation, you can [configure Logstash](#configuring-dlq) to write unsuccessful events to a dead letter queue instead of dropping them.
|
||||
|
||||
::::{note}
|
||||
The dead letter queue is currently supported only for the [{{es}} output](/reference/plugins-outputs-elasticsearch.md) and [conditional statements evaluation](/reference/event-dependent-configuration.md#conditionals). The dead letter queue is used for documents with response codes of 400 or 404, both of which indicate an event that cannot be retried. It’s also used when a conditional evaluation encounter an error.
|
||||
::::
|
||||
|
||||
|
||||
Each event written to the dead letter queue includes the original event, metadata that describes the reason the event could not be processed, information about the plugin that wrote the event, and the timestamp when the event entered the dead letter queue.
|
||||
|
||||
To process events in the dead letter queue, create a Logstash pipeline configuration that uses the [`dead_letter_queue` input plugin](/reference/plugins-inputs-dead_letter_queue.md) to read from the queue. See [Processing events in the dead letter queue](#processing-dlq-events) for more information.
|
||||
|
||||
:::{image} ../images/dead_letter_queue.png
|
||||
:alt: Diagram showing pipeline reading from the dead letter queue
|
||||
:::
|
||||
|
||||
|
||||
## {{es}} processing and the dead letter queue [es-proc-dlq]
|
||||
|
||||
**HTTP request failure.** If the HTTP request fails (because {{es}} is unreachable or because it returned an HTTP error code), the {{es}} output retries the entire request indefinitely. In these scenarios, the dead letter queue has no opportunity to intercept.
|
||||
|
||||
**HTTP request success.** The [{{es}} Bulk API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-bulk) can perform multiple actions using the same request. If the Bulk API request is successful, it returns `200 OK`, even if some documents in the batch have [failed](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-bulk#bulk-failures-ex). In this situation, the `errors` flag for the request will be `true`.
|
||||
|
||||
The response body can include metadata indicating that one or more specific actions in the bulk request could not be performed, along with an HTTP-style status code per entry to indicate why the action could not be performed. If the DLQ is configured, individual indexing failures are routed there.
|
||||
|
||||
Even if you regularly process events, events remain in the dead letter queue. The dead letter queue requires [manual intervention](#dlq-clear) to clear it.
|
||||
|
||||
|
||||
## Conditional statements and the dead letter queue [conditionals-dlq]
|
||||
|
||||
When a conditional statement reaches an error in processing an event, such as comparing string and integer values, the event, as it is at the time of evaluation, is inserted into the dead letter queue.
|
||||
|
||||
|
||||
## Configuring {{ls}} to use dead letter queues [configuring-dlq]
|
||||
|
||||
Dead letter queues are disabled by default. To enable dead letter queues, set the `dead_letter_queue_enable` option in the `logstash.yml` [settings file](/reference/logstash-settings-file.md):
|
||||
|
||||
```yaml
|
||||
dead_letter_queue.enable: true
|
||||
```
|
||||
|
||||
Dead letter queues are stored as files in the local directory of the Logstash instance. By default, the dead letter queue files are stored in `path.data/dead_letter_queue`. Each pipeline has a separate queue. For example, the dead letter queue for the `main` pipeline is stored in `LOGSTASH_HOME/data/dead_letter_queue/main` by default. The queue files are numbered sequentially: `1.log`, `2.log`, and so on.
|
||||
|
||||
You can set `path.dead_letter_queue` in the `logstash.yml` file to specify a different path for the files:
|
||||
|
||||
```yaml
|
||||
path.dead_letter_queue: "path/to/data/dead_letter_queue"
|
||||
```
|
||||
|
||||
::::{tip}
|
||||
Use the local filesystem for data integrity and performance. Network File System (NFS) is not supported.
|
||||
::::
|
||||
|
||||
|
||||
Dead letter queue entries are written to a temporary file, which is then renamed to a dead letter queue segment file, which is then eligible for ingestion. The rename happens either when this temporary file is considered *full*, or when a period of time has elapsed since the last dead letter queue eligible event was written to the temporary file.
|
||||
|
||||
This length of time can be set using the `dead_letter_queue.flush_interval` setting. This setting is in milliseconds, and defaults to 5000ms. A low value here will mean in the event of infrequent writes to the dead letter queue more, smaller, queue files may be written, while a larger value will introduce more latency between items being "written" to the dead letter queue, and being made available for reading by the dead_letter_queue input.
|
||||
|
||||
```
|
||||
Note that this value cannot be set to lower than 1000ms.
|
||||
```
|
||||
```yaml
|
||||
dead_letter_queue.flush_interval: 5000
|
||||
```
|
||||
|
||||
::::{note}
|
||||
You may not use the same `dead_letter_queue` path for two different Logstash instances.
|
||||
::::
|
||||
|
||||
|
||||
### File rotation [file-rotation]
|
||||
|
||||
Dead letter queues have a built-in file rotation policy that manages the file size of the queue. When the file size reaches a preconfigured threshold, a new file is created automatically.
|
||||
|
||||
|
||||
### Size management [size-management]
|
||||
|
||||
By default, the maximum size of each dead letter queue is set to 1024mb. To change this setting, use the `dead_letter_queue.max_bytes` option. Entries will be dropped if they would increase the size of the dead letter queue beyond this setting. Use the `dead_letter_queue.storage_policy` option to control which entries should be dropped to avoid exceeding the size limit. Set the value to `drop_newer` (default) to stop accepting new values that would push the file size over the limit. Set the value to `drop_older` to remove the oldest events to make space for new ones.
|
||||
|
||||
#### Age policy [age-policy]
|
||||
|
||||
You can use the age policy to automatically control the volume of events in the dead letter queue. Use the `dead_letter_queue.retain.age` setting (in `logstash.yml` or `pipelines.yml`) to have {{ls}} remove events that are older than a value you define. Available time units are `d`, `h`, `m`, `s` respectively for days, hours, minutes and seconds. There is no default time unit, so you need to specify it.
|
||||
|
||||
```yaml
|
||||
dead_letter_queue.retain.age: 2d
|
||||
```
|
||||
|
||||
The age policy is verified and applied on event writes and during pipeline shutdown. For that reason, your dead-letter-queue folder may store expired events for longer than specified, and the reader pipeline could possibly encounter outdated events.
|
||||
|
||||
|
||||
|
||||
### Automatic cleaning of consumed events [auto-clean]
|
||||
|
||||
By default, the dead letter queue input plugin does not remove the events that it consumes. Instead, it commits a reference to avoid re-processing events. Use the `clean_consumed` setting in the dead letter queue input plugin in order to remove segments that have been fully consumed, freeing space while processing.
|
||||
|
||||
```yaml
|
||||
input {
|
||||
dead_letter_queue {
|
||||
path => "/path/to/data/dead_letter_queue"
|
||||
pipeline_id => "main"
|
||||
clean_consumed => true
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
|
||||
## Processing events in the dead letter queue [processing-dlq-events]
|
||||
|
||||
When you are ready to process events in the dead letter queue, you create a pipeline that uses the [`dead_letter_queue` input plugin](/reference/plugins-inputs-dead_letter_queue.md) to read from the dead letter queue. The pipeline configuration that you use depends, of course, on what you need to do. For example, if the dead letter queue contains events that resulted from a mapping error in Elasticsearch, you can create a pipeline that reads the "dead" events, removes the field that caused the mapping issue, and re-indexes the clean events into Elasticsearch.
|
||||
|
||||
The following example shows a simple pipeline that reads events from the dead letter queue and writes the events, including metadata, to standard output:
|
||||
|
||||
```yaml
|
||||
input {
|
||||
dead_letter_queue {
|
||||
path => "/path/to/data/dead_letter_queue" <1>
|
||||
commit_offsets => true <2>
|
||||
pipeline_id => "main" <3>
|
||||
}
|
||||
}
|
||||
|
||||
output {
|
||||
stdout {
|
||||
codec => rubydebug { metadata => true }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
1. The path to the top-level directory containing the dead letter queue. This directory contains a separate folder for each pipeline that writes to the dead letter queue. To find the path to this directory, look at the `logstash.yml` [settings file](/reference/logstash-settings-file.md). By default, Logstash creates the `dead_letter_queue` directory under the location used for persistent storage (`path.data`), for example, `LOGSTASH_HOME/data/dead_letter_queue`. However, if `path.dead_letter_queue` is set, it uses that location instead.
|
||||
2. When `true`, saves the offset. When the pipeline restarts, it will continue reading from the position where it left off rather than reprocessing all the items in the queue. You can set `commit_offsets` to `false` when you are exploring events in the dead letter queue and want to iterate over the events multiple times.
|
||||
3. The ID of the pipeline that’s writing to the dead letter queue. The default is `"main"`.
|
||||
|
||||
|
||||
For another example, see [Example: Processing data that has mapping errors](#dlq-example).
|
||||
|
||||
When the pipeline has finished processing all the events in the dead letter queue, it will continue to run and process new events as they stream into the queue. This means that you do not need to stop your production system to handle events in the dead letter queue.
|
||||
|
||||
::::{note}
|
||||
Events emitted from the [`dead_letter_queue` input plugin](/reference/plugins-inputs-dead_letter_queue.md) plugin will not be resubmitted to the dead letter queue if they cannot be processed correctly.
|
||||
::::
|
||||
|
||||
|
||||
|
||||
## Reading from a timestamp [dlq-timestamp]
|
||||
|
||||
When you read from the dead letter queue, you might not want to process all the events in the queue, especially if there are a lot of old events in the queue. You can start processing events at a specific point in the queue by using the `start_timestamp` option. This option configures the pipeline to start processing events based on the timestamp of when they entered the queue:
|
||||
|
||||
```yaml
|
||||
input {
|
||||
dead_letter_queue {
|
||||
path => "/path/to/data/dead_letter_queue"
|
||||
start_timestamp => "2017-06-06T23:40:37"
|
||||
pipeline_id => "main"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
For this example, the pipeline starts reading all events that were delivered to the dead letter queue on or after June 6, 2017, at 23:40:37.
|
||||
|
||||
|
||||
## Example: Processing data that has mapping errors [dlq-example]
|
||||
|
||||
In this example, the user attempts to index a document that includes geo_ip data, but the data cannot be processed because it contains a mapping error:
|
||||
|
||||
```json
|
||||
{"geoip":{"location":"home"}}
|
||||
```
|
||||
|
||||
Indexing fails because the Logstash output plugin expects a `geo_point` object in the `location` field, but the value is a string. The failed event is written to the dead letter queue, along with metadata about the error that caused the failure:
|
||||
|
||||
```json
|
||||
{
|
||||
"@metadata" => {
|
||||
"dead_letter_queue" => {
|
||||
"entry_time" => #<Java::OrgLogstash::Timestamp:0x5b5dacd5>,
|
||||
"plugin_id" => "fb80f1925088497215b8d037e622dec5819b503e-4",
|
||||
"plugin_type" => "elasticsearch",
|
||||
"reason" => "Could not index event to Elasticsearch. status: 400, action: [\"index\", {:_id=>nil, :_index=>\"logstash-2017.06.22\", :_type=>\"doc\", :_routing=>nil}, 2017-06-22T01:29:29.804Z My-MacBook-Pro-2.local {\"geoip\":{\"location\":\"home\"}}], response: {\"index\"=>{\"_index\"=>\"logstash-2017.06.22\", \"_type\"=>\"doc\", \"_id\"=>\"AVzNayPze1iR9yDdI2MD\", \"status\"=>400, \"error\"=>{\"type\"=>\"mapper_parsing_exception\", \"reason\"=>\"failed to parse\", \"caused_by\"=>{\"type\"=>\"illegal_argument_exception\", \"reason\"=>\"illegal latitude value [266.30859375] for geoip.location\"}}}}"
|
||||
}
|
||||
},
|
||||
"@timestamp" => 2017-06-22T01:29:29.804Z,
|
||||
"@version" => "1",
|
||||
"geoip" => {
|
||||
"location" => "home"
|
||||
},
|
||||
"host" => "My-MacBook-Pro-2.local",
|
||||
"message" => "{\"geoip\":{\"location\":\"home\"}}"
|
||||
}
|
||||
```
|
||||
|
||||
To process the failed event, you create the following pipeline that reads from the dead letter queue and removes the mapping problem:
|
||||
|
||||
```json
|
||||
input {
|
||||
dead_letter_queue {
|
||||
path => "/path/to/data/dead_letter_queue/" <1>
|
||||
}
|
||||
}
|
||||
filter {
|
||||
mutate {
|
||||
remove_field => "[geoip][location]" <2>
|
||||
}
|
||||
}
|
||||
output {
|
||||
elasticsearch{
|
||||
hosts => [ "localhost:9200" ] <3>
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
1. The [`dead_letter_queue` input](/reference/plugins-inputs-dead_letter_queue.md) reads from the dead letter queue.
|
||||
2. The `mutate` filter removes the problem field called `location`.
|
||||
3. The clean event is sent to Elasticsearch, where it can be indexed because the mapping issue is resolved.
|
||||
|
||||
|
||||
|
||||
## Track dead letter queue size [dlq-size]
|
||||
|
||||
Monitor the size of the dead letter queue before it becomes a problem. By checking it periodically, you can determine the maximum queue size that makes sense for each pipeline.
|
||||
|
||||
The size of the DLQ for each pipeline is available in the node stats API.
|
||||
|
||||
```txt
|
||||
pipelines.${pipeline_id}.dead_letter_queue.queue_size_in_bytes.
|
||||
```
|
||||
|
||||
Where `{{pipeline_id}}` is the name of a pipeline with DLQ enabled.
|
||||
|
||||
|
||||
## Clear the dead letter queue [dlq-clear]
|
||||
|
||||
The dead letter queue cannot be cleared with the upstream pipeline running.
|
||||
|
||||
The dead letter queue is a directory of pages. To clear it, stop the pipeline and delete location/<file-name>.
|
||||
|
||||
```txt
|
||||
${path.data}/dead_letter_queue/${pipeline_id}
|
||||
```
|
||||
|
||||
Where `{{pipeline_id}}` is the name of a pipeline with DLQ enabled.
|
||||
|
||||
The pipeline creates a new dead letter queue when it starts again.
|
||||
165
docs/reference/deploying-scaling-logstash.md
Normal file
165
docs/reference/deploying-scaling-logstash.md
Normal file
|
|
@ -0,0 +1,165 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/deploying-and-scaling.html
|
||||
---
|
||||
|
||||
# Deploying and scaling Logstash [deploying-and-scaling]
|
||||
|
||||
The Elastic Stack is used for tons of use cases, from operational log and metrics analytics, to enterprise and application search. Making sure your data gets scalably, durably, and securely transported to Elasticsearch is extremely important, especially for mission critical environments.
|
||||
|
||||
The goal of this document is to highlight the most common architecture patterns for Logstash and how to effectively scale as your deployment grows. The focus will be around the operational log, metrics, and security analytics use cases because they tend to require larger scale deployments. The deploying and scaling recommendations provided here may vary based on your own requirements.
|
||||
|
||||
|
||||
## Getting Started [deploying-getting-started]
|
||||
|
||||
For first time users, if you simply want to tail a log file to grasp the power of the Elastic Stack, we recommend trying [Filebeat Modules](beats://reference/filebeat/filebeat-modules-overview.md). Filebeat Modules enable you to quickly collect, parse, and index popular log types and view pre-built Kibana dashboards within minutes. [Metricbeat Modules](beats://reference/metricbeat/metricbeat-modules.md) provide a similar experience, but with metrics data. In this context, Beats will ship data directly to Elasticsearch where [Ingest Nodes](docs-content://manage-data/ingest/transform-enrich/ingest-pipelines.md) will process and index your data.
|
||||
|
||||
:::{image} ../images/deploy1.png
|
||||
:alt: deploy1
|
||||
:::
|
||||
|
||||
|
||||
### Introducing Logstash [_introducing_logstash]
|
||||
|
||||
What are the main benefits for integrating Logstash into your architecture?
|
||||
|
||||
* Scale through ingestion spikes - Logstash has an adaptive disk-based buffering system that will absorb incoming throughput, therefore mitigating backpressure
|
||||
* Ingest from other data sources like databases, S3, or messaging queues
|
||||
* Emit data to multiple destinations like S3, HDFS, or write to a file
|
||||
* Compose more sophisticated processing pipelines with conditional dataflow logic
|
||||
|
||||
|
||||
## Scaling Ingest [scaling-ingest]
|
||||
|
||||
Beats and Logstash make ingest awesome. Together, they provide a comprehensive solution that is scalable and resilient. What can you expect?
|
||||
|
||||
* Horizontal scalability, high availability, and variable load handling
|
||||
* Message durability with at-least-once delivery guarantees
|
||||
* End-to-end secure transport with authentication and wire encryption
|
||||
|
||||
|
||||
### Beats and Logstash [_beats_and_logstash]
|
||||
|
||||
Beats run across thousands of edge host servers, collecting, tailing, and shipping logs to Logstash. Logstash serves as the centralized streaming engine for data unification and enrichment. The [Beats input plugin](/reference/plugins-inputs-beats.md) exposes a secure, acknowledgement-based endpoint for Beats to send data to Logstash.
|
||||
|
||||
:::{image} ../images/deploy2.png
|
||||
:alt: deploy2
|
||||
:::
|
||||
|
||||
::::{note}
|
||||
Enabling persistent queues is strongly recommended, and these architecture characteristics assume that they are enabled. We encourage you to review the [Persistent queues (PQ)](/reference/persistent-queues.md) documentation for feature benefits and more details on resiliency.
|
||||
::::
|
||||
|
||||
|
||||
|
||||
### Scalability [_scalability]
|
||||
|
||||
Logstash is horizontally scalable and can form groups of nodes running the same pipeline. Logstash’s adaptive buffering capabilities will facilitate smooth streaming even through variable throughput loads. If the Logstash layer becomes an ingestion bottleneck, simply add more nodes to scale out. Here are a few general recommendations:
|
||||
|
||||
* Beats should [load balance](beats://reference/filebeat/elasticsearch-output.md#_loadbalance) across a group of Logstash nodes.
|
||||
* A minimum of two Logstash nodes are recommended for high availability.
|
||||
* It’s common to deploy just one Beats input per Logstash node, but multiple Beats inputs can also be deployed per Logstash node to expose independent endpoints for different data sources.
|
||||
|
||||
|
||||
### Resiliency [_resiliency]
|
||||
|
||||
When using [Filebeat](https://www.elastic.co/products/beats/filebeat) or [Winlogbeat](https://www.elastic.co/products/beats/winlogbeat) for log collection within this ingest flow, **at-least-once delivery** is guaranteed. Both the communication protocols, from Filebeat or Winlogbeat to Logstash, and from Logstash to Elasticsearch, are synchronous and support acknowledgements. The other Beats don’t yet have support for acknowledgements.
|
||||
|
||||
Logstash persistent queues provide protection across node failures. For disk-level resiliency in Logstash, it’s important to ensure disk redundancy. For on-premise deployments, it’s recommended that you configure RAID. When running in the cloud or a containerized environment, it’s recommended that you use persistent disks with replication strategies that reflect your data SLAs.
|
||||
|
||||
::::{note}
|
||||
Make sure `queue.checkpoint.writes: 1` is set for at-least-once guarantees. For more details, see the [persistent queue durability](/reference/persistent-queues.md#durability-persistent-queues) documentation.
|
||||
::::
|
||||
|
||||
|
||||
|
||||
### Processing [_processing]
|
||||
|
||||
Logstash will commonly extract fields with [grok](/reference/plugins-filters-grok.md) or [dissect](/reference/plugins-filters-dissect.md), augment [geographical](/reference/plugins-filters-geoip.md) info, and can further enrich events with [file](/reference/plugins-filters-translate.md), [database](/reference/plugins-filters-jdbc_streaming.md), or [Elasticsearch](/reference/plugins-filters-elasticsearch.md) lookup datasets. Be aware that processing complexity can affect overall throughput and CPU utilization. Make sure to check out the other [available filter plugins](/reference/filter-plugins.md).
|
||||
|
||||
|
||||
### Secure Transport [_secure_transport]
|
||||
|
||||
Enterprise-grade security is available across the entire delivery chain.
|
||||
|
||||
* Wire encryption is recommended for both the transport from [Beats to Logstash](beats://reference/filebeat/configuring-ssl-logstash.md) and from [Logstash to Elasticsearch](/reference/secure-connection.md).
|
||||
* There’s a wealth of security options when communicating with Elasticsearch including basic authentication, TLS, PKI, LDAP, AD, and other custom realms. To enable Elasticsearch security, see [Secure a cluster](docs-content://deploy-manage/security.md).
|
||||
|
||||
|
||||
### Monitoring [_monitoring]
|
||||
|
||||
When running Logstash 5.2 or greater, the [Monitoring UI](https://www.elastic.co/products/x-pack/monitoring) provides deep visibility into your deployment metrics, helping observe performance and alleviate bottlenecks as you scale. Monitoring is an X-Pack feature under the Basic License and is therefore **free to use**. To get started, see [Monitoring Logstash](https://www.elastic.co/docs/api/doc/logstash/).
|
||||
|
||||
If external monitoring is preferred, there are [monitoring APIs](monitoring-logstash.md) that return point-in-time metrics snapshots.
|
||||
|
||||
|
||||
## Adding Other Popular Sources [adding-other-sources]
|
||||
|
||||
Users may have other mechanisms of collecting logging data, and it’s easy to integrate and centralize them into the Elastic Stack. Let’s walk through a few scenarios:
|
||||
|
||||
:::{image} ../images/deploy3.png
|
||||
:alt: deploy3
|
||||
:::
|
||||
|
||||
|
||||
### TCP, UDP, and HTTP Protocols [_tcp_udp_and_http_protocols]
|
||||
|
||||
The TCP, UDP, and HTTP protocols are common ways to feed data into Logstash. Logstash can expose endpoint listeners with the respective [TCP](/reference/plugins-inputs-tcp.md), [UDP](/reference/plugins-inputs-udp.md), and [HTTP](/reference/plugins-inputs-http.md) input plugins. The data sources enumerated below are typically ingested through one of these three protocols.
|
||||
|
||||
::::{note}
|
||||
The TCP and UDP protocols do not support application-level acknowledgements, so connectivity issues may result in data loss.
|
||||
::::
|
||||
|
||||
|
||||
For high availability scenarios, a third-party hardware or software load balancer, like HAProxy, should be added to fan out traffic to a group of Logstash nodes.
|
||||
|
||||
|
||||
### Network and Security Data [_network_and_security_data]
|
||||
|
||||
Although Beats may already satisfy your data ingest use case, network and security datasets come in a variety of forms. Let’s touch on a few other ingestion points.
|
||||
|
||||
* Network wire data - collect and analyze network traffic with [Packetbeat](https://www.elastic.co/products/beats/packetbeat).
|
||||
* Netflow v5/v9/v10 - Logstash understands data from Netflow/IPFIX exporters with the [Netflow codec](/reference/plugins-codecs-netflow.md).
|
||||
* Nmap - Logstash accepts and parses Nmap XML data with the [Nmap codec](/reference/plugins-codecs-nmap.md).
|
||||
* SNMP trap - Logstash has a native [SNMP trap input](/reference/plugins-inputs-snmptrap.md).
|
||||
* CEF - Logstash accepts and parses CEF data from systems like Arcsight SmartConnectors with the [CEF codec](/reference/plugins-codecs-cef.md).
|
||||
|
||||
|
||||
### Centralized Syslog Servers [_centralized_syslog_servers]
|
||||
|
||||
Existing syslog server technologies like rsyslog and syslog-ng generally send syslog over to Logstash TCP or UDP endpoints for extraction, processing, and persistence. If the data format conforms to RFC3164, it can be fed directly to the [Logstash syslog input](/reference/plugins-inputs-syslog.md).
|
||||
|
||||
|
||||
### Infrastructure & Application Data and IoT [_infrastructure_application_data_and_iot]
|
||||
|
||||
Infrastructure and application metrics can be collected with [Metricbeat](https://www.elastic.co/products/beats/metricbeat), but applications can also send webhooks to a Logstash HTTP input or have metrics polled from an HTTP endpoint with the [HTTP poller input plugin](/reference/plugins-inputs-http_poller.md).
|
||||
|
||||
For applications that log with log4j2, it’s recommended to use the SocketAppender to send JSON to the Logstash TCP input. Alternatively, log4j2 can also log to a file for collection with FIlebeat. Usage of the log4j1 SocketAppender is not recommended.
|
||||
|
||||
IoT devices like Raspberry Pis, smartphones, and connected vehicles often send telemetry data through one of these protocols.
|
||||
|
||||
|
||||
## Integrating with Messaging Queues [integrating-with-messaging-queues]
|
||||
|
||||
If you are leveraging message queuing technologies as part of your existing infrastructure, getting that data into the Elastic Stack is easy. For existing users who are utilizing an external queuing layer like Redis or RabbitMQ just for data buffering with Logstash, it’s recommended to use Logstash persistent queues instead of an external queuing layer. This will help with overall ease of management by removing an unnecessary layer of complexity in your ingest architecture.
|
||||
|
||||
For users who want to integrate data from existing Kafka deployments or require the underlying usage of ephemeral storage, Kafka can serve as a data hub where Beats can persist to and Logstash nodes can consume from.
|
||||
|
||||
:::{image} ../images/deploy4.png
|
||||
:alt: deploy4
|
||||
:::
|
||||
|
||||
The other TCP, UDP, and HTTP sources can persist to Kafka with Logstash as a conduit to achieve high availability in lieu of a load balancer. A group of Logstash nodes can then consume from topics with the [Kafka input](/reference/plugins-inputs-kafka.md) to further transform and enrich the data in transit.
|
||||
|
||||
|
||||
### Resiliency and Recovery [_resiliency_and_recovery]
|
||||
|
||||
When Logstash consumes from Kafka, persistent queues should be enabled and will add transport resiliency to mitigate the need for reprocessing during Logstash node failures. In this context, it’s recommended to use the default persistent queue disk allocation size `queue.max_bytes: 1GB`.
|
||||
|
||||
If Kafka is configured to retain data for an extended period of time, data can be reprocessed from Kafka in the case of disaster recovery and reconciliation.
|
||||
|
||||
|
||||
### Other Messaging Queue Integrations [_other_messaging_queue_integrations]
|
||||
|
||||
Although an additional queuing layer is not required, Logstash can consume from a myriad of other message queuing technologies like [RabbitMQ](/reference/plugins-inputs-rabbitmq.md) and [Redis](/reference/plugins-inputs-redis.md). It also supports ingestion from hosted queuing services like [Pub/Sub](/reference/plugins-inputs-google_pubsub.md), [Kinesis](/reference/plugins-inputs-kinesis.md), and [SQS](/reference/plugins-inputs-sqs.md).
|
||||
|
||||
59
docs/reference/dir-layout.md
Normal file
59
docs/reference/dir-layout.md
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/dir-layout.html
|
||||
---
|
||||
|
||||
# Logstash Directory Layout [dir-layout]
|
||||
|
||||
This section describes the default directory structure that is created when you unpack the Logstash installation packages.
|
||||
|
||||
## Directory Layout of `.zip` and `.tar.gz` Archives [zip-targz-layout]
|
||||
|
||||
The `.zip` and `.tar.gz` packages are entirely self-contained. All files and directories are, by default, contained within the home directory — the directory created when unpacking the archive.
|
||||
|
||||
This is very convenient because you don’t have to create any directories to start using Logstash, and uninstalling Logstash is as easy as removing the home directory. However, it is advisable to change the default locations of the config and the logs directories so that you do not delete important data later on.
|
||||
|
||||
| Type | Description | Default Location | Setting |
|
||||
| --- | --- | --- | --- |
|
||||
| home | Home directory of the Logstash installation. | ``{extract.path}`- Directory created by unpacking the archive` | |
|
||||
| bin | Binary scripts, including `logstash` to start Logstash and `logstash-plugin` to install plugins | ``{extract.path}/bin`` | |
|
||||
| settings | Configuration files, including `logstash.yml` and `jvm.options` | ``{extract.path}/config`` | ``path.settings`` |
|
||||
| logs | Log files | ``{extract.path}/logs`` | ``path.logs`` |
|
||||
| plugins | Local, non Ruby-Gem plugin files. Each plugin is contained in a subdirectory. Recommended for development only. | ``{extract.path}/plugins`` | ``path.plugins`` |
|
||||
| data | Data files used by logstash and its plugins for any persistence needs. | ``{extract.path}/data`` | ``path.data`` |
|
||||
|
||||
|
||||
## Directory Layout of Debian and RPM Packages [deb-layout]
|
||||
|
||||
The Debian package and the RPM package each place config files, logs, and the settings files in the appropriate locations for the system:
|
||||
|
||||
| Type | Description | Default Location | Setting |
|
||||
| --- | --- | --- | --- |
|
||||
| home | Home directory of the Logstash installation. | ``/usr/share/logstash`` | |
|
||||
| bin | Binary scripts including `logstash` to start Logstash and `logstash-plugin` to install plugins | ``/usr/share/logstash/bin`` | |
|
||||
| settings | Configuration files, including `logstash.yml` and `jvm.options` | ``/etc/logstash`` | ``path.settings`` |
|
||||
| conf | Logstash pipeline configuration files | ``/etc/logstash/conf.d/*.conf`` | `See `/etc/logstash/pipelines.yml`` |
|
||||
| logs | Log files | ``/var/log/logstash`` | ``path.logs`` |
|
||||
| plugins | Local, non Ruby-Gem plugin files. Each plugin is contained in a subdirectory. Recommended for development only. | ``/usr/share/logstash/plugins`` | ``path.plugins`` |
|
||||
| data | Data files used by logstash and its plugins for any persistence needs. | ``/var/lib/logstash`` | ``path.data`` |
|
||||
|
||||
|
||||
## Directory Layout of Docker Images [docker-layout]
|
||||
|
||||
The Docker images are created from the `.tar.gz` packages, and follow a similar directory layout.
|
||||
|
||||
| Type | Description | Default Location | Setting |
|
||||
| --- | --- | --- | --- |
|
||||
| home | Home directory of the Logstash installation. | ``/usr/share/logstash`` | |
|
||||
| bin | Binary scripts, including `logstash` to start Logstash and `logstash-plugin` to install plugins | ``/usr/share/logstash/bin`` | |
|
||||
| settings | Configuration files, including `logstash.yml` and `jvm.options` | ``/usr/share/logstash/config`` | ``path.settings`` |
|
||||
| conf | Logstash pipeline configuration files | ``/usr/share/logstash/pipeline`` | ``path.config`` |
|
||||
| plugins | Local, non Ruby-Gem plugin files. Each plugin is contained in a subdirectory. Recommended for development only. | ``/usr/share/logstash/plugins`` | ``path.plugins`` |
|
||||
| data | Data files used by logstash and its plugins for any persistence needs. | ``/usr/share/logstash/data`` | ``path.data`` |
|
||||
|
||||
::::{note}
|
||||
Logstash Docker containers do not create log files by default. They log to standard output.
|
||||
::::
|
||||
|
||||
|
||||
|
||||
130
docs/reference/docker-config.md
Normal file
130
docs/reference/docker-config.md
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/docker-config.html
|
||||
---
|
||||
|
||||
# Configuring Logstash for Docker [docker-config]
|
||||
|
||||
Logstash differentiates between two types of configuration: [Settings and Pipeline Configuration](/reference/config-setting-files.md).
|
||||
|
||||
## Pipeline Configuration [_pipeline_configuration]
|
||||
|
||||
It is essential to place your pipeline configuration where it can be found by Logstash. By default, the container will look in `/usr/share/logstash/pipeline/` for pipeline configuration files.
|
||||
|
||||
In this example we use a bind-mounted volume to provide the configuration via the `docker run` command:
|
||||
|
||||
```sh
|
||||
docker run --rm -it -v ~/pipeline/:/usr/share/logstash/pipeline/ docker.elastic.co/logstash/logstash:9.0.0
|
||||
```
|
||||
|
||||
Every file in the host directory `~/pipeline/` will then be parsed by Logstash as pipeline configuration.
|
||||
|
||||
If you don’t provide configuration to Logstash, it will run with a minimal config that listens for messages from the [Beats input plugin](/reference/plugins-inputs-beats.md) and echoes any that are received to `stdout`. In this case, the startup logs will be similar to the following:
|
||||
|
||||
```text
|
||||
Sending Logstash logs to /usr/share/logstash/logs which is now configured via log4j2.properties.
|
||||
[2016-10-26T05:11:34,992][INFO ][logstash.inputs.beats ] Beats inputs: Starting input listener {:address=>"0.0.0.0:5044"}
|
||||
[2016-10-26T05:11:35,068][INFO ][logstash.pipeline ] Starting pipeline {"id"=>"main", "pipeline.workers"=>4, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>5, "pipeline.max_inflight"=>500}
|
||||
[2016-10-26T05:11:35,078][INFO ][org.logstash.beats.Server] Starting server on port: 5044
|
||||
[2016-10-26T05:11:35,078][INFO ][logstash.pipeline ] Pipeline main started
|
||||
[2016-10-26T05:11:35,105][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}
|
||||
```
|
||||
|
||||
This is the default configuration for the image, defined in `/usr/share/logstash/pipeline/logstash.conf`. If this is the behaviour that you are observing, ensure that your pipeline configuration is being picked up correctly, and that you are replacing either `logstash.conf` or the entire `pipeline` directory.
|
||||
|
||||
|
||||
## Settings [_settings]
|
||||
|
||||
The image provides several methods for configuring settings. The conventional approach is to provide a custom `logstash.yml` file, but it’s also possible to use environment variables to define settings.
|
||||
|
||||
### Bind-mounted settings files [docker-bind-mount-settings]
|
||||
|
||||
Settings files can also be provided through bind-mounts. Logstash expects to find them at `/usr/share/logstash/config/`.
|
||||
|
||||
It’s possible to provide an entire directory containing all needed files:
|
||||
|
||||
```sh
|
||||
docker run --rm -it -v ~/settings/:/usr/share/logstash/config/ docker.elastic.co/logstash/logstash:9.0.0
|
||||
```
|
||||
|
||||
Alternatively, a single file can be mounted:
|
||||
|
||||
```sh
|
||||
docker run --rm -it -v ~/settings/logstash.yml:/usr/share/logstash/config/logstash.yml docker.elastic.co/logstash/logstash:9.0.0
|
||||
```
|
||||
|
||||
::::{note}
|
||||
Bind-mounted configuration files will retain the same permissions and ownership within the container that they have on the host system. Be sure to set permissions such that the files will be readable and, ideally, not writeable by the container’s `logstash` user (UID 1000).
|
||||
::::
|
||||
|
||||
|
||||
|
||||
### Custom Images [_custom_images]
|
||||
|
||||
Bind-mounted configuration is not the only option, naturally. If you prefer the *Immutable Infrastructure* approach, you can prepare a custom image containing your configuration by using a `Dockerfile` like this one:
|
||||
|
||||
```dockerfile
|
||||
FROM docker.elastic.co/logstash/logstash:9.0.0
|
||||
RUN rm -f /usr/share/logstash/pipeline/logstash.conf
|
||||
COPY pipeline/ /usr/share/logstash/pipeline/
|
||||
COPY config/ /usr/share/logstash/config/
|
||||
```
|
||||
|
||||
Be sure to replace or delete `logstash.conf` in your custom image, so that you don’t retain the example config from the base image.
|
||||
|
||||
|
||||
### Environment variable configuration [docker-env-config]
|
||||
|
||||
Under Docker, Logstash settings can be configured via environment variables. When the container starts, a helper process checks the environment for variables that can be mapped to Logstash settings. Settings that are found in the environment override those in the `logstash.yml` as the container starts up.
|
||||
|
||||
For compatibility with container orchestration systems, these environment variables are written in all capitals, with underscores as word separators.
|
||||
|
||||
Some example translations are shown here:
|
||||
|
||||
**Environment Variable**
|
||||
: **Logstash Setting**
|
||||
|
||||
`PIPELINE_WORKERS`
|
||||
: `pipeline.workers`
|
||||
|
||||
`LOG_LEVEL`
|
||||
: `log.level`
|
||||
|
||||
`MONITORING_ENABLED`
|
||||
: `monitoring.enabled`
|
||||
|
||||
In general, any setting listed in the [settings documentation](/reference/logstash-settings-file.md) can be configured with this technique.
|
||||
|
||||
::::{note}
|
||||
Defining settings with environment variables causes `logstash.yml` to be modified in place. This behaviour is likely undesirable if `logstash.yml` was bind-mounted from the host system. Thus, it is not recommended to combine the bind-mount technique with the environment variable technique. It is best to choose a single method for defining Logstash settings.
|
||||
::::
|
||||
|
||||
|
||||
|
||||
|
||||
## Docker defaults [_docker_defaults]
|
||||
|
||||
The following settings have different default values when using the Docker images:
|
||||
|
||||
`api.http.host`
|
||||
: `0.0.0.0`
|
||||
|
||||
`monitoring.elasticsearch.hosts`
|
||||
: `http://elasticsearch:9200`
|
||||
|
||||
::::{note}
|
||||
The setting `monitoring.elasticsearch.hosts` is not defined in the `-oss` image.
|
||||
::::
|
||||
|
||||
|
||||
These settings are defined in the default `logstash.yml`. They can be overridden with a [custom `logstash.yml`](#docker-bind-mount-settings) or via [environment variables](#docker-env-config).
|
||||
|
||||
::::{important}
|
||||
If replacing `logstash.yml` with a custom version, be sure to copy the above defaults to the custom file if you want to retain them. If not, they will be "masked" by the new file.
|
||||
::::
|
||||
|
||||
|
||||
|
||||
## Logging Configuration [_logging_configuration]
|
||||
|
||||
Under Docker, Logstash logs go to standard output by default. To change this behaviour, use any of the techniques above to replace the file at `/usr/share/logstash/config/log4j2.properties`.
|
||||
31
docs/reference/docker.md
Normal file
31
docs/reference/docker.md
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/docker.html
|
||||
---
|
||||
|
||||
# Running Logstash on Docker [docker]
|
||||
|
||||
Docker images for Logstash are available from the Elastic Docker registry. The base image is [Red Hat Universal Base Image 9 Minimal](https://catalog.redhat.com/software/containers/ubi9/ubi-minimal/61832888c0d15aff4912fe0d).
|
||||
|
||||
A list of all published Docker images and tags is available at [www.docker.elastic.co](https://www.docker.elastic.co). The source code is in [GitHub](https://github.com/elastic/logstash/tree/master).
|
||||
|
||||
These images are free to use under the Elastic license. They contain open source and free commercial features and access to paid commercial features. [Start a 30-day trial](docs-content://deploy-manage/license/manage-your-license-in-self-managed-cluster.md) to try out all of the paid commercial features. See the [Subscriptions](https://www.elastic.co/subscriptions) page for information about Elastic license levels.
|
||||
|
||||
## Pulling the image [_pulling_the_image]
|
||||
|
||||
Obtaining Logstash for Docker is as simple as issuing a `docker pull` command against the Elastic Docker registry.
|
||||
|
||||
However, version 9.0.0 of Logstash has not yet been released, so no Docker image is currently available for this version.
|
||||
|
||||
|
||||
## Verifying the image [_verifying_the_image]
|
||||
|
||||
Although it’s optional, we highly recommend verifying the signatures included with your downloaded Docker images to ensure that the images are valid.
|
||||
|
||||
Elastic images are signed with [Cosign](https://docs.sigstore.dev/cosign/overview/) which is part of the [Sigstore](https://www.sigstore.dev/) project. Cosign supports container signing, verification, and storage in an OCI registry. Install the appropriate [Cosign application](https://docs.sigstore.dev/cosign/installation/) for your operating system.
|
||||
|
||||
Run the following commands to verify the container image signature for {{ls}} v9.0.0-beta1:
|
||||
|
||||
Version 9.0.0 of Logstash has not yet been released, so no Docker image is currently available for this version.
|
||||
|
||||
|
||||
78
docs/reference/ecs-ls.md
Normal file
78
docs/reference/ecs-ls.md
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/ecs-ls.html
|
||||
---
|
||||
|
||||
# ECS in Logstash [ecs-ls]
|
||||
|
||||
The [Elastic Common Schema (ECS)][Elastic Common Schema (ECS)](ecs://reference/index.md)) is an open source specification, developed with support from the Elastic user community. ECS defines a common set of fields to be used for storing event data, such as logs and metrics, in {{es}}. With ECS, users can normalize event data to better analyze, visualize, and correlate the data represented in their events.
|
||||
|
||||
## ECS compatibility [ecs-compatibility]
|
||||
|
||||
Many plugins implement an ECS-compatibility mode, which causes them to produce and manipulate events in a manner that is compatible with the Elastic Common Schema (ECS).
|
||||
|
||||
Any plugin that supports this mode will also have an `ecs_compatibility` option, which allows you to configure which mode the individual plugin instance should operate in. If left unspecified for an individual plugin, the pipeline’s `pipeline.ecs_compatibility` setting will be observed. This allows you to configure plugins to use a specific version of ECS or to use their legacy non-ECS behavior.
|
||||
|
||||
ECS compatibility modes do not prevent you from explicitly configuring a plugin in a manner that conflicts with ECS. Instead, they ensure that *implicit* configuration avoids conflicts.
|
||||
|
||||
### Configuring ECS [ecs-configuration]
|
||||
|
||||
In {{ls}} 8, all plugins are run in ECS compatibility v8 mode by default, but you can opt out at the plugin, pipeline, or system level to maintain legacy behavior. This can be helpful if you have very complex pipelines that were defined pre-ECS, to allow you to either upgrade them or to avoid doing so independently of your {{ls}} 8.x upgrade.
|
||||
|
||||
#### Specific plugin instance [_specific_plugin_instance]
|
||||
|
||||
Use a plugin’s `ecs_compatibility` option to override the default value on the plugin instance.
|
||||
|
||||
For example, if you want a specific instance of the GeoIP Filter to behave without ECS compatibility, you can adjust its definition in your pipeline without affecting any other plugin instances.
|
||||
|
||||
```text
|
||||
filter {
|
||||
geoip {
|
||||
source => "[host][ip]"
|
||||
ecs_compatibility => disabled
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Alternatively, if you had a UDP input with a CEF codec, and wanted both to use an ECS mode while still running {{ls}} 7, you can adjust their definitions to specify the major version of ECS to use.
|
||||
|
||||
```text
|
||||
input {
|
||||
udp {
|
||||
port => 1234
|
||||
ecs_compatibility => v8
|
||||
codec => cef {
|
||||
ecs_compatibility => v8
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
#### All plugins in a given pipeline [ecs-configuration-pipeline]
|
||||
|
||||
If you wish to provide a specific default value for `ecs_compatibility` to *all* plugins in a pipeline, you can do so with the `pipeline.ecs_compatibility` setting in your pipeline definition in `config/pipelines.yml` or Central Management. This setting will be used unless overridden by a specific plugin instance. If unspecified for an individual pipeline, the global value will be used.
|
||||
|
||||
For example, setting `pipeline.ecs_compatibility: disabled` for a pipeline *locks in* that pipeline’s pre-{{ls}} 8 behavior.
|
||||
|
||||
```yaml
|
||||
- pipeline.id: my-legacy-pipeline
|
||||
path.config: "/etc/path/to/legacy-pipeline.config"
|
||||
pipeline.ecs_compatibility: disabled
|
||||
- pipeline.id: my-ecs-pipeline
|
||||
path.config: "/etc/path/to/ecs-pipeline.config"
|
||||
pipeline.ecs_compatibility: v8
|
||||
```
|
||||
|
||||
|
||||
#### All plugins in all pipelines [ecs-configuration-all]
|
||||
|
||||
Similarly, you can set the default value for the whole {{ls}} process by setting the `pipeline.ecs_compatibility` value in `config/logstash.yml`.
|
||||
|
||||
```yaml
|
||||
pipeline.ecs_compatibility: disabled
|
||||
```
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -1,119 +1,114 @@
|
|||
[[environment-variables]]
|
||||
=== Using environment variables
|
||||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/environment-variables.html
|
||||
---
|
||||
|
||||
==== Overview
|
||||
# Using environment variables [environment-variables]
|
||||
|
||||
## Overview [_overview]
|
||||
|
||||
* You can set environment variable references in the configuration for Logstash plugins by using `${var}`.
|
||||
* At Logstash startup, each reference is replaced by the value of the environment variable.
|
||||
* The replacement is case-sensitive.
|
||||
* References to undefined variables raise a Logstash configuration error.
|
||||
* You can give a default value by using the form `${var:default value}`. Logstash uses the default value if the
|
||||
environment variable is undefined.
|
||||
* You can give a default value by using the form `${var:default value}`. Logstash uses the default value if the environment variable is undefined.
|
||||
* You can add environment variable references in any plugin option type: string, number, boolean, array, or hash.
|
||||
* Environment variables for list-type URI parameters can support lists of space-delimited values. Currently, other non-URI based options do not support lists of values. See <<plugin-concepts>>
|
||||
* Environment variables are immutable. If you update the environment variable, you'll have to restart Logstash to pick up the updated value.
|
||||
* References to environment variables in `config.string` comments are evaluated during configuration parsing, and are therefore discouraged.
|
||||
Remove the `$` sign to avoid pipeline loading failures.
|
||||
* Environment variables for list-type URI parameters can support lists of space-delimited values. Currently, other non-URI based options do not support lists of values. See [Cross-plugin concepts and features](/reference/plugin-concepts.md)
|
||||
* Environment variables are immutable. If you update the environment variable, you’ll have to restart Logstash to pick up the updated value.
|
||||
* References to environment variables in `config.string` comments are evaluated during configuration parsing, and are therefore discouraged. Remove the `$` sign to avoid pipeline loading failures.
|
||||
|
||||
==== Examples
|
||||
|
||||
These examples show you how to use environment variables to set the values of some commonly used
|
||||
configuration options.
|
||||
## Examples [_examples]
|
||||
|
||||
===== Setting the TCP port
|
||||
These examples show you how to use environment variables to set the values of some commonly used configuration options.
|
||||
|
||||
Here's an example that uses an environment variable to set the TCP port:
|
||||
### Setting the TCP port [_setting_the_tcp_port]
|
||||
|
||||
[source,ruby]
|
||||
----------------------------------
|
||||
Here’s an example that uses an environment variable to set the TCP port:
|
||||
|
||||
```ruby
|
||||
input {
|
||||
tcp {
|
||||
port => "${TCP_PORT}"
|
||||
}
|
||||
}
|
||||
----------------------------------
|
||||
```
|
||||
|
||||
Now let's set the value of `TCP_PORT`:
|
||||
Now let’s set the value of `TCP_PORT`:
|
||||
|
||||
[source,shell]
|
||||
----
|
||||
```shell
|
||||
export TCP_PORT=12345
|
||||
----
|
||||
```
|
||||
|
||||
At startup, Logstash uses this configuration:
|
||||
|
||||
[source,ruby]
|
||||
----------------------------------
|
||||
```ruby
|
||||
input {
|
||||
tcp {
|
||||
port => 12345
|
||||
}
|
||||
}
|
||||
----------------------------------
|
||||
```
|
||||
|
||||
If the `TCP_PORT` environment variable is not set, Logstash returns a configuration error.
|
||||
|
||||
You can fix this problem by specifying a default value:
|
||||
|
||||
[source,ruby]
|
||||
----
|
||||
```ruby
|
||||
input {
|
||||
tcp {
|
||||
port => "${TCP_PORT:54321}"
|
||||
}
|
||||
}
|
||||
----
|
||||
```
|
||||
|
||||
Now, instead of returning a configuration error if the variable is undefined, Logstash uses the default:
|
||||
|
||||
[source,ruby]
|
||||
----
|
||||
```ruby
|
||||
input {
|
||||
tcp {
|
||||
port => 54321
|
||||
}
|
||||
}
|
||||
----
|
||||
```
|
||||
|
||||
If the environment variable is defined, Logstash uses the value specified for the variable instead of the default.
|
||||
|
||||
===== Setting the value of a tag
|
||||
|
||||
Here's an example that uses an environment variable to set the value of a tag:
|
||||
### Setting the value of a tag [_setting_the_value_of_a_tag]
|
||||
|
||||
[source,ruby]
|
||||
----
|
||||
Here’s an example that uses an environment variable to set the value of a tag:
|
||||
|
||||
```ruby
|
||||
filter {
|
||||
mutate {
|
||||
add_tag => [ "tag1", "${ENV_TAG}" ]
|
||||
}
|
||||
}
|
||||
----
|
||||
```
|
||||
|
||||
Let's set the value of `ENV_TAG`:
|
||||
Let’s set the value of `ENV_TAG`:
|
||||
|
||||
[source,shell]
|
||||
----
|
||||
```shell
|
||||
export ENV_TAG="tag2"
|
||||
----
|
||||
```
|
||||
|
||||
At startup, Logstash uses this configuration:
|
||||
|
||||
[source,ruby]
|
||||
----
|
||||
```ruby
|
||||
filter {
|
||||
mutate {
|
||||
add_tag => [ "tag1", "tag2" ]
|
||||
}
|
||||
}
|
||||
----
|
||||
```
|
||||
|
||||
===== Setting a file path
|
||||
|
||||
Here's an example that uses an environment variable to set the path to a log file:
|
||||
### Setting a file path [_setting_a_file_path]
|
||||
|
||||
[source,ruby]
|
||||
----
|
||||
Here’s an example that uses an environment variable to set the path to a log file:
|
||||
|
||||
```ruby
|
||||
filter {
|
||||
mutate {
|
||||
add_field => {
|
||||
|
|
@ -121,19 +116,17 @@ filter {
|
|||
}
|
||||
}
|
||||
}
|
||||
----
|
||||
```
|
||||
|
||||
Let's set the value of `HOME`:
|
||||
Let’s set the value of `HOME`:
|
||||
|
||||
[source,shell]
|
||||
----
|
||||
```shell
|
||||
export HOME="/path"
|
||||
----
|
||||
```
|
||||
|
||||
At startup, Logstash uses the following configuration:
|
||||
|
||||
[source,ruby]
|
||||
----
|
||||
```ruby
|
||||
filter {
|
||||
mutate {
|
||||
add_field => {
|
||||
|
|
@ -141,5 +134,7 @@ filter {
|
|||
}
|
||||
}
|
||||
}
|
||||
----
|
||||
```
|
||||
|
||||
|
||||
|
||||
105
docs/reference/event-api.md
Normal file
105
docs/reference/event-api.md
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/event-api.html
|
||||
---
|
||||
|
||||
# Event API [event-api]
|
||||
|
||||
This section is targeted for plugin developers and users of Logstash’s Ruby filter. Below we document recent changes (starting with version 5.0) in the way users have been accessing Logstash’s event based data in custom plugins and in the Ruby filter. Note that [Accessing event data and fields](/reference/event-dependent-configuration.md) data flow in Logstash’s config files — using [Field references](/reference/event-dependent-configuration.md#logstash-config-field-references) — is not affected by this change, and will continue to use existing syntax.
|
||||
|
||||
|
||||
## Event Object [_event_object]
|
||||
|
||||
Event is the main object that encapsulates data flow internally in Logstash and provides an API for the plugin developers to interact with the event’s content. Typically, this API is used in plugins and in a Ruby filter to retrieve data and use it for transformations. Event object contains the original data sent to Logstash and any additional fields created during Logstash’s filter stages.
|
||||
|
||||
In 5.0, we’ve re-implemented the Event class and its supporting classes in pure Java. Since Event is a critical component in data processing, a rewrite in Java improves performance and provides efficient serialization when storing data on disk. For the most part, this change aims at keeping backward compatibility and is transparent to the users. To this extent we’ve updated and published most of the plugins in Logstash’s ecosystem to adhere to the new API changes. However, if you are maintaining a custom plugin, or have a Ruby filter, this change will affect you. The aim of this guide is to describe the new API and provide examples to migrate to the new changes.
|
||||
|
||||
|
||||
## Event API [_event_api]
|
||||
|
||||
Prior to version 5.0, developers could access and manipulate event data by directly using Ruby hash syntax. For example, `event[field] = foo`. While this is powerful, our goal is to abstract the internal implementation details and provide well-defined getter and setter APIs.
|
||||
|
||||
**Get API**
|
||||
|
||||
The getter is a read-only access of field-based data in an Event.
|
||||
|
||||
**Syntax:** `event.get(field)`
|
||||
|
||||
**Returns:** Value for this field or nil if the field does not exist. Returned values could be a string, numeric or timestamp scalar value.
|
||||
|
||||
`field` is a structured field sent to Logstash or created after the transformation process. `field` can also be a nested [field reference](https://www.elastic.co/guide/en/logstash/current/field-references-deepdive.html) such as `[field][bar]`.
|
||||
|
||||
Examples:
|
||||
|
||||
```ruby
|
||||
event.get("foo" ) # => "baz"
|
||||
event.get("[foo]") # => "zab"
|
||||
event.get("[foo][bar]") # => 1
|
||||
event.get("[foo][bar]") # => 1.0
|
||||
event.get("[foo][bar]") # => [1, 2, 3]
|
||||
event.get("[foo][bar]") # => {"a" => 1, "b" => 2}
|
||||
event.get("[foo][bar]") # => {"a" => 1, "b" => 2, "c" => [1, 2]}
|
||||
```
|
||||
|
||||
Accessing @metadata
|
||||
|
||||
```ruby
|
||||
event.get("[@metadata][foo]") # => "baz"
|
||||
```
|
||||
|
||||
**Set API**
|
||||
|
||||
This API can be used to mutate data in an Event.
|
||||
|
||||
**Syntax:** `event.set(field, value)`
|
||||
|
||||
**Returns:** The current Event after the mutation, which can be used for chainable calls.
|
||||
|
||||
Examples:
|
||||
|
||||
```ruby
|
||||
event.set("foo", "baz")
|
||||
event.set("[foo]", "zab")
|
||||
event.set("[foo][bar]", 1)
|
||||
event.set("[foo][bar]", 1.0)
|
||||
event.set("[foo][bar]", [1, 2, 3])
|
||||
event.set("[foo][bar]", {"a" => 1, "b" => 2})
|
||||
event.set("[foo][bar]", {"a" => 1, "b" => 2, "c" => [1, 2]})
|
||||
event.set("[@metadata][foo]", "baz")
|
||||
```
|
||||
|
||||
Mutating a collection after setting it in the Event has an undefined behaviour and is not allowed.
|
||||
|
||||
```ruby
|
||||
h = {"a" => 1, "b" => 2, "c" => [1, 2]}
|
||||
event.set("[foo][bar]", h)
|
||||
|
||||
h["c"] = [3, 4]
|
||||
event.get("[foo][bar][c]") # => undefined
|
||||
|
||||
Suggested way of mutating collections:
|
||||
|
||||
h = {"a" => 1, "b" => 2, "c" => [1, 2]}
|
||||
event.set("[foo][bar]", h)
|
||||
|
||||
h["c"] = [3, 4]
|
||||
event.set("[foo][bar]", h)
|
||||
|
||||
# Alternatively,
|
||||
event.set("[foo][bar][c]", [3, 4])
|
||||
```
|
||||
|
||||
|
||||
## Ruby Filter [_ruby_filter]
|
||||
|
||||
The [Ruby Filter](/reference/plugins-filters-ruby.md) can be used to execute any ruby code and manipulate event data using the API described above. For example, using the new API:
|
||||
|
||||
```ruby
|
||||
filter {
|
||||
ruby {
|
||||
code => 'event.set("lowercase_field", event.get("message").downcase)'
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
This filter will lowercase the `message` field, and set it to a new field called `lowercase_field`
|
||||
371
docs/reference/event-dependent-configuration.md
Normal file
371
docs/reference/event-dependent-configuration.md
Normal file
|
|
@ -0,0 +1,371 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/event-dependent-configuration.html
|
||||
---
|
||||
|
||||
# Accessing event data and fields [event-dependent-configuration]
|
||||
|
||||
A Logstash pipeline usually has three stages: inputs → filters → outputs. Inputs generate events, filters modify them, and outputs ship them elsewhere.
|
||||
|
||||
All events have properties. For example, an Apache access log has properties like status code (200, 404), request path ("/", "index.html"), HTTP verb (GET, POST), client IP address, and so forth. Logstash calls these properties "fields".
|
||||
|
||||
Some configuration options in Logstash require the existence of fields in order to function. Because inputs generate events, there are no fields to evaluate within the input block—they do not exist yet!
|
||||
|
||||
::::{important}
|
||||
[Field references](#logstash-config-field-references), [sprintf format](#sprintf), and [conditionals](#conditionals) do not work in input blocks. These configuration options depend on events and fields, and therefore, work only within filter and output blocks.
|
||||
::::
|
||||
|
||||
|
||||
|
||||
## Field references [logstash-config-field-references]
|
||||
|
||||
When you need to refer to a field by name, you can use the Logstash [field reference syntax](https://www.elastic.co/guide/en/logstash/current/field-references-deepdive.html).
|
||||
|
||||
The basic syntax to access a field is `[fieldname]`. If you are referring to a **top-level field**, you can omit the `[]` and simply use `fieldname`. To refer to a **nested field**, specify the full path to that field: `[top-level field][nested field]`.
|
||||
|
||||
For example, this event has five top-level fields (agent, ip, request, response, ua) and three nested fields (status, bytes, os).
|
||||
|
||||
```js
|
||||
{
|
||||
"agent": "Mozilla/5.0 (compatible; MSIE 9.0)",
|
||||
"ip": "192.168.24.44",
|
||||
"request": "/index.html"
|
||||
"response": {
|
||||
"status": 200,
|
||||
"bytes": 52353
|
||||
},
|
||||
"ua": {
|
||||
"os": "Windows 7"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
To reference the `os` field, specify `[ua][os]`. To reference a top-level field such as `request`, you can simply specify the field name.
|
||||
|
||||
For more detailed information, see [*Field References Deep Dive*](https://www.elastic.co/guide/en/logstash/current/field-references-deepdive.html).
|
||||
|
||||
|
||||
## sprintf format [sprintf]
|
||||
|
||||
The field reference format is also used in what Logstash calls *sprintf format*. This format enables you to embed field values in other strings. For example, the statsd output has an *increment* setting that enables you to keep a count of apache logs by status code:
|
||||
|
||||
```js
|
||||
output {
|
||||
statsd {
|
||||
increment => "apache.%{[response][status]}"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Similarly, you can convert the UTC timestamp in the `@timestamp` field into a string.
|
||||
|
||||
Instead of specifying a field name inside the curly braces, use the `%{{FORMAT}}` syntax where `FORMAT` is a [java time format](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/time/format/DateTimeFormatter.html#patterns).
|
||||
|
||||
For example, if you want to use the file output to write logs based on the event’s UTC date and hour and the `type` field:
|
||||
|
||||
```js
|
||||
output {
|
||||
file {
|
||||
path => "/var/log/%{type}.%{{yyyy.MM.dd.HH}}"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
::::{note}
|
||||
The sprintf format continues to support [deprecated joda time format](http://joda-time.sourceforge.net/apidocs/org/joda/time/format/DateTimeFormat.html) strings as well using the `%{+FORMAT}` syntax. These formats are not directly interchangeable, and we advise you to begin using the more modern Java Time format.
|
||||
::::
|
||||
|
||||
|
||||
::::{note}
|
||||
A Logstash timestamp represents an instant on the UTC-timeline, so using sprintf formatters will produce results that may not align with your machine-local timezone.
|
||||
::::
|
||||
|
||||
|
||||
You can generate a fresh timestamp by using `%{{TIME_NOW}}` syntax instead of relying on the value in `@timestamp`. This is particularly useful when you need to estimate the time span of each plugin.
|
||||
|
||||
```js
|
||||
input {
|
||||
heartbeat {
|
||||
add_field => { "heartbeat_time" => "%{{TIME_NOW}}" }
|
||||
}
|
||||
}
|
||||
filter {
|
||||
mutate {
|
||||
add_field => { "mutate_time" => "%{{TIME_NOW}}" }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
## Conditionals [conditionals]
|
||||
|
||||
Sometimes you want to filter or output an event only under certain conditions. For that, you can use a conditional.
|
||||
|
||||
Conditionals in Logstash look and act the same way they do in programming languages. Conditionals support `if`, `else if` and `else` statements and can be nested.
|
||||
|
||||
The conditional syntax is:
|
||||
|
||||
```js
|
||||
if EXPRESSION {
|
||||
...
|
||||
} else if EXPRESSION {
|
||||
...
|
||||
} else {
|
||||
...
|
||||
}
|
||||
```
|
||||
|
||||
What’s an expression? Comparison tests, boolean logic, and so on!
|
||||
|
||||
You can use these comparison operators:
|
||||
|
||||
* equality: `==`, `!=`, `<`, `>`, `<=`, `>=`
|
||||
* regexp: `=~`, `!~` (checks a pattern on the right against a string value on the left)
|
||||
* inclusion: `in`, `not in`
|
||||
|
||||
Supported boolean operators are:
|
||||
|
||||
* `and`, `or`, `nand`, `xor`
|
||||
|
||||
Supported unary operators are:
|
||||
|
||||
* `!`
|
||||
|
||||
Expressions can be long and complex. Expressions can contain other expressions, you can negate expressions with `!`, and you can group them with parentheses `(...)`.
|
||||
|
||||
For example, this conditional uses the mutate filter to remove the field `secret` if the field `action` has a value of `login`:
|
||||
|
||||
```js
|
||||
filter {
|
||||
if [action] == "login" {
|
||||
mutate { remove_field => "secret" }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
If an expression generates an error when it is evaluated, event processing stops and a warning message is written to the log. For example, comparing integer value `100` with string value `"100"` cannot be evaluated with certainty, and so processing stops and the error is logged.
|
||||
|
||||
To capture the full content of the message at the time the error occurs, set the log level to `debug`. Check out [Logging](/reference/logging.md) for more information about how to configure logging and available log levels.
|
||||
|
||||
You can specify multiple expressions in a single condition:
|
||||
|
||||
```js
|
||||
output {
|
||||
# Send production errors to pagerduty
|
||||
if [loglevel] == "ERROR" and [deployment] == "production" {
|
||||
pagerduty {
|
||||
...
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
You can use the `in` operator to test whether a field contains a specific string, key, or list element. Note that the semantic meaning of `in` can vary, based on the target type. For example, when applied to a string. `in` means "is a substring of". When applied to a collection type, `in` means "collection contains the exact value".
|
||||
|
||||
```js
|
||||
filter {
|
||||
if [foo] in [foobar] {
|
||||
mutate { add_tag => "field in field" }
|
||||
}
|
||||
if [foo] in "foo" {
|
||||
mutate { add_tag => "field in string" }
|
||||
}
|
||||
if "hello" in [greeting] {
|
||||
mutate { add_tag => "string in field" }
|
||||
}
|
||||
if [foo] in ["hello", "world", "foo"] {
|
||||
mutate { add_tag => "field in list" }
|
||||
}
|
||||
if [missing] in [alsomissing] {
|
||||
mutate { add_tag => "shouldnotexist" }
|
||||
}
|
||||
if !("foo" in ["hello", "world"]) {
|
||||
mutate { add_tag => "shouldexist" }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
You use the `not in` conditional the same way. For example, you could use `not in` to only route events to Elasticsearch when `grok` is successful:
|
||||
|
||||
```js
|
||||
output {
|
||||
if "_grokparsefailure" not in [tags] {
|
||||
elasticsearch { ... }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
You can check for the existence of a specific field, but there’s currently no way to differentiate between a field that doesn’t exist versus a field that’s simply false. The expression `if [foo]` returns `false` when:
|
||||
|
||||
* `[foo]` doesn’t exist in the event,
|
||||
* `[foo]` exists in the event, but is false, or
|
||||
* `[foo]` exists in the event, but is null
|
||||
|
||||
For more complex examples, see [Using Conditionals](/reference/config-examples.md#using-conditionals).
|
||||
|
||||
::::{note}
|
||||
Sprintf date/time format in conditionals is not currently supported. A workaround using the `@metadata` field is available. See [sprintf date/time format in conditionals](#date-time) for more details and an example.
|
||||
::::
|
||||
|
||||
|
||||
|
||||
## The @metadata field [metadata]
|
||||
|
||||
In Logstash, there is a special field called `@metadata`. The contents of `@metadata` are not part of any of your events at output time, which makes it great to use for conditionals, or extending and building event fields with field reference and `sprintf` formatting.
|
||||
|
||||
This configuration file yields events from STDIN. Whatever you type becomes the `message` field in the event. The `mutate` events in the filter block add a few fields, some nested in the `@metadata` field.
|
||||
|
||||
```ruby
|
||||
input { stdin { } }
|
||||
|
||||
filter {
|
||||
mutate { add_field => { "show" => "This data will be in the output" } }
|
||||
mutate { add_field => { "[@metadata][test]" => "Hello" } }
|
||||
mutate { add_field => { "[@metadata][no_show]" => "This data will not be in the output" } }
|
||||
}
|
||||
|
||||
output {
|
||||
if [@metadata][test] == "Hello" {
|
||||
stdout { codec => rubydebug }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Let’s see what comes out:
|
||||
|
||||
```ruby
|
||||
$ bin/logstash -f ../test.conf
|
||||
Pipeline main started
|
||||
asdf
|
||||
{
|
||||
"@timestamp" => 2016-06-30T02:42:51.496Z,
|
||||
"@version" => "1",
|
||||
"host" => "example.com",
|
||||
"show" => "This data will be in the output",
|
||||
"message" => "asdf"
|
||||
}
|
||||
```
|
||||
|
||||
The "asdf" typed in became the `message` field contents, and the conditional successfully evaluated the contents of the `test` field nested within the `@metadata` field. But the output did not show a field called `@metadata`, or its contents.
|
||||
|
||||
The `rubydebug` codec allows you to reveal the contents of the `@metadata` field if you add a config flag, `metadata => true`:
|
||||
|
||||
```ruby
|
||||
stdout { codec => rubydebug { metadata => true } }
|
||||
```
|
||||
|
||||
Let’s see what the output looks like with this change:
|
||||
|
||||
```ruby
|
||||
$ bin/logstash -f ../test.conf
|
||||
Pipeline main started
|
||||
asdf
|
||||
{
|
||||
"@timestamp" => 2016-06-30T02:46:48.565Z,
|
||||
"@metadata" => {
|
||||
"test" => "Hello",
|
||||
"no_show" => "This data will not be in the output"
|
||||
},
|
||||
"@version" => "1",
|
||||
"host" => "example.com",
|
||||
"show" => "This data will be in the output",
|
||||
"message" => "asdf"
|
||||
}
|
||||
```
|
||||
|
||||
Now you can see the `@metadata` field and its sub-fields.
|
||||
|
||||
::::{important}
|
||||
Only the `rubydebug` codec allows you to show the contents of the `@metadata` field.
|
||||
::::
|
||||
|
||||
|
||||
Make use of the `@metadata` field any time you need a temporary field but do not want it to be in the final output.
|
||||
|
||||
Perhaps one of the most common use cases for this new field is with the `date` filter and having a temporary timestamp.
|
||||
|
||||
This configuration file has been simplified, but uses the timestamp format common to Apache and Nginx web servers. In the past, you’d have to delete the timestamp field yourself, after using it to overwrite the `@timestamp` field. With the `@metadata` field, this is no longer necessary:
|
||||
|
||||
```ruby
|
||||
input { stdin { } }
|
||||
|
||||
filter {
|
||||
grok { match => [ "message", "%{HTTPDATE:[@metadata][timestamp]}" ] }
|
||||
date { match => [ "[@metadata][timestamp]", "dd/MMM/yyyy:HH:mm:ss Z" ] }
|
||||
}
|
||||
|
||||
output {
|
||||
stdout { codec => rubydebug }
|
||||
}
|
||||
```
|
||||
|
||||
Notice that this configuration puts the extracted date into the `[@metadata][timestamp]` field in the `grok` filter. Let’s feed this configuration a sample date string and see what comes out:
|
||||
|
||||
```ruby
|
||||
$ bin/logstash -f ../test.conf
|
||||
Pipeline main started
|
||||
02/Mar/2014:15:36:43 +0100
|
||||
{
|
||||
"@timestamp" => 2014-03-02T14:36:43.000Z,
|
||||
"@version" => "1",
|
||||
"host" => "example.com",
|
||||
"message" => "02/Mar/2014:15:36:43 +0100"
|
||||
}
|
||||
```
|
||||
|
||||
That’s it! No extra fields in the output, and a cleaner config file because you do not have to delete a "timestamp" field after conversion in the `date` filter.
|
||||
|
||||
Another use case is the [CouchDB Changes input plugin](https://github.com/logstash-plugins/logstash-input-couchdb_changes). This plugin automatically captures CouchDB document field metadata into the `@metadata` field within the input plugin itself. When the events pass through to be indexed by Elasticsearch, the Elasticsearch output plugin allows you to specify the `action` (delete, update, insert, etc.) and the `document_id`, like this:
|
||||
|
||||
```ruby
|
||||
output {
|
||||
elasticsearch {
|
||||
action => "%{[@metadata][action]}"
|
||||
document_id => "%{[@metadata][_id]}"
|
||||
hosts => ["example.com"]
|
||||
index => "index_name"
|
||||
protocol => "http"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
### sprintf date/time format in conditionals [date-time]
|
||||
|
||||
Sprintf date/time format in conditionals is not currently supported, but a workaround is available. Put the date calculation in a field so that you can use the field reference in a conditional.
|
||||
|
||||
**Example**
|
||||
|
||||
Using sprintf time format directly to add a field based on ingestion time *will not work*:
|
||||
|
||||
```
|
||||
----------
|
||||
# non-working example
|
||||
filter{
|
||||
if "%{+HH}:%{+mm}" < "16:30" {
|
||||
mutate {
|
||||
add_field => { "string_compare" => "%{+HH}:%{+mm} is before 16:30" }
|
||||
}
|
||||
}
|
||||
}
|
||||
----------
|
||||
```
|
||||
|
||||
This workaround gives you the intended results:
|
||||
|
||||
```js
|
||||
filter {
|
||||
mutate{
|
||||
add_field => {
|
||||
"[@metadata][time]" => "%{+HH}:%{+mm}"
|
||||
}
|
||||
}
|
||||
if [@metadata][time] < "16:30" {
|
||||
mutate {
|
||||
add_field => {
|
||||
"string_compare" => "%{+HH}:%{+mm} is before 16:30"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
13
docs/reference/execution-model.md
Normal file
13
docs/reference/execution-model.md
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/execution-model.html
|
||||
---
|
||||
|
||||
# Execution Model [execution-model]
|
||||
|
||||
The Logstash event processing pipeline coordinates the execution of inputs, filters, and outputs.
|
||||
|
||||
Each input stage in the Logstash pipeline runs in its own thread. Inputs write events to a central queue that is either in memory (default) or on disk. Each pipeline worker thread takes a batch of events off this queue, runs the batch of events through the configured filters, and then runs the filtered events through any outputs. The size of the batch and number of pipeline worker threads are configurable (see [Tuning and profiling logstash pipeline performance](/reference/tuning-logstash.md)).
|
||||
|
||||
By default, Logstash uses in-memory bounded queues between pipeline stages (input → filter and filter → output) to buffer events. If Logstash terminates unsafely, any events that are stored in memory will be lost. To help prevent data loss, you can enable Logstash to persist in-flight events to disk. See [Persistent queues (PQ)](/reference/persistent-queues.md) for more information.
|
||||
|
||||
102
docs/reference/field-extraction.md
Normal file
102
docs/reference/field-extraction.md
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/field-extraction.html
|
||||
---
|
||||
|
||||
# Extracting Fields and Wrangling Data [field-extraction]
|
||||
|
||||
The plugins described in this section are useful for extracting fields and parsing unstructured data into fields.
|
||||
|
||||
[dissect filter](/reference/plugins-filters-dissect.md)
|
||||
: Extracts unstructured event data into fields by using delimiters. The dissect filter does not use regular expressions and is very fast. However, if the structure of the data varies from line to line, the grok filter is more suitable.
|
||||
|
||||
For example, let’s say you have a log that contains the following message:
|
||||
|
||||
```json
|
||||
Apr 26 12:20:02 localhost systemd[1]: Starting system activity accounting tool...
|
||||
```
|
||||
|
||||
The following config dissects the message:
|
||||
|
||||
```json
|
||||
filter {
|
||||
dissect {
|
||||
mapping => { "message" => "%{ts} %{+ts} %{+ts} %{src} %{prog}[%{pid}]: %{msg}" }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
After the dissect filter is applied, the event will be dissected into the following fields:
|
||||
|
||||
```json
|
||||
{
|
||||
"msg" => "Starting system activity accounting tool...",
|
||||
"@timestamp" => 2017-04-26T19:33:39.257Z,
|
||||
"src" => "localhost",
|
||||
"@version" => "1",
|
||||
"host" => "localhost.localdomain",
|
||||
"pid" => "1",
|
||||
"message" => "Apr 26 12:20:02 localhost systemd[1]: Starting system activity accounting tool...",
|
||||
"type" => "stdin",
|
||||
"prog" => "systemd",
|
||||
"ts" => "Apr 26 12:20:02"
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
[kv filter](/reference/plugins-filters-kv.md)
|
||||
: Parses key-value pairs.
|
||||
|
||||
For example, let’s say you have a log message that contains the following key-value pairs:
|
||||
|
||||
```json
|
||||
ip=1.2.3.4 error=REFUSED
|
||||
```
|
||||
|
||||
The following config parses the key-value pairs into fields:
|
||||
|
||||
```json
|
||||
filter {
|
||||
kv { }
|
||||
}
|
||||
```
|
||||
|
||||
After the filter is applied, the event in the example will have these fields:
|
||||
|
||||
* `ip: 1.2.3.4`
|
||||
* `error: REFUSED`
|
||||
|
||||
|
||||
[grok filter](/reference/plugins-filters-grok.md)
|
||||
: Parses unstructured event data into fields. This tool is perfect for syslog logs, Apache and other webserver logs, MySQL logs, and in general, any log format that is generally written for humans and not computer consumption. Grok works by combining text patterns into something that matches your logs.
|
||||
|
||||
For example, let’s say you have an HTTP request log that contains the following message:
|
||||
|
||||
```json
|
||||
55.3.244.1 GET /index.html 15824 0.043
|
||||
```
|
||||
|
||||
The following config parses the message into fields:
|
||||
|
||||
```json
|
||||
filter {
|
||||
grok {
|
||||
match => { "message" => "%{IP:client} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes} %{NUMBER:duration}" }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
After the filter is applied, the event in the example will have these fields:
|
||||
|
||||
* `client: 55.3.244.1`
|
||||
* `method: GET`
|
||||
* `request: /index.html`
|
||||
* `bytes: 15824`
|
||||
* `duration: 0.043`
|
||||
|
||||
|
||||
::::{tip}
|
||||
If you need help building grok patterns, try the [Grok Debugger](docs-content://explore-analyze/query-filter/tools/grok-debugger.md). The Grok Debugger is an {{xpack}} feature under the Basic License and is therefore **free to use**.
|
||||
::::
|
||||
|
||||
|
||||
113
docs/reference/filter-plugins.md
Normal file
113
docs/reference/filter-plugins.md
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/filter-plugins.html
|
||||
---
|
||||
|
||||
# Filter plugins [filter-plugins]
|
||||
|
||||
A filter plugin performs intermediary processing on an event. Filters are often applied conditionally depending on the characteristics of the event.
|
||||
|
||||
The following filter plugins are available below. For a list of Elastic supported plugins, please consult the [Support Matrix](https://www.elastic.co/support/matrix#show_logstash_plugins).
|
||||
|
||||
| | | |
|
||||
| --- | --- | --- |
|
||||
| Plugin | Description | Github repository |
|
||||
| [age](/reference/plugins-filters-age.md) | Calculates the age of an event by subtracting the event timestamp from the current timestamp | [logstash-filter-age](https://github.com/logstash-plugins/logstash-filter-age) |
|
||||
| [aggregate](/reference/plugins-filters-aggregate.md) | Aggregates information from several events originating with a single task | [logstash-filter-aggregate](https://github.com/logstash-plugins/logstash-filter-aggregate) |
|
||||
| [alter](/reference/plugins-filters-alter.md) | Performs general alterations to fields that the `mutate` filter does not handle | [logstash-filter-alter](https://github.com/logstash-plugins/logstash-filter-alter) |
|
||||
| [bytes](/reference/plugins-filters-bytes.md) | Parses string representations of computer storage sizes, such as "123 MB" or "5.6gb", into their numeric value in bytes | [logstash-filter-bytes](https://github.com/logstash-plugins/logstash-filter-bytes) |
|
||||
| [cidr](/reference/plugins-filters-cidr.md) | Checks IP addresses against a list of network blocks | [logstash-filter-cidr](https://github.com/logstash-plugins/logstash-filter-cidr) |
|
||||
| [cipher](/reference/plugins-filters-cipher.md) | Applies or removes a cipher to an event | [logstash-filter-cipher](https://github.com/logstash-plugins/logstash-filter-cipher) |
|
||||
| [clone](/reference/plugins-filters-clone.md) | Duplicates events | [logstash-filter-clone](https://github.com/logstash-plugins/logstash-filter-clone) |
|
||||
| [csv](/reference/plugins-filters-csv.md) | Parses comma-separated value data into individual fields | [logstash-filter-csv](https://github.com/logstash-plugins/logstash-filter-csv) |
|
||||
| [date](/reference/plugins-filters-date.md) | Parses dates from fields to use as the Logstash timestamp for an event | [logstash-filter-date](https://github.com/logstash-plugins/logstash-filter-date) |
|
||||
| [de_dot](/reference/plugins-filters-de_dot.md) | Computationally expensive filter that removes dots from a field name | [logstash-filter-de_dot](https://github.com/logstash-plugins/logstash-filter-de_dot) |
|
||||
| [dissect](/reference/plugins-filters-dissect.md) | Extracts unstructured event data into fields using delimiters | [logstash-filter-dissect](https://github.com/logstash-plugins/logstash-filter-dissect) |
|
||||
| [dns](/reference/plugins-filters-dns.md) | Performs a standard or reverse DNS lookup | [logstash-filter-dns](https://github.com/logstash-plugins/logstash-filter-dns) |
|
||||
| [drop](/reference/plugins-filters-drop.md) | Drops all events | [logstash-filter-drop](https://github.com/logstash-plugins/logstash-filter-drop) |
|
||||
| [elapsed](/reference/plugins-filters-elapsed.md) | Calculates the elapsed time between a pair of events | [logstash-filter-elapsed](https://github.com/logstash-plugins/logstash-filter-elapsed) |
|
||||
| [elastic_integration](/reference/plugins-filters-elastic_integration.md) | Provides additional {{ls}} processing on data from Elastic integrations | [logstash-filter-elastic_integration](https://github.com/elastic/logstash-filter-elastic_integration) |
|
||||
| [elasticsearch](/reference/plugins-filters-elasticsearch.md) | Copies fields from previous log events in Elasticsearch to current events | [logstash-filter-elasticsearch](https://github.com/logstash-plugins/logstash-filter-elasticsearch) |
|
||||
| [environment](/reference/plugins-filters-environment.md) | Stores environment variables as metadata sub-fields | [logstash-filter-environment](https://github.com/logstash-plugins/logstash-filter-environment) |
|
||||
| [extractnumbers](/reference/plugins-filters-extractnumbers.md) | Extracts numbers from a string | [logstash-filter-extractnumbers](https://github.com/logstash-plugins/logstash-filter-extractnumbers) |
|
||||
| [fingerprint](/reference/plugins-filters-fingerprint.md) | Fingerprints fields by replacing values with a consistent hash | [logstash-filter-fingerprint](https://github.com/logstash-plugins/logstash-filter-fingerprint) |
|
||||
| [geoip](/reference/plugins-filters-geoip.md) | Adds geographical information about an IP address | [logstash-filter-geoip](https://github.com/logstash-plugins/logstash-filter-geoip) |
|
||||
| [grok](/reference/plugins-filters-grok.md) | Parses unstructured event data into fields | [logstash-filter-grok](https://github.com/logstash-plugins/logstash-filter-grok) |
|
||||
| [http](/reference/plugins-filters-http.md) | Provides integration with external web services/REST APIs | [logstash-filter-http](https://github.com/logstash-plugins/logstash-filter-http) |
|
||||
| [i18n](/reference/plugins-filters-i18n.md) | Removes special characters from a field | [logstash-filter-i18n](https://github.com/logstash-plugins/logstash-filter-i18n) |
|
||||
| [java_uuid](/reference/plugins-filters-java_uuid.md) | Generates a UUID and adds it to each processed event | [core plugin](https://github.com/elastic/logstash/blob/master/logstash-core/src/main/java/org/logstash/plugins/filters/Uuid.java) |
|
||||
| [jdbc_static](/reference/plugins-filters-jdbc_static.md) | Enriches events with data pre-loaded from a remote database | [logstash-integration-jdbc](https://github.com/logstash-plugins/logstash-integration-jdbc) |
|
||||
| [jdbc_streaming](/reference/plugins-filters-jdbc_streaming.md) | Enrich events with your database data | [logstash-integration-jdbc](https://github.com/logstash-plugins/logstash-integration-jdbc) |
|
||||
| [json](/reference/plugins-filters-json.md) | Parses JSON events | [logstash-filter-json](https://github.com/logstash-plugins/logstash-filter-json) |
|
||||
| [json_encode](/reference/plugins-filters-json_encode.md) | Serializes a field to JSON | [logstash-filter-json_encode](https://github.com/logstash-plugins/logstash-filter-json_encode) |
|
||||
| [kv](/reference/plugins-filters-kv.md) | Parses key-value pairs | [logstash-filter-kv](https://github.com/logstash-plugins/logstash-filter-kv) |
|
||||
| [memcached](/reference/plugins-filters-memcached.md) | Provides integration with external data in Memcached | [logstash-filter-memcached](https://github.com/logstash-plugins/logstash-filter-memcached) |
|
||||
| [metricize](/reference/plugins-filters-metricize.md) | Takes complex events containing a number of metrics and splits these up into multiple events, each holding a single metric | [logstash-filter-metricize](https://github.com/logstash-plugins/logstash-filter-metricize) |
|
||||
| [metrics](/reference/plugins-filters-metrics.md) | Aggregates metrics | [logstash-filter-metrics](https://github.com/logstash-plugins/logstash-filter-metrics) |
|
||||
| [mutate](/reference/plugins-filters-mutate.md) | Performs mutations on fields | [logstash-filter-mutate](https://github.com/logstash-plugins/logstash-filter-mutate) |
|
||||
| [prune](/reference/plugins-filters-prune.md) | Prunes event data based on a list of fields to blacklist or whitelist | [logstash-filter-prune](https://github.com/logstash-plugins/logstash-filter-prune) |
|
||||
| [range](/reference/plugins-filters-range.md) | Checks that specified fields stay within given size or length limits | [logstash-filter-range](https://github.com/logstash-plugins/logstash-filter-range) |
|
||||
| [ruby](/reference/plugins-filters-ruby.md) | Executes arbitrary Ruby code | [logstash-filter-ruby](https://github.com/logstash-plugins/logstash-filter-ruby) |
|
||||
| [sleep](/reference/plugins-filters-sleep.md) | Sleeps for a specified time span | [logstash-filter-sleep](https://github.com/logstash-plugins/logstash-filter-sleep) |
|
||||
| [split](/reference/plugins-filters-split.md) | Splits multi-line messages, strings, or arrays into distinct events | [logstash-filter-split](https://github.com/logstash-plugins/logstash-filter-split) |
|
||||
| [syslog_pri](/reference/plugins-filters-syslog_pri.md) | Parses the `PRI` (priority) field of a `syslog` message | [logstash-filter-syslog_pri](https://github.com/logstash-plugins/logstash-filter-syslog_pri) |
|
||||
| [threats_classifier](/reference/plugins-filters-threats_classifier.md) | Enriches security logs with information about the attacker’s intent | [logstash-filter-threats_classifier](https://github.com/empow/logstash-filter-threats_classifier) |
|
||||
| [throttle](/reference/plugins-filters-throttle.md) | Throttles the number of events | [logstash-filter-throttle](https://github.com/logstash-plugins/logstash-filter-throttle) |
|
||||
| [tld](/reference/plugins-filters-tld.md) | Replaces the contents of the default message field with whatever you specify in the configuration | [logstash-filter-tld](https://github.com/logstash-plugins/logstash-filter-tld) |
|
||||
| [translate](/reference/plugins-filters-translate.md) | Replaces field contents based on a hash or YAML file | [logstash-filter-translate](https://github.com/logstash-plugins/logstash-filter-translate) |
|
||||
| [truncate](/reference/plugins-filters-truncate.md) | Truncates fields longer than a given length | [logstash-filter-truncate](https://github.com/logstash-plugins/logstash-filter-truncate) |
|
||||
| [urldecode](/reference/plugins-filters-urldecode.md) | Decodes URL-encoded fields | [logstash-filter-urldecode](https://github.com/logstash-plugins/logstash-filter-urldecode) |
|
||||
| [useragent](/reference/plugins-filters-useragent.md) | Parses user agent strings into fields | [logstash-filter-useragent](https://github.com/logstash-plugins/logstash-filter-useragent) |
|
||||
| [uuid](/reference/plugins-filters-uuid.md) | Adds a UUID to events | [logstash-filter-uuid](https://github.com/logstash-plugins/logstash-filter-uuid) |
|
||||
| [wurfl_device_detection](/reference/plugins-filters-wurfl_device_detection.md) | Enriches logs with device information such as brand, model, OS | [logstash-filter-wurfl_device_detection](https://github.com/WURFL/logstash-filter-wurfl_device_detection) |
|
||||
| [xml](/reference/plugins-filters-xml.md) | Parses XML into fields | [logstash-filter-xml](https://github.com/logstash-plugins/logstash-filter-xml) |
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
70
docs/reference/first-event.md
Normal file
70
docs/reference/first-event.md
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/first-event.html
|
||||
---
|
||||
|
||||
# Stashing Your First Event [first-event]
|
||||
|
||||
First, let’s test your Logstash installation by running the most basic *Logstash pipeline*.
|
||||
|
||||
A Logstash pipeline has two required elements, `input` and `output`, and one optional element, `filter`. The input plugins consume data from a source, the filter plugins modify the data as you specify, and the output plugins write the data to a destination.
|
||||
|
||||
:::{image} ../images/basic_logstash_pipeline.png
|
||||
:alt: basic logstash pipeline
|
||||
:::
|
||||
|
||||
To test your Logstash installation, run the most basic Logstash pipeline.
|
||||
|
||||
**MacOS, Linux**
|
||||
|
||||
```sh
|
||||
cd logstash-9.0.0
|
||||
bin/logstash -e 'input { stdin { } } output { stdout {} }'
|
||||
```
|
||||
|
||||
**Windows**
|
||||
|
||||
```sh
|
||||
cd logstash-9.0.0
|
||||
.\bin\logstash.bat -e "input { stdin { } } output { stdout {} }"
|
||||
```
|
||||
|
||||
The command might vary slightly, depending on the terminal or shell you are using.
|
||||
|
||||
::::{note}
|
||||
The location of the `bin` directory varies by platform. See [Directory layout](/reference/dir-layout.md) to find the location of `bin\logstash` on your system.
|
||||
::::
|
||||
|
||||
|
||||
::::{admonition} macOS Gatekeeper warnings
|
||||
:class: important
|
||||
|
||||
Apple’s rollout of stricter notarization requirements affected the notarization of the 9.0.0-beta1 {{ls}} artifacts. If macOS Catalina displays a dialog when you first run {{ls}} that interrupts it, you will need to take an action to allow it to run. To prevent Gatekeeper checks on the {{ls}} files, run the following command on the downloaded `.tar.gz` archive or the directory to which was extracted:
|
||||
|
||||
```sh
|
||||
xattr -d -r com.apple.quarantine <archive-or-directory>
|
||||
```
|
||||
|
||||
For example, if the `.tar.gz` file was extracted to the default logstash-9.0.0-beta1 directory, the command is:
|
||||
|
||||
```sh
|
||||
xattr -d -r com.apple.quarantine logstash-9.0.0-beta1
|
||||
```
|
||||
|
||||
Alternatively, you can add a security override if a Gatekeeper popup appears by following the instructions in the *How to open an app that hasn’t been notarized or is from an unidentified developer* section of [Safely open apps on your Mac](https://support.apple.com/en-us/HT202491).
|
||||
|
||||
::::
|
||||
|
||||
|
||||
The `-e` flag enables you to specify a configuration directly from the command line. Specifying configurations at the command line lets you quickly test configurations without having to edit a file between iterations. The pipeline in the example takes input from the standard input, `stdin`, and moves that input to the standard output, `stdout`, in a structured format.
|
||||
|
||||
After starting Logstash, wait until you see "Pipeline main started" and then enter `hello world` at the command prompt:
|
||||
|
||||
```shell
|
||||
hello world
|
||||
2013-11-21T01:22:14.405+0000 0.0.0.0 hello world
|
||||
```
|
||||
|
||||
Logstash adds timestamp and IP address information to the message. Exit Logstash by issuing a **CTRL-D** command in the shell where Logstash is running.
|
||||
|
||||
Congratulations! You’ve created and run a basic Logstash pipeline. Next, you learn how to create a more realistic pipeline.
|
||||
121
docs/reference/getting-started-with-logstash.md
Normal file
121
docs/reference/getting-started-with-logstash.md
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/getting-started-with-logstash.html
|
||||
---
|
||||
|
||||
# Getting started with Logstash [getting-started-with-logstash]
|
||||
|
||||
This section guides you through the process of installing Logstash and verifying that everything is running properly. After learning how to stash your first event, you go on to create a more advanced pipeline that takes Apache web logs as input, parses the logs, and writes the parsed data to an Elasticsearch cluster. Then you learn how to stitch together multiple input and output plugins to unify data from a variety of disparate sources.
|
||||
|
||||
This section includes the following topics:
|
||||
|
||||
* [Java (JVM) version](#ls-jvm)
|
||||
* [Installing Logstash](/reference/installing-logstash.md)
|
||||
* [Stashing Your First Event](/reference/first-event.md)
|
||||
* [Parsing Logs with Logstash](/reference/advanced-pipeline.md)
|
||||
* [Stitching Together Multiple Input and Output Plugins](/reference/multiple-input-output-plugins.md)
|
||||
|
||||
|
||||
### Java (JVM) version [ls-jvm]
|
||||
|
||||
{{ls}} requires one of these versions:
|
||||
|
||||
* Java 17 (default). Check out [Using JDK 17](#jdk17-upgrade) for settings info.
|
||||
* Java 21
|
||||
|
||||
Use the [official Oracle distribution](http://www.oracle.com/technetwork/java/javase/downloads/index.html) or an open-source distribution, such as [OpenJDK](http://openjdk.java.net/). See the [Elastic Support Matrix](https://www.elastic.co/support/matrix#matrix_jvm) for the official word on supported versions across releases.
|
||||
|
||||
::::{admonition} Bundled JDK
|
||||
:class: note
|
||||
|
||||
:name: bundled-jdk
|
||||
|
||||
{{ls}} offers architecture-specific [downloads](https://www.elastic.co/downloads/logstash) that include Adoptium Eclipse Temurin 17, a long term support (LTS) release of the JDK.
|
||||
|
||||
Use the LS_JAVA_HOME environment variable if you want to use a JDK other than the version that is bundled. If you have the LS_JAVA_HOME environment variable set to use a custom JDK, Logstash will continue to use the JDK version you have specified, even after you upgrade.
|
||||
|
||||
::::
|
||||
|
||||
|
||||
|
||||
#### Check your Java version [check-jvm]
|
||||
|
||||
Run the following command:
|
||||
|
||||
```shell
|
||||
java -version
|
||||
```
|
||||
|
||||
On systems with Java installed, this command produces output similar to the following:
|
||||
|
||||
```shell
|
||||
openjdk version "17.0.12" 2024-07-16
|
||||
OpenJDK Runtime Environment Temurin-17.0.12+7 (build 17.0.12+7)
|
||||
OpenJDK 64-Bit Server VM Temurin-17.0.12+7 (build 17.0.12+7, mixed mode)
|
||||
```
|
||||
|
||||
|
||||
#### `LS_JAVA_HOME` [java-home]
|
||||
|
||||
{{ls}} includes a bundled JDK which has been verified to work with each specific version of {{ls}}, and generally provides the best performance and reliability. If you need to use a JDK other than the bundled version, then set the `LS_JAVA_HOME` environment variable to the version you want to use.
|
||||
|
||||
On some Linux systems, you may need to have the `LS_JAVA_HOME` environment exported before installing {{ls}}, particularly if you installed Java from a tarball. {{ls}} uses Java during installation to automatically detect your environment and install the correct startup method (SysV init scripts, Upstart, or systemd). If {{ls}} is unable to find the `LS_JAVA_HOME` environment variable during package installation, you may get an error message, and {{ls}} will not start properly.
|
||||
|
||||
|
||||
#### Using JDK 17 [jdk17-upgrade]
|
||||
|
||||
{{ls}} uses JDK 17 by default, but you need to update settings in `jvm.options` and `log4j2.properties` if you are upgrading from {{ls}} 7.11.x (or earlier) to 7.12 or later.
|
||||
|
||||
|
||||
##### Updates to `jvm.options` [_updates_to_jvm_options]
|
||||
|
||||
In the `config/jvm.options` file, remove all CMS related flags:
|
||||
|
||||
```shell
|
||||
## GC configuration
|
||||
-XX:+UseConcMarkSweepGC
|
||||
-XX:CMSInitiatingOccupancyFraction=75
|
||||
-XX:+UseCMSInitiatingOccupancyOnly
|
||||
```
|
||||
|
||||
For more information about how to use `jvm.options`, please refer to [JVM settings](/reference/jvm-settings.md).
|
||||
|
||||
|
||||
##### Updates to `log4j2.properties` [_updates_to_log4j2_properties]
|
||||
|
||||
In the `config/log4j2.properties`:
|
||||
|
||||
* Replace properties that start with `appender.rolling.avoid_pipelined_filter.*` with:
|
||||
|
||||
```shell
|
||||
appender.rolling.avoid_pipelined_filter.type = PipelineRoutingFilter
|
||||
```
|
||||
|
||||
* Replace properties that start with `appender.json_rolling.avoid_pipelined_filter.*` with:
|
||||
|
||||
```shell
|
||||
appender.json_rolling.avoid_pipelined_filter.type = PipelineRoutingFilter
|
||||
```
|
||||
|
||||
* Replace properties that start with `appender.routing.*` with:
|
||||
|
||||
```shell
|
||||
appender.routing.type = PipelineRouting
|
||||
appender.routing.name = pipeline_routing_appender
|
||||
appender.routing.pipeline.type = RollingFile
|
||||
appender.routing.pipeline.name = appender-${ctx:pipeline.id}
|
||||
appender.routing.pipeline.fileName = ${sys:ls.logs}/pipeline_${ctx:pipeline.id}.log
|
||||
appender.routing.pipeline.filePattern = ${sys:ls.logs}/pipeline_${ctx:pipeline.id}.%i.log.gz
|
||||
appender.routing.pipeline.layout.type = PatternLayout
|
||||
appender.routing.pipeline.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %m%n
|
||||
appender.routing.pipeline.policy.type = SizeBasedTriggeringPolicy
|
||||
appender.routing.pipeline.policy.size = 100MB
|
||||
appender.routing.pipeline.strategy.type = DefaultRolloverStrategy
|
||||
appender.routing.pipeline.strategy.max = 30
|
||||
```
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
44
docs/reference/glob-support.md
Normal file
44
docs/reference/glob-support.md
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/glob-support.html
|
||||
---
|
||||
|
||||
# Glob Pattern Support [glob-support]
|
||||
|
||||
Logstash supports the following patterns wherever glob patterns are allowed:
|
||||
|
||||
**`*`**
|
||||
: Match any file. You can also use an `*` to restrict other values in the glob. For example, `*conf` matches all files that end in `conf`. `*apache*` matches any files with `apache` in the name. This pattern does not match hidden files (dot files) on Unix-like operating systems. To match dot files, use a pattern like `{*,.*}`.
|
||||
|
||||
**`**`**
|
||||
: Match directories recursively.
|
||||
|
||||
**`?`**
|
||||
: Match any one character.
|
||||
|
||||
**`[set]`**
|
||||
: Match any one character in a set. For example, `[a-z]`. Also supports set negation (`[^a-z]`).
|
||||
|
||||
**`{p,q}`**
|
||||
: Match either literal `p` or literal `q`. The matching literal can be more than one character, and you can specify more than two literals. This pattern is the equivalent to using alternation with the vertical bar in regular expressions (`foo|bar`).
|
||||
|
||||
**`\`**
|
||||
: Escape the next metacharacter. This means that you cannot use a backslash in Windows as part of a glob. The pattern `c:\foo*` will not work, so use `foo*` instead.
|
||||
|
||||
|
||||
## Example Patterns [example-glob-patterns]
|
||||
|
||||
Here are some common examples of glob patterns:
|
||||
|
||||
`"/path/to/*.conf"`
|
||||
: Matches config files ending in `.conf` in the specified path.
|
||||
|
||||
`"/var/log/*.log"`
|
||||
: Matches log files ending in `.log` in the specified path.
|
||||
|
||||
`"/var/log/**/*.log"`
|
||||
: Matches log files ending in `.log` in subdirectories under the specified path.
|
||||
|
||||
`"/path/to/logs/{app1,app2,app3}/data.log"`
|
||||
: Matches app log files in the `app1`, `app2`, and `app3` subdirectories under the specified path.
|
||||
|
||||
59
docs/reference/how-logstash-works.md
Normal file
59
docs/reference/how-logstash-works.md
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/pipeline.html
|
||||
---
|
||||
|
||||
# How Logstash Works [pipeline]
|
||||
|
||||
The Logstash event processing pipeline has three stages: inputs → filters → outputs. Inputs generate events, filters modify them, and outputs ship them elsewhere. Inputs and outputs support codecs that enable you to encode or decode the data as it enters or exits the pipeline without having to use a separate filter.
|
||||
|
||||
|
||||
## Inputs [_inputs]
|
||||
|
||||
You use inputs to get data into Logstash. Some of the more commonly-used inputs are:
|
||||
|
||||
* **file**: reads from a file on the filesystem, much like the UNIX command `tail -0F`
|
||||
* **syslog**: listens on the well-known port 514 for syslog messages and parses according to the RFC3164 format
|
||||
* **redis**: reads from a redis server, using both redis channels and redis lists. Redis is often used as a "broker" in a centralized Logstash installation, which queues Logstash events from remote Logstash "shippers".
|
||||
* **beats**: processes events sent by [Beats](https://www.elastic.co/downloads/beats).
|
||||
|
||||
For more information about the available inputs, see [Input Plugins](/reference/input-plugins.md).
|
||||
|
||||
|
||||
## Filters [_filters]
|
||||
|
||||
Filters are intermediary processing devices in the Logstash pipeline. You can combine filters with conditionals to perform an action on an event if it meets certain criteria. Some useful filters include:
|
||||
|
||||
* **grok**: parse and structure arbitrary text. Grok is currently the best way in Logstash to parse unstructured log data into something structured and queryable. With 120 patterns built-in to Logstash, it’s more than likely you’ll find one that meets your needs!
|
||||
* **mutate**: perform general transformations on event fields. You can rename, remove, replace, and modify fields in your events.
|
||||
* **drop**: drop an event completely, for example, *debug* events.
|
||||
* **clone**: make a copy of an event, possibly adding or removing fields.
|
||||
* **geoip**: add information about geographical location of IP addresses (also displays amazing charts in Kibana!)
|
||||
|
||||
For more information about the available filters, see [Filter Plugins](/reference/filter-plugins.md).
|
||||
|
||||
|
||||
## Outputs [_outputs]
|
||||
|
||||
Outputs are the final phase of the Logstash pipeline. An event can pass through multiple outputs, but once all output processing is complete, the event has finished its execution. Some commonly used outputs include:
|
||||
|
||||
* **elasticsearch**: send event data to Elasticsearch. If you’re planning to save your data in an efficient, convenient, and easily queryable format… Elasticsearch is the way to go. Period. Yes, we’re biased :)
|
||||
* **file**: write event data to a file on disk.
|
||||
* **graphite**: send event data to graphite, a popular open source tool for storing and graphing metrics. [http://graphite.readthedocs.io/en/latest/](http://graphite.readthedocs.io/en/latest/)
|
||||
* **statsd**: send event data to statsd, a service that "listens for statistics, like counters and timers, sent over UDP and sends aggregates to one or more pluggable backend services". If you’re already using statsd, this could be useful for you!
|
||||
|
||||
For more information about the available outputs, see [Output Plugins](/reference/output-plugins.md).
|
||||
|
||||
|
||||
## Codecs [_codecs]
|
||||
|
||||
Codecs are basically stream filters that can operate as part of an input or output. Codecs enable you to easily separate the transport of your messages from the serialization process. Popular codecs include `json`, `msgpack`, and `plain` (text).
|
||||
|
||||
* **json**: encode or decode data in the JSON format.
|
||||
* **multiline**: merge multiple-line text events such as java exception and stacktrace messages into a single event.
|
||||
|
||||
For more information about the available codecs, see [Codec Plugins](/reference/codec-plugins.md).
|
||||
|
||||
|
||||
|
||||
|
||||
8
docs/reference/index.md
Normal file
8
docs/reference/index.md
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/index.html
|
||||
- https://www.elastic.co/guide/en/logstash/current/introduction.html
|
||||
- https://www.elastic.co/guide/en/serverless/current/elasticsearch-ingest-data-through-logstash.html
|
||||
---
|
||||
|
||||
# Logstash
|
||||
129
docs/reference/input-plugins.md
Normal file
129
docs/reference/input-plugins.md
Normal file
|
|
@ -0,0 +1,129 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/input-plugins.html
|
||||
---
|
||||
|
||||
# Input plugins [input-plugins]
|
||||
|
||||
An input plugin enables a specific source of events to be read by Logstash.
|
||||
|
||||
The following input plugins are available below. For a list of Elastic supported plugins, please consult the [Support Matrix](https://www.elastic.co/support/matrix#show_logstash_plugins).
|
||||
|
||||
| | | |
|
||||
| --- | --- | --- |
|
||||
| Plugin | Description | Github repository |
|
||||
| [azure_event_hubs](/reference/plugins-inputs-azure_event_hubs.md) | Receives events from Azure Event Hubs | [azure_event_hubs](https://github.com/logstash-plugins/logstash-input-azure_event_hubs) |
|
||||
| [beats](/reference/plugins-inputs-beats.md) | Receives events from the Elastic Beats framework | [logstash-input-beats](https://github.com/logstash-plugins/logstash-input-beats) |
|
||||
| [cloudwatch](/reference/plugins-inputs-cloudwatch.md) | Pulls events from the Amazon Web Services CloudWatch API | [logstash-input-cloudwatch](https://github.com/logstash-plugins/logstash-input-cloudwatch) |
|
||||
| [couchdb_changes](/reference/plugins-inputs-couchdb_changes.md) | Streams events from CouchDB’s `_changes` URI | [logstash-input-couchdb_changes](https://github.com/logstash-plugins/logstash-input-couchdb_changes) |
|
||||
| [dead_letter_queue](/reference/plugins-inputs-dead_letter_queue.md) | read events from Logstash’s dead letter queue | [logstash-input-dead_letter_queue](https://github.com/logstash-plugins/logstash-input-dead_letter_queue) |
|
||||
| [elastic_agent](/reference/plugins-inputs-elastic_agent.md) | Receives events from the Elastic Agent framework | [logstash-input-beats](https://github.com/logstash-plugins/logstash-input-beats) (shared) |
|
||||
| [elastic_serverless_forwarder](/reference/plugins-inputs-elastic_serverless_forwarder.md) | Accepts events from Elastic Serverless Forwarder | [logstash-input-elastic_serverless_forwarder](https://github.com/logstash-plugins/logstash-input-elastic_serverless_forwarder) |
|
||||
| [elasticsearch](/reference/plugins-inputs-elasticsearch.md) | Reads query results from an Elasticsearch cluster | [logstash-input-elasticsearch](https://github.com/logstash-plugins/logstash-input-elasticsearch) |
|
||||
| [exec](/reference/plugins-inputs-exec.md) | Captures the output of a shell command as an event | [logstash-input-exec](https://github.com/logstash-plugins/logstash-input-exec) |
|
||||
| [file](/reference/plugins-inputs-file.md) | Streams events from files | [logstash-input-file](https://github.com/logstash-plugins/logstash-input-file) |
|
||||
| [ganglia](/reference/plugins-inputs-ganglia.md) | Reads Ganglia packets over UDP | [logstash-input-ganglia](https://github.com/logstash-plugins/logstash-input-ganglia) |
|
||||
| [gelf](/reference/plugins-inputs-gelf.md) | Reads GELF-format messages from Graylog2 as events | [logstash-input-gelf](https://github.com/logstash-plugins/logstash-input-gelf) |
|
||||
| [generator](/reference/plugins-inputs-generator.md) | Generates random log events for test purposes | [logstash-input-generator](https://github.com/logstash-plugins/logstash-input-generator) |
|
||||
| [github](/reference/plugins-inputs-github.md) | Reads events from a GitHub webhook | [logstash-input-github](https://github.com/logstash-plugins/logstash-input-github) |
|
||||
| [google_cloud_storage](/reference/plugins-inputs-google_cloud_storage.md) | Extract events from files in a Google Cloud Storage bucket | [logstash-input-google_cloud_storage](https://github.com/logstash-plugins/logstash-input-google_cloud_storage) |
|
||||
| [google_pubsub](/reference/plugins-inputs-google_pubsub.md) | Consume events from a Google Cloud PubSub service | [logstash-input-google_pubsub](https://github.com/logstash-plugins/logstash-input-google_pubsub) |
|
||||
| [graphite](/reference/plugins-inputs-graphite.md) | Reads metrics from the `graphite` tool | [logstash-input-graphite](https://github.com/logstash-plugins/logstash-input-graphite) |
|
||||
| [heartbeat](/reference/plugins-inputs-heartbeat.md) | Generates heartbeat events for testing | [logstash-input-heartbeat](https://github.com/logstash-plugins/logstash-input-heartbeat) |
|
||||
| [http](/reference/plugins-inputs-http.md) | Receives events over HTTP or HTTPS | [logstash-input-http](https://github.com/logstash-plugins/logstash-input-http) |
|
||||
| [http_poller](/reference/plugins-inputs-http_poller.md) | Decodes the output of an HTTP API into events | [logstash-input-http_poller](https://github.com/logstash-plugins/logstash-input-http_poller) |
|
||||
| [imap](/reference/plugins-inputs-imap.md) | Reads mail from an IMAP server | [logstash-input-imap](https://github.com/logstash-plugins/logstash-input-imap) |
|
||||
| [irc](/reference/plugins-inputs-irc.md) | Reads events from an IRC server | [logstash-input-irc](https://github.com/logstash-plugins/logstash-input-irc) |
|
||||
| [java_generator](/reference/plugins-inputs-java_generator.md) | Generates synthetic log events | [core plugin](https://github.com/elastic/logstash/blob/master/logstash-core/src/main/java/org/logstash/plugins/inputs/Generator.java) |
|
||||
| [java_stdin](/reference/plugins-inputs-java_stdin.md) | Reads events from standard input | [core plugin](https://github.com/elastic/logstash/blob/master/logstash-core/src/main/java/org/logstash/plugins/inputs/Stdin.java) |
|
||||
| [jdbc](/reference/plugins-inputs-jdbc.md) | Creates events from JDBC data | [logstash-integration-jdbc](https://github.com/logstash-plugins/logstash-integration-jdbc) |
|
||||
| [jms](/reference/plugins-inputs-jms.md) | Reads events from a Jms Broker | [logstash-input-jms](https://github.com/logstash-plugins/logstash-input-jms) |
|
||||
| [jmx](/reference/plugins-inputs-jmx.md) | Retrieves metrics from remote Java applications over JMX | [logstash-input-jmx](https://github.com/logstash-plugins/logstash-input-jmx) |
|
||||
| [kafka](/reference/plugins-inputs-kafka.md) | Reads events from a Kafka topic | [logstash-integration-kafka](https://github.com/logstash-plugins/logstash-integration-kafka) |
|
||||
| [kinesis](/reference/plugins-inputs-kinesis.md) | Receives events through an AWS Kinesis stream | [logstash-input-kinesis](https://github.com/logstash-plugins/logstash-input-kinesis) |
|
||||
| [logstash](/reference/plugins-inputs-logstash.md) | Reads from {{ls}} output of another {{ls}} instance | [logstash-integration-logstash](https://github.com/logstash-plugins/logstash-integration-logstash) |
|
||||
| [log4j](/reference/plugins-inputs-log4j.md) | Reads events over a TCP socket from a Log4j `SocketAppender` object | [logstash-input-log4j](https://github.com/logstash-plugins/logstash-input-log4j) |
|
||||
| [lumberjack](/reference/plugins-inputs-lumberjack.md) | Receives events using the Lumberjack protocl | [logstash-input-lumberjack](https://github.com/logstash-plugins/logstash-input-lumberjack) |
|
||||
| [meetup](/reference/plugins-inputs-meetup.md) | Captures the output of command line tools as an event | [logstash-input-meetup](https://github.com/logstash-plugins/logstash-input-meetup) |
|
||||
| [pipe](/reference/plugins-inputs-pipe.md) | Streams events from a long-running command pipe | [logstash-input-pipe](https://github.com/logstash-plugins/logstash-input-pipe) |
|
||||
| [puppet_facter](/reference/plugins-inputs-puppet_facter.md) | Receives facts from a Puppet server | [logstash-input-puppet_facter](https://github.com/logstash-plugins/logstash-input-puppet_facter) |
|
||||
| [rabbitmq](/reference/plugins-inputs-rabbitmq.md) | Pulls events from a RabbitMQ exchange | [logstash-integration-rabbitmq](https://github.com/logstash-plugins/logstash-integration-rabbitmq) |
|
||||
| [redis](/reference/plugins-inputs-redis.md) | Reads events from a Redis instance | [logstash-input-redis](https://github.com/logstash-plugins/logstash-input-redis) |
|
||||
| [relp](/reference/plugins-inputs-relp.md) | Receives RELP events over a TCP socket | [logstash-input-relp](https://github.com/logstash-plugins/logstash-input-relp) |
|
||||
| [rss](/reference/plugins-inputs-rss.md) | Captures the output of command line tools as an event | [logstash-input-rss](https://github.com/logstash-plugins/logstash-input-rss) |
|
||||
| [s3](/reference/plugins-inputs-s3.md) | Streams events from files in a S3 bucket | [logstash-input-s3](https://github.com/logstash-plugins/logstash-input-s3) |
|
||||
| [s3-sns-sqs](/reference/plugins-inputs-s3-sns-sqs.md) | Reads logs from AWS S3 buckets using sqs | [logstash-input-s3-sns-sqs](https://github.com/cherweg/logstash-input-s3-sns-sqs) |
|
||||
| [salesforce](/reference/plugins-inputs-salesforce.md) | Creates events based on a Salesforce SOQL query | [logstash-input-salesforce](https://github.com/logstash-plugins/logstash-input-salesforce) |
|
||||
| [snmp](/reference/plugins-inputs-snmp.md) | Polls network devices using Simple Network Management Protocol (SNMP) | [logstash-integration-snmp](https://github.com/logstash-plugins/logstash-integration-snmp) |
|
||||
| [snmptrap](/reference/plugins-inputs-snmptrap.md) | Creates events based on SNMP trap messages | [logstash-integration-snmp](https://github.com/logstash-plugins/logstash-integration-snmp) |
|
||||
| [sqlite](/reference/plugins-inputs-sqlite.md) | Creates events based on rows in an SQLite database | [logstash-input-sqlite](https://github.com/logstash-plugins/logstash-input-sqlite) |
|
||||
| [sqs](/reference/plugins-inputs-sqs.md) | Pulls events from an Amazon Web Services Simple Queue Service queue | [logstash-input-sqs](https://github.com/logstash-plugins/logstash-input-sqs) |
|
||||
| [stdin](/reference/plugins-inputs-stdin.md) | Reads events from standard input | [logstash-input-stdin](https://github.com/logstash-plugins/logstash-input-stdin) |
|
||||
| [stomp](/reference/plugins-inputs-stomp.md) | Creates events received with the STOMP protocol | [logstash-input-stomp](https://github.com/logstash-plugins/logstash-input-stomp) |
|
||||
| [syslog](/reference/plugins-inputs-syslog.md) | Reads syslog messages as events | [logstash-input-syslog](https://github.com/logstash-plugins/logstash-input-syslog) |
|
||||
| [tcp](/reference/plugins-inputs-tcp.md) | Reads events from a TCP socket | [logstash-input-tcp](https://github.com/logstash-plugins/logstash-input-tcp) |
|
||||
| [twitter](/reference/plugins-inputs-twitter.md) | Reads events from the Twitter Streaming API | [logstash-input-twitter](https://github.com/logstash-plugins/logstash-input-twitter) |
|
||||
| [udp](/reference/plugins-inputs-udp.md) | Reads events over UDP | [logstash-input-udp](https://github.com/logstash-plugins/logstash-input-udp) |
|
||||
| [unix](/reference/plugins-inputs-unix.md) | Reads events over a UNIX socket | [logstash-input-unix](https://github.com/logstash-plugins/logstash-input-unix) |
|
||||
| [varnishlog](/reference/plugins-inputs-varnishlog.md) | Reads from the `varnish` cache shared memory log | [logstash-input-varnishlog](https://github.com/logstash-plugins/logstash-input-varnishlog) |
|
||||
| [websocket](/reference/plugins-inputs-websocket.md) | Reads events from a websocket | [logstash-input-websocket](https://github.com/logstash-plugins/logstash-input-websocket) |
|
||||
| [wmi](/reference/plugins-inputs-wmi.md) | Creates events based on the results of a WMI query | [logstash-input-wmi](https://github.com/logstash-plugins/logstash-input-wmi) |
|
||||
| [xmpp](/reference/plugins-inputs-xmpp.md) | Receives events over the XMPP/Jabber protocol | [logstash-input-xmpp](https://github.com/logstash-plugins/logstash-input-xmpp) |
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
63
docs/reference/installing-logstash.md
Normal file
63
docs/reference/installing-logstash.md
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/installing-logstash.html
|
||||
---
|
||||
|
||||
# Installing Logstash [installing-logstash]
|
||||
|
||||
|
||||
## Installing from a Downloaded Binary [installing-binary]
|
||||
|
||||
The {{ls}} binaries are available from [https://www.elastic.co/downloads](https://www.elastic.co/downloads/logstash). Download the Logstash installation file for your host environment—TAR.GZ, DEB, ZIP, or RPM.
|
||||
|
||||
Unpack the file. Do not install Logstash into a directory path that contains colon (:) characters.
|
||||
|
||||
::::{note}
|
||||
These packages are free to use under the Elastic license. They contain open source and free commercial features and access to paid commercial features. [Start a 30-day trial](docs-content://deploy-manage/license/manage-your-license-in-self-managed-cluster.md) to try out all of the paid commercial features. See the [Subscriptions](https://www.elastic.co/subscriptions) page for information about Elastic license levels.
|
||||
|
||||
Alternatively, you can download an `oss` package, which contains only features that are available under the Apache 2.0 license.
|
||||
|
||||
::::
|
||||
|
||||
|
||||
On supported Linux operating systems, you can use a package manager to install Logstash.
|
||||
|
||||
|
||||
## Installing from Package Repositories [package-repositories]
|
||||
|
||||
We also have repositories available for APT and YUM based distributions. Note that we only provide binary packages, but no source packages, as the packages are created as part of the Logstash build.
|
||||
|
||||
We have split the Logstash package repositories by version into separate urls to avoid accidental upgrades across major versions. For all 9.x.y releases use 9.x as version number.
|
||||
|
||||
We use the PGP key [D88E42B4](https://pgp.mit.edu/pks/lookup?op=vindex&search=0xD27D666CD88E42B4), Elastic’s Signing Key, with fingerprint
|
||||
|
||||
```
|
||||
4609 5ACC 8548 582C 1A26 99A9 D27D 666C D88E 42B4
|
||||
```
|
||||
to sign all our packages. It is available from [https://pgp.mit.edu](https://pgp.mit.edu).
|
||||
|
||||
::::{note}
|
||||
When installing from a package repository (or from the DEB or RPM installation file), you will need to run Logstash as a service. Please refer to [Running Logstash as a Service](/reference/running-logstash.md) for more information.
|
||||
|
||||
For testing purposes, you may still run Logstash from the command line, but you may need to define the default setting options (described in [Logstash Directory Layout](/reference/dir-layout.md)) manually. Please refer to [Running Logstash from the Command Line](/reference/running-logstash-command-line.md) for more information.
|
||||
|
||||
::::
|
||||
|
||||
|
||||
|
||||
### APT [_apt]
|
||||
|
||||
Version 9.0.0 of Logstash has not yet been released.
|
||||
|
||||
|
||||
### YUM [_yum]
|
||||
|
||||
Version 9.0.0 of Logstash has not yet been released.
|
||||
|
||||
|
||||
### Docker [_docker]
|
||||
|
||||
Images are available for running Logstash as a Docker container. They are available from the Elastic Docker registry.
|
||||
|
||||
See [Running Logstash on Docker](/reference/docker.md) for details on how to configure and run Logstash Docker containers.
|
||||
|
||||
27
docs/reference/integration-plugins.md
Normal file
27
docs/reference/integration-plugins.md
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/plugin-integrations.html
|
||||
---
|
||||
|
||||
# Integration plugins [plugin-integrations]
|
||||
|
||||
Integration plugins combine related plugins—inputs, outputs, and sometimes filters and codecs—into one package.
|
||||
|
||||
| | | |
|
||||
| --- | --- | --- |
|
||||
| Integration Plugin | Description | Github repository |
|
||||
| [aws](/reference/plugins-integrations-aws.md) | Plugins for use with Amazon Web Services (AWS). | [logstash-integration-aws](https://github.com/logstash-plugins/logstash-integration-aws) |
|
||||
| [elastic_enterprise_search (deprecated) ](/reference/plugins-integrations-elastic_enterprise_search.md) | [deprecated at {{stack}} version 9.0.0 and plugin version 3.0.1] Plugins for use with Elastic Enterprise Search. | [logstash-integration-elastic_enterprise_search](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search) |
|
||||
| [jdbc](/reference/plugins-integrations-jdbc.md) | Plugins for use with databases that provide JDBC drivers. | [logstash-integration-jdbc](https://github.com/logstash-plugins/logstash-integration-jdbc) |
|
||||
| [kafka](/reference/plugins-integrations-kafka.md) | Plugins for use with the Kafka distributed streaming platform. | [logstash-integration-kafka](https://github.com/logstash-plugins/logstash-integration-kafka) |
|
||||
| [logstash](/reference/plugins-integrations-logstash.md) | Plugins to enable {{ls}}-to-{{ls}} communication. | [logstash-integration-logstash](https://github.com/logstash-plugins/logstash-integration-logstash) |
|
||||
| [rabbitmq](/reference/plugins-integrations-rabbitmq.md) | Plugins for processing events to or from a RabbitMQ broker. | [logstash-integration-rabbitmq](https://github.com/logstash-plugins/logstash-integration-rabbitmq) |
|
||||
| [snmp](/reference/plugins-integrations-snmp.md) | Plugins for polling devices using Simple Network Management Protocol (SNMP) or creating events from SNMPtrap messages. | [logstash-integration-snmp](https://github.com/logstash-plugins/logstash-integration-snmp) |
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
152
docs/reference/jvm-settings.md
Normal file
152
docs/reference/jvm-settings.md
Normal file
|
|
@ -0,0 +1,152 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/jvm-settings.html
|
||||
---
|
||||
|
||||
# JVM settings [jvm-settings]
|
||||
|
||||
Configure JVM settings in the `jvm.options` [settings file](/reference/config-setting-files.md#settings-files). JVM settings can also be set via the [`LS_JAVA_OPTS`](#ls-java-opts) environment variable.
|
||||
|
||||
This file contains a line-delimited list of JVM arguments following a special syntax:
|
||||
|
||||
* lines consisting of whitespace only are ignored
|
||||
* lines beginning with `#` are treated as comments and are ignored
|
||||
|
||||
```text
|
||||
# this is a comment
|
||||
```
|
||||
|
||||
* lines beginning with a `-` are treated as a JVM option that applies independent of the version of the JVM
|
||||
|
||||
```text
|
||||
-Xmx2g
|
||||
```
|
||||
|
||||
* lines beginning with a number followed by a `:` followed by a `-` are treated as a JVM option that applies only if the version of the JVM matches the number
|
||||
|
||||
```text
|
||||
8:-Xmx2g
|
||||
```
|
||||
|
||||
* lines beginning with a number followed by a `-` followed by a `:` are treated as a JVM option that applies only if the version of the JVM is greater than or equal to the number
|
||||
|
||||
```text
|
||||
8-:-Xmx2g
|
||||
```
|
||||
|
||||
* lines beginning with a number followed by a `-` followed by a number followed by a `:` are treated as a JVM option that applies only if the version of the JVM falls in the inclusive range of the two numbers
|
||||
|
||||
```text
|
||||
8-9:-Xmx2g
|
||||
```
|
||||
|
||||
* all other lines are rejected
|
||||
|
||||
## Setting the memory size [memory-size]
|
||||
|
||||
The memory of the JVM executing {{ls}} can be divided in two zones: heap and off-heap memory. In the heap refers to Java heap, which contains all the Java objects created by {{ls}} during its operation, see [Setting the JVM heap size](#heap-size) for description on how to size it. What’s not part of the heap is named off-heap and consists of memory that can be used and controlled by {{ls}}, generally thread stacks, direct memory and memory mapped pages, check [Setting the off-heap size](#off-heap-size) for comprehensive descriptions. In off-heap space there is some space which is used by JVM and contains all the data structures functional to the execution of the virtual machine. This memory can’t be controlled by {{ls}} and the settings are rarely customized.
|
||||
|
||||
### Setting the JVM heap size [heap-size]
|
||||
|
||||
Here are some tips for adjusting the JVM heap size:
|
||||
|
||||
* The recommended heap size for typical ingestion scenarios should be no less than 4GB and no more than 8GB.
|
||||
* CPU utilization can increase unnecessarily if the heap size is too low, resulting in the JVM constantly garbage collecting. You can check for this issue by doubling the heap size to see if performance improves.
|
||||
* Do not increase the heap size past the amount of physical memory. Some memory must be left to run the OS and other processes. As a general guideline for most installations, don’t exceed 50-75% of physical memory. The more memory you have, the higher percentage you can use.
|
||||
* Set the minimum (Xms) and maximum (Xmx) heap allocation size to the same value to prevent the heap from resizing at runtime, which is a very costly process.
|
||||
* You can make more accurate measurements of the JVM heap by using either the `jmap` command line utility distributed with Java or by using VisualVM. For more info, see [Profiling the heap](/reference/tuning-logstash.md#profiling-the-heap).
|
||||
|
||||
|
||||
### Setting the off-heap size [off-heap-size]
|
||||
|
||||
The operating system, persistent queue mmap pages, direct memory, and other processes require memory in addition to memory allocated to heap size.
|
||||
|
||||
Internal JVM data structures, thread stacks, memory mapped files and direct memory for input/output (IO) operations are all parts of the off-heap JVM memory. Memory mapped files are not part of the Logstash’s process off-heap memory, but consume RAM when paging files from disk. These mapped files speed up the access to Persistent Queues pages, a performance improvement - or trade off - to reduce expensive disk operations such as read, write, and seek. Some network I/O operations also resort to in-process direct memory usage to avoid, for example, copying of buffers between network sockets. Input plugins such as Elastic Agent, Beats, TCP, and HTTP inputs, use direct memory. The zone for Thread stacks contains the list of stack frames for each Java thread created by the JVM; each frame keeps the local arguments passed during method calls. Read on [Setting the JVM stack size](#stacks-size) if the size needs to be adapted to the processing needs.
|
||||
|
||||
Plugins, depending on their type (inputs, filters, and outputs), have different thread models. Every input plugin runs in its own thread and can potentially spawn others. For example, each JDBC input plugin launches a scheduler thread. Netty based plugins like TCP, Beats or HTTP input spawn a thread pool with 2 * number_of_cores threads. Output plugins may also start helper threads, such as a connection management thread for each {{es}} output instance. Every pipeline, also, has its own thread responsible to manage the pipeline lifecycle.
|
||||
|
||||
To summarize, we have 3 categories of memory usage, where 2 can be limited by the JVM and the other relies on available, free memory:
|
||||
|
||||
| Memory Type | Configured using | Used by |
|
||||
| --- | --- | --- |
|
||||
| JVM Heap | -Xmx | any normal object allocation |
|
||||
| JVM direct memory | -XX:MaxDirectMemorySize | beats, tcp and http inputs |
|
||||
| Native memory | N/A | Persistent Queue Pages, Thread Stacks |
|
||||
|
||||
Keep these memory requirements in mind as you calculate your ideal memory allocation.
|
||||
|
||||
|
||||
### Buffer Allocation types [off-heap-buffers-allocation]
|
||||
|
||||
Input plugins such as {{agent}}, {{beats}}, TCP, and HTTP allocate buffers in Java heap memory to read events from the network. Heap memory is the preferred allocation method, as it facilitates debugging memory usage problems (such as leaks and Out of Memory errors) through the analysis of heap dumps.
|
||||
|
||||
Before version 9.0.0, {{ls}} defaulted to direct memory instead of heap for this purpose. To re-enable the previous behavior {{ls}} provides a `pipeline.buffer.type` setting in [logstash.yml](/reference/logstash-settings-file.md) that lets you control where to allocate memory buffers for plugins that use them.
|
||||
|
||||
Performance should not be noticeably affected if you switch between `direct` and `heap`. While copying bytes from OS buffers to direct memory buffers is faster, {{ls}} Event objects produced by these plugins are allocated on the Java Heap, incurring the cost of copying from direct memory to heap memory, regardless of the setting.
|
||||
|
||||
|
||||
### Memory sizing [memory-size-calculation]
|
||||
|
||||
Total JVM memory allocation must be estimated and is controlled indirectly using Java heap and direct memory settings. By default, a JVM’s off-heap direct memory limit is the same as the heap size. Check out [beats input memory usage](/reference/plugins-inputs-beats.md#plugins-inputs-beats-memory). Consider setting `-XX:MaxDirectMemorySize` to half of the heap size or any value that can accommodate the load you expect these plugins to handle.
|
||||
|
||||
As you make your capacity calculations, keep in mind that the JVM can’t consume the total amount of the host’s memory available, as the Operating System and other processes will require memory too.
|
||||
|
||||
For a {{ls}} instance with persistent queue (PQ) enabled on multiple pipelines, we could estimate memory consumption using:
|
||||
|
||||
```text
|
||||
pipelines number * (pipeline threads * stack size + 2 * PQ page size) + direct memory + Java heap
|
||||
```
|
||||
|
||||
::::{note}
|
||||
Each Persistent Queue requires that at least head and tail pages are present and accessible in memory. The default page size is 64 MB so each PQ requires at least 128 MB of heap memory, which can be a significant source of memory consumption per pipeline. Note that the size of memory mapped file can’t be limited with an upper bound.
|
||||
::::
|
||||
|
||||
|
||||
::::{note}
|
||||
Stack size is a setting that depends on the JVM used, but could be customized with `-Xss` setting.
|
||||
::::
|
||||
|
||||
|
||||
::::{note}
|
||||
Direct memory space by default is big as much as Java heap, but can be customized with the `-XX:MaxDirectMemorySize` setting.
|
||||
::::
|
||||
|
||||
|
||||
**Example**
|
||||
|
||||
Consider a {{ls}} instance running 10 pipelines, with simple input and output plugins that doesn’t start additional threads, it has 1 pipelines thread, 1 input plugin thread and 12 workers, summing up to 14. Keep in mind that, by default, JVM allocates direct memory equal to memory allocated for Java heap.
|
||||
|
||||
The calculation results in:
|
||||
|
||||
* native memory: 1.4Gb [derived from 10 * (14 * 1Mb + 128Mb)]
|
||||
* direct memory: 4Gb
|
||||
* Java heap: 4Gb
|
||||
|
||||
|
||||
|
||||
## Setting the JVM stack size [stacks-size]
|
||||
|
||||
Large configurations may require additional JVM stack memory. If you see a stack overflow error, try increasing the JVM stack size. Add an entry similar to this one in the `jvm.options` [settings file](/reference/config-setting-files.md#settings-files):
|
||||
|
||||
```sh
|
||||
-Xss4M
|
||||
```
|
||||
|
||||
Note that the default stack size is different per platform and per OS flavor. You can find out what the default is by running:
|
||||
|
||||
```sh
|
||||
java -XX:+PrintFlagsFinal -version | grep ThreadStackSize
|
||||
```
|
||||
|
||||
Depending on the default stack size, start by multiplying by 4x, then 8x, and then 16x until the overflow error resolves.
|
||||
|
||||
|
||||
## Using `LS_JAVA_OPTS` [ls-java-opts]
|
||||
|
||||
The `LS_JAVA_OPTS` environment variable can also be used to override JVM settings in the `jvm.options` file [settings file](/reference/config-setting-files.md#settings-files). The content of this variable is additive to options configured in the `jvm.options` file, and will override any settings that exist in both places.
|
||||
|
||||
For example to set a different locale to launch {{ls}} instance:
|
||||
|
||||
```sh
|
||||
LS_JAVA_OPTS="-Duser.country=DE -Duser.language=de" bin/logstash -e 'input { stdin { codec => json } }'
|
||||
```
|
||||
159
docs/reference/keystore.md
Normal file
159
docs/reference/keystore.md
Normal file
|
|
@ -0,0 +1,159 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/keystore.html
|
||||
---
|
||||
|
||||
# Secrets keystore for secure settings [keystore]
|
||||
|
||||
When you configure Logstash, you might need to specify sensitive settings or configuration, such as passwords. Rather than relying on file system permissions to protect these values, you can use the Logstash keystore to securely store secret values for use in configuration settings.
|
||||
|
||||
After adding a key and its secret value to the keystore, you can use the key in place of the secret value when you configure sensitive settings.
|
||||
|
||||
The syntax for referencing keys is identical to the syntax for [environment variables](/reference/environment-variables.md):
|
||||
|
||||
```txt
|
||||
${KEY}
|
||||
```
|
||||
|
||||
Where KEY is the name of the key.
|
||||
|
||||
**Example**
|
||||
|
||||
Imagine that the keystore contains a key called `ES_PWD` with the value `yourelasticsearchpassword`.
|
||||
|
||||
In configuration files, use:
|
||||
|
||||
```shell
|
||||
output { elasticsearch {...password => "${ES_PWD}" } } }
|
||||
```
|
||||
|
||||
In `logstash.yml`, use:
|
||||
|
||||
```shell
|
||||
xpack.management.elasticsearch.password: ${ES_PWD}
|
||||
```
|
||||
|
||||
Notice that the Logstash keystore differs from the Elasticsearch keystore. Whereas the Elasticsearch keystore lets you store `elasticsearch.yml` values by name, the Logstash keystore lets you specify arbitrary names that you can reference in the Logstash configuration.
|
||||
|
||||
::::{note}
|
||||
There are some configuration fields that have no secret meaning, so not every field could leverage the secret store for variables substitution. Plugin’s `id` field is a field of this kind
|
||||
::::
|
||||
|
||||
|
||||
::::{note}
|
||||
Referencing keystore data from `pipelines.yml` or the command line (`-e`) is not currently supported.
|
||||
::::
|
||||
|
||||
|
||||
::::{note}
|
||||
Referencing keystore data from [centralized pipeline management](/reference/logstash-centralized-pipeline-management.md) requires each Logstash deployment to have a local copy of the keystore.
|
||||
::::
|
||||
|
||||
|
||||
::::{note}
|
||||
The {{ls}} keystore needs to be protected, but the {{ls}} user must have access to the file. While most things in {{ls}} can be protected with `chown -R root:root <foo>`, the keystore itself must be accessible from the {{ls}} user. Use `chown logstash:root <keystore> && chmod 0600 <keystore>`.
|
||||
::::
|
||||
|
||||
|
||||
When Logstash parses the settings (`logstash.yml`) or configuration (`/etc/logstash/conf.d/*.conf`), it resolves keys from the keystore before resolving environment variables.
|
||||
|
||||
|
||||
## Keystore password [keystore-password]
|
||||
|
||||
You can protect access to the Logstash keystore by storing a password in an environment variable called `LOGSTASH_KEYSTORE_PASS`. If you create the Logstash keystore after setting this variable, the keystore will be password protected. This means that the environment variable needs to be accessible to the running instance of Logstash. This environment variable must also be correctly set for any users who need to issue keystore commands (add, list, remove, etc.).
|
||||
|
||||
Using a keystore password is recommended, but optional. The data will be encrypted even if you do not set a password. However, it is highly recommended to configure the keystore password and grant restrictive permissions to any files that may contain the environment variable value. If you choose not to set a password, then you can skip the rest of this section.
|
||||
|
||||
For example:
|
||||
|
||||
```sh
|
||||
set +o history
|
||||
export LOGSTASH_KEYSTORE_PASS=mypassword
|
||||
set -o history
|
||||
bin/logstash-keystore create
|
||||
```
|
||||
|
||||
This setup requires the user running Logstash to have the environment variable `LOGSTASH_KEYSTORE_PASS=mypassword` defined. If the environment variable is not defined, Logstash cannot access the keystore.
|
||||
|
||||
When you run Logstash from an RPM or DEB package installation, the environment variables are sourced from `/etc/sysconfig/logstash`.
|
||||
|
||||
::::{note}
|
||||
You might need to create `/etc/sysconfig/logstash`. This file should be owned by `root` with `600` permissions. The expected format of `/etc/sysconfig/logstash` is `ENVIRONMENT_VARIABLE=VALUE`, with one entry per line.
|
||||
::::
|
||||
|
||||
|
||||
For other distributions, such as Docker or ZIP, see the documentation for your runtime environment (Windows, Docker, etc) to learn how to set the environment variable for the user that runs Logstash. Ensure that the environment variable (and thus the password) is only accessible to that user.
|
||||
|
||||
|
||||
## Keystore location [keystore-location]
|
||||
|
||||
The keystore must be located in the Logstash `path.settings` directory. This is the same directory that contains the `logstash.yml` file. When performing any operation against the keystore, it is recommended to set `path.settings` for the keystore command. For example, to create a keystore on a RPM/DEB installation:
|
||||
|
||||
```sh
|
||||
set +o history
|
||||
export LOGSTASH_KEYSTORE_PASS=mypassword
|
||||
set -o history
|
||||
sudo -E /usr/share/logstash/bin/logstash-keystore --path.settings /etc/logstash create
|
||||
```
|
||||
|
||||
See [Logstash Directory Layout](/reference/dir-layout.md) for more about the default directory locations.
|
||||
|
||||
::::{note}
|
||||
You will see a warning if the `path.settings` is not pointed to the same directory as the `logstash.yml`.
|
||||
::::
|
||||
|
||||
|
||||
|
||||
## Create or overwrite a keystore [creating-keystore]
|
||||
|
||||
The `create` command creates a new keystore or overwrites an existing keystore:
|
||||
|
||||
```sh
|
||||
bin/logstash-keystore create
|
||||
```
|
||||
|
||||
Creates the keystore in the directory defined in the `path.settings` setting.
|
||||
|
||||
::::{important}
|
||||
If a keystore already exists, the `create` command can overwrite it (after a Y/N prompt). Selecting `Y` clears all keys and secrets that were previously stored.
|
||||
::::
|
||||
|
||||
|
||||
::::{tip}
|
||||
Set a [keystore password](#keystore-password) when you create the keystore.
|
||||
::::
|
||||
|
||||
|
||||
|
||||
## Add keys [add-keys-to-keystore]
|
||||
|
||||
To store sensitive values, such as authentication credentials for Elasticsearch, use the `add` command:
|
||||
|
||||
```sh
|
||||
bin/logstash-keystore add ES_USER ES_PWD
|
||||
```
|
||||
|
||||
When prompted, enter a value for each key.
|
||||
|
||||
::::{note}
|
||||
Key values are limited to ASCII characters. It includes digits, letters, and a few special symbols.
|
||||
::::
|
||||
|
||||
|
||||
|
||||
## List keys [list-settings]
|
||||
|
||||
To list the keys defined in the keystore, use:
|
||||
|
||||
```sh
|
||||
bin/logstash-keystore list
|
||||
```
|
||||
|
||||
|
||||
## Remove keys [remove-settings]
|
||||
|
||||
To remove keys from the keystore, use:
|
||||
|
||||
```sh
|
||||
bin/logstash-keystore remove ES_USER ES_PWD
|
||||
```
|
||||
227
docs/reference/logging.md
Normal file
227
docs/reference/logging.md
Normal file
|
|
@ -0,0 +1,227 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/logging.html
|
||||
---
|
||||
|
||||
# Logging [logging]
|
||||
|
||||
Logstash emits internal logs during its operation, which are placed in `LS_HOME/logs` (or `/var/log/logstash` for DEB/RPM). The default logging level is `INFO`. Logstash’s logging framework is based on [Log4j 2 framework](http://logging.apache.org/log4j/2.x/), and much of its functionality is exposed directly to users.
|
||||
|
||||
You can configure logging for a particular subsystem, module, or plugin.
|
||||
|
||||
When you need to debug problems, particularly problems with plugins, consider increasing the logging level to `DEBUG` to get more verbose messages. For example, if you are debugging issues with Elasticsearch Output, you can increase log levels just for that component. This approach reduces noise from excessive logging and helps you focus on the problem area.
|
||||
|
||||
You can configure logging using the `log4j2.properties` file or the Logstash API.
|
||||
|
||||
* **`log4j2.properties` file.** Changes made through the `log4j2.properties` file require you to restart Logstash for the changes to take effect. Changes **persist** through subsequent restarts.
|
||||
* **Logging API.** Changes made through the Logging API are effective immediately without a restart. The changes **do not persist** after Logstash is restarted.
|
||||
|
||||
## Log4j2 configuration [log4j2]
|
||||
|
||||
Logstash ships with a `log4j2.properties` file with out-of-the-box settings, including logging to console. You can modify this file to change the rotation policy, type, and other [log4j2 configuration](https://logging.apache.org/log4j/2.x/manual/configuration.html#Loggers).
|
||||
|
||||
You must restart Logstash to apply any changes that you make to this file. Changes to `log4j2.properties` persist after Logstash is restarted.
|
||||
|
||||
Here’s an example using `outputs.elasticsearch`:
|
||||
|
||||
```yaml
|
||||
logger.elasticsearchoutput.name = logstash.outputs.elasticsearch
|
||||
logger.elasticsearchoutput.level = debug
|
||||
```
|
||||
|
||||
The previous example defines a name and level for the logger `logstash.outputs.elasticsearch`. The logger is usually identified by a Java class name, such as `org.logstash.dissect.Dissector`, for example. It can also be a partial package path as in `org.logstash.dissect`. For Ruby classes, like `LogStash::Outputs::Elasticsearch`, the logger name is obtained by lowercasing the full class name and replacing double colons with a single dot.
|
||||
|
||||
::::{note}
|
||||
Consider using the default log4j configuration that is shipped with {{ls}}, as it is configured to work well for most deployments. The next section describes how the rolling strategy works in case you need to make adjustments.
|
||||
::::
|
||||
|
||||
|
||||
### Rollover settings [rollover]
|
||||
|
||||
The `log4j2.properties` file has three appenders for writing to log files: one for plain text, one with json format, and one to split log lines on per pipeline basis when you set the `pipeline.separate_logs` value.
|
||||
|
||||
These appenders define:
|
||||
|
||||
* **triggering policies** that determine *if* a rollover should be performed, and
|
||||
* **rollover strategy** to defines *how* the rollover should be done.
|
||||
|
||||
By default, two triggering policies are defined—time and size.
|
||||
|
||||
* The **time** policy creates one file per day.
|
||||
* The **size** policy forces the creation of a new file after the file size surpasses 100 MB.
|
||||
|
||||
The default strategy also performs file rollovers based on a **maximum number of files**. When the limit of 30 files has been reached, the first (oldest) file is removed to create space for the new file. Subsequent files are renumbered accordingly.
|
||||
|
||||
Each file has a date, and files older than 7 days (default) are removed during rollover.
|
||||
|
||||
```text
|
||||
appender.rolling.type = RollingFile <1>
|
||||
appender.rolling.name = plain_rolling
|
||||
appender.rolling.fileName = ${sys:ls.logs}/logstash-plain.log <2>
|
||||
appender.rolling.filePattern = ${sys:ls.logs}/logstash-plain-%d{yyyy-MM-dd}-%i.log.gz <3>
|
||||
appender.rolling.policies.type = Policies
|
||||
appender.rolling.policies.time.type = TimeBasedTriggeringPolicy <4>
|
||||
appender.rolling.policies.time.interval = 1
|
||||
appender.rolling.policies.time.modulate = true
|
||||
appender.rolling.layout.type = PatternLayout
|
||||
appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c]%notEmpty{[%X{pipeline.id}]}%notEmpty{[%X{plugin.id}]} %m%n
|
||||
appender.rolling.policies.size.type = SizeBasedTriggeringPolicy <5>
|
||||
appender.rolling.policies.size.size = 100MB
|
||||
appender.rolling.strategy.type = DefaultRolloverStrategy
|
||||
appender.rolling.strategy.max = 30 <6>
|
||||
appender.rolling.strategy.action.type = Delete <7>
|
||||
appender.rolling.strategy.action.basepath = ${sys:ls.logs}
|
||||
appender.rolling.strategy.action.condition.type = IfFileName
|
||||
appender.rolling.strategy.action.condition.glob = logstash-plain-* <8>
|
||||
appender.rolling.strategy.action.condition.nested_condition.type = IfLastModified
|
||||
appender.rolling.strategy.action.condition.nested_condition.age = 7D <9>
|
||||
```
|
||||
|
||||
1. The appender type, which rolls older log files.
|
||||
2. Name of the current log file.
|
||||
3. Name’s format definition of the rolled files, in this case a date followed by an incremental number, up to 30 (by default).
|
||||
4. Time policy to trigger a rollover at the end of the day.
|
||||
5. Size policy to trigger a rollover once the plain text file reaches the size of 100 MB.
|
||||
6. Rollover strategy defines a maximum of 30 files.
|
||||
7. Action to execute during the rollover.
|
||||
8. The file set to consider by the action.
|
||||
9. Condition to execute the rollover action: older than 7 days.
|
||||
|
||||
|
||||
The rollover action can also enforce a disk usage limit, deleting older files to match the requested condition, as an example:
|
||||
|
||||
```text
|
||||
appender.rolling.type = RollingFile
|
||||
...
|
||||
appender.rolling.strategy.action.condition.glob = pipeline_${ctx:pipeline.id}.*.log.gz
|
||||
appender.rolling.strategy.action.condition.nested_condition.type = IfAccumulatedFileSize
|
||||
appender.rolling.strategy.action.condition.nested_condition.exceeds = 5MB <1>
|
||||
```
|
||||
|
||||
1. Deletes files if total accumulated compressed file size is over 5MB.
|
||||
|
||||
|
||||
|
||||
|
||||
## Logging APIs [_logging_apis]
|
||||
|
||||
For temporary logging changes, modifying the `log4j2.properties` file and restarting Logstash leads to unnecessary downtime. Instead, you can dynamically update logging levels through the logging API. These settings are effective immediately and do not need a restart.
|
||||
|
||||
::::{note}
|
||||
By default, the logging API attempts to bind to `tcp:9600`. If this port is already in use by another Logstash instance, you need to launch Logstash with the `--api.http.port` flag specified to bind to a different port. See [Command-Line Flags](/reference/running-logstash-command-line.md#command-line-flags) for more information.
|
||||
::::
|
||||
|
||||
|
||||
### Retrieve list of logging configurations [_retrieve_list_of_logging_configurations]
|
||||
|
||||
To retrieve a list of logging subsystems available at runtime, you can do a `GET` request to `_node/logging`
|
||||
|
||||
```js
|
||||
curl -XGET 'localhost:9600/_node/logging?pretty'
|
||||
```
|
||||
|
||||
Example response:
|
||||
|
||||
```js
|
||||
{
|
||||
...
|
||||
"loggers" : {
|
||||
"logstash.agent" : "INFO",
|
||||
"logstash.api.service" : "INFO",
|
||||
"logstash.basepipeline" : "INFO",
|
||||
"logstash.codecs.plain" : "INFO",
|
||||
"logstash.codecs.rubydebug" : "INFO",
|
||||
"logstash.filters.grok" : "INFO",
|
||||
"logstash.inputs.beats" : "INFO",
|
||||
"logstash.instrument.periodicpoller.jvm" : "INFO",
|
||||
"logstash.instrument.periodicpoller.os" : "INFO",
|
||||
"logstash.instrument.periodicpoller.persistentqueue" : "INFO",
|
||||
"logstash.outputs.stdout" : "INFO",
|
||||
"logstash.pipeline" : "INFO",
|
||||
"logstash.plugins.registry" : "INFO",
|
||||
"logstash.runner" : "INFO",
|
||||
"logstash.shutdownwatcher" : "INFO",
|
||||
"org.logstash.Event" : "INFO",
|
||||
"slowlog.logstash.codecs.plain" : "TRACE",
|
||||
"slowlog.logstash.codecs.rubydebug" : "TRACE",
|
||||
"slowlog.logstash.filters.grok" : "TRACE",
|
||||
"slowlog.logstash.inputs.beats" : "TRACE",
|
||||
"slowlog.logstash.outputs.stdout" : "TRACE"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
### Update logging levels [_update_logging_levels]
|
||||
|
||||
Prepend the name of the subsystem, module, or plugin with `logger.`.
|
||||
|
||||
Here is an example using `outputs.elasticsearch`:
|
||||
|
||||
```js
|
||||
curl -XPUT 'localhost:9600/_node/logging?pretty' -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"logger.logstash.outputs.elasticsearch" : "DEBUG"
|
||||
}
|
||||
'
|
||||
```
|
||||
|
||||
While this setting is in effect, Logstash emits DEBUG-level logs for *all* the Elasticsearch outputs specified in your configuration. Please note this new setting is transient and will not survive a restart.
|
||||
|
||||
::::{note}
|
||||
If you want logging changes to persist after a restart, add them to `log4j2.properties` instead.
|
||||
::::
|
||||
|
||||
|
||||
|
||||
### Reset dynamic logging levels [_reset_dynamic_logging_levels]
|
||||
|
||||
To reset any logging levels that may have been dynamically changed via the logging API, send a `PUT` request to `_node/logging/reset`. All logging levels will revert to the values specified in the `log4j2.properties` file.
|
||||
|
||||
```js
|
||||
curl -XPUT 'localhost:9600/_node/logging/reset?pretty'
|
||||
```
|
||||
|
||||
|
||||
|
||||
## Log file location [_log_file_location]
|
||||
|
||||
You can specify the log file location using `--path.logs` setting.
|
||||
|
||||
|
||||
## Slowlog [_slowlog]
|
||||
|
||||
Slowlog for Logstash adds the ability to log when a specific event takes an abnormal amount of time to make its way through the pipeline. Just like the normal application log, you can find slowlogs in your `--path.logs` directory. Slowlog is configured in the `logstash.yml` settings file with the following options:
|
||||
|
||||
```yaml
|
||||
slowlog.threshold.warn (default: -1)
|
||||
slowlog.threshold.info (default: -1)
|
||||
slowlog.threshold.debug (default: -1)
|
||||
slowlog.threshold.trace (default: -1)
|
||||
```
|
||||
|
||||
Slowlog is disabled by default. The default threshold values are set to `-1nanos` to represent an infinite threshold. No slowlog will be invoked.
|
||||
|
||||
### Enable slowlog [_enable_slowlog]
|
||||
|
||||
The `slowlog.threshold` fields use a time-value format which enables a wide range of trigger intervals. You can specify ranges using the following time units: `nanos` (nanoseconds), `micros` (microseconds), `ms` (milliseconds), `s` (second), `m` (minute), `h` (hour), `d` (day).
|
||||
|
||||
Slowlog becomes more sensitive and logs more events as you raise the log level.
|
||||
|
||||
Example:
|
||||
|
||||
```yaml
|
||||
slowlog.threshold.warn: 2s
|
||||
slowlog.threshold.info: 1s
|
||||
slowlog.threshold.debug: 500ms
|
||||
slowlog.threshold.trace: 100ms
|
||||
```
|
||||
|
||||
In this example:
|
||||
|
||||
* If the log level is set to `warn`, the log shows events that took longer than 2s to process.
|
||||
* If the log level is set to `info`, the log shows events that took longer than 1s to process.
|
||||
* If the log level is set to `debug`, the log shows events that took longer than 500ms to process.
|
||||
* If the log level is set to `trace`, the log shows events that took longer than 100ms to process.
|
||||
|
||||
The logs include the full event and filter configuration that are responsible for the slowness.
|
||||
78
docs/reference/logstash-centralized-pipeline-management.md
Normal file
78
docs/reference/logstash-centralized-pipeline-management.md
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/logstash-centralized-pipeline-management.html
|
||||
---
|
||||
|
||||
# Centralized Pipeline Management [logstash-centralized-pipeline-management]
|
||||
|
||||
The pipeline management feature centralizes the creation and management of Logstash configuration pipelines in {{kib}}.
|
||||
|
||||
::::{note}
|
||||
Centralized pipeline management is a subscription feature. If you want to try the full set of features, you can activate a free 30-day trial. To view the status of your license, start a trial, or install a new license, open the {{kib}} main menu and click **Stack Management > License Management**. For more information, see [https://www.elastic.co/subscriptions](https://www.elastic.co/subscriptions) and [License Management](docs-content://deploy-manage/license/manage-your-license-in-self-managed-cluster.md).
|
||||
::::
|
||||
|
||||
|
||||
You can control multiple Logstash instances from the pipeline management UI in {{kib}}. You can add, edit, and delete pipeline configurations. On the Logstash side, you simply need to enable configuration management and register Logstash to use the centrally managed pipeline configurations.
|
||||
|
||||
::::{important}
|
||||
After you configure {{ls}} to use centralized pipeline management, you can no longer specify local pipeline configurations. The `pipelines.yml` file and settings such as `path.config` and `config.string` are inactive when centralized pipeline management is enabled.
|
||||
::::
|
||||
|
||||
|
||||
## Manage pipelines [_manage_pipelines]
|
||||
|
||||
Before using the pipeline management UI, you must:
|
||||
|
||||
* [Configure centralized pipeline management](/reference/configuring-centralized-pipelines.md).
|
||||
* If {{kib}} is protected with basic authentication, make sure your {{kib}} user has the `logstash_admin` role as well as the `logstash_writer` role that you created when you [configured Logstash to use basic authentication](/reference/secure-connection.md). Additionally, in order to view (as read-only) non-centrally-managed pipelines in the pipeline management UI, make sure your {{kib}} user has the `monitoring_user` role as well.
|
||||
|
||||
To manage Logstash pipelines in {{kib}}:
|
||||
|
||||
1. Open {{kib}} in your browser and go to the Management tab. If you’ve set up configuration management correctly, you’ll see an area for managing Logstash.
|
||||
|
||||
:::{image} ../images/centralized_config.png
|
||||
:alt: centralized config
|
||||
:::
|
||||
|
||||
2. Click the **Pipelines** link.
|
||||
3. To add a new pipeline, click **Create pipeline** and specify values.
|
||||
|
||||
Pipeline ID
|
||||
: A name that uniquely identifies the pipeline. This is the ID that you used when you [configured centralized pipeline management](/reference/configuring-centralized-pipelines.md) and specified a list of pipeline IDs in the `xpack.management.pipeline.id` setting.
|
||||
|
||||
Description
|
||||
: A description of the pipeline configuration. This information is for your use.
|
||||
|
||||
Pipeline
|
||||
: The pipeline configuration. You can treat the editor in the pipeline management UI like any other editor. You don’t have to worry about whitespace or indentation.
|
||||
|
||||
Pipeline workers
|
||||
: The number of parallel workers used to run the filter and output stages of the pipeline.
|
||||
|
||||
Pipeline batch size
|
||||
: The maximum number of events an individual worker thread collects before executing filters and outputs.
|
||||
|
||||
Pipeline batch delay
|
||||
: Time in milliseconds to wait for each event before sending an undersized batch to pipeline workers.
|
||||
|
||||
Queue type
|
||||
: The internal queueing model for event buffering. Options are **memory** for in-memory queueing, or **persisted** for disk-based acknowledged queueing.
|
||||
|
||||
Queue max bytes
|
||||
: The total capacity of the queue when persistent queues are enabled.
|
||||
|
||||
Queue checkpoint writes
|
||||
: The maximum number of events written before a checkpoint is forced when persistent queues are enabled.
|
||||
|
||||
|
||||
### Pipeline behavior [_pipeline_behavior]
|
||||
|
||||
* The pipeline configurations and metadata are stored in Elasticsearch. Any changes that you make to a pipeline definition are picked up and loaded automatically by all Logstash instances registered to use the pipeline. The changes are applied immediately. If Logstash is registered to use the pipeline, you do not have to restart Logstash to pick up the changes.
|
||||
* The pipeline runs on all Logstash instances that are registered to use the pipeline. {{kib}} saves the new configuration, and Logstash will attempt to load it. There is no validation done at the UI level.
|
||||
* You need to check the local Logstash logs for configuration errors. If you’re using the Logstash monitoring feature in {{kib}}, use the Monitoring tab to check the status of your Logstash nodes.
|
||||
* You can specify multiple pipeline configurations that run in parallel on the same Logstash node.
|
||||
* If you edit and save a pipeline configuration, Logstash reloads the configuration in the background and continues processing events.
|
||||
* If you try to delete a pipeline that is running (for example, `apache`) in {{kib}}, Logstash will attempt to stop the pipeline. Logstash waits until all events have been fully processed by the pipeline. Before you delete a pipeline, make sure you understand your data sources. Stopping a pipeline may lead to data loss.
|
||||
|
||||
|
||||
|
||||
78
docs/reference/logstash-geoip-database-management.md
Normal file
78
docs/reference/logstash-geoip-database-management.md
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/logstash-geoip-database-management.html
|
||||
---
|
||||
|
||||
# GeoIP Database Management [logstash-geoip-database-management]
|
||||
|
||||
Logstash provides a mechanism for provisioning and maintaining GeoIP databases, which plugins can use to ensure that they have access to an always-up-to-date and EULA-compliant database for geo enrichment. This mechanism requires internet access or a network route to an Elastic GeoIP database service.
|
||||
|
||||
If the database manager is enabled in `logstash.yml` (as it is by default), a plugin may subscribe to a database, triggering a download if a valid database is not already available. Logstash checks for updates every day. When an updated database is discovered, it is downloaded in the background and made available to the plugins that rely on it.
|
||||
|
||||
The GeoIP databases are separately-licensed from MaxMind under the terms of an End User License Agreement, which prohibits a database from being used after an update has been available for more than 30 days. When Logstash cannot reach the database service for 30 days or more to validate that a managed database is up-to-date, that database is deleted and made unavailable to the plugins that subscribed to it.
|
||||
|
||||
::::{note}
|
||||
GeoIP database management is a licensed feature of Logstash, and is only available in the Elastic-licensed complete distribution of Logstash.
|
||||
::::
|
||||
|
||||
|
||||
## Database Metrics [logstash-geoip-database-management-metrics]
|
||||
|
||||
You can monitor the managed database’s status through the [Node Stats API](https://www.elastic.co/docs/api/doc/logstash/operation/operation-nodestats).
|
||||
|
||||
The following request returns a JSON document containing database manager stats, including:
|
||||
|
||||
* database status and freshness
|
||||
|
||||
* `geoip_download_manager.database.*.status`
|
||||
|
||||
* `init` : initial CC database status
|
||||
* `up_to_date` : using up-to-date EULA database
|
||||
* `to_be_expired` : 25 days without calling service
|
||||
* `expired` : 30 days without calling service
|
||||
|
||||
* `fail_check_in_days` : number of days Logstash fails to call service since the last success
|
||||
|
||||
* info about download successes and failures
|
||||
|
||||
* `geoip_download_manager.download_stats.successes` number of successful checks and downloads
|
||||
* `geoip_download_manager.download_stats.failures` number of failed check or download
|
||||
* `geoip_download_manager.download_stats.status`
|
||||
|
||||
* `updating` : check and download at the moment
|
||||
* `succeeded` : last download succeed
|
||||
* `failed` : last download failed
|
||||
|
||||
|
||||
```js
|
||||
curl -XGET 'localhost:9600/_node/stats/geoip_download_manager?pretty'
|
||||
```
|
||||
|
||||
Example response:
|
||||
|
||||
```js
|
||||
{
|
||||
"geoip_download_manager" : {
|
||||
"database" : {
|
||||
"ASN" : {
|
||||
"status" : "up_to_date",
|
||||
"fail_check_in_days" : 0,
|
||||
"last_updated_at": "2021-06-21T16:06:54+02:00"
|
||||
},
|
||||
"City" : {
|
||||
"status" : "up_to_date",
|
||||
"fail_check_in_days" : 0,
|
||||
"last_updated_at": "2021-06-21T16:06:54+02:00"
|
||||
}
|
||||
},
|
||||
"download_stats" : {
|
||||
"successes" : 15,
|
||||
"failures" : 1,
|
||||
"last_checked_at" : "2021-06-21T16:07:03+02:00",
|
||||
"status" : "succeeded"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
28
docs/reference/logstash-monitoring-ui.md
Normal file
28
docs/reference/logstash-monitoring-ui.md
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/logstash-monitoring-ui.html
|
||||
---
|
||||
|
||||
# Monitoring UI [logstash-monitoring-ui]
|
||||
|
||||
Use the {{stack}} {{monitor-features}} to view metrics and gain insight into how your {{ls}} deployment is running. In the overview dashboard, you can see all events received and sent by Logstash, plus info about memory usage and uptime:
|
||||
|
||||
:::{image} ../images/overviewstats.png
|
||||
:alt: Logstash monitoring overview dashboard in Kibana
|
||||
:::
|
||||
|
||||
Then you can drill down to see stats about a specific node:
|
||||
|
||||
:::{image} ../images/nodestats.png
|
||||
:alt: Logstash monitoring node stats dashboard in Kibana
|
||||
:::
|
||||
|
||||
::::{note}
|
||||
A {{ls}} node is considered unique based on its persistent UUID, which is written to the [`path.data`](/reference/logstash-settings-file.md) directory when the node starts.
|
||||
::::
|
||||
|
||||
|
||||
Before you can use the monitoring UI, [configure Logstash monitoring](/reference/monitoring-logstash-legacy.md).
|
||||
|
||||
For information about using the Monitoring UI, see [{{monitoring}} in {{kib}}](docs-content://deploy-manage/monitor/monitoring-data/visualizing-monitoring-data.md).
|
||||
|
||||
55
docs/reference/logstash-pipeline-viewer.md
Normal file
55
docs/reference/logstash-pipeline-viewer.md
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/logstash-pipeline-viewer.html
|
||||
---
|
||||
|
||||
# Pipeline Viewer UI [logstash-pipeline-viewer]
|
||||
|
||||
The pipeline viewer UI offers additional visibility into the behavior and performance of complex pipeline configurations. Use the pipeline viewer to visualize and monitor the behavior of complex Logstash pipeline configurations. You can see and interact with a tree view that illustrates the pipeline topology, data flow, and branching logic.
|
||||
|
||||
The pipeline viewer highlights CPU% and event latency in cases where the values are anomalous. This information helps you quickly identify processing that is disproportionately slow.
|
||||
|
||||
:::{image} ../images/pipeline-tree.png
|
||||
:alt: Pipeline Viewer
|
||||
:class: screenshot
|
||||
:::
|
||||
|
||||
|
||||
## Prerequisites [_prerequisites]
|
||||
|
||||
Before using the pipeline viewer:
|
||||
|
||||
* [Configure Logstash monitoring](monitoring-logstash.md).
|
||||
* Start the Logstash pipeline that you want to monitor.
|
||||
|
||||
Logstash begins shipping metrics to the monitoring cluster.
|
||||
|
||||
|
||||
## View the pipeline [_view_the_pipeline]
|
||||
|
||||
To view the pipeline:
|
||||
|
||||
* Kibana → Monitoring → Logstash → Pipelines
|
||||
|
||||
Each pipeline is identified by a pipeline ID (`main` by default). For each pipeline, you see the pipeline’s throughput and the number of nodes on which the pipeline is running during the selected time range.
|
||||
|
||||
Many elements in the tree are clickable. For example, you can click the plugin name to expand the detail view.
|
||||
|
||||
:::{image} ../images/pipeline-input-detail.png
|
||||
:alt: Pipeline Input Detail
|
||||
:class: screenshot
|
||||
:::
|
||||
|
||||
Click the arrow beside a branch name to collapse or expand it.
|
||||
|
||||
|
||||
## Notes and best practices [_notes_and_best_practices]
|
||||
|
||||
**Use semantic IDs.** Specify semantic IDs when you configure the stages in your Logstash pipeline. Otherwise, Logstash generates them for you. Semantic IDs help you identify configurations that are causing bottlenecks. For example, you may have several grok filters running in your pipeline. If you have specified semantic IDs, you can tell at a glance which filters are slow. Semantic IDs, such as `apacheParsingGrok` and `cloudwatchGrok`, point you to the grok filters that are causing bottlenecks.
|
||||
|
||||
**Outliers.** Values and stats that are anomalously slow or otherwise out of line are highlighted. This doesn’t necessarily indicate a problem, but it highlights potential bottle necks so that you can find them quickly.
|
||||
|
||||
Some plugins are slower than others due to the nature of the work they do. For instance, you may find that a grok filter that uses a complicated regexp runs a lot slower than a mutate filter that simply adds a field. The grok filter might be highlighted in this case, though it may not be possible to further optimize its work.
|
||||
|
||||
**Versioning.** Version information is available from the dropdown list beside the pipeline ID. Logstash generates a new version each time you modify a pipeline, and stores multiple versions of the pipeline stats. Use this information to see how changes over time affect throughput and other metrics. Logstash does not store multiple versions of the pipeline configurations.
|
||||
|
||||
96
docs/reference/logstash-settings-file.md
Normal file
96
docs/reference/logstash-settings-file.md
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/logstash-settings-file.html
|
||||
---
|
||||
|
||||
# logstash.yml [logstash-settings-file]
|
||||
|
||||
You can set options in the Logstash settings file, `logstash.yml`, to control Logstash execution. For example, you can specify pipeline settings, the location of configuration files, logging options, and other settings. Most of the settings in the `logstash.yml` file are also available as [command-line flags](/reference/running-logstash-command-line.md#command-line-flags) when you run Logstash. Any flags that you set at the command line override the corresponding settings in the `logstash.yml` file.
|
||||
|
||||
The `logstash.yml` file is written in [YAML](http://yaml.org/). Its location varies by platform (see [Logstash Directory Layout](/reference/dir-layout.md)). You can specify settings in hierarchical form or use flat keys. For example, to use hierarchical form to set the pipeline batch size and batch delay, you specify:
|
||||
|
||||
```yaml
|
||||
pipeline:
|
||||
batch:
|
||||
size: 125
|
||||
delay: 50
|
||||
```
|
||||
|
||||
To express the same values as flat keys, you specify:
|
||||
|
||||
```yaml
|
||||
pipeline.batch.size: 125
|
||||
pipeline.batch.delay: 50
|
||||
```
|
||||
|
||||
The `logstash.yml` file also supports bash-style interpolation of environment variables and keystore secrets in setting values.
|
||||
|
||||
```yaml
|
||||
pipeline:
|
||||
batch:
|
||||
size: ${BATCH_SIZE}
|
||||
delay: ${BATCH_DELAY:50}
|
||||
node:
|
||||
name: "node_${LS_NODE_NAME}"
|
||||
path:
|
||||
queue: "/tmp/${QUEUE_DIR:queue}"
|
||||
```
|
||||
|
||||
Note that the `${VAR_NAME:default_value}` notation is supported, setting a default batch delay of `50` and a default `path.queue` of `/tmp/queue` in the above example.
|
||||
|
||||
The `logstash.yml` file includes these settings.
|
||||
|
||||
| Setting | Description | Default value |
|
||||
| --- | --- | --- |
|
||||
| `node.name` | A descriptive name for the node. | Machine’s hostname |
|
||||
| `path.data` | The directory that Logstash and its plugins use for any persistent needs. | `LOGSTASH_HOME/data` |
|
||||
| `pipeline.id` | The ID of the pipeline. | `main` |
|
||||
| `pipeline.workers` | The number of workers that will, in parallel, execute the filter and outputstages of the pipeline. This setting uses the[`java.lang.Runtime.getRuntime.availableProcessors`](https://docs.oracle.com/javase/7/docs/api/java/lang/Runtime.md#availableProcessors())value as a default if not overridden by `pipeline.workers` in `pipelines.yml` or`pipeline.workers` from `logstash.yml`. If you have modified this setting andsee that events are backing up, or that the CPU is not saturated, considerincreasing this number to better utilize machine processing power. | Number of the host’s CPU cores |
|
||||
| `pipeline.batch.size` | The maximum number of events an individual worker thread will collect from inputs before attempting to execute its filters and outputs. Larger batch sizes are generally more efficient, but come at the cost of increased memory overhead. You may need to increase JVM heap space in the `jvm.options` config file. See [Logstash Configuration Files](/reference/config-setting-files.md) for more info. | `125` |
|
||||
| `pipeline.batch.delay` | When creating pipeline event batches, how long in milliseconds to wait for each event before dispatching an undersized batch to pipeline workers. | `50` |
|
||||
| `pipeline.unsafe_shutdown` | When set to `true`, forces Logstash to exit during shutdown even if there are still inflight events in memory. By default, Logstash will refuse to quit until all received events have been pushed to the outputs. Enabling this option can lead to data loss during shutdown. | `false` |
|
||||
| `pipeline.plugin_classloaders` | (Beta) Load Java plugins in independent classloaders to isolate their dependencies. | `false` |
|
||||
| `pipeline.ordered` | Set the pipeline event ordering. Valid options are:<br><br>* `auto`. Automatically enables ordering if the `pipeline.workers` setting is `1`, and disables otherwise.<br>* `true`. Enforces ordering on the pipeline and prevents Logstash from starting if there are multiple workers.<br>* `false`. Disables the processing required to preserve order. Ordering will not be guaranteed, but you save the processing cost of preserving order.<br> | `auto` |
|
||||
| `pipeline.ecs_compatibility` | Sets the pipeline’s default value for `ecs_compatibility`, a setting that is available to plugins that implement an ECS compatibility mode for use with the Elastic Common Schema. Possible values are:<br><br>* `disabled`<br>* `v1`<br>* `v8`<br><br>This option allows the [early opt-in (or preemptive opt-out) of ECS compatibility](/reference/ecs-ls.md) modes in plugins, which is scheduled to be on-by-default in a future major release of {{ls}}.<br><br>Values other than `disabled` are currently considered BETA, and may produce unintended consequences when upgrading {{ls}}.<br> | `disabled` |
|
||||
| `path.config` | The path to the Logstash config for the main pipeline. If you specify a directory or wildcard, config files are read from the directory in alphabetical order. | Platform-specific. See [Logstash Directory Layout](/reference/dir-layout.md). |
|
||||
| `config.string` | A string that contains the pipeline configuration to use for the main pipeline. Use the same syntax as the config file. | *N/A* |
|
||||
| `config.test_and_exit` | When set to `true`, checks that the configuration is valid and then exits. Note that grok patterns are not checked for correctness with this setting. Logstash can read multiple config files from a directory. If you combine this setting with `log.level: debug`, Logstash will log the combined config file, annotating each config block with the source file it came from. | `false` |
|
||||
| `config.reload.automatic` | When set to `true`, periodically checks if the configuration has changed and reloads the configuration whenever it is changed. This can also be triggered manually through the SIGHUP signal. | `false` |
|
||||
| `config.reload.interval` | How often in seconds Logstash checks the config files for changes. Note that the unit qualifier (`s`) is required. | `3s` |
|
||||
| `config.debug` | When set to `true`, shows the fully compiled configuration as a debug log message. You must also set `log.level: debug`. WARNING: The log message will include any *password* options passed to plugin configs as plaintext, and may result in plaintext passwords appearing in your logs! | `false` |
|
||||
| `config.support_escapes` | When set to `true`, quoted strings will process the following escape sequences: `\n` becomes a literal newline (ASCII 10). `\r` becomes a literal carriage return (ASCII 13). `\t` becomes a literal tab (ASCII 9). `\\` becomes a literal backslash `\`. `\"` becomes a literal double quotation mark. `\'` becomes a literal quotation mark. | `false` |
|
||||
| `config.field_reference.escape_style` | Provides a way to reference fields that contain [field reference special characters](https://www.elastic.co/guide/en/logstash/current/field-references-deepdive.html#formal-grammar-escape-sequences) `[` and `]`.<br><br>::::{note} <br>This feature is in **technical preview** and may change in the future.<br>::::<br><br><br>Current options are:<br><br>* `percent`: URI-style `%`+`HH` hexadecimal encoding of UTF-8 bytes (`[` → `%5B`; `]` → `%5D`)<br>* `ampersand`: HTML-style `&#`+`DD`+`;` encoding of decimal Unicode code-points (`[` → `[`; `]` → `]`)<br>* `none`: field names containing special characters *cannot* be referenced.<br> | `none` |
|
||||
| `queue.type` | The internal queuing model to use for event buffering. Specify `memory` for legacy in-memory based queuing, or `persisted` for disk-based ACKed queueing ([persistent queues](/reference/persistent-queues.md)). | `memory` |
|
||||
| `path.queue` | The directory path where the data files will be stored when persistent queues are enabled (`queue.type: persisted`). | `path.data/queue` |
|
||||
| `queue.page_capacity` | The size of the page data files used when persistent queues are enabled (`queue.type: persisted`). The queue data consists of append-only data files separated into pages. | 64mb |
|
||||
| `queue.max_events` | The maximum number of unread events in the queue when persistent queues are enabled (`queue.type: persisted`). | 0 (unlimited) |
|
||||
| `queue.max_bytes` | The total capacity of the queue (`queue.type: persisted`) in number of bytes. Make sure the capacity of your disk drive is greater than the value you specify here. If both `queue.max_events` and `queue.max_bytes` are specified, Logstash uses whichever criteria is reached first. | 1024mb (1g) |
|
||||
| `queue.checkpoint.acks` | The maximum number of ACKed events before forcing a checkpoint when persistent queues are enabled (`queue.type: persisted`). Specify `queue.checkpoint.acks: 0` to set this value to unlimited. | 1024 |
|
||||
| `queue.checkpoint.writes` | The maximum number of written events before forcing a checkpoint when persistent queues are enabled (`queue.type: persisted`). Specify `queue.checkpoint.writes: 0` to set this value to unlimited. | 1024 |
|
||||
| `queue.checkpoint.retry` | When enabled, Logstash will retry four times per attempted checkpoint write for any checkpoint writes that fail. Any subsequent errors are not retried. This is a workaround for failed checkpoint writes that have been seen only on Windows platform, filesystems with non-standard behavior such as SANs and is not recommended except in those specific circumstances. (`queue.type: persisted`) | `true` |
|
||||
| `queue.drain` | When enabled, Logstash waits until the persistent queue (`queue.type: persisted`) is drained before shutting down. | `false` |
|
||||
| `dead_letter_queue.enable` | Flag to instruct Logstash to enable the DLQ feature supported by plugins. | `false` |
|
||||
| `dead_letter_queue.max_bytes` | The maximum size of each dead letter queue. Entries will be dropped if they would increase the size of the dead letter queue beyond this setting. | `1024mb` |
|
||||
| `dead_letter_queue.storage_policy` | Defines the action to take when the dead_letter_queue.max_bytes setting is reached: `drop_newer` stops accepting new values that would push the file size over the limit, and `drop_older` removes the oldest events to make space for new ones. | `drop_newer` |
|
||||
| `path.dead_letter_queue` | The directory path where the data files will be stored for the dead-letter queue. | `path.data/dead_letter_queue` |
|
||||
| `api.enabled` | The HTTP API is enabled by default. It can be disabled, but features that rely on it will not work as intended. | `true` |
|
||||
| `api.environment` | The API returns the provided string as a part of its response. Setting your environment may help to disambiguate between similarly-named nodes in production vs test environments. | `production` |
|
||||
| `api.http.host` | The bind address for the HTTP API endpoint. By default, the {{ls}} HTTP API binds only to the local loopback interface. When configured securely (`api.ssl.enabled: true` and `api.auth.type: basic`), the HTTP API binds to *all* available interfaces. | `"127.0.0.1"` |
|
||||
| `api.http.port` | The bind port for the HTTP API endpoint. | `9600-9700` |
|
||||
| `api.ssl.enabled` | Set to `true` to enable SSL on the HTTP API. Doing so requires both `api.ssl.keystore.path` and `api.ssl.keystore.password` to be set. | `false` |
|
||||
| `api.ssl.keystore.path` | The path to a valid JKS or PKCS12 keystore for use in securing the {{ls}} API. The keystore must be password-protected, and must contain a single certificate chain and a private key. This setting is ignored unless `api.ssl.enabled` is set to `true`. | *N/A* |
|
||||
| `api.ssl.keystore.password` | The password to the keystore provided with `api.ssl.keystore.path`. This setting is ignored unless `api.ssl.enabled` is set to `true`. | *N/A* |
|
||||
| `api.ssl.supported_protocols` | List of allowed SSL/TLS versions to use when establishing a secure connection. The availability of protocols depends on the JVM version. Certain protocols are disabled by default and need to be enabled manually by changing `jdk.tls.disabledAlgorithms` in the **$JDK_HOME/conf/security/java.security** configuration file. Possible values are:<br><br>* `TLSv1`<br>* `TLSv1.1`<br>* `TLSv1.2`<br>* `TLSv1.3`<br> | *N/A* |
|
||||
| `api.auth.type` | Set to `basic` to require HTTP Basic auth on the API using the credentials supplied with `api.auth.basic.username` and `api.auth.basic.password`. | `none` |
|
||||
| `api.auth.basic.username` | The username to require for HTTP Basic auth Ignored unless `api.auth.type` is set to `basic`. | *N/A* |
|
||||
| `api.auth.basic.password` | The password to require for HTTP Basic auth. Ignored unless `api.auth.type` is set to `basic`. It should meet default password policy which requires non-empty minimum 8 char string that includes a digit, upper case letter and lower case letter. The default password policy can be customized by following options:<br><br>* Set `api.auth.basic.password_policy.include.digit` `REQUIRED` (default) to accept only passwords that include at least one digit or `OPTIONAL` to exclude from requirement.<br>* Set `api.auth.basic.password_policy.include.upper` `REQUIRED` (default) to accept only passwords that include at least one upper case letter or `OPTIONAL` to exclude from requirement.<br>* Set `api.auth.basic.password_policy.include.lower` `REQUIRED` (default) to accept only passwords that include at least one lower case letter or `OPTIONAL` to exclude from requirement.<br>* Set `api.auth.basic.password_policy.include.symbol` `REQUIRED` to accept only passwords that include at least one special character or `OPTIONAL` (default) to exclude from requirement.<br>* Set `api.auth.basic.password_policy.length.minimum` to a value from 9 to 1024 if you want to require more than the 8 character default setting for passwords.<br> | *N/A* |
|
||||
| `api.auth.basic.password_policy.mode` | Raises either `WARN` or `ERROR` message when password requirements are not met.Ignored unless `api.auth.type` is set to `basic`. | `WARN` |
|
||||
| `log.level` | The log level. Valid options are:<br><br>* `fatal`<br>* `error`<br>* `warn`<br>* `info`<br>* `debug`<br>* `trace`<br> | `info` |
|
||||
| `log.format` | The log format. Set to `json` to log in JSON format, or `plain` to use `Object#.inspect`. | `plain` |
|
||||
| `log.format.json.fix_duplicate_message_fields` | When the log format is `json` avoid collision of field names in log lines. | `true` |
|
||||
| `path.logs` | The directory where Logstash will write its log to. | `LOGSTASH_HOME/logs` |
|
||||
| `pipeline.separate_logs` | This a boolean setting to enable separation of logs per pipeline in different log files. If enabled Logstash will create a different log file for each pipeline,using the pipeline.id as name of the file. The destination directory is taken from the `path.log`s setting. When there are many pipelines configured in Logstash,separating each log lines per pipeline could be helpful in case you need to troubleshoot what’s happening in a single pipeline, without interference of the other ones. | `false` |
|
||||
| `path.plugins` | Where to find custom plugins. You can specify this setting multiple times to include multiple paths. Plugins are expected to be in a specific directory hierarchy: `PATH/logstash/TYPE/NAME.rb` where `TYPE` is `inputs`, `filters`, `outputs`, or `codecs`, and `NAME` is the name of the plugin. | Platform-specific. See [Logstash Directory Layout](/reference/dir-layout.md). |
|
||||
| `allow_superuser` | Setting to `true` to allow or `false` to block running Logstash as a superuser. | `false` |
|
||||
| `pipeline.buffer.type` | Determine where to allocate memory buffers, for plugins that leverage them.Defaults to `heap` but can be switched to `direct` to instruct Logstash to prefer allocation of buffers in direct memory. | `heap` Check out [Buffer Allocation types](/reference/jvm-settings.md#off-heap-buffers-allocation) for more info. |
|
||||
|
||||
42
docs/reference/logstash-to-logstash-communications.md
Normal file
42
docs/reference/logstash-to-logstash-communications.md
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/ls-to-ls.html
|
||||
---
|
||||
|
||||
# Logstash-to-Logstash communications [ls-to-ls]
|
||||
|
||||
{{ls}}-to-{{ls}} communication is available if you need to have one {{ls}} instance communicate with another {{ls}} instance. Implementing Logstash-to-Logstash communication can add complexity to your environment, but you may need it if the data path crosses network or firewall boundaries. However, we suggest you don’t implement unless it is strictly required.
|
||||
|
||||
::::{note}
|
||||
If you are looking for information on connecting multiple pipelines within one Logstash instance, see [Pipeline-to-pipeline communication](/reference/pipeline-to-pipeline.md).
|
||||
::::
|
||||
|
||||
|
||||
Logstash-to-Logstash communication can be achieved in one of two ways:
|
||||
|
||||
* [Logstash output to Logstash Input](#native-considerations)
|
||||
* [Lumberjack output to Beats input](#lumberjack-considerations)
|
||||
|
||||
$$$native-considerations$$$**Logstash to Logstash considerations**
|
||||
|
||||
This is the preferred method to implement Logstash-to-Logstash. It replaces [Logstash-to-Logstash: HTTP output to HTTP input](/reference/ls-to-ls-http.md) and has these considerations:
|
||||
|
||||
* It relies on HTTP as the communication protocol between the Input and Output.
|
||||
* It supports multiple hosts, providing high availability by load balancing equally amongst the multiple destination hosts.
|
||||
* No connection information is added to events.
|
||||
|
||||
Ready to see more configuration details? See [Logstash-to-Logstash: Output to Input](/reference/ls-to-ls-native.md).
|
||||
|
||||
$$$lumberjack-considerations$$$**Lumberjack-Beats considerations**
|
||||
|
||||
Lumberjack output to Beats input has been our standard approach for {{ls}}-to-{{ls}} communication, but our recommended approach is now [Logstash-to-Logstash: Output to Input](/reference/ls-to-ls-native.md). Before you implement the Lumberjack to Beats configuration, keep these points in mind:
|
||||
|
||||
* Lumberjack to Beats provides high availability, but does not provide load balancing. The Lumberjack output plugin allows defining multiple output hosts for high availability, but instead of load-balancing between all output hosts, it falls back to one host on the list in the case of failure.
|
||||
* If you need a proxy between the Logstash instances, TCP proxy is the only option.
|
||||
* There’s no explicit way to exert back pressure back to the beats input.
|
||||
|
||||
Ready to see more configuration details? See [Logstash-to-Logstash: Lumberjack output to Beats input](/reference/ls-to-ls-lumberjack.md).
|
||||
|
||||
|
||||
|
||||
|
||||
236
docs/reference/lookup-enrichment.md
Normal file
236
docs/reference/lookup-enrichment.md
Normal file
|
|
@ -0,0 +1,236 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/lookup-enrichment.html
|
||||
---
|
||||
|
||||
# Enriching Data with Lookups [lookup-enrichment]
|
||||
|
||||
These plugins can help you enrich data with additional info, such as GeoIP and user agent info:
|
||||
|
||||
* [dns filter](#dns-def)
|
||||
* [elasticsearch filter](#es-def)
|
||||
* [geoip filter](#geoip-def)
|
||||
* [http filter](#http-def)
|
||||
* [jdbc_static filter](#jdbc-static-def)
|
||||
* [jdbc_streaming filter](#jdbc-stream-def)
|
||||
* [memcached filter](#memcached-def)
|
||||
* [translate filter](#translate-def)
|
||||
* [useragent filter](#useragent-def)
|
||||
|
||||
|
||||
## Lookup plugins [lookup-plugins]
|
||||
|
||||
$$$dns-def$$$dns filter
|
||||
: The [dns filter plugin](/reference/plugins-filters-dns.md) performs a standard or reverse DNS lookup.
|
||||
|
||||
The following config performs a reverse lookup on the address in the `source_host` field and replaces it with the domain name:
|
||||
|
||||
```json
|
||||
filter {
|
||||
dns {
|
||||
reverse => [ "source_host" ]
|
||||
action => "replace"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
$$$es-def$$$elasticsearch filter
|
||||
: The [elasticsearch filter](/reference/plugins-filters-elasticsearch.md) copies fields from previous log events in Elasticsearch to current events.
|
||||
|
||||
The following config shows a complete example of how this filter might be used. Whenever Logstash receives an "end" event, it uses this Elasticsearch filter to find the matching "start" event based on some operation identifier. Then it copies the `@timestamp` field from the "start" event into a new field on the "end" event. Finally, using a combination of the date filter and the ruby filter, the code in the example calculates the time duration in hours between the two events.
|
||||
|
||||
```json
|
||||
if [type] == "end" {
|
||||
elasticsearch {
|
||||
hosts => ["es-server"]
|
||||
query => "type:start AND operation:%{[opid]}"
|
||||
fields => { "@timestamp" => "started" }
|
||||
}
|
||||
date {
|
||||
match => ["[started]", "ISO8601"]
|
||||
target => "[started]"
|
||||
}
|
||||
ruby {
|
||||
code => 'event.set("duration_hrs", (event.get("@timestamp") - event.get("started")) / 3600) rescue nil'
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
$$$geoip-def$$$geoip filter
|
||||
: The [geoip filter](/reference/plugins-filters-geoip.md) adds geographical information about the location of IP addresses. For example:
|
||||
|
||||
```json
|
||||
filter {
|
||||
geoip {
|
||||
source => "clientip"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
After the geoip filter is applied, the event will be enriched with geoip fields. For example:
|
||||
|
||||
```json
|
||||
filter {
|
||||
geoip {
|
||||
source => "clientip"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
$$$http-def$$$http filter
|
||||
: The [http filter](/reference/plugins-filters-http.md) integrates with external web services/REST APIs, and enables lookup enrichment against any HTTP service or endpoint. This plugin is well suited for many enrichment use cases, such as social APIs, sentiment APIs, security feed APIs, and business service APIs.
|
||||
|
||||
$$$jdbc-static-def$$$jdbc_static filter
|
||||
: The [jdbc_static filter](/reference/plugins-filters-jdbc_static.md) enriches events with data pre-loaded from a remote database.
|
||||
|
||||
The following example fetches data from a remote database, caches it in a local database, and uses lookups to enrich events with data cached in the local database.
|
||||
|
||||
```json
|
||||
filter {
|
||||
jdbc_static {
|
||||
loaders => [ <1>
|
||||
{
|
||||
id => "remote-servers"
|
||||
query => "select ip, descr from ref.local_ips order by ip"
|
||||
local_table => "servers"
|
||||
},
|
||||
{
|
||||
id => "remote-users"
|
||||
query => "select firstname, lastname, userid from ref.local_users order by userid"
|
||||
local_table => "users"
|
||||
}
|
||||
]
|
||||
local_db_objects => [ <2>
|
||||
{
|
||||
name => "servers"
|
||||
index_columns => ["ip"]
|
||||
columns => [
|
||||
["ip", "varchar(15)"],
|
||||
["descr", "varchar(255)"]
|
||||
]
|
||||
},
|
||||
{
|
||||
name => "users"
|
||||
index_columns => ["userid"]
|
||||
columns => [
|
||||
["firstname", "varchar(255)"],
|
||||
["lastname", "varchar(255)"],
|
||||
["userid", "int"]
|
||||
]
|
||||
}
|
||||
]
|
||||
local_lookups => [ <3>
|
||||
{
|
||||
id => "local-servers"
|
||||
query => "select descr as description from servers WHERE ip = :ip"
|
||||
parameters => {ip => "[from_ip]"}
|
||||
target => "server"
|
||||
},
|
||||
{
|
||||
id => "local-users"
|
||||
query => "select firstname, lastname from users WHERE userid = :id"
|
||||
parameters => {id => "[loggedin_userid]"}
|
||||
target => "user" <4>
|
||||
}
|
||||
]
|
||||
# using add_field here to add & rename values to the event root
|
||||
add_field => { server_name => "%{[server][0][description]}" }
|
||||
add_field => { user_firstname => "%{[user][0][firstname]}" } <5>
|
||||
add_field => { user_lastname => "%{[user][0][lastname]}" }
|
||||
remove_field => ["server", "user"]
|
||||
jdbc_user => "logstash"
|
||||
jdbc_password => "example"
|
||||
jdbc_driver_class => "org.postgresql.Driver"
|
||||
jdbc_driver_library => "/tmp/logstash/vendor/postgresql-42.1.4.jar"
|
||||
jdbc_connection_string => "jdbc:postgresql://remotedb:5432/ls_test_2"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
1. Queries an external database to fetch the dataset that will be cached locally.
|
||||
2. Defines the columns, types, and indexes used to build the local database structure. The column names and types should match the external database.
|
||||
3. Performs lookup queries on the local database to enrich the events.
|
||||
4. Specifies the event field that will store the looked-up data. If the lookup returns multiple columns, the data is stored as a JSON object within the field.
|
||||
5. Takes data from the JSON object and stores it in top-level event fields for easier analysis in Kibana.
|
||||
|
||||
|
||||
$$$jdbc-stream-def$$$jdbc_streaming filter
|
||||
: The [jdbc_streaming filter](/reference/plugins-filters-jdbc_streaming.md) enriches events with database data.
|
||||
|
||||
The following example executes a SQL query and stores the result set in a field called `country_details`:
|
||||
|
||||
```json
|
||||
filter {
|
||||
jdbc_streaming {
|
||||
jdbc_driver_library => "/path/to/mysql-connector-java-5.1.34-bin.jar"
|
||||
jdbc_driver_class => "com.mysql.jdbc.Driver"
|
||||
jdbc_connection_string => "jdbc:mysql://localhost:3306/mydatabase"
|
||||
jdbc_user => "me"
|
||||
jdbc_password => "secret"
|
||||
statement => "select * from WORLD.COUNTRY WHERE Code = :code"
|
||||
parameters => { "code" => "country_code"}
|
||||
target => "country_details"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
$$$memcached-def$$$memcached filter
|
||||
: The [memcached filter](/reference/plugins-filters-memcached.md) enables key/value lookup enrichment against a Memcached object caching system. It supports both read (GET) and write (SET) operations. It is a notable addition for security analytics use cases.
|
||||
|
||||
$$$translate-def$$$translate filter
|
||||
: The [translate filter](/reference/plugins-filters-translate.md) replaces field contents based on replacement values specified in a hash or file. Currently supports these file types: YAML, JSON, and CSV.
|
||||
|
||||
The following example takes the value of the `response_code` field, translates it to a description based on the values specified in the dictionary, and then removes the `response_code` field from the event:
|
||||
|
||||
```json
|
||||
filter {
|
||||
translate {
|
||||
field => "response_code"
|
||||
destination => "http_response"
|
||||
dictionary => {
|
||||
"200" => "OK"
|
||||
"403" => "Forbidden"
|
||||
"404" => "Not Found"
|
||||
"408" => "Request Timeout"
|
||||
}
|
||||
remove_field => "response_code"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
$$$useragent-def$$$useragent filter
|
||||
: The [useragent filter](/reference/plugins-filters-useragent.md) parses user agent strings into fields.
|
||||
|
||||
The following example takes the user agent string in the `agent` field, parses it into user agent fields, and adds the user agent fields to a new field called `user_agent`. It also removes the original `agent` field:
|
||||
|
||||
```json
|
||||
filter {
|
||||
useragent {
|
||||
source => "agent"
|
||||
target => "user_agent"
|
||||
remove_field => "agent"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
After the filter is applied, the event will be enriched with user agent fields. For example:
|
||||
|
||||
```json
|
||||
"user_agent": {
|
||||
"os": "Mac OS X 10.12",
|
||||
"major": "50",
|
||||
"minor": "0",
|
||||
"os_minor": "12",
|
||||
"os_major": "10",
|
||||
"name": "Firefox",
|
||||
"os_name": "Mac OS X",
|
||||
"device": "Other"
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
137
docs/reference/ls-to-ls-http.md
Normal file
137
docs/reference/ls-to-ls-http.md
Normal file
|
|
@ -0,0 +1,137 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/ls-to-ls-http.html
|
||||
---
|
||||
|
||||
# Logstash-to-Logstash: HTTP output to HTTP input [ls-to-ls-http]
|
||||
|
||||
HTTP output to HTTP input is an alternative to the Lumberjack output to Beats input approach for Logstash-to-Logstash communication. This approach relies on the use of [http output](/reference/plugins-outputs-http.md) to [http input](/reference/plugins-inputs-http.md) plugins.
|
||||
|
||||
::::{note}
|
||||
{{ls}}-to-{{ls}} using HTTP input/output plugins is now being deprecated in favor of [Logstash-to-Logstash: Output to Input](/reference/ls-to-ls-native.md).
|
||||
::::
|
||||
|
||||
|
||||
## Configuration overview [overview-http-http]
|
||||
|
||||
To use the HTTP protocol to connect two Logstash instances:
|
||||
|
||||
1. Configure the downstream (server) Logstash to use HTTP input
|
||||
2. Configure the upstream (client) Logstash to use HTTP output
|
||||
3. Secure the communication between HTTP input and HTTP output
|
||||
|
||||
### Configure the downstream Logstash to use HTTP input [configure-downstream-logstash-http-input]
|
||||
|
||||
Configure the HTTP input on the downstream (receiving) Logstash to receive connections. The minimum configuration requires these options:
|
||||
|
||||
* `port` - To set a custom port.
|
||||
* `additional_codecs` - To set `application/json` to be `json_lines`.
|
||||
|
||||
```json
|
||||
input {
|
||||
http {
|
||||
port => 8080
|
||||
additional_codecs => { "application/json" => "json_lines" }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
### Configure the upstream Logstash to use HTTP output [configure-upstream-logstash-http-output]
|
||||
|
||||
In order to obtain the best performance when sending data from one Logstash to another, the data needs to be batched and compressed. As such, the upstream Logstash (the sending Logstash) needs to be configured with these options:
|
||||
|
||||
* `url` - The receiving Logstash.
|
||||
* `http_method` - Set to `post`.
|
||||
* `retry_non_idempotent` - Set to `true`, in order to retry failed events.
|
||||
* `format` - Set to `json_batch` to batch the data.
|
||||
* `http_compression` - Set to `true` to ensure the data is compressed.
|
||||
|
||||
```json
|
||||
output {
|
||||
http {
|
||||
url => '<protocol>://<downstream-logstash>:<port>'
|
||||
http_method => post
|
||||
retry_non_idempotent => true
|
||||
format => json_batch
|
||||
http_compression => true
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
### Secure Logstash to Logstash [securing-logstash-to-logstash-http]
|
||||
|
||||
It is important that you secure the communication between Logstash instances. Use SSL/TLS mutual authentication in order to ensure that the upstream Logstash instance sends encrypted data to a trusted downstream Logstash instance, and vice versa.
|
||||
|
||||
1. Create a certificate authority (CA) in order to sign the certificates that you plan to use between Logstash instances. Creating a correct SSL/TLS infrastructure is outside the scope of this document.
|
||||
|
||||
::::{tip}
|
||||
We recommend you use the [elasticsearch-certutil](elasticsearch://reference/elasticsearch/command-line-tools/certutil.md) tool to generate your certificates.
|
||||
::::
|
||||
|
||||
2. Configure the downstream (receiving) Logstash to use SSL. Add these settings to the HTTP Input configuration:
|
||||
|
||||
* `ssl`: When set to `true`, it enables Logstash use of SSL/TLS
|
||||
* `ssl_key`: Specifies the key that Logstash uses to authenticate with the client.
|
||||
* `ssl_certificate`: Specifies the certificate that Logstash uses to authenticate with the client.
|
||||
* `ssl_certificate_authorities`: Configures Logstash to trust any certificates signed by the specified CA.
|
||||
* `ssl_verify_mode`: Specifies whether Logstash server verifies the client certificate against the CA.
|
||||
|
||||
For example:
|
||||
|
||||
```json
|
||||
input {
|
||||
http {
|
||||
...
|
||||
|
||||
ssl => true
|
||||
ssl_key => "server.key.pk8"
|
||||
ssl_certificate => "server.crt"
|
||||
ssl_certificate_authorities => "ca.crt"
|
||||
ssl_verify_mode => force_peer
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
3. Configure the upstream (sending) Logstash to use SSL. Add these settings to the HTTP output configuration:
|
||||
|
||||
* `cacert`: Configures the Logstash client to trust any certificates signed by the specified CA.
|
||||
* `client_key`: Specifies the key the Logstash client uses to authenticate with the Logstash server.
|
||||
* `client_cert`: Specifies the certificate that the Logstash client uses to authenticate to the Logstash server.
|
||||
|
||||
For example:
|
||||
|
||||
```json
|
||||
output {
|
||||
http {
|
||||
...
|
||||
|
||||
cacert => "ca.crt"
|
||||
client_key => "client.key.pk8"
|
||||
client_cert => "client.crt"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
4. If you would like an additional authentication step, you can also use basic user/password authentication in both Logstash instances:
|
||||
|
||||
* `user`: Sets the username to use for authentication.
|
||||
* `password`: Sets the password to use for authentication.
|
||||
|
||||
For example, you would need to add the following to both Logstash instances:
|
||||
|
||||
```json
|
||||
...
|
||||
http {
|
||||
...
|
||||
|
||||
user => "your-user"
|
||||
password => "your-secret"
|
||||
}
|
||||
...
|
||||
```
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -1,34 +1,39 @@
|
|||
[[ls-to-ls-lumberjack]]
|
||||
=== Logstash-to-Logstash: Lumberjack output to Beats input
|
||||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/ls-to-ls-lumberjack.html
|
||||
---
|
||||
|
||||
You can set up communication between two Logstash machines by connecting the Lumberjack output to the Beats input.
|
||||
# Logstash-to-Logstash: Lumberjack output to Beats input [ls-to-ls-lumberjack]
|
||||
|
||||
Logstash-to-Logstash using Lumberjack and Beats has been our standard approach for {ls}-to-{ls}, and may still be the best option for more robust use cases.
|
||||
You can set up communication between two Logstash machines by connecting the Lumberjack output to the Beats input.
|
||||
|
||||
NOTE: Check out these <<lumberjack-considerations,considerations>> before you implement Logstash-to-Logstash using Lumberjack and Beats.
|
||||
Logstash-to-Logstash using Lumberjack and Beats has been our standard approach for {{ls}}-to-{{ls}}, and may still be the best option for more robust use cases.
|
||||
|
||||
==== Configuration overview
|
||||
::::{note}
|
||||
Check out these [considerations](/reference/logstash-to-logstash-communications.md#lumberjack-considerations) before you implement Logstash-to-Logstash using Lumberjack and Beats.
|
||||
::::
|
||||
|
||||
|
||||
## Configuration overview [_configuration_overview]
|
||||
|
||||
Use the Lumberjack protocol to connect two Logstash machines.
|
||||
|
||||
. Generate a trusted SSL certificate (required by the lumberjack protocol).
|
||||
. Copy the SSL certificate to the upstream Logstash machine.
|
||||
. Copy the SSL certificate and key to the downstream Logstash machine.
|
||||
. Set the upstream Logstash machine to use the Lumberjack output to send data.
|
||||
. Set the downstream Logstash machine to listen for incoming Lumberjack connections through the Beats input.
|
||||
. Test it.
|
||||
1. Generate a trusted SSL certificate (required by the lumberjack protocol).
|
||||
2. Copy the SSL certificate to the upstream Logstash machine.
|
||||
3. Copy the SSL certificate and key to the downstream Logstash machine.
|
||||
4. Set the upstream Logstash machine to use the Lumberjack output to send data.
|
||||
5. Set the downstream Logstash machine to listen for incoming Lumberjack connections through the Beats input.
|
||||
6. Test it.
|
||||
|
||||
[[generate-self-signed-cert]]
|
||||
===== Generate a self-signed SSL certificate and key
|
||||
### Generate a self-signed SSL certificate and key [generate-self-signed-cert]
|
||||
|
||||
Use the `openssl req` command to generate a self-signed certificate and key. The `openssl req` command is available with some operating systems. You may need to install the openssl command line program for others.
|
||||
|
||||
Run the following command:
|
||||
|
||||
[source,shell]
|
||||
----
|
||||
```shell
|
||||
openssl req -x509 -batch -nodes -newkey rsa:2048 -keyout lumberjack.key -out lumberjack.cert -subj /CN=localhost
|
||||
----
|
||||
```
|
||||
|
||||
where:
|
||||
|
||||
|
|
@ -36,55 +41,50 @@ where:
|
|||
* `lumberjack.cert` is the name of the SSL certificate to be created
|
||||
* `localhost` is the name of the upstream Logstash computer
|
||||
|
||||
|
||||
This command produces output similar to the following:
|
||||
|
||||
[source,shell]
|
||||
----
|
||||
```shell
|
||||
Generating a 2048 bit RSA private key
|
||||
.................................+++
|
||||
....................+++
|
||||
writing new private key to 'lumberjack.key'
|
||||
----
|
||||
```
|
||||
|
||||
[[copy-cert-key]]
|
||||
===== Copy the SSL certificate and key
|
||||
|
||||
### Copy the SSL certificate and key [copy-cert-key]
|
||||
|
||||
Copy the SSL certificate to the upstream Logstash machine.
|
||||
|
||||
Copy the SSL certificate and key to the downstream Logstash machine.
|
||||
|
||||
[[save-cert-ls1]]
|
||||
===== Start the upstream Logstash instance
|
||||
|
||||
### Start the upstream Logstash instance [save-cert-ls1]
|
||||
|
||||
Start Logstash and generate test events:
|
||||
|
||||
[source,shell]
|
||||
----
|
||||
```shell
|
||||
bin/logstash -e 'input { generator { count => 5 } } output { lumberjack { codec => json hosts => "mydownstreamhost" ssl_certificate => "lumberjack.cert" port => 5000 } }'
|
||||
----
|
||||
```
|
||||
|
||||
This sample command sends five events to mydownstreamhost:5000 using the SSL certificate provided.
|
||||
|
||||
[[save-cert-ls2]]
|
||||
===== Start the downstream Logstash instance
|
||||
|
||||
### Start the downstream Logstash instance [save-cert-ls2]
|
||||
|
||||
Start the downstream instance of Logstash:
|
||||
|
||||
[source,shell]
|
||||
----
|
||||
```shell
|
||||
bin/logstash -e 'input { beats { codec => json port => 5000 ssl_enabled => true ssl_certificate => "lumberjack.cert" ssl_key => "lumberjack.key"} }'
|
||||
----
|
||||
```
|
||||
|
||||
This sample command sets port 5000 to listen for incoming Beats input.
|
||||
|
||||
[[test-ls-to-ls]]
|
||||
===== Verify the communication
|
||||
|
||||
### Verify the communication [test-ls-to-ls]
|
||||
|
||||
Watch the downstream Logstash machine for the incoming events. You should see five incrementing events similar to the following:
|
||||
|
||||
[source,shell]
|
||||
----
|
||||
```shell
|
||||
{
|
||||
"@timestamp" => 2018-02-07T12:16:39.415Z,
|
||||
"sequence" => 0
|
||||
|
|
@ -95,5 +95,9 @@ Watch the downstream Logstash machine for the incoming events. You should see fi
|
|||
"@version" => "1",
|
||||
"host" => "ls1.semicomplete.com"
|
||||
}
|
||||
----
|
||||
```
|
||||
|
||||
If you see all five events with consistent fields and formatting, incrementing by one, then your configuration is correct.
|
||||
|
||||
|
||||
|
||||
133
docs/reference/ls-to-ls-native.md
Normal file
133
docs/reference/ls-to-ls-native.md
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/ls-to-ls-native.html
|
||||
---
|
||||
|
||||
# Logstash-to-Logstash: Output to Input [ls-to-ls-native]
|
||||
|
||||
The Logstash output to Logstash input is the default approach for Logstash-to-Logstash communication.
|
||||
|
||||
::::{note}
|
||||
Check out these [considerations](/reference/logstash-to-logstash-communications.md#native-considerations) before you implement {{ls}}-to-{{ls}}.
|
||||
::::
|
||||
|
||||
|
||||
## Configuration overview [overview-ls-ls]
|
||||
|
||||
To connect two Logstash instances:
|
||||
|
||||
1. Configure the downstream (server) Logstash to use Logstash input
|
||||
2. Configure the upstream (client) Logstash to use Logstash output
|
||||
3. Secure the communication between Logstash input and Logstash output
|
||||
|
||||
### Configure the downstream Logstash to use Logstash input [configure-downstream-logstash-input]
|
||||
|
||||
Configure the Logstash input on the downstream (receiving) Logstash to receive connections. The minimum configuration requires this option:
|
||||
|
||||
* `port` - To set a custom port. The default is 9800 if none is provided.
|
||||
|
||||
```json
|
||||
input {
|
||||
logstash {
|
||||
port => 9800
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
### Configure the upstream Logstash to use Logstash output [configure-upstream-logstash-output]
|
||||
|
||||
In order to obtain the best performance when sending data from one Logstash to another, the data is batched and compressed. As such, the upstream Logstash (the sending Logstash) only needs to be concerned about configuring the receiving endpoint with these options:
|
||||
|
||||
* `hosts` - The receiving one or more Logstash host and port pairs. If no port specified, 9800 will be used.
|
||||
|
||||
::::{note}
|
||||
{{ls}} load balances batched events to *all* of its configured downstream hosts. Any failures caused by network issues, back-pressure or other conditions, will result in the downstream host being isolated from load balancing for at least 60 seconds.
|
||||
::::
|
||||
|
||||
|
||||
```json
|
||||
output {
|
||||
logstash {
|
||||
hosts => ["10.0.0.123", "10.0.1.123:9800"]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
### Secure Logstash to Logstash [securing-logstash-to-logstash]
|
||||
|
||||
It is important that you secure the communication between Logstash instances. Use SSL/TLS mutual authentication in order to ensure that the upstream Logstash instance sends encrypted data to a trusted downstream Logstash instance, and vice versa.
|
||||
|
||||
1. Create a certificate authority (CA) in order to sign the certificates that you plan to use between Logstash instances. Creating a correct SSL/TLS infrastructure is outside the scope of this document.
|
||||
|
||||
::::{tip}
|
||||
We recommend you use the [elasticsearch-certutil](elasticsearch://reference/elasticsearch/command-line-tools/certutil.md) tool to generate your certificates.
|
||||
::::
|
||||
|
||||
2. Configure the downstream (receiving) Logstash to use SSL. Add these settings to the Logstash input configuration:
|
||||
|
||||
* `ssl_enabled`: When set to `true`, it enables Logstash use of SSL/TLS
|
||||
* `ssl_key`: Specifies the key that Logstash uses to authenticate with the client.
|
||||
* `ssl_certificate`: Specifies the certificate that Logstash uses to authenticate with the client.
|
||||
* `ssl_certificate_authorities`: Configures Logstash to trust any certificates signed by the specified CA.
|
||||
* `ssl_client_authentication`: Specifies whether Logstash server verifies the client certificate against the CA.
|
||||
|
||||
For example:
|
||||
|
||||
```json
|
||||
input {
|
||||
logstash {
|
||||
...
|
||||
|
||||
ssl_enabled => true
|
||||
ssl_key => "server.pkcs8.key"
|
||||
ssl_certificate => "server.crt"
|
||||
ssl_certificate_authorities => "ca.crt"
|
||||
ssl_client_authentication => required
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
3. Configure the upstream (sending) Logstash to use SSL. Add these settings to the Logstash output configuration:
|
||||
|
||||
* `ssl_key`: Specifies the key the Logstash client uses to authenticate with the Logstash server.
|
||||
* `ssl_certificate`: Specifies the certificate that the Logstash client uses to authenticate to the Logstash server.
|
||||
* `ssl_certificate_authorities`: Configures the Logstash client to trust any certificates signed by the specified CA.
|
||||
|
||||
For example:
|
||||
|
||||
```json
|
||||
output {
|
||||
logstash {
|
||||
...
|
||||
|
||||
ssl_enabled => true
|
||||
ssl_key => "client.pkcs8.key"
|
||||
ssl_certificate => "client.crt"
|
||||
ssl_certificate_authorities => "ca.crt"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
4. If you would like an additional authentication step, you can also use basic user/password authentication in both Logstash instances:
|
||||
|
||||
* `username`: Sets the username to use for authentication.
|
||||
* `password`: Sets the password to use for authentication.
|
||||
|
||||
For example, you would need to add the following to both Logstash instances:
|
||||
|
||||
```json
|
||||
...
|
||||
logstash {
|
||||
...
|
||||
|
||||
username => "your-user"
|
||||
password => "your-secret"
|
||||
}
|
||||
...
|
||||
```
|
||||
|
||||
|
||||
|
||||
|
||||
14
docs/reference/managing-geoip-databases.md
Normal file
14
docs/reference/managing-geoip-databases.md
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/geoip-database-management.html
|
||||
---
|
||||
|
||||
# Managing GeoIP databases [geoip-database-management]
|
||||
|
||||
Logstash provides GeoIP database management features to make it easier for you to use plugins that require an up-to-date database to enrich events with geographic data.
|
||||
|
||||
* [Feature Overview](/reference/logstash-geoip-database-management.md)
|
||||
* [Configuration Guide](/reference/configuring-geoip-database-management.md)
|
||||
|
||||
|
||||
|
||||
13
docs/reference/managing-logstash.md
Normal file
13
docs/reference/managing-logstash.md
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/config-management.html
|
||||
---
|
||||
|
||||
# Managing Logstash [config-management]
|
||||
|
||||
Logstash provides configuration management features to make it easier for you to manage updates to your configuration over time.
|
||||
|
||||
The topics in this section describe Logstash configuration management features only. For information about other config management tools, such as Puppet and Chef, see the documentation for those projects.
|
||||
|
||||
|
||||
|
||||
65
docs/reference/memory-queue.md
Normal file
65
docs/reference/memory-queue.md
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/memory-queue.html
|
||||
---
|
||||
|
||||
# Memory queue [memory-queue]
|
||||
|
||||
By default, Logstash uses in-memory bounded queues between pipeline stages (inputs → pipeline workers) to buffer events. If Logstash experiences a temporary machine failure, the contents of the memory queue will be lost. Temporary machine failures are scenarios where Logstash or its host machine are terminated abnormally, but are capable of being restarted.
|
||||
|
||||
## Benefits of memory queues [mem-queue-benefits]
|
||||
|
||||
The memory queue might be a good choice if you value throughput over data resiliency.
|
||||
|
||||
* Easier configuration
|
||||
* Easier management and administration
|
||||
* Faster throughput
|
||||
|
||||
|
||||
## Limitations of memory queues [mem-queue-limitations]
|
||||
|
||||
* Can lose data in abnormal termination
|
||||
* Don’t do well handling sudden bursts of data, where extra capacity in needed for {{ls}} to catch up
|
||||
|
||||
::::{tip}
|
||||
Consider using [persistent queues](/reference/persistent-queues.md) to avoid these limitations.
|
||||
::::
|
||||
|
||||
|
||||
|
||||
## Memory queue size [sizing-mem-queue]
|
||||
|
||||
Memory queue size is not configured directly. Instead, it depends on how you have Logstash tuned.
|
||||
|
||||
Its upper bound is defined by `pipeline.workers` (default: number of CPUs) times the `pipeline.batch.size` (default: 125) events. This value, called the "inflight count," determines maximum number of events that can be held in each memory queue.
|
||||
|
||||
Doubling the number of workers OR doubling the batch size will effectively double the memory queue’s capacity (and memory usage). Doubling both will *quadruple* the capacity (and usage).
|
||||
|
||||
::::{important}
|
||||
Each pipeline has its own queue.
|
||||
::::
|
||||
|
||||
|
||||
See [Tuning and profiling logstash pipeline performance](/reference/tuning-logstash.md) for more info on the effects of adjusting `pipeline.batch.size` and `pipeline.workers`.
|
||||
|
||||
If you need to absorb bursts of traffic, consider using [persistent queues](/reference/persistent-queues.md) instead. Persistent queues are bound to allocated capacity on disk.
|
||||
|
||||
### Settings that affect queue size [mq-settings]
|
||||
|
||||
These values can be configured in `logstash.yml` and `pipelines.yml`.
|
||||
|
||||
pipeline.batch.size
|
||||
: Number events to retrieve from inputs before sending to filters+workers The default is 125.
|
||||
|
||||
pipelines.workers
|
||||
: Number of workers that will, in parallel, execute the filters+outputs stage of the pipeline. This value defaults to the number of the host’s CPU cores.
|
||||
|
||||
|
||||
|
||||
## Back pressure [backpressure-mem-queue]
|
||||
|
||||
When the queue is full, Logstash puts back pressure on the inputs to stall data flowing into Logstash. This mechanism helps Logstash control the rate of data flow at the input stage without overwhelming outputs like Elasticsearch.
|
||||
|
||||
Each input handles back pressure independently.
|
||||
|
||||
|
||||
276
docs/reference/monitoring-internal-collection-legacy.md
Normal file
276
docs/reference/monitoring-internal-collection-legacy.md
Normal file
|
|
@ -0,0 +1,276 @@
|
|||
---
|
||||
navigation_title: "Legacy collection (deprecated)"
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/monitoring-internal-collection-legacy.html
|
||||
---
|
||||
|
||||
# Collect {{ls}} monitoring data using legacy collectors [monitoring-internal-collection-legacy]
|
||||
|
||||
|
||||
::::{warning}
|
||||
Deprecated in 7.9.0.
|
||||
::::
|
||||
|
||||
|
||||
::::{note}
|
||||
Starting from version 9.0, legacy internal collection is behind a feature flag and is turned off by default. Set `xpack.monitoring.allow_legacy_collection` to `true` to allow access to the feature.
|
||||
::::
|
||||
|
||||
|
||||
Using [{{agent}} for monitoring](/reference/monitoring-logstash-with-elastic-agent.md) is a better alternative for most {{ls}} deployments.
|
||||
|
||||
## Components for legacy collection [_components_for_legacy_collection]
|
||||
|
||||
Monitoring {{ls}} with legacy collection uses these components:
|
||||
|
||||
* [Collectors](#logstash-monitoring-collectors-legacy)
|
||||
* [Output](#logstash-monitoring-output-legacy)
|
||||
|
||||
These pieces live outside of the default Logstash pipeline in a dedicated monitoring pipeline. This configuration ensures that all data and processing has a minimal impact on ordinary Logstash processing. Existing Logstash features, such as the [`elasticsearch` output](/reference/plugins-outputs-elasticsearch.md), can be reused to benefit from its retry policies.
|
||||
|
||||
::::{note}
|
||||
The `elasticsearch` output that is used for monitoring {{ls}} is configured exclusively through settings found in `logstash.yml`. It is not configured by using anything from the Logstash configurations that might also be using their own separate `elasticsearch` outputs.
|
||||
::::
|
||||
|
||||
|
||||
The production {{es}} cluster should be configured to receive {{ls}} monitoring data. This configuration enables the production {{es}} cluster to add metadata (for example, its cluster UUID) to the Logstash monitoring data and then route it to the monitoring clusters. For more information about typical monitoring architectures, see [How monitoring works](docs-content://deploy-manage/monitor/stack-monitoring.md) in the [Elasticsearch Reference](docs-content://get-started/index.md).
|
||||
|
||||
|
||||
#### Collectors [logstash-monitoring-collectors-legacy]
|
||||
|
||||
Collectors, as their name implies, collect things. In monitoring for Logstash, collectors are just [Inputs](/reference/how-logstash-works.md) in the same way that ordinary Logstash configurations provide inputs.
|
||||
|
||||
Like monitoring for {{es}}, each collector can create zero or more monitoring documents. As it is currently implemented, each Logstash node runs two types of collectors: one for node stats and one for pipeline stats.
|
||||
|
||||
| Collector | Data Types | Description |
|
||||
| --- | --- | --- |
|
||||
| Node Stats | `logstash_stats` | Gathers details about the running node, such as memory utilization and CPUusage (for example, `GET /_stats`).<br>This runs on every Logstash node with monitoring enabled. One commonfailure is that Logstash directories are copied with their `path.data` directoryincluded (`./data` by default), which copies the persistent UUID of the Logstashnode along with it. As a result, it generally appears that one or more Logstashnodes are failing to collect monitoring data, when in fact they are all reallymisreporting as the *same* Logstash node. Re-use `path.data` directories onlywhen upgrading Logstash, such that upgraded nodes replace the previous versions. |
|
||||
| Pipeline Stats | `logstash_state` | Gathers details about the node’s running pipelines, which powers theMonitoring Pipeline UI. |
|
||||
|
||||
Per collection interval, which defaults to 10 seconds (`10s`), each collector is run. The failure of an individual collector does not impact any other collector. Each collector, as an ordinary Logstash input, creates a separate Logstash event in its isolated monitoring pipeline. The Logstash output then sends the data.
|
||||
|
||||
The collection interval can be configured dynamically and you can also disable data collection. For more information about the configuration options for the collectors, see [Monitoring Settings](#monitoring-settings-legacy).
|
||||
|
||||
::::{warning}
|
||||
Unlike {{es}} and {{kib}} monitoring, there is no `xpack.monitoring.collection.enabled` setting on Logstash. You must use the `xpack.monitoring.enabled` setting to enable and disable data collection.
|
||||
::::
|
||||
|
||||
|
||||
If gaps exist in the monitoring charts in {{kib}}, it is typically because either a collector failed or the monitoring cluster did not receive the data (for example, it was being restarted). In the event that a collector fails, a logged error should exist on the node that attempted to perform the collection.
|
||||
|
||||
|
||||
### Output [logstash-monitoring-output-legacy]
|
||||
|
||||
Like all Logstash pipelines, the purpose of the dedicated monitoring pipeline is to send events to outputs. In the case of monitoring for Logstash, the output is always an `elasticsearch` output. However, unlike ordinary Logstash pipelines, the output is configured within the `logstash.yml` settings file via the `xpack.monitoring.elasticsearch.*` settings.
|
||||
|
||||
Other than its unique manner of configuration, this `elasticsearch` output behaves like all `elasticsearch` outputs, including its ability to pause data collection when issues exist with the output.
|
||||
|
||||
::::{important}
|
||||
It is critical that all Logstash nodes share the same setup. Otherwise, monitoring data might be routed in different ways or to different places.
|
||||
::::
|
||||
|
||||
|
||||
|
||||
#### Default Configuration [logstash-monitoring-default-legacy]
|
||||
|
||||
If a Logstash node does not explicitly define a monitoring output setting, the following default configuration is used:
|
||||
|
||||
```yaml
|
||||
xpack.monitoring.elasticsearch.hosts: [ "http://localhost:9200" ]
|
||||
```
|
||||
|
||||
All data produced by monitoring for Logstash is indexed in the monitoring cluster by using the `.monitoring-logstash` template, which is managed by the [exporters](docs-content://deploy-manage/monitor/stack-monitoring/es-monitoring-exporters.md) within {{es}}.
|
||||
|
||||
If you are working with a cluster that has {{security}} enabled, extra steps are necessary to properly configure Logstash. For more information, see [*Monitoring {{ls}} (legacy)*](/reference/monitoring-logstash-legacy.md).
|
||||
|
||||
::::{important}
|
||||
When discussing security relative to the `elasticsearch` output, it is critical to remember that all users are managed on the production cluster, which is identified in the `xpack.monitoring.elasticsearch.hosts` setting. This is particularly important to remember when you move from development environments to production environments, where you often have dedicated monitoring clusters.
|
||||
::::
|
||||
|
||||
|
||||
For more information about the configuration options for the output, see [Monitoring Settings](#monitoring-settings-legacy).
|
||||
|
||||
|
||||
## Configure {{ls}} monitoring with legacy collectors [configure-internal-collectors-legacy]
|
||||
|
||||
|
||||
To monitor Logstash nodes:
|
||||
|
||||
1. Specify where to send monitoring data. This cluster is often referred to as the *production cluster*. For examples of typical monitoring architectures, see [How monitoring works](docs-content://deploy-manage/monitor/stack-monitoring.md).
|
||||
|
||||
::::{important}
|
||||
To visualize Logstash as part of the Elastic Stack (as shown in Step 6), send metrics to your *production* cluster. Sending metrics to a dedicated monitoring cluster will show the Logstash metrics under the *monitoring* cluster.
|
||||
::::
|
||||
|
||||
2. Verify that the `xpack.monitoring.allow_legacy_collection` and `xpack.monitoring.collection.enabled` settings are `true` on the production cluster. If that setting is `false`, the collection of monitoring data is disabled in {{es}} and data is ignored from all other sources.
|
||||
3. Configure your Logstash nodes to send metrics by setting `xpack.monitoring.enabled` to `true` and specifying the destination {{es}} node(s) as `xpack.monitoring.elasticsearch.hosts` in `logstash.yml`. If {{security-features}} are enabled, you also need to specify the credentials for the [built-in `logstash_system` user](docs-content://deploy-manage/users-roles/cluster-or-deployment-auth/built-in-users.md). For more information about these settings, see [Monitoring Settings](#monitoring-settings-legacy).
|
||||
|
||||
```yaml
|
||||
xpack.monitoring.enabled: true
|
||||
xpack.monitoring.elasticsearch.hosts: ["http://es-prod-node-1:9200", "http://es-prod-node-2:9200"] <1>
|
||||
xpack.monitoring.elasticsearch.username: "logstash_system"
|
||||
xpack.monitoring.elasticsearch.password: "changeme"
|
||||
```
|
||||
|
||||
1. If SSL/TLS is enabled on the production cluster, you must connect through HTTPS. As of v5.2.1, you can specify multiple Elasticsearch hosts as an array as well as specifying a single host as a string. If multiple URLs are specified, Logstash can round-robin requests to these production nodes.
|
||||
|
||||
4. If SSL/TLS is enabled on the production {{es}} cluster, specify the trusted CA certificates that will be used to verify the identity of the nodes in the cluster.
|
||||
|
||||
To add a CA certificate to a Logstash node’s trusted certificates, you can specify the location of the PEM encoded certificate with the `certificate_authority` setting:
|
||||
|
||||
```yaml
|
||||
xpack.monitoring.elasticsearch.ssl.certificate_authority: /path/to/ca.crt
|
||||
```
|
||||
|
||||
To add a CA without having it loaded on disk, you can specify a hex-encoded SHA 256 fingerprint of the DER-formatted CA with the `ca_trusted_fingerprint` setting:
|
||||
|
||||
```yaml
|
||||
xpack.monitoring.elasticsearch.ssl.ca_trusted_fingerprint: 2cfe62e474fb381cc7773c84044c28c9785ac5d1940325f942a3d736508de640
|
||||
```
|
||||
|
||||
::::{note}
|
||||
A self-secured Elasticsearch cluster will provide the fingerprint of its CA to the console during setup.
|
||||
|
||||
You can also get the SHA256 fingerprint of an Elasticsearch’s CA using the `openssl` command-line utility on the Elasticsearch host:
|
||||
|
||||
```shell
|
||||
openssl x509 -fingerprint -sha256 -in $ES_HOME/config/certs/http_ca.crt
|
||||
```
|
||||
|
||||
::::
|
||||
|
||||
|
||||
Alternatively, you can configure trusted certificates using a truststore (a Java Keystore file that contains the certificates):
|
||||
|
||||
```yaml
|
||||
xpack.monitoring.elasticsearch.ssl.truststore.path: /path/to/file
|
||||
xpack.monitoring.elasticsearch.ssl.truststore.password: password
|
||||
```
|
||||
|
||||
Also, optionally, you can set up client certificate using a keystore (a Java Keystore file that contains the certificate) or using a certificate and key file pair:
|
||||
|
||||
```yaml
|
||||
xpack.monitoring.elasticsearch.ssl.keystore.path: /path/to/file
|
||||
xpack.monitoring.elasticsearch.ssl.keystore.password: password
|
||||
```
|
||||
|
||||
```yaml
|
||||
xpack.monitoring.elasticsearch.ssl.certificate: /path/to/certificate
|
||||
xpack.monitoring.elasticsearch.ssl.key: /path/to/key
|
||||
```
|
||||
|
||||
Set sniffing to `true` to enable discovery of other nodes of the {{es}} cluster. It defaults to `false`.
|
||||
|
||||
```yaml
|
||||
xpack.monitoring.elasticsearch.sniffing: false
|
||||
```
|
||||
|
||||
5. Restart your Logstash nodes.
|
||||
6. To verify your monitoring configuration, point your web browser at your {{kib}} host, and select **Stack Monitoring** from the side navigation. If this is an initial setup, select **set up with self monitoring** and click **Turn on monitoring**. Metrics reported from your Logstash nodes should be visible in the Logstash section. When security is enabled, to view the monitoring dashboards you must log in to {{kib}} as a user who has the `kibana_user` and `monitoring_user` roles.
|
||||
|
||||
:::{image} ../images/monitoring-ui.png
|
||||
:alt: Monitoring
|
||||
:::
|
||||
|
||||
|
||||
|
||||
## Monitoring settings for legacy collection [monitoring-settings-legacy]
|
||||
|
||||
|
||||
You can set the following `xpack.monitoring` settings in `logstash.yml` to control how monitoring data is collected from your Logstash nodes. However, the defaults work best in most circumstances. For more information about configuring Logstash, see [logstash.yml](/reference/logstash-settings-file.md).
|
||||
|
||||
### General monitoring settings [monitoring-general-settings-legacy]
|
||||
|
||||
`xpack.monitoring.enabled`
|
||||
: Monitoring is disabled by default. Set to `true` to enable {{xpack}} monitoring.
|
||||
|
||||
`xpack.monitoring.elasticsearch.hosts`
|
||||
: The {{es}} instances that you want to ship your Logstash metrics to. This might be the same {{es}} instance specified in the `outputs` section in your Logstash configuration, or a different one. This is **not** the URL of your dedicated monitoring cluster. Even if you are using a dedicated monitoring cluster, the Logstash metrics must be routed through your production cluster. You can specify a single host as a string, or specify multiple hosts as an array. Defaults to `http://localhost:9200`.
|
||||
|
||||
::::{note}
|
||||
If your Elasticsearch cluster is configured with dedicated master-eligible nodes, Logstash metrics should *not* be routed to these nodes, as doing so can create resource contention and impact the stability of the Elasticsearch cluster. Therefore, do not include such nodes in `xpack.monitoring.elasticsearch.hosts`.
|
||||
::::
|
||||
|
||||
|
||||
`xpack.monitoring.elasticsearch.proxy`
|
||||
: The monitoring {{es}} instance and monitored Logstash can be separated by a proxy. To enable Logstash to connect to a proxied {{es}}, set this value to the URI of the intermediate proxy using the standard URI format, `<protocol>://<host>` for example `http://192.168.1.1`. An empty string is treated as if proxy was not set.
|
||||
|
||||
`xpack.monitoring.elasticsearch.username` and `xpack.monitoring.elasticsearch.password`
|
||||
: If your {{es}} is protected with basic authentication, these settings provide the username and password that the Logstash instance uses to authenticate for shipping monitoring data.
|
||||
|
||||
|
||||
### Monitoring collection settings [monitoring-collection-settings-legacy]
|
||||
|
||||
`xpack.monitoring.collection.interval`
|
||||
: Controls how often data samples are collected and shipped on the Logstash side. Defaults to `10s`. If you modify the collection interval, set the `xpack.monitoring.min_interval_seconds` option in `kibana.yml` to the same value.
|
||||
|
||||
|
||||
### Monitoring TLS/SSL settings [monitoring-ssl-settings-legacy]
|
||||
|
||||
You can configure the following Transport Layer Security (TLS) or Secure Sockets Layer (SSL) settings. For more information, see [Configuring credentials for {{ls}} monitoring](/reference/secure-connection.md#ls-monitoring-user).
|
||||
|
||||
`xpack.monitoring.elasticsearch.ssl.ca_trusted_fingerprint`
|
||||
: Optional setting that enables you to specify the hex-encoded SHA-256 fingerprint of the certificate authority for your {{es}} instance.
|
||||
|
||||
::::{note}
|
||||
A self-secured Elasticsearch cluster will provide the fingerprint of its CA to the console during setup.
|
||||
|
||||
You can also get the SHA256 fingerprint of an Elasticsearch’s CA using the `openssl` command-line utility on the Elasticsearch host:
|
||||
|
||||
```shell
|
||||
openssl x509 -fingerprint -sha256 -in $ES_HOME/config/certs/http_ca.crt
|
||||
```
|
||||
|
||||
::::
|
||||
|
||||
|
||||
`xpack.monitoring.elasticsearch.ssl.certificate_authority`
|
||||
: Optional setting that enables you to specify a path to the `.pem` file for the certificate authority for your {{es}} instance.
|
||||
|
||||
`xpack.monitoring.elasticsearch.ssl.truststore.path`
|
||||
: Optional settings that provide the paths to the Java keystore (JKS) to validate the server’s certificate.
|
||||
|
||||
`xpack.monitoring.elasticsearch.ssl.truststore.password`
|
||||
: Optional settings that provide the password to the truststore.
|
||||
|
||||
`xpack.monitoring.elasticsearch.ssl.keystore.path`
|
||||
: Optional settings that provide the paths to the Java keystore (JKS) to validate the client’s certificate.
|
||||
|
||||
`xpack.monitoring.elasticsearch.ssl.keystore.password`
|
||||
: Optional settings that provide the password to the keystore.
|
||||
|
||||
`xpack.monitoring.elasticsearch.ssl.certificate`
|
||||
: Optional setting that provides the path to an SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file.
|
||||
|
||||
::::{note}
|
||||
This setting can be used only if `xpack.monitoring.elasticsearch.ssl.key` is set.
|
||||
::::
|
||||
|
||||
|
||||
`xpack.monitoring.elasticsearch.ssl.key`
|
||||
: Optional setting that provides the path to an OpenSSL-style RSA private key that corresponds to the `xpack.monitoring.elasticsearch.ssl.certificate`.
|
||||
|
||||
::::{note}
|
||||
This setting can be used only if `xpack.monitoring.elasticsearch.ssl.certificate` is set.
|
||||
::::
|
||||
|
||||
|
||||
`xpack.monitoring.elasticsearch.ssl.verification_mode`
|
||||
: Option to validate the server’s certificate. Defaults to `full`. To disable, set to `none`. Disabling this severely compromises security.
|
||||
|
||||
`xpack.monitoring.elasticsearch.ssl.cipher_suites`
|
||||
: Optional setting that provides the list of cipher suites to use, listed by priorities. Supported cipher suites vary depending on the Java and protocol versions.
|
||||
|
||||
|
||||
### Additional settings [monitoring-additional-settings-legacy]
|
||||
|
||||
`xpack.monitoring.elasticsearch.cloud_id`
|
||||
: If you’re using {{es}} in {{ecloud}}, you should specify the identifier here. This setting is an alternative to `xpack.monitoring.elasticsearch.hosts`. If `cloud_id` is configured, `xpack.monitoring.elasticsearch.hosts` should not be used. The {{es}} instances that you want to ship your Logstash metrics to. This might be the same {{es}} instance specified in the `outputs` section in your Logstash configuration, or a different one.
|
||||
|
||||
`xpack.monitoring.elasticsearch.cloud_auth`
|
||||
: If you’re using {{es}} in {{ecloud}}, you can set your auth credentials here. This setting is an alternative to both `xpack.monitoring.elasticsearch.username` and `xpack.monitoring.elasticsearch.password`. If `cloud_auth` is configured, those settings should not be used.
|
||||
|
||||
`xpack.monitoring.elasticsearch.api_key`
|
||||
: Authenticate using an Elasticsearch API key. Note that this option also requires using SSL.
|
||||
|
||||
The API key Format is `id:api_key` where `id` and `api_key` are as returned by the Elasticsearch [Create API key API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-security-create-api-key).
|
||||
23
docs/reference/monitoring-logstash-legacy.md
Normal file
23
docs/reference/monitoring-logstash-legacy.md
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/configuring-logstash.html
|
||||
---
|
||||
|
||||
# Monitoring Logstash (Legacy) [configuring-logstash]
|
||||
|
||||
Use the {{stack}} {{monitor-features}} to gain insight into the health of {{ls}} instances running in your environment. For an introduction to monitoring your Elastic stack, see [Monitoring a cluster](docs-content://deploy-manage/monitor.md) in the [Elasticsearch Reference](docs-content://get-started/index.md). Then, make sure that monitoring is enabled on your {{es}} cluster.
|
||||
|
||||
These options for collecting {{ls}} metrics for stack monitoring have been available for a while:
|
||||
|
||||
* [{{metricbeat}} collection](/reference/monitoring-with-metricbeat.md). Metricbeat collects monitoring data from your {{ls}} instance and sends it directly to your monitoring cluster. The benefit of Metricbeat collection is that the monitoring agent remains active even if the {{ls}} instance does not.
|
||||
* [Legacy collection (deprecated)](/reference/monitoring-internal-collection-legacy.md). Legacy collectors send monitoring data to your production cluster.
|
||||
|
||||
For more features, dependability, and easier management, consider using:
|
||||
|
||||
* [{{agent}} collection for Stack Monitoring](/reference/monitoring-with-elastic-agent.md). {{agent}} collects monitoring data from your {{ls}} instance and sends it directly to your monitoring cluster, and shows the data in {{ls}} Dashboards. The benefit of {{agent}} collection is that the monitoring agent remains active even if the {{ls}} instance does not, you can manage all your monitoring agents from a central location in {{fleet}}.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
20
docs/reference/monitoring-logstash-with-elastic-agent.md
Normal file
20
docs/reference/monitoring-logstash-with-elastic-agent.md
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
---
|
||||
mapped_pages:
|
||||
- https://www.elastic.co/guide/en/logstash/current/monitoring-with-ea.html
|
||||
---
|
||||
|
||||
# Monitoring Logstash with Elastic Agent [monitoring-with-ea]
|
||||
|
||||
You can use {{agent}} to collect data about {{ls}} and ship it to the monitoring cluster. When you use {{agent}} collection, the monitoring agent remains active even if the {{ls}} instance does not. Plus you have the option to manage all of your monitoring agents from a central location in {{fleet}}.
|
||||
|
||||
{{agent}} gives you a single, unified way to add monitoring for logs, metrics, and other types of data to a host. Each agent has a single policy you can update to add integrations for new data sources, security protections, and more.
|
||||
|
||||
You can use {{agent}} to collect {{ls}} monitoring data on:
|
||||
|
||||
* [{{ecloud}} or self-managed dashboards](/reference/dashboard-monitoring-with-elastic-agent.md).<br> {{agent}} collects monitoring data from your {{ls}} instance, sends it directly to your monitoring cluster, and shows the data in {{ls}} dashboards. {{ls}} dashboards include an extended range of metrics, including plugin drilldowns, and plugin specific dashboards for the dissect filter, the grok filter, and the elasticsearch output.
|
||||
* [{{ecloud}} dashboards (serverless)](/reference/serverless-monitoring-with-elastic-agent.md).<br> {{agent}} collects monitoring data from your {{ls}} instance, sends it to [Elastic serverless](docs-content://deploy-manage/deploy/elastic-cloud/serverless.md), and shows the data in {{ls}} dashboards in [Elastic Observability](docs-content://solutions/observability.md). {{ls}} dashboards include an extended range of metrics, including plugin drilldowns, and plugin specific dashboards for the dissect filter, the grok filter, and the elasticsearch output.
|
||||
* [{{stack}} monitoring](/reference/monitoring-with-elastic-agent.md).<br> Use the Elastic Stack monitoring features to gain insight into the health of {{ls}} instances running in your environment.
|
||||
|
||||
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue