mirror of
https://github.com/elastic/logstash.git
synced 2025-04-22 05:37:21 -04:00
708c156c1b
Clean backport of #12304 This commit changes the DLQ writer to write to a temporary file which will be renamed on "completion", to avoid the possibility of the DLQ reader reading an incomplete DLQ segment. The temp file will be renamed and made available, either when the capacity of this segment is reached, or if a configurable 'flush interval' has elapsed since the last event reached the dead letter queue. This commit fixes #8022, #10275, #10967 This commit replaces #11127
86 lines
3.6 KiB
YAML
86 lines
3.6 KiB
YAML
# List of pipelines to be loaded by Logstash
|
|
#
|
|
# This document must be a list of dictionaries/hashes, where the keys/values are pipeline settings.
|
|
# Default values for omitted settings are read from the `logstash.yml` file.
|
|
# When declaring multiple pipelines, each MUST have its own `pipeline.id`.
|
|
#
|
|
# Example of two pipelines:
|
|
#
|
|
# - pipeline.id: test
|
|
# pipeline.workers: 1
|
|
# pipeline.batch.size: 1
|
|
# config.string: "input { generator {} } filter { sleep { time => 1 } } output { stdout { codec => dots } }"
|
|
# - pipeline.id: another_test
|
|
# queue.type: persisted
|
|
# path.config: "/tmp/logstash/*.config"
|
|
#
|
|
# Available options:
|
|
#
|
|
# # name of the pipeline
|
|
# pipeline.id: mylogs
|
|
#
|
|
# # The configuration string to be used by this pipeline
|
|
# config.string: "input { generator {} } filter { sleep { time => 1 } } output { stdout { codec => dots } }"
|
|
#
|
|
# # The path from where to read the configuration text
|
|
# path.config: "/etc/conf.d/logstash/myconfig.cfg"
|
|
#
|
|
# # How many worker threads execute the Filters+Outputs stage of the pipeline
|
|
# pipeline.workers: 1 (actually defaults to number of CPUs)
|
|
#
|
|
# # How many events to retrieve from inputs before sending to filters+workers
|
|
# pipeline.batch.size: 125
|
|
#
|
|
# # How long to wait in milliseconds while polling for the next event
|
|
# # before dispatching an undersized batch to filters+outputs
|
|
# pipeline.batch.delay: 50
|
|
#
|
|
# # Internal queuing model, "memory" for legacy in-memory based queuing and
|
|
# # "persisted" for disk-based acked queueing. Defaults is memory
|
|
# queue.type: memory
|
|
#
|
|
# # If using queue.type: persisted, the page data files size. The queue data consists of
|
|
# # append-only data files separated into pages. Default is 64mb
|
|
# queue.page_capacity: 64mb
|
|
#
|
|
# # If using queue.type: persisted, the maximum number of unread events in the queue.
|
|
# # Default is 0 (unlimited)
|
|
# queue.max_events: 0
|
|
#
|
|
# # If using queue.type: persisted, the total capacity of the queue in number of bytes.
|
|
# # Default is 1024mb or 1gb
|
|
# queue.max_bytes: 1024mb
|
|
#
|
|
# # If using queue.type: persisted, the maximum number of acked events before forcing a checkpoint
|
|
# # Default is 1024, 0 for unlimited
|
|
# queue.checkpoint.acks: 1024
|
|
#
|
|
# # If using queue.type: persisted, the maximum number of written events before forcing a checkpoint
|
|
# # Default is 1024, 0 for unlimited
|
|
# queue.checkpoint.writes: 1024
|
|
#
|
|
# # If using queue.type: persisted, the interval in milliseconds when a checkpoint is forced on the head page
|
|
# # Default is 1000, 0 for no periodic checkpoint.
|
|
# queue.checkpoint.interval: 1000
|
|
#
|
|
# # Enable Dead Letter Queueing for this pipeline.
|
|
# dead_letter_queue.enable: false
|
|
#
|
|
# If using dead_letter_queue.enable: true, the maximum size of dead letter queue for this pipeline. Entries
|
|
# will be dropped if they would increase the size of the dead letter queue beyond this setting.
|
|
# Default is 1024mb
|
|
# dead_letter_queue.max_bytes: 1024mb
|
|
#
|
|
# If using dead_letter_queue.enable: true, the interval in milliseconds where if no further events eligible for the DLQ
|
|
# have been created, a dead letter queue file will be written. A low value here will mean that more, smaller, queue files
|
|
# may be written, while a larger value will introduce more latency between items being "written" to the dead letter queue, and
|
|
# being available to be read by the dead_letter_queue input when items are are written infrequently.
|
|
# Default is 5000.
|
|
#
|
|
# dead_letter_queue.flush_interval: 5000
|
|
|
|
#
|
|
# If using dead_letter_queue.enable: true, the directory path where the data files will be stored.
|
|
# Default is path.data/dead_letter_queue
|
|
#
|
|
# path.dead_letter_queue:
|