diff --git a/salt/logstash/files/logstash.yml b/salt/logstash/files/logstash.yml index 8677b5f53..7414e4e01 100644 --- a/salt/logstash/files/logstash.yml +++ b/salt/logstash/files/logstash.yml @@ -1,6 +1,227 @@ -path.config: /usr/share/logstash/pipeline +# Settings file in YAML +# +# Settings can be specified either in hierarchical form, e.g.: +# +# pipeline: +# batch: +# size: 125 +# delay: 5 +# +# Or as flat keys: +# +# pipeline.batch.size: 125 +# pipeline.batch.delay: 5 +# +# ------------ Node identity ------------ +# +# Use a descriptive name for the node: +# +# node.name: test +# +# If omitted the node name will default to the machine's host name +# +# ------------ Data path ------------------ +# +# Which directory should be used by logstash and its plugins +# for any persistent needs. Defaults to LOGSTASH_HOME/data +# +# path.data: +# +# ------------ Pipeline Settings -------------- +# +# The ID of the pipeline. +# +# pipeline.id: main +# +# Set the number of workers that will, in parallel, execute the filters+outputs +# stage of the pipeline. +# +# This defaults to the number of the host's CPU cores. +# +# pipeline.workers: 2 +# +# How many events to retrieve from inputs before sending to filters+workers +# +# pipeline.batch.size: 125 +# +# How long to wait in milliseconds while polling for the next event +# before dispatching an undersized batch to filters+outputs +# +# pipeline.batch.delay: 50 +# +# Force Logstash to exit during shutdown even if there are still inflight +# events in memory. By default, logstash will refuse to quit until all +# received events have been pushed to the outputs. +# +# WARNING: enabling this can lead to data loss during shutdown +# +# pipeline.unsafe_shutdown: false +# +# ------------ Pipeline Configuration Settings -------------- +# +# Where to fetch the pipeline configuration for the main pipeline +# +# path.config: +# /etc/logstash/conf.d is mapped to /usr/share/logstash/pipeline in the Docker image +path.config: /usr/share/logstash/pipeline/*.conf + +# Special Docker path +# path.config: /usr/share/logstash/pipeline + +# +# Pipeline configuration string for the main pipeline +# +# config.string: +# +# At startup, test if the configuration is valid and exit (dry run) +# +# config.test_and_exit: false +# +# Periodically check if the configuration has changed and reload the pipeline +# This can also be triggered manually through the SIGHUP signal +# +# config.reload.automatic: false +# +# How often to check if the pipeline configuration has changed (in seconds) +# +# config.reload.interval: 3s +# +# Show fully compiled configuration as debug log message +# NOTE: --log.level must be 'debug' +# +# config.debug: false +# +# When enabled, process escaped characters such as \n and \" in strings in the +# pipeline configuration files. +# +# config.support_escapes: false +# +# ------------ Module Settings --------------- +# Define modules here. Modules definitions must be defined as an array. +# The simple way to see this is to prepend each `name` with a `-`, and keep +# all associated variables under the `name` they are associated with, and +# above the next, like this: +# +# modules: +# - name: MODULE_NAME +# var.PLUGINTYPE1.PLUGINNAME1.KEY1: VALUE +# var.PLUGINTYPE1.PLUGINNAME1.KEY2: VALUE +# var.PLUGINTYPE2.PLUGINNAME1.KEY1: VALUE +# var.PLUGINTYPE3.PLUGINNAME3.KEY1: VALUE +# +# Module variable names must be in the format of +# +# var.PLUGIN_TYPE.PLUGIN_NAME.KEY +# +# modules: +# +# ------------ Cloud Settings --------------- +# Define Elastic Cloud settings here. +# Format of cloud.id is a base64 value e.g. dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy +# and it may have an label prefix e.g. staging:dXMtZ... +# This will overwrite 'var.elasticsearch.hosts' and 'var.kibana.host' +# cloud.id: +# +# Format of cloud.auth is: : +# This is optional +# If supplied this will overwrite 'var.elasticsearch.username' and 'var.elasticsearch.password' +# If supplied this will overwrite 'var.kibana.username' and 'var.kibana.password' +# cloud.auth: elastic: +# +# ------------ Queuing Settings -------------- +# +# Internal queuing model, "memory" for legacy in-memory based queuing and +# "persisted" for disk-based acked queueing. Defaults is memory +# +# queue.type: memory +# +# If using queue.type: persisted, the directory path where the data files will be stored. +# Default is path.data/queue +# +# path.queue: +# +# If using queue.type: persisted, the page data files size. The queue data consists of +# append-only data files separated into pages. Default is 64mb +# +# queue.page_capacity: 64mb +# +# If using queue.type: persisted, the maximum number of unread events in the queue. +# Default is 0 (unlimited) +# +# queue.max_events: 0 +# +# If using queue.type: persisted, the total capacity of the queue in number of bytes. +# If you would like more unacked events to be buffered in Logstash, you can increase the +# capacity using this setting. Please make sure your disk drive has capacity greater than +# the size specified here. If both max_bytes and max_events are specified, Logstash will pick +# whichever criteria is reached first +# Default is 1024mb or 1gb +# +# queue.max_bytes: 1024mb +# +# If using queue.type: persisted, the maximum number of acked events before forcing a checkpoint +# Default is 1024, 0 for unlimited +# +# queue.checkpoint.acks: 1024 +# +# If using queue.type: persisted, the maximum number of written events before forcing a checkpoint +# Default is 1024, 0 for unlimited +# +# queue.checkpoint.writes: 1024 +# +# If using queue.type: persisted, the interval in milliseconds when a checkpoint is forced on the head page +# Default is 1000, 0 for no periodic checkpoint. +# +# queue.checkpoint.interval: 1000 +# +# ------------ Dead-Letter Queue Settings -------------- +# Flag to turn on dead-letter queue. +# +# dead_letter_queue.enable: false + +# If using dead_letter_queue.enable: true, the maximum size of each dead letter queue. Entries +# will be dropped if they would increase the size of the dead letter queue beyond this setting. +# Default is 1024mb +# dead_letter_queue.max_bytes: 1024mb + +# If using dead_letter_queue.enable: true, the directory path where the data files will be stored. +# Default is path.data/dead_letter_queue +# +# path.dead_letter_queue: +# +# ------------ Metrics Settings -------------- +# +# Bind address for the metrics REST endpoint +# +# http.host: "127.0.0.1" http.host: 0.0.0.0 -queue.type: persisted -queue.max_bytes: 1gb -pipeline.workers: {{ pipeline.workers }} +# +# Bind port for the metrics REST endpoint, this option also accept a range +# (9600-9700) and logstash will pick up the first available ports. +# +# http.port: 9600-9700 +# +# ------------ Debugging Settings -------------- +# +# Options for log.level: +# * fatal +# * error +# * warn +# * info (default) +# * debug +# * trace +# +# log.level: info +# path.logs: path.logs: /var/log/logstash +# +# ------------ Other Settings -------------- +# +# Where to find custom plugins +# path.plugins: [] + +{% set pipeline.workers = salt['pillar.get']('master:ls_pipeline_workers', '1') %} +{% set pipeline.batch = salt['pillar.get']('master:ls_pipeline_batch_size', '125') %} + +pipeline.workers: {{ pipeline.workers }} +pipeline.batch.size: {{ pipeline.batch }}