mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2026-09-24 08:34:29 +02:00
Josh Patterson
52fc0cb828
The Logstash log level was hardcoded to info in log4j2.properties, and logstash.yml carried no log.level key, so the only way to raise verbosity for troubleshooting was to hand-edit a file that the next highstate overwrites. Add log_x_level and log_x_format to logstash:config so both render into logstash.yml, annotated as advanced per-node settings with the value sets Logstash 9.3.7 accepts. log4j2.properties gains jinja, so it moves to log4j2.properties.jinja and is rendered by a discrete lslog4j2 state rather than the lsetcsync recurse, which cannot rename. The recurse exclude_pat now matches both names so it neither copies the template verbatim nor lets clean: True delete the rendered file, matching how pipelines.yml is already handled. The appender layout is selected at render time so log.format actually changes the log output instead of being a dead setting, keeping the existing file name so nothing downstream moves. rootLogger.level now follows ls.log.level rather than claiming info regardless of the configured level.
165 lines
6.9 KiB
YAML+Jinja
165 lines
6.9 KiB
YAML+Jinja
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
|
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
|
# https://securityonion.net/license; you may not use this file except in compliance with the
|
|
# Elastic License 2.0.
|
|
|
|
{% from 'allowed_states.map.jinja' import allowed_states %}
|
|
{% if sls.split('.')[0] in allowed_states %}
|
|
{% from 'vars/globals.map.jinja' import GLOBALS %}
|
|
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
|
|
{% from 'logstash/map.jinja' import LOGSTASH_MERGED %}
|
|
{% from 'logstash/map.jinja' import LOGSTASH_NODES %}
|
|
{% set lsheap = LOGSTASH_MERGED.settings.lsheap %}
|
|
|
|
include:
|
|
- ca
|
|
{% if GLOBALS.role not in ['so-receiver','so-fleet'] %}
|
|
- elasticsearch.ca
|
|
{% endif %}
|
|
{# Kafka ca runs on nodes that can run logstash for Kafka input / output. Only when Kafka is global pipeline #}
|
|
{% if GLOBALS.role in ['so-searchnode', 'so-manager', 'so-managerhype', 'so-managersearch', 'so-receiver', 'so-standalone'] and GLOBALS.pipeline == 'KAFKA' %}
|
|
- kafka.ca
|
|
- kafka.ssl
|
|
{% endif %}
|
|
- logstash.ssl
|
|
- logstash.config
|
|
- logstash.sostatus
|
|
|
|
so-logstash:
|
|
docker_container.running:
|
|
- image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-logstash:{{ GLOBALS.so_version }}
|
|
- restart_policy: unless-stopped
|
|
- hostname: so-logstash
|
|
- name: so-logstash
|
|
- networks:
|
|
- sobridge:
|
|
- ipv4_address: {{ DOCKERMERGED.containers['so-logstash'].ip }}
|
|
- user: "931:0"
|
|
- extra_hosts:
|
|
{% for node in LOGSTASH_NODES %}
|
|
{% for hostname, ip in node.items() %}
|
|
- {{hostname}}:{{ip}}
|
|
{% endfor %}
|
|
{% endfor %}
|
|
{% if DOCKERMERGED.containers['so-logstash'].extra_hosts %}
|
|
{% for XTRAHOST in DOCKERMERGED.containers['so-logstash'].extra_hosts %}
|
|
- {{ XTRAHOST }}
|
|
{% endfor %}
|
|
{% endif %}
|
|
- environment:
|
|
- LS_JAVA_OPTS=-Xms{{ lsheap }} -Xmx{{ lsheap }}
|
|
{% if DOCKERMERGED.containers['so-logstash'].extra_env %}
|
|
{% for XTRAENV in DOCKERMERGED.containers['so-logstash'].extra_env %}
|
|
- {{ XTRAENV }}
|
|
{% endfor %}
|
|
{% endif %}
|
|
- port_bindings:
|
|
{% for BINDING in DOCKERMERGED.containers['so-logstash'].port_bindings %}
|
|
- {{ BINDING }}
|
|
{% endfor %}
|
|
- binds:
|
|
- /opt/so/conf/elasticsearch/templates/:/templates/:ro
|
|
- /opt/so/conf/logstash/etc/:/usr/share/logstash/config/:ro
|
|
- /opt/so/conf/logstash/pipelines:/usr/share/logstash/pipelines:ro
|
|
- /opt/so/rules:/etc/nsm/rules:ro
|
|
- /nsm/import:/nsm/import:ro
|
|
- /nsm/logstash:/usr/share/logstash/data:rw
|
|
- /opt/so/log/logstash:/var/log/logstash:rw
|
|
- /sys/fs/cgroup:/sys/fs/cgroup:ro
|
|
- /opt/so/conf/logstash/etc/certs:/usr/share/logstash/certs:ro
|
|
- /etc/pki/tls/certs/intca.crt:/usr/share/filebeat/ca.crt:ro
|
|
{% if GLOBALS.is_manager or GLOBALS.role in ['so-fleet', 'so-heavynode', 'so-receiver'] %}
|
|
- /etc/pki/elasticfleet-logstash.crt:/usr/share/logstash/elasticfleet-logstash.crt:ro
|
|
- /etc/pki/elasticfleet-logstash.key:/usr/share/logstash/elasticfleet-logstash.key:ro
|
|
- /etc/pki/elasticfleet-lumberjack.crt:/usr/share/logstash/elasticfleet-lumberjack.crt:ro
|
|
- /etc/pki/elasticfleet-lumberjack.key:/usr/share/logstash/elasticfleet-lumberjack.key:ro
|
|
{% if GLOBALS.role != 'so-fleet' %}
|
|
- /etc/pki/filebeat.crt:/usr/share/logstash/filebeat.crt:ro
|
|
- /etc/pki/filebeat.p8:/usr/share/logstash/filebeat.key:ro
|
|
{% endif %}
|
|
{% endif %}
|
|
{% if GLOBALS.role not in ['so-receiver','so-fleet'] %}
|
|
- /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro
|
|
- /opt/so/conf/ca/tls-ca-bundle.pem:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem:ro
|
|
{% endif %}
|
|
{% if GLOBALS.pipeline == "KAFKA" and GLOBALS.role in ['so-manager', 'so-managerhype', 'so-managersearch', 'so-standalone', 'so-searchnode'] %}
|
|
- /etc/pki/kafka-logstash.p12:/usr/share/logstash/kafka-logstash.p12:ro
|
|
- /opt/so/conf/kafka/kafka-truststore.jks:/etc/pki/kafka-truststore.jks:ro
|
|
{% endif %}
|
|
{% if GLOBALS.role == 'so-eval' %}
|
|
- /nsm/zeek:/nsm/zeek:ro
|
|
- /nsm/suricata:/suricata:ro
|
|
- /opt/so/log/fleet/:/osquery/logs:ro
|
|
- /opt/so/log/strelka:/strelka:ro
|
|
{% endif %}
|
|
{% if DOCKERMERGED.containers['so-logstash'].custom_bind_mounts %}
|
|
{% for BIND in DOCKERMERGED.containers['so-logstash'].custom_bind_mounts %}
|
|
- {{ BIND }}
|
|
{% endfor %}
|
|
{% endif %}
|
|
{% if DOCKERMERGED.containers['so-logstash'].ulimits %}
|
|
- ulimits:
|
|
{% for ULIMIT in DOCKERMERGED.containers['so-logstash'].ulimits %}
|
|
- {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
|
|
{% endfor %}
|
|
{% endif %}
|
|
- watch:
|
|
- file: lsetcsync
|
|
- file: lslog4j2
|
|
- file: trusttheca
|
|
{% if GLOBALS.is_manager %}
|
|
- file: elasticsearch_cacerts
|
|
- file: elasticsearch_capems
|
|
{% endif %}
|
|
{% if GLOBALS.is_manager or GLOBALS.role in ['so-fleet', 'so-heavynode', 'so-receiver'] %}
|
|
- x509: etc_elasticfleet_logstash_crt
|
|
- x509: etc_elasticfleet_logstash_key
|
|
- x509: etc_elasticfleetlumberjack_crt
|
|
- x509: etc_elasticfleetlumberjack_key
|
|
{% if GLOBALS.role != 'so-fleet' %}
|
|
- x509: etc_filebeat_crt
|
|
- file: logstash_filebeat_p8
|
|
{% endif %}
|
|
{% endif %}
|
|
{% for assigned_pipeline in LOGSTASH_MERGED.assigned_pipelines.roles[GLOBALS.role.split('-')[1]] %}
|
|
- file: ls_pipeline_{{assigned_pipeline}}
|
|
{% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %}
|
|
- file: ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}
|
|
{% endfor %}
|
|
{% endfor %}
|
|
{% if GLOBALS.pipeline == 'KAFKA' and GLOBALS.role in ['so-manager', 'so-managerhype', 'so-managersearch', 'so-standalone', 'so-searchnode'] %}
|
|
- file: kafkacertz
|
|
{% endif %}
|
|
- require:
|
|
- file: trusttheca
|
|
{% if GLOBALS.is_manager %}
|
|
- file: elasticsearch_cacerts
|
|
- file: elasticsearch_capems
|
|
{% endif %}
|
|
{% if GLOBALS.is_manager or GLOBALS.role in ['so-fleet', 'so-heavynode', 'so-receiver'] %}
|
|
- x509: etc_elasticfleet_logstash_crt
|
|
- x509: etc_elasticfleet_logstash_key
|
|
- x509: etc_elasticfleetlumberjack_crt
|
|
- x509: etc_elasticfleetlumberjack_key
|
|
{% if GLOBALS.role != 'so-fleet' %}
|
|
- x509: etc_filebeat_crt
|
|
- file: logstash_filebeat_p8
|
|
{% endif %}
|
|
{% endif %}
|
|
{% if GLOBALS.pipeline == 'KAFKA' and GLOBALS.role in ['so-manager', 'so-managerhype', 'so-managersearch', 'so-standalone', 'so-searchnode'] %}
|
|
- file: kafkacertz
|
|
{% endif %}
|
|
|
|
delete_so-logstash_so-status.disabled:
|
|
file.uncomment:
|
|
- name: /opt/so/conf/so-status/so-status.conf
|
|
- regex: ^so-logstash$
|
|
|
|
{% else %}
|
|
|
|
{{sls}}_state_not_allowed:
|
|
test.fail_without_changes:
|
|
- name: {{sls}}_state_not_allowed
|
|
|
|
{% endif %}
|