From 52fc0cb828f40f9f13e909653af6b44a86e660ce Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Fri, 21 Aug 2026 09:37:21 -0400 Subject: [PATCH] Expose Logstash log.level and log.format in SOC The Logstash log level was hardcoded to info in log4j2.properties, and logstash.yml carried no log.level key, so the only way to raise verbosity for troubleshooting was to hand-edit a file that the next highstate overwrites. Add log_x_level and log_x_format to logstash:config so both render into logstash.yml, annotated as advanced per-node settings with the value sets Logstash 9.3.7 accepts. log4j2.properties gains jinja, so it moves to log4j2.properties.jinja and is rendered by a discrete lslog4j2 state rather than the lsetcsync recurse, which cannot rename. The recurse exclude_pat now matches both names so it neither copies the template verbatim nor lets clean: True delete the rendered file, matching how pipelines.yml is already handled. The appender layout is selected at render time so log.format actually changes the log output instead of being a dead setting, keeping the existing file name so nothing downstream moves. rootLogger.level now follows ls.log.level rather than claiming info regardless of the configured level. --- salt/logstash/config.sls | 14 ++++++++- salt/logstash/defaults.yaml | 2 ++ salt/logstash/enabled.sls | 1 + ...4j2.properties => log4j2.properties.jinja} | 11 +++++-- salt/logstash/soc_logstash.yaml | 29 +++++++++++++++++++ 5 files changed, 53 insertions(+), 4 deletions(-) rename salt/logstash/etc/{log4j2.properties => log4j2.properties.jinja} (83%) diff --git a/salt/logstash/config.sls b/salt/logstash/config.sls index 47feba42c..b32a71a9a 100644 --- a/salt/logstash/config.sls +++ b/salt/logstash/config.sls @@ -125,6 +125,14 @@ lspipelinesyml: - defaults: ASSIGNED_PIPELINES: {{ ASSIGNED_PIPELINES }} +lslog4j2: + file.managed: + - name: /opt/so/conf/logstash/etc/log4j2.properties + - source: salt://logstash/etc/log4j2.properties.jinja + - template: jinja + - user: 931 + - group: 939 + lsetcsync: file.recurse: - name: /opt/so/conf/logstash/etc @@ -133,7 +141,11 @@ lsetcsync: - group: 939 - template: jinja - clean: True - - exclude_pat: pipelines* +{#- both names are matched: the .jinja source so the recurse does not copy it verbatim, + and the rendered file so clean: True does not delete what lslog4j2 wrote #} + - exclude_pat: + - pipelines* + - log4j2.properties* - defaults: LOGSTASH_MERGED: {{ LOGSTASH_MERGED }} diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml index db5e4ee58..41b960eca 100644 --- a/salt/logstash/defaults.yaml +++ b/salt/logstash/defaults.yaml @@ -64,6 +64,8 @@ logstash: lsheap: 500m config: api_x_http_x_host: 0.0.0.0 + log_x_level: info + log_x_format: plain path_x_logs: /var/log/logstash pipeline_x_workers: 1 pipeline_x_batch_x_size: 125 diff --git a/salt/logstash/enabled.sls b/salt/logstash/enabled.sls index 80e40d78d..39ff79539 100644 --- a/salt/logstash/enabled.sls +++ b/salt/logstash/enabled.sls @@ -105,6 +105,7 @@ so-logstash: {% endif %} - watch: - file: lsetcsync + - file: lslog4j2 - file: trusttheca {% if GLOBALS.is_manager %} - file: elasticsearch_cacerts diff --git a/salt/logstash/etc/log4j2.properties b/salt/logstash/etc/log4j2.properties.jinja similarity index 83% rename from salt/logstash/etc/log4j2.properties rename to salt/logstash/etc/log4j2.properties.jinja index 750a6e316..fd13e1bf2 100644 --- a/salt/logstash/etc/log4j2.properties +++ b/salt/logstash/etc/log4j2.properties.jinja @@ -1,3 +1,4 @@ +{%- from 'logstash/map.jinja' import LOGSTASH_MERGED -%} status = error name = LogstashPropertiesConfig @@ -16,8 +17,14 @@ name = LogstashPropertiesConfig appender.rolling.type = RollingFile appender.rolling.name = rolling appender.rolling.fileName = /var/log/logstash/logstash.log +{%- if LOGSTASH_MERGED.config.get('log_x_format', 'plain') == 'json' %} +appender.rolling.layout.type = JSONLayout +appender.rolling.layout.compact = true +appender.rolling.layout.eventEol = true +{%- else %} appender.rolling.layout.type = PatternLayout appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %.10000m%n +{%- endif %} appender.rolling.filePattern = /var/log/logstash/logstash-%d{yyyy-MM-dd}.log.gz appender.rolling.policies.type = Policies appender.rolling.policies.time.type = TimeBasedTriggeringPolicy @@ -32,7 +39,5 @@ appender.rolling.strategy.action.condition.type = IfFileName appender.rolling.strategy.action.condition.glob = *.gz appender.rolling.strategy.action.condition.nested_condition.type = IfLastModified appender.rolling.strategy.action.condition.nested_condition.age = 7D -rootLogger.level = info +rootLogger.level = ${sys:ls.log.level} rootLogger.appenderRef.rolling.ref = rolling -#rootLogger.level = ${sys:ls.log.level} -#rootLogger.appenderRef.console.ref = ${sys:ls.log.format}_console diff --git a/salt/logstash/soc_logstash.yaml b/salt/logstash/soc_logstash.yaml index 40794afe4..c195959e9 100644 --- a/salt/logstash/soc_logstash.yaml +++ b/salt/logstash/soc_logstash.yaml @@ -62,6 +62,35 @@ logstash: helpLink: logstash readonly: True advanced: True + log_x_level: + description: >- + Verbosity of the Logstash log at /opt/so/log/logstash/logstash.log. debug and trace produce + a very large volume of log data on a busy node and should be used only while troubleshooting; + the log rotates at 1GB and rotated files are deleted after 7 days. Setting this to debug is + also what makes the per-pipeline config.debug setting emit anything. + title: log.level + options: + - 'fatal' + - 'error' + - 'warn' + - 'info' + - 'debug' + - 'trace' + advanced: True + global: False + helpLink: logstash + log_x_format: + description: >- + Layout of the Logstash log. plain writes human readable lines; json writes one JSON object + per line, which is easier to parse but harder to read directly. The file name and location + do not change. + title: log.format + options: + - 'plain' + - 'json' + advanced: True + global: False + helpLink: logstash path_x_logs: description: Path inside the container to wrote logs. helpLink: logstash