Merge pull request #16182 from Security-Onion-Solutions/feature/logstash-log-level

Expose Logstash log.level and log.format in SOC

salt/logstash/config.sls

@@ -133,6 +133,14 @@ lspipelinesyml:
     - defaults:
         ASSIGNED_PIPELINES: {{ ASSIGNED_PIPELINES }}
 
+lslog4j2:
+  file.managed:
+    - name: /opt/so/conf/logstash/etc/log4j2.properties
+    - source: salt://logstash/etc/log4j2.properties.jinja
+    - template: jinja
+    - user: 931
+    - group: 939
+
 lsetcsync:
   file.recurse:
     - name: /opt/so/conf/logstash/etc
@@ -141,7 +149,11 @@ lsetcsync:
     - group: 939
     - template: jinja
     - clean: True
-    - exclude_pat: pipelines*
+{#- both names are matched: the .jinja source so the recurse does not copy it verbatim,
+    and the rendered file so clean: True does not delete what lslog4j2 wrote #}
+    - exclude_pat:
+      - pipelines*
+      - log4j2.properties*
     - defaults:
         LOGSTASH_MERGED: {{ LOGSTASH_MERGED }}
 

salt/logstash/defaults.yaml

@@ -462,6 +462,8 @@ logstash:
     lsheap: 500m
   config:
     api_x_http_x_host: 0.0.0.0
+    log_x_level: info
+    log_x_format: plain
     path_x_logs: /var/log/logstash
     pipeline_x_workers: 1
     pipeline_x_batch_x_size: 125

salt/logstash/enabled.sls

@@ -105,6 +105,7 @@ so-logstash:
     {% endif %}
     - watch:
       - file: lsetcsync
+      - file: lslog4j2
       - file: lspipelinesyml
       - file: trusttheca
       {% if GLOBALS.is_manager %}

salt/logstash/etc/log4j2.properties.jinja

@@ -1,3 +1,4 @@
+{%- from 'logstash/map.jinja' import LOGSTASH_MERGED -%}
 status = error
 name = LogstashPropertiesConfig
 
@@ -16,8 +17,14 @@ name = LogstashPropertiesConfig
 appender.rolling.type = RollingFile
 appender.rolling.name = rolling
 appender.rolling.fileName = /var/log/logstash/logstash.log
+{%- if LOGSTASH_MERGED.config.get('log_x_format', 'plain') == 'json' %}
+appender.rolling.layout.type = JSONLayout
+appender.rolling.layout.compact = true
+appender.rolling.layout.eventEol = true
+{%- else %}
 appender.rolling.layout.type = PatternLayout
 appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %.10000m%n
+{%- endif %}
 appender.rolling.filePattern = /var/log/logstash/logstash-%d{yyyy-MM-dd}.log.gz
 appender.rolling.policies.type = Policies
 appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
@@ -32,7 +39,5 @@ appender.rolling.strategy.action.condition.type = IfFileName
 appender.rolling.strategy.action.condition.glob = *.gz
 appender.rolling.strategy.action.condition.nested_condition.type = IfLastModified
 appender.rolling.strategy.action.condition.nested_condition.age = 7D
-rootLogger.level = info
+rootLogger.level = ${sys:ls.log.level}
 rootLogger.appenderRef.rolling.ref = rolling
-#rootLogger.level = ${sys:ls.log.level}
-#rootLogger.appenderRef.console.ref = ${sys:ls.log.format}_console

salt/logstash/soc_logstash.yaml

@@ -413,6 +413,35 @@ logstash:
       helpLink: logstash
       readonly: True
       advanced: True
+    log_x_level:
+      description: >-
+        Verbosity of the Logstash log at /opt/so/log/logstash/logstash.log. debug and trace produce
+        a very large volume of log data on a busy node and should be used only while troubleshooting;
+        the log rotates at 1GB and rotated files are deleted after 7 days. Setting this to debug is
+        also what makes the per-pipeline config.debug setting emit anything.
+      title: log.level
+      options:
+      - 'fatal'
+      - 'error'
+      - 'warn'
+      - 'info'
+      - 'debug'
+      - 'trace'
+      advanced: True
+      global: False
+      helpLink: logstash
+    log_x_format:
+      description: >-
+        Layout of the Logstash log. plain writes human readable lines; json writes one JSON object
+        per line, which is easier to parse but harder to read directly. The file name and location
+        do not change.
+      title: log.format
+      options:
+      - 'plain'
+      - 'json'
+      advanced: True
+      global: False
+      helpLink: logstash
     path_x_logs: 
       description: Path inside the container to wrote logs.
       helpLink: logstash