Merge branch '3/dev' into feature/logstash-log-level

Merge pull request #16180 from Security-Onion-Solutions/feature/logstash-pipeline-settings-discrete
FEATURE: Allow for tuning multiple Logstash pipelines in SOC
2026-08-22 15:38:19 +02:00 · 2026-08-21 11:45:00 -04:00 · 2026-08-21 11:33:57 -04:00 · 2026-08-21 09:47:09 -04:00 · 2026-08-20 09:45:17 -04:00 · 2026-08-20 09:31:56 -04:00
6 changed files with 55 additions and 6 deletions
@@ -133,6 +133,14 @@ lspipelinesyml:
    - defaults:
        ASSIGNED_PIPELINES: {{ ASSIGNED_PIPELINES }}

+lslog4j2:
+  file.managed:
+    - name: /opt/so/conf/logstash/etc/log4j2.properties
+    - source: salt://logstash/etc/log4j2.properties.jinja
+    - template: jinja
+    - user: 931
+    - group: 939
+
 lsetcsync:
  file.recurse:
    - name: /opt/so/conf/logstash/etc
@@ -141,7 +149,11 @@ lsetcsync:
    - group: 939
    - template: jinja
    - clean: True
-    - exclude_pat: pipelines*
+{#- both names are matched: the .jinja source so the recurse does not copy it verbatim,
+    and the rendered file so clean: True does not delete what lslog4j2 wrote #}
+    - exclude_pat:
+      - pipelines*
+      - log4j2.properties*
    - defaults:
        LOGSTASH_MERGED: {{ LOGSTASH_MERGED }}

@@ -462,6 +462,8 @@ logstash:
    lsheap: 500m
  config:
    api_x_http_x_host: 0.0.0.0
+    log_x_level: info
+    log_x_format: plain
    path_x_logs: /var/log/logstash
    pipeline_x_workers: 1
    pipeline_x_batch_x_size: 125
@@ -105,6 +105,7 @@ so-logstash:
    {% endif %}
    - watch:
      - file: lsetcsync
+      - file: lslog4j2
      - file: lspipelinesyml
      - file: trusttheca
      {% if GLOBALS.is_manager %}
@@ -1,3 +1,4 @@
+{%- from 'logstash/map.jinja' import LOGSTASH_MERGED -%}
 status = error
 name = LogstashPropertiesConfig

@@ -16,8 +17,14 @@ name = LogstashPropertiesConfig
 appender.rolling.type = RollingFile
 appender.rolling.name = rolling
 appender.rolling.fileName = /var/log/logstash/logstash.log
+{%- if LOGSTASH_MERGED.config.get('log_x_format', 'plain') == 'json' %}
+appender.rolling.layout.type = JSONLayout
+appender.rolling.layout.compact = true
+appender.rolling.layout.eventEol = true
+{%- else %}
 appender.rolling.layout.type = PatternLayout
 appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %.10000m%n
+{%- endif %}
 appender.rolling.filePattern = /var/log/logstash/logstash-%d{yyyy-MM-dd}.log.gz
 appender.rolling.policies.type = Policies
 appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
@@ -32,7 +39,5 @@ appender.rolling.strategy.action.condition.type = IfFileName
 appender.rolling.strategy.action.condition.glob = *.gz
 appender.rolling.strategy.action.condition.nested_condition.type = IfLastModified
 appender.rolling.strategy.action.condition.nested_condition.age = 7D
-rootLogger.level = info
+rootLogger.level = ${sys:ls.log.level}
 rootLogger.appenderRef.rolling.ref = rolling
-#rootLogger.level = ${sys:ls.log.level}
-#rootLogger.appenderRef.console.ref = ${sys:ls.log.format}_console
@@ -413,6 +413,35 @@ logstash:
      helpLink: logstash
      readonly: True
      advanced: True
+    log_x_level:
+      description: >-
+        Verbosity of the Logstash log at /opt/so/log/logstash/logstash.log. debug and trace produce
+        a very large volume of log data on a busy node and should be used only while troubleshooting;
+        the log rotates at 1GB and rotated files are deleted after 7 days. Setting this to debug is
+        also what makes the per-pipeline config.debug setting emit anything.
+      title: log.level
+      options:
+      - 'fatal'
+      - 'error'
+      - 'warn'
+      - 'info'
+      - 'debug'
+      - 'trace'
+      advanced: True
+      global: False
+      helpLink: logstash
+    log_x_format:
+      description: >-
+        Layout of the Logstash log. plain writes human readable lines; json writes one JSON object
+        per line, which is easier to parse but harder to read directly. The file name and location
+        do not change.
+      title: log.format
+      options:
+      - 'plain'
+      - 'json'
+      advanced: True
+      global: False
+      helpLink: logstash
    path_x_logs: 
      description: Path inside the container to wrote logs.
      helpLink: logstash
@@ -2671,7 +2671,7 @@ soc:
              # The id (UUIDv4) is pregenerated and can safely be used.
              # Click "Convert" to convert the Sigma rule to use Security Onion field mappings within an EQL query
              #
-              # Rule Creation Guide: https://github.com/SigmaHQ/sigma/wiki/Rule-Creation-Guide
+              # Rule Creation Guide: https://github.com/SigmaHQ/sigma/wiki/Rule-Creation-High%E2%80%90Level-Guide
              # Logsources: https://sigmahq.io/docs/basics/log-sources.html

              title: 'A Short Capitalized Title With Less Than 50 Characters'
@@ -2683,7 +2683,7 @@ soc:
              references:
                - 'https://local.invalid'
              author: '@SecurityOnion'
-              date: 'YYYY/MM/DD'
+              date: '[today]'
              tags:
                - detection.threat_hunting
                - attack.technique_id
Author	SHA1	Message	Date
Josh Patterson	f566a8965d	Merge branch '3/dev' into feature/logstash-log-level	2026-08-21 11:45:00 -04:00
Josh Patterson	905cc1c0dd	Merge pull request #16180 from Security-Onion-Solutions/feature/logstash-pipeline-settings-discrete FEATURE: Allow for tuning multiple Logstash pipelines in SOC	2026-08-21 11:33:57 -04:00
Josh Patterson	52fc0cb828	Expose Logstash log.level and log.format in SOC The Logstash log level was hardcoded to info in log4j2.properties, and logstash.yml carried no log.level key, so the only way to raise verbosity for troubleshooting was to hand-edit a file that the next highstate overwrites. Add log_x_level and log_x_format to logstash:config so both render into logstash.yml, annotated as advanced per-node settings with the value sets Logstash 9.3.7 accepts. log4j2.properties gains jinja, so it moves to log4j2.properties.jinja and is rendered by a discrete lslog4j2 state rather than the lsetcsync recurse, which cannot rename. The recurse exclude_pat now matches both names so it neither copies the template verbatim nor lets clean: True delete the rendered file, matching how pipelines.yml is already handled. The appender layout is selected at render time so log.format actually changes the log output instead of being a dead setting, keeping the existing file name so nothing downstream moves. rootLogger.level now follows ls.log.level rather than claiming info regardless of the configured level.	2026-08-21 09:47:09 -04:00
Josh Brower	d2ff29b7a8	Merge pull request #16175 from Security-Onion-Solutions/fix/defaultsigma Update Sigma template	2026-08-20 09:45:17 -04:00
Josh Brower	7bdaf9338e	Update Sigma template	2026-08-20 09:31:56 -04:00