Expose Logstash 9.3.7 pipeline settings per pipeline in SOC

Add logstash:pipeline_settings carrying the 27 pipeline-scoped settings
Logstash 9.3.7 accepts, annotated individually per pipeline and rendered
into pipelines.yml. A blank setting inherits from logstash.yml. Restart
logstash when pipelines.yml changes, and add the missing managerhype
annotation.

Fixes #15090

salt/common/tools/sbin/so-log-check

@@ -154,6 +154,8 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|id.orig_h"                    # false positive (zeek test data)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|emerging-all.rules"           # false positive (error in rulename)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|invalid query input"          # false positive (Invalid user input in hunt query)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|no data available for the requested dates" # false positive (pcap cypress test submits a job with an empty time frame)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|no job processor"             # false positive (same empty-time-frame job on import nodes, where no pcap processor runs)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|example"                      # false positive (example test data)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|status 200"                   # false positive (request successful, contained error string in content)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|app_layer.error"              # false positive (suricata 7) in stats.log e.g. app_layer.error.imap.parser | Total | 0

salt/logstash/config.sls

@@ -81,6 +81,10 @@ ls_custom_pipeline_conf_{{assigned_pipeline}}_{{pipeline}}:
 
 
 {% for assigned_pipeline in ASSIGNED_PIPELINES %}
+{# a blank per-pipeline setting falls back to the global logstash.yml value #}
+{% set PIPELINE_OVERRIDES = LOGSTASH_MERGED.get('pipeline_settings', {}).get(assigned_pipeline, {}) %}
+{% set THREADS = PIPELINE_OVERRIDES.get('pipeline_x_workers') or LOGSTASH_MERGED.config.pipeline_x_workers %}
+{% set BATCH = PIPELINE_OVERRIDES.get('pipeline_x_batch_x_size') or LOGSTASH_MERGED.config.pipeline_x_batch_x_size %}
     {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %}
 ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}:
   file.managed:
@@ -92,8 +96,8 @@ ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_")
         GLOBALS: {{ GLOBALS }}
         ES_USER: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') }}"
         ES_PASS: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') }}"
-        THREADS: {{ LOGSTASH_MERGED.config.pipeline_x_workers }}
-        BATCH: {{ LOGSTASH_MERGED.config.pipeline_x_batch_x_size }}
+        THREADS: {{ THREADS }}
+        BATCH: {{ BATCH }}
       {% else %}
     - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{CONFIGFILE.split('/')[1]}}
       {% endif %}

salt/logstash/defaults.yaml

@@ -60,6 +60,259 @@ logstash:
     custom008: PLACEHOLDER
     custom009: PLACEHOLDER
     custom010: PLACEHOLDER
+  pipeline_settings:
+    fleet:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    manager:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    receiver:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    search:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom0:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom1:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom2:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom3:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom4:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
   settings:
     lsheap: 500m
   config:

salt/logstash/enabled.sls

@@ -105,6 +105,7 @@ so-logstash:
     {% endif %}
     - watch:
       - file: lsetcsync
+      - file: lspipelinesyml
       - file: trusttheca
       {% if GLOBALS.is_manager %}
       - file: elasticsearch_cacerts

salt/logstash/etc/pipelines.yml.jinja

@@ -1,4 +1,17 @@
+{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
+{%- set PIPELINE_SETTINGS = LOGSTASH_MERGED.get('pipeline_settings', {}) %}
 {%- for assigned_pipeline in ASSIGNED_PIPELINES %}
 - pipeline.id: {{ assigned_pipeline }}
   path.config: "/usr/share/logstash/pipelines/{{ assigned_pipeline }}/"
+{%-   set extra = PIPELINE_SETTINGS.get(assigned_pipeline, {}) %}
+{%-   if extra is mapping %}
+{#-     values are emitted unquoted so yaml re-infers the type logstash expects:
+        4 as an integer, false as a boolean, 1024mb and auto as strings #}
+{%-     for key, value in extra | dictsort %}
+{%-       set rendered = key | replace('_x_', '.') %}
+{%-       if value not in ['', None] and rendered not in ['pipeline.id', 'path.config'] %}
+  {{ rendered }}: {{ value }}
+{%-       endif %}
+{%-     endfor %}
+{%-   endif %}
 {% endfor -%}

salt/logstash/soc_logstash.yaml

@@ -16,6 +16,7 @@ logstash:
       heavynode: *assigned_pipelines
       searchnode: *assigned_pipelines
       manager: *assigned_pipelines
+      managerhype: *assigned_pipelines
       managersearch: *assigned_pipelines
       fleet: *assigned_pipelines
   defined_pipelines:
@@ -51,6 +52,322 @@ logstash:
     custom008: *pipeline_config
     custom009: *pipeline_config
     custom010: *pipeline_config
+  pipeline_settings:
+    manager: &pipeline_settings
+      pipeline_x_workers:
+        description: >-
+          Number of worker threads that run filters and outputs for this pipeline. May be set higher
+          than the CPU core count when outputs spend time waiting on I/O. Leave blank to use the value
+          from logstash.yml.
+        title: pipeline.workers
+        regex: '^$|^[1-9][0-9]*$'
+        regexFailureMessage: Must be blank, or a positive whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      pipeline_x_batch_x_size:
+        description: >-
+          Maximum number of events an individual worker thread collects before running filters and
+          outputs. Larger batches are more efficient but increase heap use; total in-flight events is
+          workers multiplied by batch size. Leave blank to use the value from logstash.yml.
+        title: pipeline.batch.size
+        regex: '^$|^[1-9][0-9]*$'
+        regexFailureMessage: Must be blank, or a positive whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      pipeline_x_batch_x_delay:
+        description: >-
+          Milliseconds a worker waits for the next event before running a batch that is not yet full.
+          Leave blank to use the value from logstash.yml.
+        title: pipeline.batch.delay
+        regex: '^$|^[0-9]+$'
+        regexFailureMessage: Must be blank, or a whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      pipeline_x_batch_x_metrics_x_sampling_mode:
+        description: >-
+          How much batch size metering this pipeline records. Fuller sampling helps size batches but
+          consumes additional heap. Leave blank to use the value from logstash.yml.
+        title: pipeline.batch.metrics.sampling_mode
+        options:
+        - ''
+        - 'disabled'
+        - 'minimal'
+        - 'full'
+        advanced: True
+        global: False
+        helpLink: logstash
+      pipeline_x_ordered:
+        description: >-
+          Whether event order is preserved through this pipeline. auto enables ordering only when the
+          pipeline runs a single worker. Leave blank to use the value from logstash.yml.
+        title: pipeline.ordered
+        options:
+        - ''
+        - 'auto'
+        - 'true'
+        - 'false'
+        advanced: True
+        global: False
+        helpLink: logstash
+      pipeline_x_ecs_compatibility:
+        description: >-
+          Elastic Common Schema compatibility mode for plugins in this pipeline. Security Onion sets
+          this globally and it should rarely be changed per pipeline. Leave blank to use the value
+          from logstash.yml.
+        title: pipeline.ecs_compatibility
+        options:
+        - ''
+        - 'disabled'
+        - 'v1'
+        - 'v8'
+        advanced: True
+        global: False
+        helpLink: logstash
+      pipeline_x_reloadable:
+        description: >-
+          Whether this pipeline may be reloaded when its configuration changes. Leave blank to use the
+          value from logstash.yml.
+        title: pipeline.reloadable
+        options:
+        - ''
+        - 'true'
+        - 'false'
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_type:
+        description: >-
+          Queue backing this pipeline. persisted buffers events to disk under /nsm/logstash so they
+          survive a restart, at some throughput cost; memory does not. Leave blank to use the value
+          from logstash.yml.
+        title: queue.type
+        options:
+        - ''
+        - 'memory'
+        - 'persisted'
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_max_bytes:
+        description: >-
+          Total size of the persistent queue for this pipeline. Only applies when queue.type is
+          persisted, and must fit the disk backing /nsm/logstash. Leave blank to use the value from
+          logstash.yml.
+        title: queue.max_bytes
+        regex: '^$|^[0-9]+(b|kb|mb|gb|tb|pb)$'
+        regexFailureMessage: Must be blank, or a size such as 512mb, 1gb.
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_page_capacity:
+        description: >-
+          Size of each page in the persistent queue for this pipeline. Leave blank to use the value
+          from logstash.yml.
+        title: queue.page_capacity
+        regex: '^$|^[0-9]+(b|kb|mb|gb|tb|pb)$'
+        regexFailureMessage: Must be blank, or a size such as 512mb, 1gb.
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_max_events:
+        description: >-
+          Maximum number of events in the persistent queue for this pipeline. 0 means unlimited. Leave
+          blank to use the value from logstash.yml.
+        title: queue.max_events
+        regex: '^$|^[0-9]+$'
+        regexFailureMessage: Must be blank, or a whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_checkpoint_x_acks:
+        description: >-
+          Number of acknowledged events before a persistent queue checkpoint is forced. 0 means
+          unlimited. Leave blank to use the value from logstash.yml.
+        title: queue.checkpoint.acks
+        regex: '^$|^[0-9]+$'
+        regexFailureMessage: Must be blank, or a whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_checkpoint_x_writes:
+        description: >-
+          Number of written events before a persistent queue checkpoint is forced. 0 means unlimited.
+          Leave blank to use the value from logstash.yml.
+        title: queue.checkpoint.writes
+        regex: '^$|^[0-9]+$'
+        regexFailureMessage: Must be blank, or a whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_checkpoint_x_interval:
+        description: >-
+          Milliseconds between persistent queue head page checkpoints. 0 disables periodic
+          checkpointing. Leave blank to use the value from logstash.yml.
+        title: queue.checkpoint.interval
+        regex: '^$|^[0-9]+$'
+        regexFailureMessage: Must be blank, or a whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_checkpoint_x_retry:
+        description: >-
+          Whether Logstash retries a failed persistent queue checkpoint write. Leave blank to use the
+          value from logstash.yml.
+        title: queue.checkpoint.retry
+        options:
+        - ''
+        - 'true'
+        - 'false'
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_compression:
+        description: >-
+          Compression applied to persistent queue pages for this pipeline, trading CPU for disk. Leave
+          blank to use the value from logstash.yml.
+        title: queue.compression
+        options:
+        - ''
+        - 'none'
+        - 'speed'
+        - 'balanced'
+        - 'size'
+        - 'disabled'
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_drain:
+        description: >-
+          Whether Logstash drains the persistent queue before shutting down this pipeline. Draining a
+          large queue makes shutdown take considerably longer. Leave blank to use the value from
+          logstash.yml.
+        title: queue.drain
+        options:
+        - ''
+        - 'true'
+        - 'false'
+        advanced: True
+        global: False
+        helpLink: logstash
+      dead_letter_queue_x_enable:
+        description: >-
+          Whether events this pipeline cannot process are written to a dead letter queue instead of
+          being dropped. Leave blank to use the value from logstash.yml.
+        title: dead_letter_queue.enable
+        options:
+        - ''
+        - 'true'
+        - 'false'
+        advanced: True
+        global: False
+        helpLink: logstash
+      dead_letter_queue_x_max_bytes:
+        description: >-
+          Total size of the dead letter queue for this pipeline. Leave blank to use the value from
+          logstash.yml.
+        title: dead_letter_queue.max_bytes
+        regex: '^$|^[0-9]+(b|kb|mb|gb|tb|pb)$'
+        regexFailureMessage: Must be blank, or a size such as 512mb, 1gb.
+        advanced: True
+        global: False
+        helpLink: logstash
+      dead_letter_queue_x_flush_interval:
+        description: >-
+          Milliseconds before a partial dead letter queue segment is flushed. Leave blank to use the
+          value from logstash.yml.
+        title: dead_letter_queue.flush_interval
+        regex: '^$|^[0-9]+$'
+        regexFailureMessage: Must be blank, or a whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      dead_letter_queue_x_flush_check_interval:
+        description: >-
+          Milliseconds between checks for a dead letter queue segment that needs flushing. Leave blank
+          to use the value from logstash.yml.
+        title: dead_letter_queue.flush_check_interval
+        regex: '^$|^[0-9]+$'
+        regexFailureMessage: Must be blank, or a whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      dead_letter_queue_x_storage_policy:
+        description: >-
+          What happens when the dead letter queue is full: drop_newer discards incoming events,
+          drop_older discards the oldest stored events. Leave blank to use the value from
+          logstash.yml.
+        title: dead_letter_queue.storage_policy
+        options:
+        - ''
+        - 'drop_newer'
+        - 'drop_older'
+        advanced: True
+        global: False
+        helpLink: logstash
+      dead_letter_queue_x_retain_x_age:
+        description: >-
+          How long an event is kept in the dead letter queue before removal, such as 5d. Leave blank
+          to use the value from logstash.yml.
+        title: dead_letter_queue.retain.age
+        regex: '^$|^[0-9]+[dhms]$'
+        regexFailureMessage: Must be blank, or a number followed by d, h, m, or s, such as 5d.
+        advanced: True
+        global: False
+        helpLink: logstash
+      path_x_queue:
+        description: >-
+          Directory inside the Logstash container holding the persistent queue for this pipeline. The
+          default lives under the /nsm/logstash bind mount; a path outside it will not survive a
+          container restart. Leave blank to use the value from logstash.yml.
+        title: path.queue
+        advanced: True
+        global: False
+        helpLink: logstash
+      path_x_dead_letter_queue:
+        description: >-
+          Directory inside the Logstash container holding the dead letter queue for this pipeline. The
+          default lives under the /nsm/logstash bind mount; a path outside it will not survive a
+          container restart. Leave blank to use the value from logstash.yml.
+        title: path.dead_letter_queue
+        advanced: True
+        global: False
+        helpLink: logstash
+      config_x_debug:
+        description: >-
+          Whether the fully compiled configuration for this pipeline is written to the log. The output
+          may contain sensitive values from the pipeline configuration. Leave blank to use the value
+          from logstash.yml.
+        title: config.debug
+        options:
+        - ''
+        - 'true'
+        - 'false'
+        advanced: True
+        global: False
+        helpLink: logstash
+      config_x_support_escapes:
+        description: >-
+          Whether escape sequences such as \n and \t in this pipeline's quoted strings are
+          interpreted. Leave blank to use the value from logstash.yml.
+        title: config.support_escapes
+        options:
+        - ''
+        - 'true'
+        - 'false'
+        advanced: True
+        global: False
+        helpLink: logstash
+    fleet: *pipeline_settings
+    receiver: *pipeline_settings
+    search: *pipeline_settings
+    custom0: *pipeline_settings
+    custom1: *pipeline_settings
+    custom2: *pipeline_settings
+    custom3: *pipeline_settings
+    custom4: *pipeline_settings
   settings:
     lsheap: 
       description: Heap size to use for logstash

salt/manager/tools/sbin/soup

@@ -344,6 +344,16 @@ check_cluster_health() {
 
 check_fleet_server() {
   echo "Checking that Elastic Fleet Server is responding."
+  # Before checking fleet health, check for and fix known issue with elastic-agent container and fs.protected_symlinks
+  local protected_symlinks=$(sysctl -b fs.protected_symlinks)
+  if [[ "$protected_symlinks" == "1" ]]; then
+    # disable fs.protected_symlinks and restart elasticfleet
+    sysctl -w fs.protected_symlinks=0
+    docker stop so-elastic-fleet; docker rm -f so-elastic-fleet
+    printf "\nUpdated sysctl fs.protected_symlinks. Restarting fleet before running health check and continuing with soup.\n"
+    salt-call state.apply elasticfleet queue=True
+  fi
+  
   # Modeled on the wait_for_so-elastic-fleet state check in elasticfleet/enabled.sls,
   # which waits for HTTP 200 from the Fleet Server status API.
   if curl -sk --fail --retry 3 --retry-delay 10 --max-time 30 "https://localhost:8220/api/status" > /dev/null 2>&1; then
@@ -1036,8 +1046,20 @@ post_to_3.2.0() {
 }
 ### 3.2.0 End ###
 
-### 3.2.0 Scripts ###
+### 3.3.0 Scripts ###
+
+ # Sets fs.protected_symlinks=0
+ #
+ # Elastic Agent docker image chowns its directory to the running UID 
+ # but does not chown the elastic-agent launcher symlink. 
+ # Preventing non-root users from following that launcher symlink.
+disable_sysctl_fs_protected_symlink() {
+  salt -C 'I@stig:enabled' state.single sysctl.present name=fs.protected_symlinks value=0 config=/etc/sysctl.conf || true
+}
+    
 up_to_3.3.0() {
+  disable_sysctl_fs_protected_symlink
+  
   INSTALLEDVERSION=3.3.0
 }
 

salt/salt/minion/boot_highstate.sls

@@ -3,6 +3,8 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+
 # Manages /etc/systemd/system/so-boot-highstate.service, a Type=oneshot
 # RemainAfterExit=yes unit that runs `salt-call state.highstate` exactly once
 # per system boot. Replaces the legacy `startup_states: highstate` minion
@@ -19,9 +21,25 @@ so_boot_highstate_unit_file:
     - onchanges_in:
       - module: systemd_reload
 
+# Non-managers never apply salt.minion during setup, so reaching this state means
+# setup is finished and the marker is safe to write unconditionally. This also
+# heals nodes installed before this fix, which have no marker and no legacy
+# startup_states line to grep for. Managers do highstate mid-setup, so they only
+# get the marker from the legacy upgrade signal; fresh installs get it from
+# mark_setup_complete in setup/so-functions.
+mark_setup_complete:
+  file.managed:
+    - name: /opt/so/state/setup-complete
+    - replace: false
+    - makedirs: True
+{% if GLOBALS.is_manager %}
+    - onlyif: "grep -qx 'startup_states: highstate' /etc/salt/minion"
+{% endif %}
+    - require_in:
+      - service: so_boot_highstate_service
+
 # Only enable once setup is complete. Until then the gate file is missing and
-# the unit's own ConditionPathExists would no-op it anyway -- this just keeps
-# `systemctl is-enabled` honest for the sync_es_users gate.
+# the unit's own ConditionPathExists would no-op it anyway.
 so_boot_highstate_service:
   service.enabled:
     - name: so-boot-highstate.service

salt/salt/minion/init.sls

@@ -87,27 +87,15 @@ set_log_levels:
 # so-boot-highstate.service (managed in salt.minion.boot_highstate), which
 # runs once per system boot only. Strip the line from /etc/salt/minion on
 # upgrade; both the commented and uncommented forms historically existed.
+# Ordered after mark_setup_complete (salt.minion.boot_highstate); the manager
+# gate there greps for this line, so it must run before we delete it.
 remove_startup_states:
   file.line:
     - name: /etc/salt/minion
     - match: 'startup_states: highstate'
     - mode: delete
-
-# Upgrade-path bridge: systems that already passed setup under the old gate
-# (`grep -x 'startup_states: highstate' /etc/salt/minion`) get a /opt/so/state/setup-complete
-# marker so so-boot-highstate.service can be enabled and the so-user_sync cron
-# in sync_es_users.sls keeps installing. Setup-in-progress systems instead get
-# the marker from `mark_setup_complete` in setup/so-functions at the right
-# moment. `replace: false` means we never overwrite a marker once written.
-mark_setup_complete_for_upgrades:
-  file.managed:
-    - name: /opt/so/state/setup-complete
-    - replace: false
-    - makedirs: True
-    - onlyif: "grep -qx 'startup_states: highstate' /etc/salt/minion"
-    - require_in:
-      - file: remove_startup_states
-      - service: so_boot_highstate_service
+    - require:
+      - file: mark_setup_complete
 
 {% endif %}
 

salt/setup/virt/setSalt.sls

@@ -8,6 +8,15 @@ set_role_grain:
     - name: role
     - value: so-{{ grains.id.split("_") | last }}
 
+# salt-cloud guests never run so-setup, so nothing else marks them setup-complete.
+# Replaces the 'startup_states: highstate' line this state used to append. No
+# GLOBALS import -- this runs before the guest's pillars exist.
+mark_setup_complete_vm_guest:
+  file.managed:
+    - name: /opt/so/state/setup-complete
+    - replace: false
+    - makedirs: True
+
 enable_salt_minion:
   service.enabled:
     - name: salt-minion

salt/stig/enabled.sls

@@ -65,6 +65,15 @@ run_remediate:
     - success_retcodes:
       - 2
 
+# Elastic Agent docker image chowns its directory to the running UID but does not
+#  chown the elastic-agent launcher symlink. fs.protected_symlinks=1 then prevents
+#  non-root users from following that launcher symlink.
+{# OSCAP rule id: xccdf_org.ssgproject.content_rule_sysctl_fs_protected_symlinks #}
+fs.protected_symlinks:
+  sysctl.present:
+    - value: 0
+    - config: /etc/sysctl.conf
+
 {# OSCAP rule id: xccdf_org.ssgproject.content_rule_disable_ctrlaltdel_burstaction #}
 disable_ctrl_alt_del_action:
   file.replace:

salt/stig/files/sos-oscap.xml

@@ -1601,7 +1601,7 @@ DISA STIG for Oracle Linux 9 V1R3.</xccdf-1.2:description>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sudoers_validate_passwd" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_crypto_fips_enabled" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_hardlinks" selected="true"/>
-        <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_symlinks" selected="true"/>
+        <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_symlinks" selected="false"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_core_pattern" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_dmesg_restrict" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_exec_shield" selected="true"/>
@@ -2202,7 +2202,7 @@ standard DISA STIG for Oracle Linux 9 profile.</xccdf-1.2:description>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sudoers_validate_passwd" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_crypto_fips_enabled" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_hardlinks" selected="true"/>
-        <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_symlinks" selected="true"/>
+        <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_symlinks" selected="false"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_core_pattern" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_dmesg_restrict" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_exec_shield" selected="true"/>

salt/zeek/defaults.yaml

@@ -15,7 +15,7 @@ zeek:
       MailHostUpDown: 0
       LogRotationInterval: 3600
       LogExpireInterval: 0
-      StatsLogEnable: 1
+      StatsLogEnable: 0
       StatsLogExpireInterval: 0
       StatusCmdShowAll: 0
       CrashExpireInterval: 0

salt/zeek/disabled.sls

@@ -23,6 +23,11 @@ zeekpacketlosscron:
     - identifier: zeekpacketlosscron
     - user: root
 
+zeekctlcron:
+  cron.absent:
+    - identifier: zeekctlcron
+    - user: root
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/zeek/enabled.sls

@@ -87,6 +87,21 @@ zeekpacketlosscron:
     - month: '*'
     - dayweek: '*'
 
+# LogExpireInterval, StatsLogExpireInterval and CrashExpireInterval are only acted on by
+# 'zeekctl cron', so run it on the interval upstream recommends. This also restarts any
+# node that died unexpectedly. Runs as root because the script needs the docker socket;
+# it drops to the zeek user inside the container.
+zeekctlcron:
+  cron.present:
+    - name: /usr/sbin/so-zeek-cron > /dev/null 2>&1
+    - identifier: zeekctlcron
+    - user: root
+    - minute: '*/5'
+    - hour: '*'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/zeek/soc_zeek.yaml

@@ -58,6 +58,86 @@ zeek:
       CompressLogs:
         description: This setting enables compression of Zeek logs. If you are seeing packet loss at the top of the hour in Zeek or PCAP you might need to disable this by seting it to 0. This will use more disk space but save IO and CPU.
         helpLink: zeek
+      LogExpireInterval:
+        description: >-
+          How long to keep rotated Zeek logs in /nsm/zeek/logs. A bare number means DAYS, so 7 means 7 days.
+          You may also give an explicit unit, such as "7 days" or "12 hr". Use 0 to keep logs forever.
+          This value must not be shorter than LogRotationInterval (3600 seconds by default), so the smallest
+          usable value is 1 hr - Zeek will fail to start if it is shorter. Expiry is applied by "zeekctl cron",
+          which runs every 5 minutes, and removes log files older than this based on their modification time.
+        regex: ^(0|[1-9][0-9]*( ?(day|hr)s?)?)$
+        regexFailureMessage: Enter 0, or a positive number optionally followed by "day" or "hr" (for example 7, "7 days", or "12 hr"). Minutes are not accepted because a log expire interval shorter than the log rotation interval prevents Zeek from starting.
+        helpLink: zeek
+        advanced: True
+      StatsLogEnable:
+        description: >-
+          Set to 1 to have "zeekctl cron" write node statistics to /nsm/zeek/logs/stats. This is
+          disabled because the CPU and memory portion depends on the "top" command, which the Zeek
+          container does not include, so every run records an error for each node instead. The
+          interface packet counters it also collects are not used anywhere in Security Onion, which
+          tracks Zeek packet loss separately through packetloss.log and Telegraf. It is read only
+          for that reason.
+        regex: ^[01]$
+        regexFailureMessage: You must enter 0 or 1.
+        helpLink: zeek
+        advanced: True
+        readonly: True
+      StatsLogExpireInterval:
+        description: >-
+          Number of days to keep entries in the Zeek stats log, or 0 to keep them forever.
+          Applied by "zeekctl cron", which runs every 5 minutes. This has no effect unless
+          StatsLogEnable is turned on, which it is not by default.
+        regex: ^[0-9]+$
+        regexFailureMessage: You must enter a whole number of days, or 0 to keep entries forever.
+        helpLink: zeek
+        advanced: True
+      CrashExpireInterval:
+        description: >-
+          Number of days to keep Zeek crash directories, or 0 to keep them forever.
+          Applied by "zeekctl cron", which runs every 5 minutes.
+        regex: ^[0-9]+$
+        regexFailureMessage: You must enter a whole number of days, or 0 to keep crash directories forever.
+        helpLink: zeek
+        advanced: True
+      MinDiskSpace:
+        description: >-
+          Percentage of free disk space below which ZeekControl reports a warning, or 0 to disable the check
+          entirely. The Zeek container does not include a mail program, so the warning is not emailed - it
+          appears in the output of "zeekctl cron" instead. This setting never deletes anything - cleanup based
+          on disk usage is handled separately by so-sensor-clean.
+        regex: ^([0-9]|[1-9][0-9]|100)$
+        regexFailureMessage: You must enter a percentage between 0 and 100.
+        helpLink: zeek
+        advanced: True
+      MailTo:
+        description: >-
+          Address that ZeekControl would send mail to, covering cron output and crash reports, and the address
+          Zeek's notice framework would use. The Zeek container does not include a mail program, and Security
+          Onion never enables the notice email action, so no mail is sent and this address is unused. It is
+          read only for that reason.
+        helpLink: zeek
+        advanced: True
+        readonly: True
+      MailConnectionSummary:
+        description: >-
+          Set to 1 to email the hourly connection summary. This only controls the emailed copy - the summary is
+          generated and archived with the other Zeek logs either way. The Zeek container does not include a mail
+          program, so no mail is sent and this setting has no effect. It is read only for that reason.
+        regex: ^[01]$
+        regexFailureMessage: You must enter 0 or 1.
+        helpLink: zeek
+        advanced: True
+        readonly: True
+      MailHostUpDown:
+        description: >-
+          Set to 1 to report when a Zeek node changes between the up and down states. The Zeek container does
+          not include a mail program, so this notification cannot be emailed. It is read only for that reason.
+          Host status detection still runs regardless of this setting - only the notification is affected.
+        regex: ^[01]$
+        regexFailureMessage: You must enter 0 or 1.
+        helpLink: zeek
+        advanced: True
+        readonly: True
   policy:
     custom:
       filters: 

salt/zeek/tools/sbin/so-zeek-cron

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+# Run zeekctl's periodic maintenance tasks. This is what actually enforces
+# LogExpireInterval, StatsLogExpireInterval and CrashExpireInterval - without a periodic
+# 'zeekctl cron' those settings are inert no matter what they are set to.
+
+# This also restarts any node that died unexpectedly, and marks it crashed so a crash report
+# is produced. That is upstream's default cron behavior and it recovers a single node in
+# place. The beacon in salt/_beacons/zeek.py is the only other recovery path, it is disabled
+# by default (healthcheck:enabled), and it removes and recreates the whole container, so
+# letting zeekctl handle a single dead worker avoids the heavier restart.
+
+if ! docker ps --filter name=so-zeek --format '{{.Names}}' | grep -q '^so-zeek$'; then
+  exit 0
+fi
+
+# Run as the zeek user so the stats logs and zeekctl-config.sh this writes stay owned by
+# uid 937 rather than root.
+docker exec so-zeek runuser -l zeek -c '/opt/zeek/bin/zeekctl cron'

setup/so-setup

@@ -833,6 +833,7 @@ if ! [[ -f $install_opt_file ]]; then
 		check_sos_appliance
 		drop_install_options
 		hypervisor_local_states
+		mark_setup_complete
 		verify_setup
 	fi