Correct pipeline_settings annotations against Logstash 9.3.7

Widen the byte-size regex, which rejected values Logstash accepts and so
blocked the save in SOC: bare-letter units (1g, 512m, 64k), decimals
(1.5gb), whitespace before the unit, and a bare integer. Allow whitespace
in dead_letter_queue.retain.age (5 d). Both stay lowercase-only, matching
byte_value.rb and AbstractPipelineExt.parseToDuration.

Fix description gaps: queue.checkpoint.retry is a Windows/SAN workaround
Elastic does not otherwise recommend, batch metrics sampling is technical
preview, queue.checkpoint.interval is deprecated in 9.1, compression makes
a queue unreadable by Logstash before 9.2, flush_check_interval has a
1000ms floor, max_events counts unread events, and the path settings are
created by Logstash but reject symlinks. Note which settings apply only to
persisted queues or an enabled DLQ.

Drop the undocumented 'disabled' value from queue.compression.

Numeric fields stay stricter than NumericSetting, which has no validator
and would accept negatives, floats and NaN in event counts and intervals.

salt/logstash/config.sls

@@ -81,6 +81,14 @@ ls_custom_pipeline_conf_{{assigned_pipeline}}_{{pipeline}}:
 
 
 {% for assigned_pipeline in ASSIGNED_PIPELINES %}
+{# a blank per-pipeline setting falls back to the global logstash.yml value #}
+{% set PARSED_OVERRIDES = LOGSTASH_MERGED.get('pipeline_settings', {}).get(assigned_pipeline, {}) %}
+{% if PARSED_OVERRIDES is not mapping %}
+{%   do salt.log.warning('logstash: ignoring malformed pipeline_settings for pipeline ' ~ assigned_pipeline ~ '; expected a set of settings') %}
+{% endif %}
+{% set PIPELINE_OVERRIDES = PARSED_OVERRIDES if PARSED_OVERRIDES is mapping else {} %}
+{% set THREADS = PIPELINE_OVERRIDES.get('pipeline_x_workers') or LOGSTASH_MERGED.config.pipeline_x_workers %}
+{% set BATCH = PIPELINE_OVERRIDES.get('pipeline_x_batch_x_size') or LOGSTASH_MERGED.config.pipeline_x_batch_x_size %}
     {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %}
 ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}:
   file.managed:
@@ -92,8 +100,8 @@ ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_")
         GLOBALS: {{ GLOBALS }}
         ES_USER: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') }}"
         ES_PASS: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') }}"
-        THREADS: {{ LOGSTASH_MERGED.config.pipeline_x_workers }}
-        BATCH: {{ LOGSTASH_MERGED.config.pipeline_x_batch_x_size }}
+        THREADS: {{ THREADS }}
+        BATCH: {{ BATCH }}
       {% else %}
     - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{CONFIGFILE.split('/')[1]}}
       {% endif %}

salt/logstash/defaults.yaml

@@ -60,6 +60,259 @@ logstash:
     custom008: PLACEHOLDER
     custom009: PLACEHOLDER
     custom010: PLACEHOLDER
+  pipeline_settings:
+    fleet:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    manager:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    receiver:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    search:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom0:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom1:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom2:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom3:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom4:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
   settings:
     lsheap: 500m
   config:

salt/logstash/enabled.sls

@@ -105,6 +105,7 @@ so-logstash:
     {% endif %}
     - watch:
       - file: lsetcsync
+      - file: lspipelinesyml
       - file: trusttheca
       {% if GLOBALS.is_manager %}
       - file: elasticsearch_cacerts

salt/logstash/etc/pipelines.yml.jinja

@@ -1,4 +1,17 @@
+{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
+{%- set PIPELINE_SETTINGS = LOGSTASH_MERGED.get('pipeline_settings', {}) %}
 {%- for assigned_pipeline in ASSIGNED_PIPELINES %}
 - pipeline.id: {{ assigned_pipeline }}
   path.config: "/usr/share/logstash/pipelines/{{ assigned_pipeline }}/"
+{%-   set extra = PIPELINE_SETTINGS.get(assigned_pipeline, {}) %}
+{%-   if extra is mapping %}
+{#-     values are emitted unquoted so yaml re-infers the type logstash expects:
+        4 as an integer, false as a boolean, 1024mb and auto as strings #}
+{%-     for key, value in extra | dictsort %}
+{%-       set rendered = key | replace('_x_', '.') %}
+{%-       if value not in ['', None] and rendered not in ['pipeline.id', 'path.config'] %}
+  {{ rendered }}: {{ value }}
+{%-       endif %}
+{%-     endfor %}
+{%-   endif %}
 {% endfor -%}

salt/logstash/soc_logstash.yaml

@@ -16,6 +16,7 @@ logstash:
       heavynode: *assigned_pipelines
       searchnode: *assigned_pipelines
       manager: *assigned_pipelines
+      managerhype: *assigned_pipelines
       managersearch: *assigned_pipelines
       fleet: *assigned_pipelines
   defined_pipelines:
@@ -51,6 +52,343 @@ logstash:
     custom008: *pipeline_config
     custom009: *pipeline_config
     custom010: *pipeline_config
+  pipeline_settings:
+    manager: &pipeline_settings
+      pipeline_x_workers:
+        description: >-
+          Number of worker threads that run filters and outputs for this pipeline. May be set higher
+          than the CPU core count when outputs spend time waiting on I/O. Leave blank to use the value
+          from logstash.yml.
+        title: pipeline.workers
+        regex: '^$|^[1-9][0-9]*$'
+        regexFailureMessage: Must be blank, or a positive whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      pipeline_x_batch_x_size:
+        description: >-
+          Maximum number of events an individual worker thread collects before running filters and
+          outputs. Larger batches are more efficient but increase heap use; total in-flight events is
+          workers multiplied by batch size. Leave blank to use the value from logstash.yml.
+        title: pipeline.batch.size
+        regex: '^$|^[1-9][0-9]*$'
+        regexFailureMessage: Must be blank, or a positive whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      pipeline_x_batch_x_delay:
+        description: >-
+          Milliseconds a worker waits for the next event before running a batch that is not yet full.
+          Leave blank to use the value from logstash.yml.
+        title: pipeline.batch.delay
+        regex: '^$|^[0-9]+$'
+        regexFailureMessage: Must be blank, or a whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      pipeline_x_batch_x_metrics_x_sampling_mode:
+        description: >-
+          Controls how often batch size metrics are collected for this pipeline, which helps tune
+          pipeline.batch.size to the batch sizes actually being processed. Fuller sampling consumes
+          additional heap. Elastic marks this setting as a technical preview that may change in a
+          future release. Leave blank to use the value from logstash.yml.
+        title: pipeline.batch.metrics.sampling_mode
+        options:
+        - ''
+        - 'disabled'
+        - 'minimal'
+        - 'full'
+        advanced: True
+        global: False
+        helpLink: logstash
+      pipeline_x_ordered:
+        description: >-
+          Whether event order is preserved through this pipeline. auto enables ordering only when the
+          pipeline runs a single worker. Leave blank to use the value from logstash.yml.
+        title: pipeline.ordered
+        options:
+        - ''
+        - 'auto'
+        - 'true'
+        - 'false'
+        advanced: True
+        global: False
+        helpLink: logstash
+      pipeline_x_ecs_compatibility:
+        description: >-
+          Elastic Common Schema compatibility mode for plugins in this pipeline. Security Onion sets
+          this globally and it should rarely be changed per pipeline. Elastic considers values other
+          than disabled to be BETA, and they may produce unintended consequences when upgrading
+          Logstash. Leave blank to use the value from logstash.yml.
+        title: pipeline.ecs_compatibility
+        options:
+        - ''
+        - 'disabled'
+        - 'v1'
+        - 'v8'
+        advanced: True
+        global: False
+        helpLink: logstash
+      pipeline_x_reloadable:
+        description: >-
+          Whether this pipeline may be reloaded when its configuration changes. Leave blank to use the
+          value from logstash.yml.
+        title: pipeline.reloadable
+        options:
+        - ''
+        - 'true'
+        - 'false'
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_type:
+        description: >-
+          Queue backing this pipeline. persisted buffers events to disk under /nsm/logstash so they
+          survive a restart, at some throughput cost; memory does not. Leave blank to use the value
+          from logstash.yml.
+        title: queue.type
+        options:
+        - ''
+        - 'memory'
+        - 'persisted'
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_max_bytes:
+        description: >-
+          Total capacity of the persistent queue for this pipeline, in bytes. Only applies when
+          queue.type is persisted. The disk backing /nsm/logstash must be larger than this value. If
+          both queue.max_events and queue.max_bytes are set, whichever is reached first applies. Leave
+          blank to use the value from logstash.yml.
+        title: queue.max_bytes
+        regex: '^$|^[0-9]+$|^[0-9]+(\.[0-9]+)?\s*(b|kb?|mb?|gb?|tb?|pb?)$'
+        regexFailureMessage: Must be blank, or a size such as 512mb, 1gb, or 64k. Units are lowercase.
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_page_capacity:
+        description: >-
+          Size of the individual append-only page data files that make up the persistent queue for
+          this pipeline. Only applies when queue.type is persisted. Leave blank to use the value from
+          logstash.yml.
+        title: queue.page_capacity
+        regex: '^$|^[0-9]+$|^[0-9]+(\.[0-9]+)?\s*(b|kb?|mb?|gb?|tb?|pb?)$'
+        regexFailureMessage: Must be blank, or a size such as 512mb, 1gb, or 64k. Units are lowercase.
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_max_events:
+        description: >-
+          Maximum number of unread events in the persistent queue for this pipeline. 0 means
+          unlimited. Only applies when queue.type is persisted. Leave blank to use the value from
+          logstash.yml.
+        title: queue.max_events
+        regex: '^$|^[0-9]+$'
+        regexFailureMessage: Must be blank, or a whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_checkpoint_x_acks:
+        description: >-
+          Maximum number of acknowledged events before a checkpoint is forced. 0 means unlimited. Only
+          applies when queue.type is persisted. Leave blank to use the value from logstash.yml.
+        title: queue.checkpoint.acks
+        regex: '^$|^[0-9]+$'
+        regexFailureMessage: Must be blank, or a whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_checkpoint_x_writes:
+        description: >-
+          Maximum number of written events before a checkpoint is forced. Setting this to 1 gives
+          maximum durability at a severe performance cost. 0 means unlimited. Only applies when
+          queue.type is persisted. Leave blank to use the value from logstash.yml.
+        title: queue.checkpoint.writes
+        regex: '^$|^[0-9]+$'
+        regexFailureMessage: Must be blank, or a whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_checkpoint_x_interval:
+        description: >-
+          Milliseconds between forced checkpoints on the persistent queue head page. 0 eliminates
+          periodic checkpoints. Deprecated by Elastic as of Logstash 9.1. Only applies when queue.type
+          is persisted. Leave blank to use the value from logstash.yml.
+        title: queue.checkpoint.interval
+        regex: '^$|^[0-9]+$'
+        regexFailureMessage: Must be blank, or a whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_checkpoint_x_retry:
+        description: >-
+          When enabled, Logstash retries four times per attempted checkpoint write that fails; later
+          errors are not retried. Elastic describes this as a workaround for failed checkpoint writes
+          seen only on Windows and on filesystems with non-standard behaviour such as SANs, and does
+          not recommend enabling it otherwise. Only applies when queue.type is persisted. Leave blank
+          to use the value from logstash.yml.
+        title: queue.checkpoint.retry
+        options:
+        - ''
+        - 'true'
+        - 'false'
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_compression:
+        description: >-
+          Compression applied to persistent queue pages for this pipeline, trading CPU for disk: speed
+          favours the fastest operation, size the smallest files, and balanced sits between them. Once
+          compressed events have been written, that queue cannot be read by Logstash releases earlier
+          than 9.2. Only applies when queue.type is persisted. Leave blank to use the value from
+          logstash.yml.
+        title: queue.compression
+        options:
+        - ''
+        - 'none'
+        - 'speed'
+        - 'balanced'
+        - 'size'
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_drain:
+        description: >-
+          When enabled, Logstash waits for the persistent queue to drain before shutting down this
+          pipeline. Draining a large queue makes shutdown take considerably longer. Only applies when
+          queue.type is persisted. Leave blank to use the value from logstash.yml.
+        title: queue.drain
+        options:
+        - ''
+        - 'true'
+        - 'false'
+        advanced: True
+        global: False
+        helpLink: logstash
+      dead_letter_queue_x_enable:
+        description: >-
+          Whether events this pipeline cannot process are written to a dead letter queue instead of
+          being dropped. Leave blank to use the value from logstash.yml.
+        title: dead_letter_queue.enable
+        options:
+        - ''
+        - 'true'
+        - 'false'
+        advanced: True
+        global: False
+        helpLink: logstash
+      dead_letter_queue_x_max_bytes:
+        description: >-
+          Total capacity of the dead letter queue for this pipeline, in bytes. Only applies when
+          dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml.
+        title: dead_letter_queue.max_bytes
+        regex: '^$|^[0-9]+$|^[0-9]+(\.[0-9]+)?\s*(b|kb?|mb?|gb?|tb?|pb?)$'
+        regexFailureMessage: Must be blank, or a size such as 512mb, 1gb, or 64k. Units are lowercase.
+        advanced: True
+        global: False
+        helpLink: logstash
+      dead_letter_queue_x_flush_interval:
+        description: >-
+          Milliseconds before an incomplete dead letter queue segment is flushed and made available to
+          the dead_letter_queue input. Lower values write more, smaller segment files; higher values
+          add latency before events can be read. Only applies when dead_letter_queue.enable is true.
+          Leave blank to use the value from logstash.yml.
+        title: dead_letter_queue.flush_interval
+        regex: '^$|^[0-9]+$'
+        regexFailureMessage: Must be blank, or a whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      dead_letter_queue_x_flush_check_interval:
+        description: >-
+          Milliseconds between checks for a stale dead letter queue segment needing a flush. Cannot be
+          set lower than 1000. Smaller values rotate segments sooner at the cost of CPU. Only applies
+          when dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml.
+        title: dead_letter_queue.flush_check_interval
+        regex: '^$|^[0-9]+$'
+        regexFailureMessage: Must be blank, or a whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      dead_letter_queue_x_storage_policy:
+        description: >-
+          Action taken when dead_letter_queue.max_bytes is reached: drop_newer stops accepting new
+          events, drop_older removes the oldest events to make room. Only applies when
+          dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml.
+        title: dead_letter_queue.storage_policy
+        options:
+        - ''
+        - 'drop_newer'
+        - 'drop_older'
+        advanced: True
+        global: False
+        helpLink: logstash
+      dead_letter_queue_x_retain_x_age:
+        description: >-
+          How long an event is kept in the dead letter queue before Logstash removes it, such as 5d.
+          Units are d, h, m and s; there is no default unit, so one must be given. Only applies when
+          dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml.
+        title: dead_letter_queue.retain.age
+        regex: '^$|^[0-9]+\s*[dhms]$'
+        regexFailureMessage: Must be blank, or a number followed by d, h, m, or s, such as 5d.
+        advanced: True
+        global: False
+        helpLink: logstash
+      path_x_queue:
+        description: >-
+          Directory inside the Logstash container holding the persistent queue for this pipeline. The
+          default lives under the /nsm/logstash bind mount; a path outside it will not survive a
+          container restart. Logstash creates the directory if it is missing, requires it to be
+          writable, and refuses to start if the path is a symlink. Only applies when queue.type is
+          persisted. Leave blank to use the value from logstash.yml.
+        title: path.queue
+        advanced: True
+        global: False
+        helpLink: logstash
+      path_x_dead_letter_queue:
+        description: >-
+          Directory inside the Logstash container holding the dead letter queue for this pipeline. The
+          default lives under the /nsm/logstash bind mount; a path outside it will not survive a
+          container restart. Logstash creates the directory if it is missing, requires it to be
+          writable, and refuses to start if the path is a symlink. Only applies when
+          dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml.
+        title: path.dead_letter_queue
+        advanced: True
+        global: False
+        helpLink: logstash
+      config_x_debug:
+        description: >-
+          Whether the fully compiled configuration for this pipeline is written to the log. The output
+          may contain sensitive values from the pipeline configuration. Leave blank to use the value
+          from logstash.yml.
+        title: config.debug
+        options:
+        - ''
+        - 'true'
+        - 'false'
+        advanced: True
+        global: False
+        helpLink: logstash
+      config_x_support_escapes:
+        description: >-
+          Whether escape sequences such as \n and \t in this pipeline's quoted strings are
+          interpreted. Leave blank to use the value from logstash.yml.
+        title: config.support_escapes
+        options:
+        - ''
+        - 'true'
+        - 'false'
+        advanced: True
+        global: False
+        helpLink: logstash
+    fleet: *pipeline_settings
+    receiver: *pipeline_settings
+    search: *pipeline_settings
+    custom0: *pipeline_settings
+    custom1: *pipeline_settings
+    custom2: *pipeline_settings
+    custom3: *pipeline_settings
+    custom4: *pipeline_settings
   settings:
     lsheap: 
       description: Heap size to use for logstash

salt/manager/tools/sbin/soup

@@ -344,6 +344,16 @@ check_cluster_health() {
 
 check_fleet_server() {
   echo "Checking that Elastic Fleet Server is responding."
+  # Before checking fleet health, check for and fix known issue with elastic-agent container and fs.protected_symlinks
+  local protected_symlinks=$(sysctl -b fs.protected_symlinks)
+  if [[ "$protected_symlinks" == "1" ]]; then
+    # disable fs.protected_symlinks and restart elasticfleet
+    sysctl -w fs.protected_symlinks=0
+    docker stop so-elastic-fleet; docker rm -f so-elastic-fleet
+    printf "\nUpdated sysctl fs.protected_symlinks. Restarting fleet before running health check and continuing with soup.\n"
+    salt-call state.apply elasticfleet queue=True
+  fi
+  
   # Modeled on the wait_for_so-elastic-fleet state check in elasticfleet/enabled.sls,
   # which waits for HTTP 200 from the Fleet Server status API.
   if curl -sk --fail --retry 3 --retry-delay 10 --max-time 30 "https://localhost:8220/api/status" > /dev/null 2>&1; then
@@ -1036,8 +1046,20 @@ post_to_3.2.0() {
 }
 ### 3.2.0 End ###
 
-### 3.2.0 Scripts ###
+### 3.3.0 Scripts ###
+
+ # Sets fs.protected_symlinks=0
+ #
+ # Elastic Agent docker image chowns its directory to the running UID 
+ # but does not chown the elastic-agent launcher symlink. 
+ # Preventing non-root users from following that launcher symlink.
+disable_sysctl_fs_protected_symlink() {
+  salt -C 'I@stig:enabled' state.single sysctl.present name=fs.protected_symlinks value=0 config=/etc/sysctl.conf || true
+}
+    
 up_to_3.3.0() {
+  disable_sysctl_fs_protected_symlink
+  
   INSTALLEDVERSION=3.3.0
 }
 

salt/salt/minion/boot_highstate.sls

@@ -3,6 +3,8 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+
 # Manages /etc/systemd/system/so-boot-highstate.service, a Type=oneshot
 # RemainAfterExit=yes unit that runs `salt-call state.highstate` exactly once
 # per system boot. Replaces the legacy `startup_states: highstate` minion
@@ -19,9 +21,25 @@ so_boot_highstate_unit_file:
     - onchanges_in:
       - module: systemd_reload
 
+# Non-managers never apply salt.minion during setup, so reaching this state means
+# setup is finished and the marker is safe to write unconditionally. This also
+# heals nodes installed before this fix, which have no marker and no legacy
+# startup_states line to grep for. Managers do highstate mid-setup, so they only
+# get the marker from the legacy upgrade signal; fresh installs get it from
+# mark_setup_complete in setup/so-functions.
+mark_setup_complete:
+  file.managed:
+    - name: /opt/so/state/setup-complete
+    - replace: false
+    - makedirs: True
+{% if GLOBALS.is_manager %}
+    - onlyif: "grep -qx 'startup_states: highstate' /etc/salt/minion"
+{% endif %}
+    - require_in:
+      - service: so_boot_highstate_service
+
 # Only enable once setup is complete. Until then the gate file is missing and
-# the unit's own ConditionPathExists would no-op it anyway -- this just keeps
-# `systemctl is-enabled` honest for the sync_es_users gate.
+# the unit's own ConditionPathExists would no-op it anyway.
 so_boot_highstate_service:
   service.enabled:
     - name: so-boot-highstate.service

salt/salt/minion/init.sls

@@ -87,27 +87,15 @@ set_log_levels:
 # so-boot-highstate.service (managed in salt.minion.boot_highstate), which
 # runs once per system boot only. Strip the line from /etc/salt/minion on
 # upgrade; both the commented and uncommented forms historically existed.
+# Ordered after mark_setup_complete (salt.minion.boot_highstate); the manager
+# gate there greps for this line, so it must run before we delete it.
 remove_startup_states:
   file.line:
     - name: /etc/salt/minion
     - match: 'startup_states: highstate'
     - mode: delete
-
-# Upgrade-path bridge: systems that already passed setup under the old gate
-# (`grep -x 'startup_states: highstate' /etc/salt/minion`) get a /opt/so/state/setup-complete
-# marker so so-boot-highstate.service can be enabled and the so-user_sync cron
-# in sync_es_users.sls keeps installing. Setup-in-progress systems instead get
-# the marker from `mark_setup_complete` in setup/so-functions at the right
-# moment. `replace: false` means we never overwrite a marker once written.
-mark_setup_complete_for_upgrades:
-  file.managed:
-    - name: /opt/so/state/setup-complete
-    - replace: false
-    - makedirs: True
-    - onlyif: "grep -qx 'startup_states: highstate' /etc/salt/minion"
-    - require_in:
-      - file: remove_startup_states
-      - service: so_boot_highstate_service
+    - require:
+      - file: mark_setup_complete
 
 {% endif %}
 

salt/setup/virt/setSalt.sls

@@ -8,6 +8,15 @@ set_role_grain:
     - name: role
     - value: so-{{ grains.id.split("_") | last }}
 
+# salt-cloud guests never run so-setup, so nothing else marks them setup-complete.
+# Replaces the 'startup_states: highstate' line this state used to append. No
+# GLOBALS import -- this runs before the guest's pillars exist.
+mark_setup_complete_vm_guest:
+  file.managed:
+    - name: /opt/so/state/setup-complete
+    - replace: false
+    - makedirs: True
+
 enable_salt_minion:
   service.enabled:
     - name: salt-minion

salt/soc/soc_soc.yaml

@@ -732,11 +732,13 @@ soc:
             global: True
             advanced: True
             forcedType: int
+            readonlyUi: True
           maxDelegationDepth:
             description: Maximum delegation nesting depth for sub-agents. For example, a value of 2 lets the main agent delegate to a sub-agent that may itself delegate one level deeper. Any deeper delegation is refused and the requesting agent continues without it. Set to 0 to disable the limit.
             global: True
             advanced: True
             forcedType: int
+            readonlyUi: True
           adapters:
             description: Configuration for AI adapters used by the Onion AI assistant. Please see documentation for help on which fields are required for which protocols.
             global: True
@@ -779,6 +781,60 @@ soc:
             description: Indicates if the Assistant Module should operate in agentic mode or not. If true, agents can work together to solve tasks.
             global: True
             forcedType: bool
+          agents:
+            description: Agent definitions for the Onion AI assistant, managed from the Agent Studio. An entry naming a system agent overrides only the fields an admin may change; everything else comes from the built-in definition.
+            global: True
+            advanced: False
+            readonlyUi: True
+            storage: db
+            forcedType: "[]{}"
+            helpLink: onion-ai
+            syntax: json
+            uiElements:
+            - field: name
+              label: Name
+              required: True
+            - field: enabled
+              label: Enabled
+              forcedType: bool
+            - field: isOrchestrator
+              label: Orchestrator
+              forcedType: bool
+            - field: model
+              label: Model
+            - field: allowedSkills
+              label: Skills
+              forcedType: "[]string"
+            - field: canDelegateTo
+              label: Delegates To
+              forcedType: "[]string"
+            - field: description
+              label: Description
+            - field: persona
+              label: Persona
+              multiline: True
+          skills:
+            description: Skill definitions for the Onion AI assistant, managed from the Agent Studio. An entry naming a system skill overrides only its enabled state and persona addendum; its tool set comes from the built-in definition.
+            global: True
+            advanced: False
+            readonlyUi: True
+            storage: db
+            forcedType: "[]{}"
+            helpLink: onion-ai
+            syntax: json
+            uiElements:
+            - field: name
+              label: Name
+              required: True
+            - field: enabled
+              label: Enabled
+              forcedType: bool
+            - field: tools
+              label: Tools
+              forcedType: "[]string"
+            - field: persona
+              label: Persona
+              multiline: True
           agentMapping:
             Orchestrator:
               description: The initial agent in most agentic conversations. This agent will delegate requests to specialized agents.

salt/stig/enabled.sls

@@ -65,6 +65,15 @@ run_remediate:
     - success_retcodes:
       - 2
 
+# Elastic Agent docker image chowns its directory to the running UID but does not
+#  chown the elastic-agent launcher symlink. fs.protected_symlinks=1 then prevents
+#  non-root users from following that launcher symlink.
+{# OSCAP rule id: xccdf_org.ssgproject.content_rule_sysctl_fs_protected_symlinks #}
+fs.protected_symlinks:
+  sysctl.present:
+    - value: 0
+    - config: /etc/sysctl.conf
+
 {# OSCAP rule id: xccdf_org.ssgproject.content_rule_disable_ctrlaltdel_burstaction #}
 disable_ctrl_alt_del_action:
   file.replace:

salt/stig/files/sos-oscap.xml

@@ -1601,7 +1601,7 @@ DISA STIG for Oracle Linux 9 V1R3.</xccdf-1.2:description>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sudoers_validate_passwd" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_crypto_fips_enabled" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_hardlinks" selected="true"/>
-        <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_symlinks" selected="true"/>
+        <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_symlinks" selected="false"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_core_pattern" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_dmesg_restrict" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_exec_shield" selected="true"/>
@@ -2202,7 +2202,7 @@ standard DISA STIG for Oracle Linux 9 profile.</xccdf-1.2:description>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sudoers_validate_passwd" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_crypto_fips_enabled" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_hardlinks" selected="true"/>
-        <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_symlinks" selected="true"/>
+        <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_symlinks" selected="false"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_core_pattern" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_dmesg_restrict" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_exec_shield" selected="true"/>

salt/zeek/defaults.yaml

@@ -15,7 +15,7 @@ zeek:
       MailHostUpDown: 0
       LogRotationInterval: 3600
       LogExpireInterval: 0
-      StatsLogEnable: 1
+      StatsLogEnable: 0
       StatsLogExpireInterval: 0
       StatusCmdShowAll: 0
       CrashExpireInterval: 0

salt/zeek/soc_zeek.yaml

@@ -69,10 +69,24 @@ zeek:
         regexFailureMessage: Enter 0, or a positive number optionally followed by "day" or "hr" (for example 7, "7 days", or "12 hr"). Minutes are not accepted because a log expire interval shorter than the log rotation interval prevents Zeek from starting.
         helpLink: zeek
         advanced: True
+      StatsLogEnable:
+        description: >-
+          Set to 1 to have "zeekctl cron" write node statistics to /nsm/zeek/logs/stats. This is
+          disabled because the CPU and memory portion depends on the "top" command, which the Zeek
+          container does not include, so every run records an error for each node instead. The
+          interface packet counters it also collects are not used anywhere in Security Onion, which
+          tracks Zeek packet loss separately through packetloss.log and Telegraf. It is read only
+          for that reason.
+        regex: ^[01]$
+        regexFailureMessage: You must enter 0 or 1.
+        helpLink: zeek
+        advanced: True
+        readonly: True
       StatsLogExpireInterval:
         description: >-
           Number of days to keep entries in the Zeek stats log, or 0 to keep them forever.
-          Applied by "zeekctl cron", which runs every 5 minutes.
+          Applied by "zeekctl cron", which runs every 5 minutes. This has no effect unless
+          StatsLogEnable is turned on, which it is not by default.
         regex: ^[0-9]+$
         regexFailureMessage: You must enter a whole number of days, or 0 to keep entries forever.
         helpLink: zeek

setup/so-setup

@@ -833,6 +833,7 @@ if ! [[ -f $install_opt_file ]]; then
 		check_sos_appliance
 		drop_install_options
 		hypervisor_local_states
+		mark_setup_complete
 		verify_setup
 	fi