From 3744c0bd6c239754df258f6a3c2093976c01d8f3 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 17 Aug 2026 15:24:05 -0500 Subject: [PATCH 01/17] fix issue with fs.protected_symlinks prior to checking for fleet health --- salt/manager/tools/sbin/soup | 24 +++++++++++++++++++++++- salt/stig/enabled.sls | 9 +++++++++ salt/stig/files/sos-oscap.xml | 4 ++-- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index b155d1bec..83d6cc438 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -341,6 +341,16 @@ check_cluster_health() { check_fleet_server() { echo "Checking that Elastic Fleet Server is responding." + # Before checking fleet health, check for and fix known issue with elastic-agent container and fs.protected_symlinks + local protected_symlinks=$(sysctl -b fs.protected_symlinks) + if [[ "$protected_symlinks" == "1" ]]; then + # disable fs.protected_symlinks and restart elasticfleet + sysctl -w fs.protected_symlinks=0 + docker stop so-elastic-fleet; docker rm -f so-elastic-fleet + printf "\nUpdated sysctl fs.protected_symlinks. Restarting fleet before running health check and continuing with soup.\n" + salt-call state.apply elasticfleet queue=True + fi + # Modeled on the wait_for_so-elastic-fleet state check in elasticfleet/enabled.sls, # which waits for HTTP 200 from the Fleet Server status API. if curl -sk --fail --retry 3 --retry-delay 10 --max-time 30 "https://localhost:8220/api/status" > /dev/null 2>&1; then @@ -1008,8 +1018,20 @@ post_to_3.2.0() { } ### 3.2.0 End ### -### 3.2.0 Scripts ### +### 3.3.0 Scripts ### + + # Sets fs.protected_symlinks=0 + # + # Elastic Agent docker image chowns its directory to the running UID + # but does not chown the elastic-agent launcher symlink. + # Preventing non-root users from following that launcher symlink. +disable_sysctl_fs_protected_symlink() { + salt -C 'I@stig:enabled' state.single sysctl.present name=fs.protected_symlinks value=0 config=/etc/sysctl.conf +} + up_to_3.3.0() { + disable_sysctl_fs_protected_symlink + INSTALLEDVERSION=3.3.0 } diff --git a/salt/stig/enabled.sls b/salt/stig/enabled.sls index 91aae7069..b53256e04 100644 --- a/salt/stig/enabled.sls +++ b/salt/stig/enabled.sls @@ -65,6 +65,15 @@ run_remediate: - success_retcodes: - 2 +# Elastic Agent docker image chowns its directory to the running UID but does not +# chown the elastic-agent launcher symlink. fs.protected_symlinks=1 then prevents +# non-root users from following that launcher symlink. +{# OSCAP rule id: xccdf_org.ssgproject.content_rule_sysctl_fs_protected_symlinks #} +fs.protected_symlinks: + sysctl.present: + - value: 0 + - config: /etc/sysctl.conf + {# OSCAP rule id: xccdf_org.ssgproject.content_rule_disable_ctrlaltdel_burstaction #} disable_ctrl_alt_del_action: file.replace: diff --git a/salt/stig/files/sos-oscap.xml b/salt/stig/files/sos-oscap.xml index aa5b2ed31..9fc8dacf8 100644 --- a/salt/stig/files/sos-oscap.xml +++ b/salt/stig/files/sos-oscap.xml @@ -1601,7 +1601,7 @@ DISA STIG for Oracle Linux 9 V1R3. - + @@ -2202,7 +2202,7 @@ standard DISA STIG for Oracle Linux 9 profile. - + From 4b74e2c320d2b3421cf03e65faef6f719bd6d9f2 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 17 Aug 2026 15:26:01 -0500 Subject: [PATCH 02/17] allow for unavailable minions --- salt/manager/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 83d6cc438..00d583ff4 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1026,7 +1026,7 @@ post_to_3.2.0() { # but does not chown the elastic-agent launcher symlink. # Preventing non-root users from following that launcher symlink. disable_sysctl_fs_protected_symlink() { - salt -C 'I@stig:enabled' state.single sysctl.present name=fs.protected_symlinks value=0 config=/etc/sysctl.conf + salt -C 'I@stig:enabled' state.single sysctl.present name=fs.protected_symlinks value=0 config=/etc/sysctl.conf || true } up_to_3.3.0() { From 2f2187f714a3eb3f8384c1e1f1b3cd8018b26400 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Tue, 18 Aug 2026 09:45:23 -0400 Subject: [PATCH 03/17] Write setup-complete marker on non-manager nodes so-boot-highstate.service was never enabled outside managers: only the manager branch of so-setup called mark_setup_complete, so the marker its service.enabled gates on never existed on sensors, search nodes, receivers, etc. Move the marker state into salt.minion.boot_highstate as the sole owner within a highstate. Non-managers never apply salt.minion during setup, so reaching it means setup is done and the marker is unconditional -- this also heals already-installed nodes. Managers keep the legacy startup_states gate, since they do highstate mid-setup. Also add the marker to setup.virt for salt-cloud guests (replacing the startup_states line removed in fabecb82) and to so-setup's non-manager branch. --- salt/salt/minion/boot_highstate.sls | 22 ++++++++++++++++++++-- salt/salt/minion/init.sls | 20 ++++---------------- salt/setup/virt/setSalt.sls | 9 +++++++++ setup/so-setup | 1 + 4 files changed, 34 insertions(+), 18 deletions(-) diff --git a/salt/salt/minion/boot_highstate.sls b/salt/salt/minion/boot_highstate.sls index eb2596dad..b97972c9f 100644 --- a/salt/salt/minion/boot_highstate.sls +++ b/salt/salt/minion/boot_highstate.sls @@ -3,6 +3,8 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. +{% from 'vars/globals.map.jinja' import GLOBALS %} + # Manages /etc/systemd/system/so-boot-highstate.service, a Type=oneshot # RemainAfterExit=yes unit that runs `salt-call state.highstate` exactly once # per system boot. Replaces the legacy `startup_states: highstate` minion @@ -19,9 +21,25 @@ so_boot_highstate_unit_file: - onchanges_in: - module: systemd_reload +# Non-managers never apply salt.minion during setup, so reaching this state means +# setup is finished and the marker is safe to write unconditionally. This also +# heals nodes installed before this fix, which have no marker and no legacy +# startup_states line to grep for. Managers do highstate mid-setup, so they only +# get the marker from the legacy upgrade signal; fresh installs get it from +# mark_setup_complete in setup/so-functions. +mark_setup_complete: + file.managed: + - name: /opt/so/state/setup-complete + - replace: false + - makedirs: True +{% if GLOBALS.is_manager %} + - onlyif: "grep -qx 'startup_states: highstate' /etc/salt/minion" +{% endif %} + - require_in: + - service: so_boot_highstate_service + # Only enable once setup is complete. Until then the gate file is missing and -# the unit's own ConditionPathExists would no-op it anyway -- this just keeps -# `systemctl is-enabled` honest for the sync_es_users gate. +# the unit's own ConditionPathExists would no-op it anyway. so_boot_highstate_service: service.enabled: - name: so-boot-highstate.service diff --git a/salt/salt/minion/init.sls b/salt/salt/minion/init.sls index fa94ec7be..a608788c2 100644 --- a/salt/salt/minion/init.sls +++ b/salt/salt/minion/init.sls @@ -87,27 +87,15 @@ set_log_levels: # so-boot-highstate.service (managed in salt.minion.boot_highstate), which # runs once per system boot only. Strip the line from /etc/salt/minion on # upgrade; both the commented and uncommented forms historically existed. +# Ordered after mark_setup_complete (salt.minion.boot_highstate); the manager +# gate there greps for this line, so it must run before we delete it. remove_startup_states: file.line: - name: /etc/salt/minion - match: 'startup_states: highstate' - mode: delete - -# Upgrade-path bridge: systems that already passed setup under the old gate -# (`grep -x 'startup_states: highstate' /etc/salt/minion`) get a /opt/so/state/setup-complete -# marker so so-boot-highstate.service can be enabled and the so-user_sync cron -# in sync_es_users.sls keeps installing. Setup-in-progress systems instead get -# the marker from `mark_setup_complete` in setup/so-functions at the right -# moment. `replace: false` means we never overwrite a marker once written. -mark_setup_complete_for_upgrades: - file.managed: - - name: /opt/so/state/setup-complete - - replace: false - - makedirs: True - - onlyif: "grep -qx 'startup_states: highstate' /etc/salt/minion" - - require_in: - - file: remove_startup_states - - service: so_boot_highstate_service + - require: + - file: mark_setup_complete {% endif %} diff --git a/salt/setup/virt/setSalt.sls b/salt/setup/virt/setSalt.sls index 59ab9e1e3..41496fea5 100644 --- a/salt/setup/virt/setSalt.sls +++ b/salt/setup/virt/setSalt.sls @@ -8,6 +8,15 @@ set_role_grain: - name: role - value: so-{{ grains.id.split("_") | last }} +# salt-cloud guests never run so-setup, so nothing else marks them setup-complete. +# Replaces the 'startup_states: highstate' line this state used to append. No +# GLOBALS import -- this runs before the guest's pillars exist. +mark_setup_complete_vm_guest: + file.managed: + - name: /opt/so/state/setup-complete + - replace: false + - makedirs: True + enable_salt_minion: service.enabled: - name: salt-minion diff --git a/setup/so-setup b/setup/so-setup index 896505ba5..79fcabfeb 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -833,6 +833,7 @@ if ! [[ -f $install_opt_file ]]; then check_sos_appliance drop_install_options hypervisor_local_states + mark_setup_complete verify_setup fi From dff3d76efd73d869bdf0532dd9d952b6a1802045 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 19 Aug 2026 16:21:28 -0400 Subject: [PATCH 04/17] Expose Logstash 9.3.7 pipeline settings per pipeline in SOC Add logstash:pipeline_settings carrying the 27 pipeline-scoped settings Logstash 9.3.7 accepts, annotated individually per pipeline and rendered into pipelines.yml. A blank setting inherits from logstash.yml. Restart logstash when pipelines.yml changes, and add the missing managerhype annotation. Fixes #15090 --- salt/logstash/config.sls | 8 +- salt/logstash/defaults.yaml | 253 ++++++++++++++++++++ salt/logstash/enabled.sls | 1 + salt/logstash/etc/pipelines.yml.jinja | 13 ++ salt/logstash/soc_logstash.yaml | 317 ++++++++++++++++++++++++++ 5 files changed, 590 insertions(+), 2 deletions(-) diff --git a/salt/logstash/config.sls b/salt/logstash/config.sls index 47feba42c..a661e96cc 100644 --- a/salt/logstash/config.sls +++ b/salt/logstash/config.sls @@ -81,6 +81,10 @@ ls_custom_pipeline_conf_{{assigned_pipeline}}_{{pipeline}}: {% for assigned_pipeline in ASSIGNED_PIPELINES %} +{# a blank per-pipeline setting falls back to the global logstash.yml value #} +{% set PIPELINE_OVERRIDES = LOGSTASH_MERGED.get('pipeline_settings', {}).get(assigned_pipeline, {}) %} +{% set THREADS = PIPELINE_OVERRIDES.get('pipeline_x_workers') or LOGSTASH_MERGED.config.pipeline_x_workers %} +{% set BATCH = PIPELINE_OVERRIDES.get('pipeline_x_batch_x_size') or LOGSTASH_MERGED.config.pipeline_x_batch_x_size %} {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %} ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}: file.managed: @@ -92,8 +96,8 @@ ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") GLOBALS: {{ GLOBALS }} ES_USER: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') }}" ES_PASS: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') }}" - THREADS: {{ LOGSTASH_MERGED.config.pipeline_x_workers }} - BATCH: {{ LOGSTASH_MERGED.config.pipeline_x_batch_x_size }} + THREADS: {{ THREADS }} + BATCH: {{ BATCH }} {% else %} - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{CONFIGFILE.split('/')[1]}} {% endif %} diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml index db5e4ee58..84bb91afd 100644 --- a/salt/logstash/defaults.yaml +++ b/salt/logstash/defaults.yaml @@ -60,6 +60,259 @@ logstash: custom008: PLACEHOLDER custom009: PLACEHOLDER custom010: PLACEHOLDER + pipeline_settings: + fleet: + pipeline_x_workers: '' + pipeline_x_batch_x_size: '' + pipeline_x_batch_x_delay: '' + pipeline_x_batch_x_metrics_x_sampling_mode: '' + pipeline_x_ordered: '' + pipeline_x_ecs_compatibility: '' + pipeline_x_reloadable: '' + queue_x_type: '' + queue_x_max_bytes: '' + queue_x_page_capacity: '' + queue_x_max_events: '' + queue_x_checkpoint_x_acks: '' + queue_x_checkpoint_x_writes: '' + queue_x_checkpoint_x_interval: '' + queue_x_checkpoint_x_retry: '' + queue_x_compression: '' + queue_x_drain: '' + dead_letter_queue_x_enable: '' + dead_letter_queue_x_max_bytes: '' + dead_letter_queue_x_flush_interval: '' + dead_letter_queue_x_flush_check_interval: '' + dead_letter_queue_x_storage_policy: '' + dead_letter_queue_x_retain_x_age: '' + path_x_queue: '' + path_x_dead_letter_queue: '' + config_x_debug: '' + config_x_support_escapes: '' + manager: + pipeline_x_workers: '' + pipeline_x_batch_x_size: '' + pipeline_x_batch_x_delay: '' + pipeline_x_batch_x_metrics_x_sampling_mode: '' + pipeline_x_ordered: '' + pipeline_x_ecs_compatibility: '' + pipeline_x_reloadable: '' + queue_x_type: '' + queue_x_max_bytes: '' + queue_x_page_capacity: '' + queue_x_max_events: '' + queue_x_checkpoint_x_acks: '' + queue_x_checkpoint_x_writes: '' + queue_x_checkpoint_x_interval: '' + queue_x_checkpoint_x_retry: '' + queue_x_compression: '' + queue_x_drain: '' + dead_letter_queue_x_enable: '' + dead_letter_queue_x_max_bytes: '' + dead_letter_queue_x_flush_interval: '' + dead_letter_queue_x_flush_check_interval: '' + dead_letter_queue_x_storage_policy: '' + dead_letter_queue_x_retain_x_age: '' + path_x_queue: '' + path_x_dead_letter_queue: '' + config_x_debug: '' + config_x_support_escapes: '' + receiver: + pipeline_x_workers: '' + pipeline_x_batch_x_size: '' + pipeline_x_batch_x_delay: '' + pipeline_x_batch_x_metrics_x_sampling_mode: '' + pipeline_x_ordered: '' + pipeline_x_ecs_compatibility: '' + pipeline_x_reloadable: '' + queue_x_type: '' + queue_x_max_bytes: '' + queue_x_page_capacity: '' + queue_x_max_events: '' + queue_x_checkpoint_x_acks: '' + queue_x_checkpoint_x_writes: '' + queue_x_checkpoint_x_interval: '' + queue_x_checkpoint_x_retry: '' + queue_x_compression: '' + queue_x_drain: '' + dead_letter_queue_x_enable: '' + dead_letter_queue_x_max_bytes: '' + dead_letter_queue_x_flush_interval: '' + dead_letter_queue_x_flush_check_interval: '' + dead_letter_queue_x_storage_policy: '' + dead_letter_queue_x_retain_x_age: '' + path_x_queue: '' + path_x_dead_letter_queue: '' + config_x_debug: '' + config_x_support_escapes: '' + search: + pipeline_x_workers: '' + pipeline_x_batch_x_size: '' + pipeline_x_batch_x_delay: '' + pipeline_x_batch_x_metrics_x_sampling_mode: '' + pipeline_x_ordered: '' + pipeline_x_ecs_compatibility: '' + pipeline_x_reloadable: '' + queue_x_type: '' + queue_x_max_bytes: '' + queue_x_page_capacity: '' + queue_x_max_events: '' + queue_x_checkpoint_x_acks: '' + queue_x_checkpoint_x_writes: '' + queue_x_checkpoint_x_interval: '' + queue_x_checkpoint_x_retry: '' + queue_x_compression: '' + queue_x_drain: '' + dead_letter_queue_x_enable: '' + dead_letter_queue_x_max_bytes: '' + dead_letter_queue_x_flush_interval: '' + dead_letter_queue_x_flush_check_interval: '' + dead_letter_queue_x_storage_policy: '' + dead_letter_queue_x_retain_x_age: '' + path_x_queue: '' + path_x_dead_letter_queue: '' + config_x_debug: '' + config_x_support_escapes: '' + custom0: + pipeline_x_workers: '' + pipeline_x_batch_x_size: '' + pipeline_x_batch_x_delay: '' + pipeline_x_batch_x_metrics_x_sampling_mode: '' + pipeline_x_ordered: '' + pipeline_x_ecs_compatibility: '' + pipeline_x_reloadable: '' + queue_x_type: '' + queue_x_max_bytes: '' + queue_x_page_capacity: '' + queue_x_max_events: '' + queue_x_checkpoint_x_acks: '' + queue_x_checkpoint_x_writes: '' + queue_x_checkpoint_x_interval: '' + queue_x_checkpoint_x_retry: '' + queue_x_compression: '' + queue_x_drain: '' + dead_letter_queue_x_enable: '' + dead_letter_queue_x_max_bytes: '' + dead_letter_queue_x_flush_interval: '' + dead_letter_queue_x_flush_check_interval: '' + dead_letter_queue_x_storage_policy: '' + dead_letter_queue_x_retain_x_age: '' + path_x_queue: '' + path_x_dead_letter_queue: '' + config_x_debug: '' + config_x_support_escapes: '' + custom1: + pipeline_x_workers: '' + pipeline_x_batch_x_size: '' + pipeline_x_batch_x_delay: '' + pipeline_x_batch_x_metrics_x_sampling_mode: '' + pipeline_x_ordered: '' + pipeline_x_ecs_compatibility: '' + pipeline_x_reloadable: '' + queue_x_type: '' + queue_x_max_bytes: '' + queue_x_page_capacity: '' + queue_x_max_events: '' + queue_x_checkpoint_x_acks: '' + queue_x_checkpoint_x_writes: '' + queue_x_checkpoint_x_interval: '' + queue_x_checkpoint_x_retry: '' + queue_x_compression: '' + queue_x_drain: '' + dead_letter_queue_x_enable: '' + dead_letter_queue_x_max_bytes: '' + dead_letter_queue_x_flush_interval: '' + dead_letter_queue_x_flush_check_interval: '' + dead_letter_queue_x_storage_policy: '' + dead_letter_queue_x_retain_x_age: '' + path_x_queue: '' + path_x_dead_letter_queue: '' + config_x_debug: '' + config_x_support_escapes: '' + custom2: + pipeline_x_workers: '' + pipeline_x_batch_x_size: '' + pipeline_x_batch_x_delay: '' + pipeline_x_batch_x_metrics_x_sampling_mode: '' + pipeline_x_ordered: '' + pipeline_x_ecs_compatibility: '' + pipeline_x_reloadable: '' + queue_x_type: '' + queue_x_max_bytes: '' + queue_x_page_capacity: '' + queue_x_max_events: '' + queue_x_checkpoint_x_acks: '' + queue_x_checkpoint_x_writes: '' + queue_x_checkpoint_x_interval: '' + queue_x_checkpoint_x_retry: '' + queue_x_compression: '' + queue_x_drain: '' + dead_letter_queue_x_enable: '' + dead_letter_queue_x_max_bytes: '' + dead_letter_queue_x_flush_interval: '' + dead_letter_queue_x_flush_check_interval: '' + dead_letter_queue_x_storage_policy: '' + dead_letter_queue_x_retain_x_age: '' + path_x_queue: '' + path_x_dead_letter_queue: '' + config_x_debug: '' + config_x_support_escapes: '' + custom3: + pipeline_x_workers: '' + pipeline_x_batch_x_size: '' + pipeline_x_batch_x_delay: '' + pipeline_x_batch_x_metrics_x_sampling_mode: '' + pipeline_x_ordered: '' + pipeline_x_ecs_compatibility: '' + pipeline_x_reloadable: '' + queue_x_type: '' + queue_x_max_bytes: '' + queue_x_page_capacity: '' + queue_x_max_events: '' + queue_x_checkpoint_x_acks: '' + queue_x_checkpoint_x_writes: '' + queue_x_checkpoint_x_interval: '' + queue_x_checkpoint_x_retry: '' + queue_x_compression: '' + queue_x_drain: '' + dead_letter_queue_x_enable: '' + dead_letter_queue_x_max_bytes: '' + dead_letter_queue_x_flush_interval: '' + dead_letter_queue_x_flush_check_interval: '' + dead_letter_queue_x_storage_policy: '' + dead_letter_queue_x_retain_x_age: '' + path_x_queue: '' + path_x_dead_letter_queue: '' + config_x_debug: '' + config_x_support_escapes: '' + custom4: + pipeline_x_workers: '' + pipeline_x_batch_x_size: '' + pipeline_x_batch_x_delay: '' + pipeline_x_batch_x_metrics_x_sampling_mode: '' + pipeline_x_ordered: '' + pipeline_x_ecs_compatibility: '' + pipeline_x_reloadable: '' + queue_x_type: '' + queue_x_max_bytes: '' + queue_x_page_capacity: '' + queue_x_max_events: '' + queue_x_checkpoint_x_acks: '' + queue_x_checkpoint_x_writes: '' + queue_x_checkpoint_x_interval: '' + queue_x_checkpoint_x_retry: '' + queue_x_compression: '' + queue_x_drain: '' + dead_letter_queue_x_enable: '' + dead_letter_queue_x_max_bytes: '' + dead_letter_queue_x_flush_interval: '' + dead_letter_queue_x_flush_check_interval: '' + dead_letter_queue_x_storage_policy: '' + dead_letter_queue_x_retain_x_age: '' + path_x_queue: '' + path_x_dead_letter_queue: '' + config_x_debug: '' + config_x_support_escapes: '' settings: lsheap: 500m config: diff --git a/salt/logstash/enabled.sls b/salt/logstash/enabled.sls index 80e40d78d..fb0852f00 100644 --- a/salt/logstash/enabled.sls +++ b/salt/logstash/enabled.sls @@ -105,6 +105,7 @@ so-logstash: {% endif %} - watch: - file: lsetcsync + - file: lspipelinesyml - file: trusttheca {% if GLOBALS.is_manager %} - file: elasticsearch_cacerts diff --git a/salt/logstash/etc/pipelines.yml.jinja b/salt/logstash/etc/pipelines.yml.jinja index 427cc9f14..7788ba601 100644 --- a/salt/logstash/etc/pipelines.yml.jinja +++ b/salt/logstash/etc/pipelines.yml.jinja @@ -1,4 +1,17 @@ +{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %} +{%- set PIPELINE_SETTINGS = LOGSTASH_MERGED.get('pipeline_settings', {}) %} {%- for assigned_pipeline in ASSIGNED_PIPELINES %} - pipeline.id: {{ assigned_pipeline }} path.config: "/usr/share/logstash/pipelines/{{ assigned_pipeline }}/" +{%- set extra = PIPELINE_SETTINGS.get(assigned_pipeline, {}) %} +{%- if extra is mapping %} +{#- values are emitted unquoted so yaml re-infers the type logstash expects: + 4 as an integer, false as a boolean, 1024mb and auto as strings #} +{%- for key, value in extra | dictsort %} +{%- set rendered = key | replace('_x_', '.') %} +{%- if value not in ['', None] and rendered not in ['pipeline.id', 'path.config'] %} + {{ rendered }}: {{ value }} +{%- endif %} +{%- endfor %} +{%- endif %} {% endfor -%} diff --git a/salt/logstash/soc_logstash.yaml b/salt/logstash/soc_logstash.yaml index 40794afe4..9dfad556b 100644 --- a/salt/logstash/soc_logstash.yaml +++ b/salt/logstash/soc_logstash.yaml @@ -16,6 +16,7 @@ logstash: heavynode: *assigned_pipelines searchnode: *assigned_pipelines manager: *assigned_pipelines + managerhype: *assigned_pipelines managersearch: *assigned_pipelines fleet: *assigned_pipelines defined_pipelines: @@ -51,6 +52,322 @@ logstash: custom008: *pipeline_config custom009: *pipeline_config custom010: *pipeline_config + pipeline_settings: + manager: &pipeline_settings + pipeline_x_workers: + description: >- + Number of worker threads that run filters and outputs for this pipeline. May be set higher + than the CPU core count when outputs spend time waiting on I/O. Leave blank to use the value + from logstash.yml. + title: pipeline.workers + regex: '^$|^[1-9][0-9]*$' + regexFailureMessage: Must be blank, or a positive whole number. + advanced: True + global: False + helpLink: logstash + pipeline_x_batch_x_size: + description: >- + Maximum number of events an individual worker thread collects before running filters and + outputs. Larger batches are more efficient but increase heap use; total in-flight events is + workers multiplied by batch size. Leave blank to use the value from logstash.yml. + title: pipeline.batch.size + regex: '^$|^[1-9][0-9]*$' + regexFailureMessage: Must be blank, or a positive whole number. + advanced: True + global: False + helpLink: logstash + pipeline_x_batch_x_delay: + description: >- + Milliseconds a worker waits for the next event before running a batch that is not yet full. + Leave blank to use the value from logstash.yml. + title: pipeline.batch.delay + regex: '^$|^[0-9]+$' + regexFailureMessage: Must be blank, or a whole number. + advanced: True + global: False + helpLink: logstash + pipeline_x_batch_x_metrics_x_sampling_mode: + description: >- + How much batch size metering this pipeline records. Fuller sampling helps size batches but + consumes additional heap. Leave blank to use the value from logstash.yml. + title: pipeline.batch.metrics.sampling_mode + options: + - '' + - 'disabled' + - 'minimal' + - 'full' + advanced: True + global: False + helpLink: logstash + pipeline_x_ordered: + description: >- + Whether event order is preserved through this pipeline. auto enables ordering only when the + pipeline runs a single worker. Leave blank to use the value from logstash.yml. + title: pipeline.ordered + options: + - '' + - 'auto' + - 'true' + - 'false' + advanced: True + global: False + helpLink: logstash + pipeline_x_ecs_compatibility: + description: >- + Elastic Common Schema compatibility mode for plugins in this pipeline. Security Onion sets + this globally and it should rarely be changed per pipeline. Leave blank to use the value + from logstash.yml. + title: pipeline.ecs_compatibility + options: + - '' + - 'disabled' + - 'v1' + - 'v8' + advanced: True + global: False + helpLink: logstash + pipeline_x_reloadable: + description: >- + Whether this pipeline may be reloaded when its configuration changes. Leave blank to use the + value from logstash.yml. + title: pipeline.reloadable + options: + - '' + - 'true' + - 'false' + advanced: True + global: False + helpLink: logstash + queue_x_type: + description: >- + Queue backing this pipeline. persisted buffers events to disk under /nsm/logstash so they + survive a restart, at some throughput cost; memory does not. Leave blank to use the value + from logstash.yml. + title: queue.type + options: + - '' + - 'memory' + - 'persisted' + advanced: True + global: False + helpLink: logstash + queue_x_max_bytes: + description: >- + Total size of the persistent queue for this pipeline. Only applies when queue.type is + persisted, and must fit the disk backing /nsm/logstash. Leave blank to use the value from + logstash.yml. + title: queue.max_bytes + regex: '^$|^[0-9]+(b|kb|mb|gb|tb|pb)$' + regexFailureMessage: Must be blank, or a size such as 512mb, 1gb. + advanced: True + global: False + helpLink: logstash + queue_x_page_capacity: + description: >- + Size of each page in the persistent queue for this pipeline. Leave blank to use the value + from logstash.yml. + title: queue.page_capacity + regex: '^$|^[0-9]+(b|kb|mb|gb|tb|pb)$' + regexFailureMessage: Must be blank, or a size such as 512mb, 1gb. + advanced: True + global: False + helpLink: logstash + queue_x_max_events: + description: >- + Maximum number of events in the persistent queue for this pipeline. 0 means unlimited. Leave + blank to use the value from logstash.yml. + title: queue.max_events + regex: '^$|^[0-9]+$' + regexFailureMessage: Must be blank, or a whole number. + advanced: True + global: False + helpLink: logstash + queue_x_checkpoint_x_acks: + description: >- + Number of acknowledged events before a persistent queue checkpoint is forced. 0 means + unlimited. Leave blank to use the value from logstash.yml. + title: queue.checkpoint.acks + regex: '^$|^[0-9]+$' + regexFailureMessage: Must be blank, or a whole number. + advanced: True + global: False + helpLink: logstash + queue_x_checkpoint_x_writes: + description: >- + Number of written events before a persistent queue checkpoint is forced. 0 means unlimited. + Leave blank to use the value from logstash.yml. + title: queue.checkpoint.writes + regex: '^$|^[0-9]+$' + regexFailureMessage: Must be blank, or a whole number. + advanced: True + global: False + helpLink: logstash + queue_x_checkpoint_x_interval: + description: >- + Milliseconds between persistent queue head page checkpoints. 0 disables periodic + checkpointing. Leave blank to use the value from logstash.yml. + title: queue.checkpoint.interval + regex: '^$|^[0-9]+$' + regexFailureMessage: Must be blank, or a whole number. + advanced: True + global: False + helpLink: logstash + queue_x_checkpoint_x_retry: + description: >- + Whether Logstash retries a failed persistent queue checkpoint write. Leave blank to use the + value from logstash.yml. + title: queue.checkpoint.retry + options: + - '' + - 'true' + - 'false' + advanced: True + global: False + helpLink: logstash + queue_x_compression: + description: >- + Compression applied to persistent queue pages for this pipeline, trading CPU for disk. Leave + blank to use the value from logstash.yml. + title: queue.compression + options: + - '' + - 'none' + - 'speed' + - 'balanced' + - 'size' + - 'disabled' + advanced: True + global: False + helpLink: logstash + queue_x_drain: + description: >- + Whether Logstash drains the persistent queue before shutting down this pipeline. Draining a + large queue makes shutdown take considerably longer. Leave blank to use the value from + logstash.yml. + title: queue.drain + options: + - '' + - 'true' + - 'false' + advanced: True + global: False + helpLink: logstash + dead_letter_queue_x_enable: + description: >- + Whether events this pipeline cannot process are written to a dead letter queue instead of + being dropped. Leave blank to use the value from logstash.yml. + title: dead_letter_queue.enable + options: + - '' + - 'true' + - 'false' + advanced: True + global: False + helpLink: logstash + dead_letter_queue_x_max_bytes: + description: >- + Total size of the dead letter queue for this pipeline. Leave blank to use the value from + logstash.yml. + title: dead_letter_queue.max_bytes + regex: '^$|^[0-9]+(b|kb|mb|gb|tb|pb)$' + regexFailureMessage: Must be blank, or a size such as 512mb, 1gb. + advanced: True + global: False + helpLink: logstash + dead_letter_queue_x_flush_interval: + description: >- + Milliseconds before a partial dead letter queue segment is flushed. Leave blank to use the + value from logstash.yml. + title: dead_letter_queue.flush_interval + regex: '^$|^[0-9]+$' + regexFailureMessage: Must be blank, or a whole number. + advanced: True + global: False + helpLink: logstash + dead_letter_queue_x_flush_check_interval: + description: >- + Milliseconds between checks for a dead letter queue segment that needs flushing. Leave blank + to use the value from logstash.yml. + title: dead_letter_queue.flush_check_interval + regex: '^$|^[0-9]+$' + regexFailureMessage: Must be blank, or a whole number. + advanced: True + global: False + helpLink: logstash + dead_letter_queue_x_storage_policy: + description: >- + What happens when the dead letter queue is full: drop_newer discards incoming events, + drop_older discards the oldest stored events. Leave blank to use the value from + logstash.yml. + title: dead_letter_queue.storage_policy + options: + - '' + - 'drop_newer' + - 'drop_older' + advanced: True + global: False + helpLink: logstash + dead_letter_queue_x_retain_x_age: + description: >- + How long an event is kept in the dead letter queue before removal, such as 5d. Leave blank + to use the value from logstash.yml. + title: dead_letter_queue.retain.age + regex: '^$|^[0-9]+[dhms]$' + regexFailureMessage: Must be blank, or a number followed by d, h, m, or s, such as 5d. + advanced: True + global: False + helpLink: logstash + path_x_queue: + description: >- + Directory inside the Logstash container holding the persistent queue for this pipeline. The + default lives under the /nsm/logstash bind mount; a path outside it will not survive a + container restart. Leave blank to use the value from logstash.yml. + title: path.queue + advanced: True + global: False + helpLink: logstash + path_x_dead_letter_queue: + description: >- + Directory inside the Logstash container holding the dead letter queue for this pipeline. The + default lives under the /nsm/logstash bind mount; a path outside it will not survive a + container restart. Leave blank to use the value from logstash.yml. + title: path.dead_letter_queue + advanced: True + global: False + helpLink: logstash + config_x_debug: + description: >- + Whether the fully compiled configuration for this pipeline is written to the log. The output + may contain sensitive values from the pipeline configuration. Leave blank to use the value + from logstash.yml. + title: config.debug + options: + - '' + - 'true' + - 'false' + advanced: True + global: False + helpLink: logstash + config_x_support_escapes: + description: >- + Whether escape sequences such as \n and \t in this pipeline's quoted strings are + interpreted. Leave blank to use the value from logstash.yml. + title: config.support_escapes + options: + - '' + - 'true' + - 'false' + advanced: True + global: False + helpLink: logstash + fleet: *pipeline_settings + receiver: *pipeline_settings + search: *pipeline_settings + custom0: *pipeline_settings + custom1: *pipeline_settings + custom2: *pipeline_settings + custom3: *pipeline_settings + custom4: *pipeline_settings settings: lsheap: description: Heap size to use for logstash From 7bdaf9338e5d70fea6164c096a6a4adcd247b456 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 20 Aug 2026 09:31:56 -0400 Subject: [PATCH 05/17] Update Sigma template --- salt/soc/defaults.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 4bb1b0f8b..c5e14d31b 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -2671,7 +2671,7 @@ soc: # The id (UUIDv4) is pregenerated and can safely be used. # Click "Convert" to convert the Sigma rule to use Security Onion field mappings within an EQL query # - # Rule Creation Guide: https://github.com/SigmaHQ/sigma/wiki/Rule-Creation-Guide + # Rule Creation Guide: https://github.com/SigmaHQ/sigma/wiki/Rule-Creation-High%E2%80%90Level-Guide # Logsources: https://sigmahq.io/docs/basics/log-sources.html title: 'A Short Capitalized Title With Less Than 50 Characters' @@ -2683,7 +2683,7 @@ soc: references: - 'https://local.invalid' author: '@SecurityOnion' - date: 'YYYY/MM/DD' + date: '[today]' tags: - detection.threat_hunting - attack.technique_id From 356da0039509ee4e256a74d8d917c06de875e593 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 20 Aug 2026 13:29:38 -0400 Subject: [PATCH 06/17] Ignore malformed logstash pipeline_settings instead of failing the state A non-mapping value under logstash:pipeline_settings: made config.sls raise "'str object' has no attribute 'get'", which failed the whole logstash.config render rather than just skipping the bad value. pipelines.yml.jinja already guarded this; config.sls now does too, and logs which pipeline was ignored. --- salt/logstash/config.sls | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/logstash/config.sls b/salt/logstash/config.sls index a661e96cc..1c5b6e9e2 100644 --- a/salt/logstash/config.sls +++ b/salt/logstash/config.sls @@ -82,7 +82,11 @@ ls_custom_pipeline_conf_{{assigned_pipeline}}_{{pipeline}}: {% for assigned_pipeline in ASSIGNED_PIPELINES %} {# a blank per-pipeline setting falls back to the global logstash.yml value #} -{% set PIPELINE_OVERRIDES = LOGSTASH_MERGED.get('pipeline_settings', {}).get(assigned_pipeline, {}) %} +{% set PARSED_OVERRIDES = LOGSTASH_MERGED.get('pipeline_settings', {}).get(assigned_pipeline, {}) %} +{% if PARSED_OVERRIDES is not mapping %} +{% do salt.log.warning('logstash: ignoring malformed pipeline_settings for pipeline ' ~ assigned_pipeline ~ '; expected a set of settings') %} +{% endif %} +{% set PIPELINE_OVERRIDES = PARSED_OVERRIDES if PARSED_OVERRIDES is mapping else {} %} {% set THREADS = PIPELINE_OVERRIDES.get('pipeline_x_workers') or LOGSTASH_MERGED.config.pipeline_x_workers %} {% set BATCH = PIPELINE_OVERRIDES.get('pipeline_x_batch_x_size') or LOGSTASH_MERGED.config.pipeline_x_batch_x_size %} {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %} From c1f256e63098503c9db36d9fc3cffd41617374ea Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 20 Aug 2026 16:35:31 -0400 Subject: [PATCH 07/17] Correct pipeline_settings annotations against Logstash 9.3.7 Widen the byte-size regex, which rejected values Logstash accepts and so blocked the save in SOC: bare-letter units (1g, 512m, 64k), decimals (1.5gb), whitespace before the unit, and a bare integer. Allow whitespace in dead_letter_queue.retain.age (5 d). Both stay lowercase-only, matching byte_value.rb and AbstractPipelineExt.parseToDuration. Fix description gaps: queue.checkpoint.retry is a Windows/SAN workaround Elastic does not otherwise recommend, batch metrics sampling is technical preview, queue.checkpoint.interval is deprecated in 9.1, compression makes a queue unreadable by Logstash before 9.2, flush_check_interval has a 1000ms floor, max_events counts unread events, and the path settings are created by Logstash but reject symlinks. Note which settings apply only to persisted queues or an enabled DLQ. Drop the undocumented 'disabled' value from queue.compression. Numeric fields stay stricter than NumericSetting, which has no validator and would accept negatives, floats and NaN in event counts and intervals. --- salt/logstash/soc_logstash.yaml | 111 +++++++++++++++++++------------- 1 file changed, 66 insertions(+), 45 deletions(-) diff --git a/salt/logstash/soc_logstash.yaml b/salt/logstash/soc_logstash.yaml index 9dfad556b..bce0e97c7 100644 --- a/salt/logstash/soc_logstash.yaml +++ b/salt/logstash/soc_logstash.yaml @@ -88,8 +88,10 @@ logstash: helpLink: logstash pipeline_x_batch_x_metrics_x_sampling_mode: description: >- - How much batch size metering this pipeline records. Fuller sampling helps size batches but - consumes additional heap. Leave blank to use the value from logstash.yml. + Controls how often batch size metrics are collected for this pipeline, which helps tune + pipeline.batch.size to the batch sizes actually being processed. Fuller sampling consumes + additional heap. Elastic marks this setting as a technical preview that may change in a + future release. Leave blank to use the value from logstash.yml. title: pipeline.batch.metrics.sampling_mode options: - '' @@ -115,8 +117,9 @@ logstash: pipeline_x_ecs_compatibility: description: >- Elastic Common Schema compatibility mode for plugins in this pipeline. Security Onion sets - this globally and it should rarely be changed per pipeline. Leave blank to use the value - from logstash.yml. + this globally and it should rarely be changed per pipeline. Elastic considers values other + than disabled to be BETA, and they may produce unintended consequences when upgrading + Logstash. Leave blank to use the value from logstash.yml. title: pipeline.ecs_compatibility options: - '' @@ -153,29 +156,32 @@ logstash: helpLink: logstash queue_x_max_bytes: description: >- - Total size of the persistent queue for this pipeline. Only applies when queue.type is - persisted, and must fit the disk backing /nsm/logstash. Leave blank to use the value from - logstash.yml. + Total capacity of the persistent queue for this pipeline, in bytes. Only applies when + queue.type is persisted. The disk backing /nsm/logstash must be larger than this value. If + both queue.max_events and queue.max_bytes are set, whichever is reached first applies. Leave + blank to use the value from logstash.yml. title: queue.max_bytes - regex: '^$|^[0-9]+(b|kb|mb|gb|tb|pb)$' - regexFailureMessage: Must be blank, or a size such as 512mb, 1gb. + regex: '^$|^[0-9]+$|^[0-9]+(\.[0-9]+)?\s*(b|kb?|mb?|gb?|tb?|pb?)$' + regexFailureMessage: Must be blank, or a size such as 512mb, 1gb, or 64k. Units are lowercase. advanced: True global: False helpLink: logstash queue_x_page_capacity: description: >- - Size of each page in the persistent queue for this pipeline. Leave blank to use the value - from logstash.yml. + Size of the individual append-only page data files that make up the persistent queue for + this pipeline. Only applies when queue.type is persisted. Leave blank to use the value from + logstash.yml. title: queue.page_capacity - regex: '^$|^[0-9]+(b|kb|mb|gb|tb|pb)$' - regexFailureMessage: Must be blank, or a size such as 512mb, 1gb. + regex: '^$|^[0-9]+$|^[0-9]+(\.[0-9]+)?\s*(b|kb?|mb?|gb?|tb?|pb?)$' + regexFailureMessage: Must be blank, or a size such as 512mb, 1gb, or 64k. Units are lowercase. advanced: True global: False helpLink: logstash queue_x_max_events: description: >- - Maximum number of events in the persistent queue for this pipeline. 0 means unlimited. Leave - blank to use the value from logstash.yml. + Maximum number of unread events in the persistent queue for this pipeline. 0 means + unlimited. Only applies when queue.type is persisted. Leave blank to use the value from + logstash.yml. title: queue.max_events regex: '^$|^[0-9]+$' regexFailureMessage: Must be blank, or a whole number. @@ -184,8 +190,8 @@ logstash: helpLink: logstash queue_x_checkpoint_x_acks: description: >- - Number of acknowledged events before a persistent queue checkpoint is forced. 0 means - unlimited. Leave blank to use the value from logstash.yml. + Maximum number of acknowledged events before a checkpoint is forced. 0 means unlimited. Only + applies when queue.type is persisted. Leave blank to use the value from logstash.yml. title: queue.checkpoint.acks regex: '^$|^[0-9]+$' regexFailureMessage: Must be blank, or a whole number. @@ -194,8 +200,9 @@ logstash: helpLink: logstash queue_x_checkpoint_x_writes: description: >- - Number of written events before a persistent queue checkpoint is forced. 0 means unlimited. - Leave blank to use the value from logstash.yml. + Maximum number of written events before a checkpoint is forced. Setting this to 1 gives + maximum durability at a severe performance cost. 0 means unlimited. Only applies when + queue.type is persisted. Leave blank to use the value from logstash.yml. title: queue.checkpoint.writes regex: '^$|^[0-9]+$' regexFailureMessage: Must be blank, or a whole number. @@ -204,8 +211,9 @@ logstash: helpLink: logstash queue_x_checkpoint_x_interval: description: >- - Milliseconds between persistent queue head page checkpoints. 0 disables periodic - checkpointing. Leave blank to use the value from logstash.yml. + Milliseconds between forced checkpoints on the persistent queue head page. 0 eliminates + periodic checkpoints. Deprecated by Elastic as of Logstash 9.1. Only applies when queue.type + is persisted. Leave blank to use the value from logstash.yml. title: queue.checkpoint.interval regex: '^$|^[0-9]+$' regexFailureMessage: Must be blank, or a whole number. @@ -214,8 +222,11 @@ logstash: helpLink: logstash queue_x_checkpoint_x_retry: description: >- - Whether Logstash retries a failed persistent queue checkpoint write. Leave blank to use the - value from logstash.yml. + When enabled, Logstash retries four times per attempted checkpoint write that fails; later + errors are not retried. Elastic describes this as a workaround for failed checkpoint writes + seen only on Windows and on filesystems with non-standard behaviour such as SANs, and does + not recommend enabling it otherwise. Only applies when queue.type is persisted. Leave blank + to use the value from logstash.yml. title: queue.checkpoint.retry options: - '' @@ -226,8 +237,11 @@ logstash: helpLink: logstash queue_x_compression: description: >- - Compression applied to persistent queue pages for this pipeline, trading CPU for disk. Leave - blank to use the value from logstash.yml. + Compression applied to persistent queue pages for this pipeline, trading CPU for disk: speed + favours the fastest operation, size the smallest files, and balanced sits between them. Once + compressed events have been written, that queue cannot be read by Logstash releases earlier + than 9.2. Only applies when queue.type is persisted. Leave blank to use the value from + logstash.yml. title: queue.compression options: - '' @@ -235,15 +249,14 @@ logstash: - 'speed' - 'balanced' - 'size' - - 'disabled' advanced: True global: False helpLink: logstash queue_x_drain: description: >- - Whether Logstash drains the persistent queue before shutting down this pipeline. Draining a - large queue makes shutdown take considerably longer. Leave blank to use the value from - logstash.yml. + When enabled, Logstash waits for the persistent queue to drain before shutting down this + pipeline. Draining a large queue makes shutdown take considerably longer. Only applies when + queue.type is persisted. Leave blank to use the value from logstash.yml. title: queue.drain options: - '' @@ -266,18 +279,20 @@ logstash: helpLink: logstash dead_letter_queue_x_max_bytes: description: >- - Total size of the dead letter queue for this pipeline. Leave blank to use the value from - logstash.yml. + Total capacity of the dead letter queue for this pipeline, in bytes. Only applies when + dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml. title: dead_letter_queue.max_bytes - regex: '^$|^[0-9]+(b|kb|mb|gb|tb|pb)$' - regexFailureMessage: Must be blank, or a size such as 512mb, 1gb. + regex: '^$|^[0-9]+$|^[0-9]+(\.[0-9]+)?\s*(b|kb?|mb?|gb?|tb?|pb?)$' + regexFailureMessage: Must be blank, or a size such as 512mb, 1gb, or 64k. Units are lowercase. advanced: True global: False helpLink: logstash dead_letter_queue_x_flush_interval: description: >- - Milliseconds before a partial dead letter queue segment is flushed. Leave blank to use the - value from logstash.yml. + Milliseconds before an incomplete dead letter queue segment is flushed and made available to + the dead_letter_queue input. Lower values write more, smaller segment files; higher values + add latency before events can be read. Only applies when dead_letter_queue.enable is true. + Leave blank to use the value from logstash.yml. title: dead_letter_queue.flush_interval regex: '^$|^[0-9]+$' regexFailureMessage: Must be blank, or a whole number. @@ -286,8 +301,9 @@ logstash: helpLink: logstash dead_letter_queue_x_flush_check_interval: description: >- - Milliseconds between checks for a dead letter queue segment that needs flushing. Leave blank - to use the value from logstash.yml. + Milliseconds between checks for a stale dead letter queue segment needing a flush. Cannot be + set lower than 1000. Smaller values rotate segments sooner at the cost of CPU. Only applies + when dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml. title: dead_letter_queue.flush_check_interval regex: '^$|^[0-9]+$' regexFailureMessage: Must be blank, or a whole number. @@ -296,9 +312,9 @@ logstash: helpLink: logstash dead_letter_queue_x_storage_policy: description: >- - What happens when the dead letter queue is full: drop_newer discards incoming events, - drop_older discards the oldest stored events. Leave blank to use the value from - logstash.yml. + Action taken when dead_letter_queue.max_bytes is reached: drop_newer stops accepting new + events, drop_older removes the oldest events to make room. Only applies when + dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml. title: dead_letter_queue.storage_policy options: - '' @@ -309,10 +325,11 @@ logstash: helpLink: logstash dead_letter_queue_x_retain_x_age: description: >- - How long an event is kept in the dead letter queue before removal, such as 5d. Leave blank - to use the value from logstash.yml. + How long an event is kept in the dead letter queue before Logstash removes it, such as 5d. + Units are d, h, m and s; there is no default unit, so one must be given. Only applies when + dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml. title: dead_letter_queue.retain.age - regex: '^$|^[0-9]+[dhms]$' + regex: '^$|^[0-9]+\s*[dhms]$' regexFailureMessage: Must be blank, or a number followed by d, h, m, or s, such as 5d. advanced: True global: False @@ -321,7 +338,9 @@ logstash: description: >- Directory inside the Logstash container holding the persistent queue for this pipeline. The default lives under the /nsm/logstash bind mount; a path outside it will not survive a - container restart. Leave blank to use the value from logstash.yml. + container restart. Logstash creates the directory if it is missing, requires it to be + writable, and refuses to start if the path is a symlink. Only applies when queue.type is + persisted. Leave blank to use the value from logstash.yml. title: path.queue advanced: True global: False @@ -330,7 +349,9 @@ logstash: description: >- Directory inside the Logstash container holding the dead letter queue for this pipeline. The default lives under the /nsm/logstash bind mount; a path outside it will not survive a - container restart. Leave blank to use the value from logstash.yml. + container restart. Logstash creates the directory if it is missing, requires it to be + writable, and refuses to start if the path is a symlink. Only applies when + dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml. title: path.dead_letter_queue advanced: True global: False From 5c3a69d74239ab4246a26be9a56f4a0c7b4493bf Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 20 Aug 2026 17:33:34 -0400 Subject: [PATCH 08/17] Warn about two pipeline_settings combinations that stop a pipeline Grid testing every permitted value on the manager pipeline surfaced two combinations the UI allows that take the pipeline down, neither of which the descriptions mentioned. pipeline.ordered: true requires pipeline.workers: 1; with more workers the pipeline fails to start with "enabling the 'pipeline.ordered' setting requires the use of a single pipeline worker". Also correct the auto wording: it only engages when workers is explicitly set to 1. queue.max_bytes larger than the free space on /nsm/logstash fails queue creation with "Unable to allocate N more bytes", rather than merely being inadvisable. --- salt/logstash/soc_logstash.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/salt/logstash/soc_logstash.yaml b/salt/logstash/soc_logstash.yaml index bce0e97c7..729b8c242 100644 --- a/salt/logstash/soc_logstash.yaml +++ b/salt/logstash/soc_logstash.yaml @@ -103,8 +103,10 @@ logstash: helpLink: logstash pipeline_x_ordered: description: >- - Whether event order is preserved through this pipeline. auto enables ordering only when the - pipeline runs a single worker. Leave blank to use the value from logstash.yml. + Whether event order is preserved through this pipeline. auto enables ordering only when + pipeline.workers is explicitly set to 1, and does nothing otherwise. Setting this to true + requires pipeline.workers to be 1 as well; with more workers this pipeline fails to start. + Leave blank to use the value from logstash.yml. title: pipeline.ordered options: - '' @@ -157,8 +159,9 @@ logstash: queue_x_max_bytes: description: >- Total capacity of the persistent queue for this pipeline, in bytes. Only applies when - queue.type is persisted. The disk backing /nsm/logstash must be larger than this value. If - both queue.max_events and queue.max_bytes are set, whichever is reached first applies. Leave + queue.type is persisted. The disk backing /nsm/logstash must have room for this much data or + the pipeline fails to start, reporting that it was unable to allocate the space. If both + queue.max_events and queue.max_bytes are set, whichever is reached first applies. Leave blank to use the value from logstash.yml. title: queue.max_bytes regex: '^$|^[0-9]+$|^[0-9]+(\.[0-9]+)?\s*(b|kb?|mb?|gb?|tb?|pb?)$' From 52fc0cb828f40f9f13e909653af6b44a86e660ce Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Fri, 21 Aug 2026 09:37:21 -0400 Subject: [PATCH 09/17] Expose Logstash log.level and log.format in SOC The Logstash log level was hardcoded to info in log4j2.properties, and logstash.yml carried no log.level key, so the only way to raise verbosity for troubleshooting was to hand-edit a file that the next highstate overwrites. Add log_x_level and log_x_format to logstash:config so both render into logstash.yml, annotated as advanced per-node settings with the value sets Logstash 9.3.7 accepts. log4j2.properties gains jinja, so it moves to log4j2.properties.jinja and is rendered by a discrete lslog4j2 state rather than the lsetcsync recurse, which cannot rename. The recurse exclude_pat now matches both names so it neither copies the template verbatim nor lets clean: True delete the rendered file, matching how pipelines.yml is already handled. The appender layout is selected at render time so log.format actually changes the log output instead of being a dead setting, keeping the existing file name so nothing downstream moves. rootLogger.level now follows ls.log.level rather than claiming info regardless of the configured level. --- salt/logstash/config.sls | 14 ++++++++- salt/logstash/defaults.yaml | 2 ++ salt/logstash/enabled.sls | 1 + ...4j2.properties => log4j2.properties.jinja} | 11 +++++-- salt/logstash/soc_logstash.yaml | 29 +++++++++++++++++++ 5 files changed, 53 insertions(+), 4 deletions(-) rename salt/logstash/etc/{log4j2.properties => log4j2.properties.jinja} (83%) diff --git a/salt/logstash/config.sls b/salt/logstash/config.sls index 47feba42c..b32a71a9a 100644 --- a/salt/logstash/config.sls +++ b/salt/logstash/config.sls @@ -125,6 +125,14 @@ lspipelinesyml: - defaults: ASSIGNED_PIPELINES: {{ ASSIGNED_PIPELINES }} +lslog4j2: + file.managed: + - name: /opt/so/conf/logstash/etc/log4j2.properties + - source: salt://logstash/etc/log4j2.properties.jinja + - template: jinja + - user: 931 + - group: 939 + lsetcsync: file.recurse: - name: /opt/so/conf/logstash/etc @@ -133,7 +141,11 @@ lsetcsync: - group: 939 - template: jinja - clean: True - - exclude_pat: pipelines* +{#- both names are matched: the .jinja source so the recurse does not copy it verbatim, + and the rendered file so clean: True does not delete what lslog4j2 wrote #} + - exclude_pat: + - pipelines* + - log4j2.properties* - defaults: LOGSTASH_MERGED: {{ LOGSTASH_MERGED }} diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml index db5e4ee58..41b960eca 100644 --- a/salt/logstash/defaults.yaml +++ b/salt/logstash/defaults.yaml @@ -64,6 +64,8 @@ logstash: lsheap: 500m config: api_x_http_x_host: 0.0.0.0 + log_x_level: info + log_x_format: plain path_x_logs: /var/log/logstash pipeline_x_workers: 1 pipeline_x_batch_x_size: 125 diff --git a/salt/logstash/enabled.sls b/salt/logstash/enabled.sls index 80e40d78d..39ff79539 100644 --- a/salt/logstash/enabled.sls +++ b/salt/logstash/enabled.sls @@ -105,6 +105,7 @@ so-logstash: {% endif %} - watch: - file: lsetcsync + - file: lslog4j2 - file: trusttheca {% if GLOBALS.is_manager %} - file: elasticsearch_cacerts diff --git a/salt/logstash/etc/log4j2.properties b/salt/logstash/etc/log4j2.properties.jinja similarity index 83% rename from salt/logstash/etc/log4j2.properties rename to salt/logstash/etc/log4j2.properties.jinja index 750a6e316..fd13e1bf2 100644 --- a/salt/logstash/etc/log4j2.properties +++ b/salt/logstash/etc/log4j2.properties.jinja @@ -1,3 +1,4 @@ +{%- from 'logstash/map.jinja' import LOGSTASH_MERGED -%} status = error name = LogstashPropertiesConfig @@ -16,8 +17,14 @@ name = LogstashPropertiesConfig appender.rolling.type = RollingFile appender.rolling.name = rolling appender.rolling.fileName = /var/log/logstash/logstash.log +{%- if LOGSTASH_MERGED.config.get('log_x_format', 'plain') == 'json' %} +appender.rolling.layout.type = JSONLayout +appender.rolling.layout.compact = true +appender.rolling.layout.eventEol = true +{%- else %} appender.rolling.layout.type = PatternLayout appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %.10000m%n +{%- endif %} appender.rolling.filePattern = /var/log/logstash/logstash-%d{yyyy-MM-dd}.log.gz appender.rolling.policies.type = Policies appender.rolling.policies.time.type = TimeBasedTriggeringPolicy @@ -32,7 +39,5 @@ appender.rolling.strategy.action.condition.type = IfFileName appender.rolling.strategy.action.condition.glob = *.gz appender.rolling.strategy.action.condition.nested_condition.type = IfLastModified appender.rolling.strategy.action.condition.nested_condition.age = 7D -rootLogger.level = info +rootLogger.level = ${sys:ls.log.level} rootLogger.appenderRef.rolling.ref = rolling -#rootLogger.level = ${sys:ls.log.level} -#rootLogger.appenderRef.console.ref = ${sys:ls.log.format}_console diff --git a/salt/logstash/soc_logstash.yaml b/salt/logstash/soc_logstash.yaml index 40794afe4..c195959e9 100644 --- a/salt/logstash/soc_logstash.yaml +++ b/salt/logstash/soc_logstash.yaml @@ -62,6 +62,35 @@ logstash: helpLink: logstash readonly: True advanced: True + log_x_level: + description: >- + Verbosity of the Logstash log at /opt/so/log/logstash/logstash.log. debug and trace produce + a very large volume of log data on a busy node and should be used only while troubleshooting; + the log rotates at 1GB and rotated files are deleted after 7 days. Setting this to debug is + also what makes the per-pipeline config.debug setting emit anything. + title: log.level + options: + - 'fatal' + - 'error' + - 'warn' + - 'info' + - 'debug' + - 'trace' + advanced: True + global: False + helpLink: logstash + log_x_format: + description: >- + Layout of the Logstash log. plain writes human readable lines; json writes one JSON object + per line, which is easier to parse but harder to read directly. The file name and location + do not change. + title: log.format + options: + - 'plain' + - 'json' + advanced: True + global: False + helpLink: logstash path_x_logs: description: Path inside the container to wrote logs. helpLink: logstash From 12744353fb95e83e7939003793d03efd70761d70 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Fri, 21 Aug 2026 10:18:13 -0400 Subject: [PATCH 10/17] Allow ten custom logstash pipelines instead of five --- salt/logstash/defaults.yaml | 145 ++++++++++++++++++++++++++++++++ salt/logstash/soc_logstash.yaml | 10 +++ 2 files changed, 155 insertions(+) diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml index 84bb91afd..fcaa2ed09 100644 --- a/salt/logstash/defaults.yaml +++ b/salt/logstash/defaults.yaml @@ -42,6 +42,11 @@ logstash: custom2: [] custom3: [] custom4: [] + custom5: [] + custom6: [] + custom7: [] + custom8: [] + custom9: [] pipeline_config: custom001: |- filter { @@ -313,6 +318,146 @@ logstash: path_x_dead_letter_queue: '' config_x_debug: '' config_x_support_escapes: '' + custom5: + pipeline_x_workers: '' + pipeline_x_batch_x_size: '' + pipeline_x_batch_x_delay: '' + pipeline_x_batch_x_metrics_x_sampling_mode: '' + pipeline_x_ordered: '' + pipeline_x_ecs_compatibility: '' + pipeline_x_reloadable: '' + queue_x_type: '' + queue_x_max_bytes: '' + queue_x_page_capacity: '' + queue_x_max_events: '' + queue_x_checkpoint_x_acks: '' + queue_x_checkpoint_x_writes: '' + queue_x_checkpoint_x_interval: '' + queue_x_checkpoint_x_retry: '' + queue_x_compression: '' + queue_x_drain: '' + dead_letter_queue_x_enable: '' + dead_letter_queue_x_max_bytes: '' + dead_letter_queue_x_flush_interval: '' + dead_letter_queue_x_flush_check_interval: '' + dead_letter_queue_x_storage_policy: '' + dead_letter_queue_x_retain_x_age: '' + path_x_queue: '' + path_x_dead_letter_queue: '' + config_x_debug: '' + config_x_support_escapes: '' + custom6: + pipeline_x_workers: '' + pipeline_x_batch_x_size: '' + pipeline_x_batch_x_delay: '' + pipeline_x_batch_x_metrics_x_sampling_mode: '' + pipeline_x_ordered: '' + pipeline_x_ecs_compatibility: '' + pipeline_x_reloadable: '' + queue_x_type: '' + queue_x_max_bytes: '' + queue_x_page_capacity: '' + queue_x_max_events: '' + queue_x_checkpoint_x_acks: '' + queue_x_checkpoint_x_writes: '' + queue_x_checkpoint_x_interval: '' + queue_x_checkpoint_x_retry: '' + queue_x_compression: '' + queue_x_drain: '' + dead_letter_queue_x_enable: '' + dead_letter_queue_x_max_bytes: '' + dead_letter_queue_x_flush_interval: '' + dead_letter_queue_x_flush_check_interval: '' + dead_letter_queue_x_storage_policy: '' + dead_letter_queue_x_retain_x_age: '' + path_x_queue: '' + path_x_dead_letter_queue: '' + config_x_debug: '' + config_x_support_escapes: '' + custom7: + pipeline_x_workers: '' + pipeline_x_batch_x_size: '' + pipeline_x_batch_x_delay: '' + pipeline_x_batch_x_metrics_x_sampling_mode: '' + pipeline_x_ordered: '' + pipeline_x_ecs_compatibility: '' + pipeline_x_reloadable: '' + queue_x_type: '' + queue_x_max_bytes: '' + queue_x_page_capacity: '' + queue_x_max_events: '' + queue_x_checkpoint_x_acks: '' + queue_x_checkpoint_x_writes: '' + queue_x_checkpoint_x_interval: '' + queue_x_checkpoint_x_retry: '' + queue_x_compression: '' + queue_x_drain: '' + dead_letter_queue_x_enable: '' + dead_letter_queue_x_max_bytes: '' + dead_letter_queue_x_flush_interval: '' + dead_letter_queue_x_flush_check_interval: '' + dead_letter_queue_x_storage_policy: '' + dead_letter_queue_x_retain_x_age: '' + path_x_queue: '' + path_x_dead_letter_queue: '' + config_x_debug: '' + config_x_support_escapes: '' + custom8: + pipeline_x_workers: '' + pipeline_x_batch_x_size: '' + pipeline_x_batch_x_delay: '' + pipeline_x_batch_x_metrics_x_sampling_mode: '' + pipeline_x_ordered: '' + pipeline_x_ecs_compatibility: '' + pipeline_x_reloadable: '' + queue_x_type: '' + queue_x_max_bytes: '' + queue_x_page_capacity: '' + queue_x_max_events: '' + queue_x_checkpoint_x_acks: '' + queue_x_checkpoint_x_writes: '' + queue_x_checkpoint_x_interval: '' + queue_x_checkpoint_x_retry: '' + queue_x_compression: '' + queue_x_drain: '' + dead_letter_queue_x_enable: '' + dead_letter_queue_x_max_bytes: '' + dead_letter_queue_x_flush_interval: '' + dead_letter_queue_x_flush_check_interval: '' + dead_letter_queue_x_storage_policy: '' + dead_letter_queue_x_retain_x_age: '' + path_x_queue: '' + path_x_dead_letter_queue: '' + config_x_debug: '' + config_x_support_escapes: '' + custom9: + pipeline_x_workers: '' + pipeline_x_batch_x_size: '' + pipeline_x_batch_x_delay: '' + pipeline_x_batch_x_metrics_x_sampling_mode: '' + pipeline_x_ordered: '' + pipeline_x_ecs_compatibility: '' + pipeline_x_reloadable: '' + queue_x_type: '' + queue_x_max_bytes: '' + queue_x_page_capacity: '' + queue_x_max_events: '' + queue_x_checkpoint_x_acks: '' + queue_x_checkpoint_x_writes: '' + queue_x_checkpoint_x_interval: '' + queue_x_checkpoint_x_retry: '' + queue_x_compression: '' + queue_x_drain: '' + dead_letter_queue_x_enable: '' + dead_letter_queue_x_max_bytes: '' + dead_letter_queue_x_flush_interval: '' + dead_letter_queue_x_flush_check_interval: '' + dead_letter_queue_x_storage_policy: '' + dead_letter_queue_x_retain_x_age: '' + path_x_queue: '' + path_x_dead_letter_queue: '' + config_x_debug: '' + config_x_support_escapes: '' settings: lsheap: 500m config: diff --git a/salt/logstash/soc_logstash.yaml b/salt/logstash/soc_logstash.yaml index 729b8c242..ea037e58a 100644 --- a/salt/logstash/soc_logstash.yaml +++ b/salt/logstash/soc_logstash.yaml @@ -35,6 +35,11 @@ logstash: custom2: *defined_pipelines custom3: *defined_pipelines custom4: *defined_pipelines + custom5: *defined_pipelines + custom6: *defined_pipelines + custom7: *defined_pipelines + custom8: *defined_pipelines + custom9: *defined_pipelines pipeline_config: custom001: &pipeline_config description: Pipeline configuration for Logstash @@ -392,6 +397,11 @@ logstash: custom2: *pipeline_settings custom3: *pipeline_settings custom4: *pipeline_settings + custom5: *pipeline_settings + custom6: *pipeline_settings + custom7: *pipeline_settings + custom8: *pipeline_settings + custom9: *pipeline_settings settings: lsheap: description: Heap size to use for logstash From 60052e0910946b89cd0e42d9ed279c00bd06ec96 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Tue, 18 Aug 2026 15:31:15 -0600 Subject: [PATCH 11/17] Memory Defaults and Annotations --- salt/soc/defaults.yaml | 11 +++++++++++ salt/soc/soc_soc.yaml | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index c5e14d31b..0b74ca332 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1537,6 +1537,17 @@ soc: Orchestrator: sonnet@SOAI Investigator: gemma@SOAI DetectionEngineer: gemma@SOAI + useMemory: true + useMemoryScanner: true + memoryProximityThreshold: 0.8 + messageProximityThreshold: 0.5 + maxUserMemoriesToInclude: 5 + maxGlobalMemoriesToInclude: 5 + maxUserMemoriesToReconcile: 20 + maxGlobalMemoriesToReconcile: 20 + memoryModel: gemma@SOAI + embedModel: gemma@SOAI + reconcileModel: gemma@SOAI onionconfig: saltstackDir: /opt/so/saltstack bypassEnabled: false diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 0dda9d090..627bb660e 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -845,6 +845,39 @@ soc: DetectionEngineer: description: This agent manages detections and their overrides, including tuning noisy rules and authoring rule content. global: True + useMemory: + description: Enables the Memory system for OnionAI + global: True + useMemoryScanner: + description: Enables the memory scanner for automatic memory extraction from historical sessions. + global: True + memoryProximityThreshold: + description: Describes how close memories need to be on a floating point scale from 0.0 to 1.0 to be considered when reconciling new memories with old ones. This value is usually higher than messageProximityThreshold. + global: True + messageProximityThreshold: + description: Describes how close a memory needs to be to a user's message on a floating point scale from 0.0 to 1.0 to be included in the context. This value is usually lower than memoryProximityThreshold. + global: True + maxUserMemoriesToInclude: + description: Specify the max number of user-specific memories to include in the prompt when a user sends a message. + global: True + maxGlobalMemoriesToInclude: + description: Specify the max number of global memories to include in the prompt when a user sends a message. + global: True + maxUserMemoriesToReconcile: + description: When reconciling new user-specific memories with existing user-specific memories, this determines how many old memories may be considered. + global: True + maxGlobalMemoriesToReconcile: + description: When reconciling new global memories with existing global memories, this determines how many old memories may be considered. + global: True + memoryModel: + description: The model to use when extracting memories from sessions. + global: True + embedModel: + description: The model to use when embedding a memory as a vector. Note that only memories embedded using the same model may be compared and only memories created with the model specified here will be considered when informing an agent of existing memories. + global: True + reconcileModel: + description: The model to use when reconciling memories that contain nearly the same content. + global: True client: assistant: enabled: From 99e1d83358394d00ecdefb3de905aa548b756e72 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Wed, 19 Aug 2026 14:16:12 -0600 Subject: [PATCH 12/17] Add Interval and Disable by Default Added `memoryScanIntervalSeconds` with a default of 5 mins. Opted to set `useMemoryScanner` to false so by default our user's sessions are not sent to the cloud before they have a chance to configure the new setting. --- salt/soc/defaults.yaml | 12 +++++++++++- salt/soc/soc_soc.yaml | 3 +++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 0b74ca332..3d7098960 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1538,7 +1538,8 @@ soc: Investigator: gemma@SOAI DetectionEngineer: gemma@SOAI useMemory: true - useMemoryScanner: true + useMemoryScanner: false + memoryScanIntervalSeconds: 300 memoryProximityThreshold: 0.8 messageProximityThreshold: 0.5 maxUserMemoriesToInclude: 5 @@ -2738,5 +2739,14 @@ soc: enabled: true adapter: SOAI charsPerTokenEstimate: 4 + - id: amazon.titan-embed-text-v2 + displayName: amazon.titan-embed-text-v2 + origin: USA + contextLimitSmall: 8192 + contextLimitLarge: 8192 + lowBalanceColorAlert: 500000 + enabled: true + adapter: SOAI + charsPerTokenEstimate: 4 diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 627bb660e..5fef5eded 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -851,6 +851,9 @@ soc: useMemoryScanner: description: Enables the memory scanner for automatic memory extraction from historical sessions. global: True + memoryScanIntervalSeconds: + description: How long to wait in seconds between attempts to scan sessions for new memories. + global: True memoryProximityThreshold: description: Describes how close memories need to be on a floating point scale from 0.0 to 1.0 to be considered when reconciling new memories with old ones. This value is usually higher than messageProximityThreshold. global: True From fcb889a30c16d7c496d383272af6b0a904b7cc1a Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Wed, 19 Aug 2026 14:17:21 -0600 Subject: [PATCH 13/17] Specify Default Embed Model --- salt/soc/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 3d7098960..c6e833d32 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1547,7 +1547,7 @@ soc: maxUserMemoriesToReconcile: 20 maxGlobalMemoriesToReconcile: 20 memoryModel: gemma@SOAI - embedModel: gemma@SOAI + embedModel: amazon.titan-embed-text-v2@SOAI reconcileModel: gemma@SOAI onionconfig: saltstackDir: /opt/so/saltstack From a127ef57140109c0eead9181408c41829e2ff15b Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Mon, 24 Aug 2026 14:09:25 -0600 Subject: [PATCH 14/17] Show Toggle in UI Must specify bool fields with `forcedType: bool` in order for them to render as toggles in the UI. --- salt/soc/soc_soc.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 5fef5eded..b329b736b 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -848,9 +848,11 @@ soc: useMemory: description: Enables the Memory system for OnionAI global: True + forcedType: bool useMemoryScanner: description: Enables the memory scanner for automatic memory extraction from historical sessions. global: True + forcedType: bool memoryScanIntervalSeconds: description: How long to wait in seconds between attempts to scan sessions for new memories. global: True @@ -878,6 +880,7 @@ soc: embedModel: description: The model to use when embedding a memory as a vector. Note that only memories embedded using the same model may be compared and only memories created with the model specified here will be considered when informing an agent of existing memories. global: True + advanced: True reconcileModel: description: The model to use when reconciling memories that contain nearly the same content. global: True From 5e9fd4a45bd610dd4a4bc65f8a3d1fbc1fe79d7f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 25 Aug 2026 09:30:01 -0400 Subject: [PATCH 15/17] Remove the stock EL9 kernel once a node is running UEK8 The UEK8 rollout installs the new kernel and flips the boot default, but leaves the stock EL9 (RHCK) packages behind: disk in /boot and a stale GRUB entry on every upgraded node. They cannot be removed in the same pass that installs UEK8. dnf's protect_running_kernel refuses to erase the booted kernel-core, so the removal has to wait until the node has rebooted onto 6.x. Waiting is the safer sequencing anyway -- the node proves it comes up on UEK8 before its fallback is deleted -- so this does not remove RHCK from the uek7 branch either, where dnf would allow it. so-kernel-upgrade grows a --cleanup mode that does only the removal and no-ops (exit 0, with a log line) on a node not yet running UEK8. Its uek8 branch, which previously reported "nothing to do", now runs that cleanup along with set_default_kernel_conf -- which also closes a gap where a node that came up on UEK8 straight from a fresh install never had DEFAULTKERNEL=kernel-uek-core written. The common highstate calls --cleanup gated on the running kernel, so the cleanup lands grid-wide as each node reboots: fresh installs reboot at the end of setup, upgraded nodes whenever the admin schedules it. The rpm check inside the script is the idempotency guard, so subsequent highstates cost an rpm query rather than a dnf transaction, and the package list is not duplicated into the state where it could drift. --- salt/common/init.sls | 14 ++++ salt/common/tools/sbin/so-kernel-upgrade | 88 ++++++++++++++++++++++-- 2 files changed, 95 insertions(+), 7 deletions(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 9618d2c67..bafd12fd6 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -141,6 +141,20 @@ pin_nic_names: - file: common_sbin - file: statedir +# Once a node is actually running UEK8, the stock EL9 (RHCK) kernel packages are dead weight. +# They can't be removed any earlier -- dnf protects the running kernel -- so the cleanup waits +# for the reboot, which makes the highstate the natural place to catch it: fresh installs +# reboot at the end of setup, and upgraded nodes reboot whenever the admin schedules it. +# so-kernel-upgrade --cleanup checks rpm before touching dnf, so this costs an rpm query on +# every highstate after the first pass. The package list lives in the script only, so there +# is nothing here to drift out of sync with it. +remove_stock_kernel: + cmd.run: + - name: /usr/sbin/so-kernel-upgrade --cleanup + - onlyif: 'uname -r | grep -qE "^6\.[0-9]+.*uek"' + - require: + - file: common_sbin + common_sbin_jinja: file.recurse: - name: /usr/sbin diff --git a/salt/common/tools/sbin/so-kernel-upgrade b/salt/common/tools/sbin/so-kernel-upgrade index e750c1a52..6ebde9ebd 100755 --- a/salt/common/tools/sbin/so-kernel-upgrade +++ b/salt/common/tools/sbin/so-kernel-upgrade @@ -5,10 +5,11 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. # -# so-kernel-upgrade — install the UEK8 (6.x) kernel and make it the boot default. +# so-kernel-upgrade — install the UEK8 (6.x) kernel, make it the boot default, and once the +# node is running it, remove the stock EL9 kernel. # # Security Onion is moving off the EL9 stock kernel (RHCK, 5.14) and UEK7 (5.15) onto UEK8 -# (6.x). Three things have to happen, and the tool has to drive each one: +# (6.x). Four things have to happen, and the tool has to drive each one: # # 1. Populate. The manager mirrors the UEK8 packages into /nsm/kernelrepo via so-repo-sync, # and serves them to the grid over https:///kernelrepo. Until that sync runs the @@ -26,10 +27,21 @@ # - From the stock EL9 kernel (RHCK, 5.14, no UEK) it is a flavor CROSS that is NOT # auto-promoted, so the box keeps booting RHCK until grubby is told otherwise. # This tool inspects the running kernel and only runs 'grubby --set-default' for RHCK. +# 4. Clean up. Once the node is actually RUNNING UEK8 the stock kernel packages are dead +# weight -- disk in /boot and a stale GRUB entry. They cannot come off any earlier: +# dnf's protect_running_kernel refuses to erase the booted kernel-core, so the removal +# has to wait for the reboot. Waiting is also the safer sequencing on its own terms -- +# the node has proven it comes up on UEK8 before its fallback is deleted. That is why +# the removal does not happen in the uek7 branch either, where dnf would allow it. # # Every one of those failure modes is silent by default. This tool handles each case and fails # loudly when it cannot, rather than reporting success while changing nothing. # +# Invocation: with no arguments it drives the whole sequence for whatever kernel the node is +# on. With --cleanup it does the step 4 removal ONLY, and no-ops on a node that isn't running +# UEK8 yet -- that is the form the common highstate calls (remove_stock_kernel in +# salt/common/init.sls) so the cleanup lands grid-wide after each node reboots. +# # Manager vs minion: only the manager owns /nsm/kernelrepo, so only the manager can populate # it. If the repo is empty here, a manager runs so-repo-sync itself; a minion has no way to # fix it and exits non-zero telling the admin to sync the manager first. @@ -49,6 +61,11 @@ KERNEL_REPO_DIR="/nsm/kernelrepo" REPOSYNC_CONF="/opt/so/conf/reposync/repodownload.conf" GLOBAL_PILLAR="/opt/so/saltstack/local/pillar/global/soc_global.sls" +# Stock EL9 (RHCK) kernel packages, removed only once the node is running UEK8 (see step 4 +# in the header). Left deliberately narrow: UEK7 kernel-uek builds age out on their own via +# installonly_limit=3, and kernel-devel/kernel-headers are not touched. +RHCK_PKGS="kernel kernel-core kernel-modules kernel-modules-core kernel-tools kernel-tools-libs" + log() { echo "[so-kernel-upgrade] $*"; } die() { echo "[so-kernel-upgrade] ERROR: $*" >&2; exit 1; } @@ -149,8 +166,13 @@ ensure_kernel_repo() { } reboot_notice() { - [ "$(uname -r)" = "$(basename "$1" | sed 's/^vmlinuz-//')" ] \ - || log "REBOOT REQUIRED to start using the UEK8 kernel (currently running $(uname -r))." + [ "$(uname -r)" = "$(basename "$1" | sed 's/^vmlinuz-//')" ] && return 0 + log "REBOOT REQUIRED to start using the UEK8 kernel (currently running $(uname -r))." + # The stock kernel can't be removed until it stops being the running one, so say when + # that will happen rather than leaving the admin to wonder if it was missed. + [ -n "$(rhck_installed)" ] \ + && log "The stock EL9 kernel is left in place until then; it is removed by the next highstate after the reboot." + return 0 } # Keep future kernel updates on the UEK line rather than falling back to RHCK. Oracle ships @@ -162,6 +184,32 @@ set_default_kernel_conf() { fi } +# Which of RHCK_PKGS are actually installed, one per line. rpm -qa treats each argument as a +# name glob and prints only what it finds, so a package that was never installed (or is +# already gone) simply doesn't appear -- no "not installed" noise and no non-zero exit. +rhck_installed() { + rpm -qa $RHCK_PKGS 2>/dev/null +} + +# Remove the stock EL9 kernel. Only ever called once the running kernel is UEK8. The rpm +# check above is the idempotency guard, so this is a cheap no-op on every highstate after +# the first one -- it costs an rpm query, not a dnf transaction. +remove_rhck() { + local installed; installed="$(rhck_installed)" + if [ -z "$installed" ]; then + log "no stock EL9 (RHCK) kernel packages installed; nothing to remove." + return 0 + fi + + log "running UEK8; removing the stock EL9 (RHCK) kernel packages:" + echo "$installed" | sed 's/^/[so-kernel-upgrade] /' + dnf -y remove $RHCK_PKGS || die "failed to remove the stock EL9 kernel packages" + + installed="$(rhck_installed)" + [ -z "$installed" ] || die "dnf reported success but these remain: $(echo $installed)" + log "stock EL9 kernel packages removed." +} + # Make sure a UEK8 kernel is installed, leaving its boot entry in INSTALLED_UEK8. If one is # already present we leave the repo alone -- it may be disabled or empty and we don't need it # just to flip the boot default. Otherwise install the explicit NEVRA, not the bare package @@ -184,12 +232,38 @@ ensure_uek8_installed() { log "installed UEK8 kernel: $INSTALLED_UEK8" } +# --cleanup does step 4 and nothing else. It exits 0 rather than failing on a node that +# isn't on UEK8 yet: the highstate gates on 'uname -r' before calling this, and a state that +# fails whenever that gate races would be worse than one that says what it's waiting for. +case "$1" in +"") + ;; +--cleanup) + if [ "$(running_flavor)" != uek8 ]; then + log "not running a UEK8 kernel yet (currently $(uname -r)); leaving the stock EL9 kernel in place." + log "Run so-kernel-upgrade with no arguments to install UEK8, then reboot." + exit 0 + fi + set_default_kernel_conf + remove_rhck + exit 0 + ;; +*) + echo "Usage: so-kernel-upgrade [--cleanup]" >&2 + echo " (no arguments) install UEK8, make it the boot default, clean up once it's running" >&2 + echo " --cleanup remove the stock EL9 kernel; no-op unless already running UEK8" >&2 + exit 1 + ;; +esac + case "$(running_flavor)" in uek8) # Already on the 6.x UEK line. A plain 'dnf update' keeps this node current within the - # lineage and auto-promotes newer builds, so there is nothing for this tool to do. - log "already running a UEK8 kernel ($(uname -r)); nothing to do." - exit 0 + # lineage and auto-promotes newer builds, so there is no install or grubby work left -- + # only the step 4 cleanup, which this is the first point in the sequence that can run it. + log "already running a UEK8 kernel ($(uname -r)); no kernel install needed." + set_default_kernel_conf + remove_rhck ;; uek7) From f6ab92fc248ca6faf408465164a0b5c5d8dec1bf Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 25 Aug 2026 09:43:56 -0400 Subject: [PATCH 16/17] add vector ext for agentic memory --- salt/postgres/files/init-db.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/postgres/files/init-db.sh b/salt/postgres/files/init-db.sh index 4d65b0c97..08488fc29 100644 --- a/salt/postgres/files/init-db.sh +++ b/salt/postgres/files/init-db.sh @@ -29,6 +29,8 @@ psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-E -- revoking CONNECT closes the soft edge entirely. REVOKE CONNECT ON DATABASE "$POSTGRES_DB" FROM PUBLIC; GRANT CONNECT ON DATABASE "$POSTGRES_DB" TO "$SO_POSTGRES_USER"; + + CREATE EXTENSION IF NOT EXISTS vector; EOSQL # Bootstrap the Telegraf metrics database. Per-minion roles + schemas are From 9f6679c0432666d64c9d9555df6446a4be8ca9f5 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 26 Aug 2026 13:42:37 -0400 Subject: [PATCH 17/17] Use argv arrays for telegraf inputs.exec commands Telegraf 1.39 deprecated bare string entries in inputs.exec commands and will drop them in 1.45, warning on every start: W! DeprecationWarning: Value "/scripts/esindexsize.sh" for option "command" of plugin "inputs.exec" deprecated since version 1.39.0 Each entry is now a single-element argv array. Array form skips shell parsing, which is fine here: every command is a bare script path from telegraf's scripts list, no args or shell metacharacters. --- salt/telegraf/etc/telegraf.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 93d0e71b3..ae1b83084 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -335,7 +335,7 @@ {%- do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('sostatus.sh') %} [[inputs.exec]] commands = [ - "/scripts/sostatus.sh" + ["/scripts/sostatus.sh"] ] data_format = "influx" timeout = "15s" @@ -346,7 +346,7 @@ [[inputs.exec]] commands = [ {%- for script in TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] %} - "/scripts/{{script}}"{% if not loop.last %},{% endif %} + ["/scripts/{{script}}"]{% if not loop.last %},{% endif %} {%- endfor %} ] data_format = "influx" @@ -375,7 +375,7 @@ {%- if GLOBALS.is_manager or GLOBALS.role == 'so-heavynode' %} [[ inputs.exec ]] commands = [ - "/scripts/esindexsize.sh" + ["/scripts/esindexsize.sh"] ] data_format = "influx" interval = "1h"