From d7e971a0fcbf305aa5fa1512ddf7920eea79714f Mon Sep 17 00:00:00 2001 From: Marco Pedrinazzi Date: Wed, 11 Mar 2026 13:36:47 +0100 Subject: [PATCH 01/20] m365 and fortigate mappings sigma --- salt/soc/files/soc/sigma_so_pipeline.yaml | 115 ++++++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/salt/soc/files/soc/sigma_so_pipeline.yaml b/salt/soc/files/soc/sigma_so_pipeline.yaml index 4462bde42..11a20ff03 100644 --- a/salt/soc/files/soc/sigma_so_pipeline.yaml +++ b/salt/soc/files/soc/sigma_so_pipeline.yaml @@ -117,6 +117,121 @@ transformations: - type: logsource product: linux service: auth + # Maps M365 audit rules to Elastic Agent O365 integration logs + - id: m365_audit_field_mappings + type: field_name_mapping + mapping: + Operation: event.action + ResultStatus: event.outcome + ApplicationId: o365.audit.ApplicationId + ObjectId: o365.audit.ObjectId + RequestType: o365.audit.RequestType + rule_conditions: + - type: logsource + product: m365 + service: audit + - id: m365_audit_add-fields + type: add_condition + conditions: + event.dataset: 'o365.audit' + event.module: 'o365' + rule_conditions: + - type: logsource + product: m365 + service: audit + # Maps M365 exchange rules to Elastic Agent O365 integration logs + - id: m365_exchange_field_mappings + type: field_name_mapping + mapping: + eventSource: event.provider + eventName: event.action + status: event.outcome + rule_conditions: + - type: logsource + product: m365 + service: exchange + - id: m365_exchange_add-fields + type: add_condition + conditions: + event.dataset: 'o365.audit' + event.module: 'o365' + rule_conditions: + - type: logsource + product: m365 + service: exchange + # Maps M365 threat_management rules to Elastic Agent O365 integration logs + - id: m365_threat_management_field_mappings + type: field_name_mapping + mapping: + eventSource: event.provider + eventName: event.action + status: event.outcome + rule_conditions: + - type: logsource + product: m365 + service: threat_management + - id: m365_threat_management_add-fields + type: add_condition + conditions: + event.dataset: 'o365.audit' + event.module: 'o365' + rule_conditions: + - type: logsource + product: m365 + service: threat_management + # Maps M365 threat_detection rules to Elastic Agent O365 integration logs + - id: m365_threat_detection_field_mappings + type: field_name_mapping + mapping: + eventSource: event.provider + eventName: event.action + status: event.outcome + rule_conditions: + - type: logsource + product: m365 + service: threat_detection + - id: m365_threat_detection_add-fields + type: add_condition + conditions: + event.dataset: 'o365.audit' + event.module: 'o365' + rule_conditions: + - type: logsource + product: m365 + service: threat_detection + # Maps FortiGate event rules to Elastic Agent Fortinet integration logs + - id: fortigate_event_field_mappings + type: field_name_mapping + mapping: + action: fortinet.firewall.action + cfgpath: fortinet.firewall.cfgpath + cfgobj: fortinet.firewall.cfgobj + cfgattr: fortinet.firewall.cfgattr + devname: observer.name + devid: observer.serial_number + logid: event.code + type: fortinet.firewall.type + subtype: fortinet.firewall.subtype + level: log.level + vd: fortinet.firewall.vd + logdesc: fortinet.firewall.desc + user: user.name + ui: fortinet.firewall.ui + cfgtid: fortinet.firewall.cfgtid + msg: message + rule_conditions: + - type: logsource + product: fortigate + service: event + - id: fortigate_event_add-fields + type: add_condition + conditions: + event.dataset: 'fortinet_fortigate.log' + event.module: 'fortinet_fortigate' + rule_conditions: + - type: logsource + product: fortigate + service: event # event.code should always be a string - id: convert_event_code_to_string type: convert_type From 3a4b7b50de09734cd248e2417bc74264c3c0c906 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 30 Apr 2026 10:15:09 -0400 Subject: [PATCH 02/20] ensure python3-pyyaml is installed before continuing setup --- setup/so-functions | 18 ++++++++++++++++++ setup/so-setup | 3 +++ 2 files changed, 21 insertions(+) diff --git a/setup/so-functions b/setup/so-functions index 3cd665076..a7afdbaa3 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1701,6 +1701,24 @@ remove_package() { fi } +ensure_pyyaml() { + title "Ensuring python3-pyyaml is installed" + if rpm -q python3-pyyaml >/dev/null 2>&1; then + info "python3-pyyaml already installed" + return 0 + fi + info "python3-pyyaml not found, attempting to install" + set -o pipefail + dnf -y install python3-pyyaml 2>&1 | tee -a "$setup_log" + local result=$? + set +o pipefail + if [[ $result -ne 0 ]] || ! rpm -q python3-pyyaml >/dev/null 2>&1; then + error "Failed to install python3-pyyaml (exit=$result)" + fail_setup + fi + info "python3-pyyaml installed successfully" +} + # When updating the salt version, also update the version in securityonion-builds/images/iso-task/Dockerfile and salt/salt/master.defaults.yaml and salt/salt/minion.defaults.yaml # CAUTION! SALT VERSION UDDATES - READ BELOW # When updating the salt version, also update the version in: diff --git a/setup/so-setup b/setup/so-setup index 7875b9c99..6c77e781c 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -66,6 +66,9 @@ set_timezone # Let's see what OS we are dealing with here detect_os +# Ensure python3-pyyaml is available before any code that may need so-yaml/PyYAML +ensure_pyyaml + # Check to see if this is the setup type of "desktop". is_desktop= From 1c6574c69423bee1d7b5f6a6ccd505f72222fc8b Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Mon, 4 May 2026 14:03:14 -0400 Subject: [PATCH 03/20] ensure minion ids --- salt/orch/delete_hypervisor.sls | 11 +++++++- salt/orch/vm_pillar_clean.sls | 11 +++++++- salt/reactor/check_hypervisor.sls | 10 ++++--- salt/reactor/createEmptyPillar.sls | 42 ++++++++++++++++++---------- salt/reactor/deleteKey.sls | 44 ++++++++++++++++++++++-------- 5 files changed, 86 insertions(+), 32 deletions(-) diff --git a/salt/orch/delete_hypervisor.sls b/salt/orch/delete_hypervisor.sls index 3f0bd02b6..784977e28 100644 --- a/salt/orch/delete_hypervisor.sls +++ b/salt/orch/delete_hypervisor.sls @@ -3,7 +3,14 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{% set hypervisor = pillar.minion_id %} +{% set hypervisor = pillar.get('minion_id', '') %} + +{% if not hypervisor|regex_match('^[A-Za-z0-9._-]{1,253}$') %} +{% do salt.log.error('delete_hypervisor_orch: refusing unsafe minion_id=' ~ hypervisor) %} +delete_hypervisor_invalid_minion_id: + test.fail_without_changes: + - name: delete_hypervisor_invalid_minion_id +{% else %} ensure_hypervisor_mine_deleted: salt.function: @@ -20,3 +27,5 @@ update_salt_cloud_profile: - sls: - salt.cloud.config - concurrent: True + +{% endif %} diff --git a/salt/orch/vm_pillar_clean.sls b/salt/orch/vm_pillar_clean.sls index ca5c16054..9b6bf1ee5 100644 --- a/salt/orch/vm_pillar_clean.sls +++ b/salt/orch/vm_pillar_clean.sls @@ -12,7 +12,14 @@ {% if 'vrt' in salt['pillar.get']('features', []) %} {% do salt.log.debug('vm_pillar_clean_orch: Running') %} -{% set vm_name = pillar.get('vm_name') %} +{% set vm_name = pillar.get('vm_name', '') %} + +{% if not vm_name|regex_match('^[A-Za-z0-9._-]{1,253}$') %} +{% do salt.log.error('vm_pillar_clean_orch: refusing unsafe vm_name=' ~ vm_name) %} +vm_pillar_clean_invalid_name: + test.fail_without_changes: + - name: vm_pillar_clean_invalid_name +{% else %} delete_adv_{{ vm_name }}_pillar: module.run: @@ -24,6 +31,8 @@ delete_{{ vm_name }}_pillar: - file.remove: - path: /opt/so/saltstack/local/pillar/minions/{{ vm_name }}.sls +{% endif %} + {% else %} {% do salt.log.error( diff --git a/salt/reactor/check_hypervisor.sls b/salt/reactor/check_hypervisor.sls index 91b7c0c02..c0fa49ddc 100644 --- a/salt/reactor/check_hypervisor.sls +++ b/salt/reactor/check_hypervisor.sls @@ -3,12 +3,15 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{% if data['id'].endswith('_hypervisor') and data['result'] == True %} +{% set hid = data['id'] %} +{% if hid|regex_match('^[A-Za-z0-9._-]{1,253}$') + and hid.endswith('_hypervisor') + and data['result'] == True %} {% if data['act'] == 'accept' %} check_and_trigger: runner.setup_hypervisor.setup_environment: - - minion_id: {{ data['id'] }} + - minion_id: {{ hid }} {% endif %} {% if data['act'] == 'delete' %} @@ -17,8 +20,7 @@ delete_hypervisor: - args: - mods: orch.delete_hypervisor - pillar: - minion_id: {{ data['id'] }} + minion_id: {{ hid }} {% endif %} {% endif %} - diff --git a/salt/reactor/createEmptyPillar.sls b/salt/reactor/createEmptyPillar.sls index c6c655bab..2076b53bd 100644 --- a/salt/reactor/createEmptyPillar.sls +++ b/salt/reactor/createEmptyPillar.sls @@ -1,7 +1,7 @@ #!py # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. @@ -9,30 +9,42 @@ import logging import os import pwd import grp +import re + +log = logging.getLogger(__name__) + +PILLAR_ROOT = '/opt/so/saltstack/local/pillar/minions/' +_VMNAME_RE = re.compile(r'^[A-Za-z0-9._-]{1,253}$') + def run(): - vm_name = data['kwargs']['name'] - logging.error("createEmptyPillar reactor: vm_name: %s" % vm_name) - pillar_root = '/opt/so/saltstack/local/pillar/minions/' + vm_name = data.get('kwargs', {}).get('name', '') + if not _VMNAME_RE.match(str(vm_name)): + log.error("createEmptyPillar reactor: refusing unsafe vm_name=%r", vm_name) + return {} + + log.info("createEmptyPillar reactor: vm_name: %s", vm_name) pillar_files = ['adv_' + vm_name + '.sls', vm_name + '.sls'] try: - # Get socore user and group IDs socore_uid = pwd.getpwnam('socore').pw_uid socore_gid = grp.getgrnam('socore').gr_gid + pillar_root_real = os.path.realpath(PILLAR_ROOT) for f in pillar_files: - full_path = pillar_root + f - if not os.path.exists(full_path): - # Create empty file - os.mknod(full_path) - # Set ownership to socore:socore - os.chown(full_path, socore_uid, socore_gid) - # Set mode to 644 (rw-r--r--) - os.chmod(full_path, 0o640) - logging.error("createEmptyPillar reactor: created %s with socore:socore ownership and mode 644" % f) + full_path = os.path.join(PILLAR_ROOT, f) + resolved = os.path.realpath(full_path) + if os.path.dirname(resolved) != pillar_root_real: + log.error("createEmptyPillar reactor: refusing path outside pillar root: %s", resolved) + continue + if os.path.exists(resolved): + continue + os.mknod(resolved) + os.chown(resolved, socore_uid, socore_gid) + os.chmod(resolved, 0o640) + log.info("createEmptyPillar reactor: created %s with socore:socore ownership and mode 0640", f) except (KeyError, OSError) as e: - logging.error("createEmptyPillar reactor: Error setting ownership/permissions: %s" % str(e)) + log.error("createEmptyPillar reactor: Error setting ownership/permissions: %s", e) return {} diff --git a/salt/reactor/deleteKey.sls b/salt/reactor/deleteKey.sls index 4d522a4b5..a57d6370c 100644 --- a/salt/reactor/deleteKey.sls +++ b/salt/reactor/deleteKey.sls @@ -1,18 +1,40 @@ +#!py + # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -remove_key: - wheel.key.delete: - - args: - - match: {{ data['name'] }} +import logging +import re -{{ data['name'] }}_pillar_clean: - runner.state.orchestrate: - - args: - - mods: orch.vm_pillar_clean - - pillar: - vm_name: {{ data['name'] }} +log = logging.getLogger(__name__) -{% do salt.log.info('deleteKey reactor: deleted minion key: %s' % data['name']) %} +_VMNAME_RE = re.compile(r'^[A-Za-z0-9._-]{1,253}$') + + +def run(): + name = data.get('name', '') + if not _VMNAME_RE.match(str(name)): + log.error("deleteKey reactor: refusing unsafe name=%r", name) + return {} + + log.info("deleteKey reactor: deleted minion key: %s", name) + + return { + 'remove_key': { + 'wheel.key.delete': [ + {'args': [ + {'match': name}, + ]}, + ], + }, + '%s_pillar_clean' % name: { + 'runner.state.orchestrate': [ + {'args': [ + {'mods': 'orch.vm_pillar_clean'}, + {'pillar': {'vm_name': name}}, + ]}, + ], + }, + } From 652ac5d61f878ebca697a6a08cb7799f7351c2e5 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Tue, 5 May 2026 14:26:04 -0400 Subject: [PATCH 04/20] fix regex --- salt/orch/delete_hypervisor.sls | 2 +- salt/orch/vm_pillar_clean.sls | 2 +- salt/reactor/check_hypervisor.sls | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/orch/delete_hypervisor.sls b/salt/orch/delete_hypervisor.sls index 784977e28..45e17a74a 100644 --- a/salt/orch/delete_hypervisor.sls +++ b/salt/orch/delete_hypervisor.sls @@ -5,7 +5,7 @@ {% set hypervisor = pillar.get('minion_id', '') %} -{% if not hypervisor|regex_match('^[A-Za-z0-9._-]{1,253}$') %} +{% if not hypervisor|regex_match('^([A-Za-z0-9._-]{1,253})$') %} {% do salt.log.error('delete_hypervisor_orch: refusing unsafe minion_id=' ~ hypervisor) %} delete_hypervisor_invalid_minion_id: test.fail_without_changes: diff --git a/salt/orch/vm_pillar_clean.sls b/salt/orch/vm_pillar_clean.sls index 9b6bf1ee5..57612bbd4 100644 --- a/salt/orch/vm_pillar_clean.sls +++ b/salt/orch/vm_pillar_clean.sls @@ -14,7 +14,7 @@ {% do salt.log.debug('vm_pillar_clean_orch: Running') %} {% set vm_name = pillar.get('vm_name', '') %} -{% if not vm_name|regex_match('^[A-Za-z0-9._-]{1,253}$') %} +{% if not vm_name|regex_match('^([A-Za-z0-9._-]{1,253})$') %} {% do salt.log.error('vm_pillar_clean_orch: refusing unsafe vm_name=' ~ vm_name) %} vm_pillar_clean_invalid_name: test.fail_without_changes: diff --git a/salt/reactor/check_hypervisor.sls b/salt/reactor/check_hypervisor.sls index c0fa49ddc..da81e0d5f 100644 --- a/salt/reactor/check_hypervisor.sls +++ b/salt/reactor/check_hypervisor.sls @@ -4,7 +4,7 @@ # Elastic License 2.0. {% set hid = data['id'] %} -{% if hid|regex_match('^[A-Za-z0-9._-]{1,253}$') +{% if hid|regex_match('^([A-Za-z0-9._-]{1,253})$') and hid.endswith('_hypervisor') and data['result'] == True %} From dceed421aede0960f6ba889539496099a25b5db4 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 5 May 2026 13:41:00 -0500 Subject: [PATCH 05/20] update grok type conversion to convert processor --- salt/elasticsearch/files/ingest/common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/ingest/common b/salt/elasticsearch/files/ingest/common index b7048cf3b..409bf5af2 100644 --- a/salt/elasticsearch/files/ingest/common +++ b/salt/elasticsearch/files/ingest/common @@ -63,7 +63,7 @@ { "set": { "if": "ctx.event?.dataset != null && !ctx.event.dataset.contains('.')", "field": "event.dataset", "value": "{{event.module}}.{{event.dataset}}" } }, { "split": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "dataset_tag_temp" } }, { "append": { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp.1}}" } }, - { "grok": { "if": "ctx.http?.response?.status_code != null", "field": "http.response.status_code", "patterns": ["%{NUMBER:http.response.status_code:long} %{GREEDYDATA}"]} }, + { "convert": { "if": "ctx.http?.response?.status_code != null", "field": "http.response.status_code", "type":"long", "ignore_missing": true } }, { "set": { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } }, { "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "dataset_tag_temp", "event.dataset_temp" ], "ignore_missing": true, "ignore_failure": true } }, { "pipeline": { "name": "global@custom", "ignore_missing_pipeline": true, "description": "[Fleet] Global pipeline for all data streams" } } From f17da4e68bd1d8d165e02724885accddb8963d39 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 5 May 2026 15:13:24 -0400 Subject: [PATCH 06/20] Add management bond setup option --- setup/so-functions | 65 +++++++++++++++++++++++++++++++++-- setup/so-whiptail | 85 ++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 145 insertions(+), 5 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 3cd665076..4d60963c5 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -556,7 +556,7 @@ check_requirements() { local req_cores local req_storage local nic_list - readarray -t nic_list <<< "$(ip link| awk -F: '$0 !~ "lo|vir|veth|br|docker|wl|^[^0-9]"{print $2}' | grep -vwe "bond0" | sed 's/ //g' | sed -r 's/(.*)(\.[0-9]+)@\1/\1\2/g')" + readarray -t nic_list <<< "$(ip link| awk -F: '$0 !~ "lo|vir|veth|br|docker|wl|^[^0-9]"{print $2}' | grep -vwe "bond0\|bond1" | sed 's/ //g' | sed -r 's/(.*)(\.[0-9]+)@\1/\1\2/g')" local num_nics=${#nic_list[@]} if [[ $is_eval ]]; then @@ -745,6 +745,56 @@ configure_network_sensor() { return $err } +configure_management_bond() { + local bond_name="bond1" + local bond_mode=${MBOND_MODE:-active-backup} + + info "Setting up $bond_name management interface with mode $bond_mode" + + if [[ ${#MBNICS[@]} -eq 0 ]]; then + error "[ERROR] No management bond NICs were selected." + fail_setup + fi + + nmcli -t -f NAME con show | grep -Fxq "$bond_name" + local found_int=$? + + if [[ $found_int != 0 ]]; then + nmcli con add type bond ifname "$bond_name" con-name "$bond_name" mode "$bond_mode" -- \ + ipv6.method ignore \ + connection.autoconnect yes >> "$setup_log" 2>&1 + else + nmcli con mod "$bond_name" \ + bond.options "mode=$bond_mode" \ + ipv6.method ignore \ + connection.autoconnect yes >> "$setup_log" 2>&1 + fi + + local err=0 + for MBNIC in "${MBNICS[@]}"; do + local slave_name="$bond_name-slave-$MBNIC" + + nmcli -t -f NAME con show | grep -Fxq "$slave_name" + found_int=$? + + if [[ $found_int != 0 ]]; then + nmcli con add type ethernet ifname "$MBNIC" con-name "$slave_name" master "$bond_name" -- \ + connection.autoconnect yes >> "$setup_log" 2>&1 + else + nmcli con mod "$slave_name" \ + connection.master "$bond_name" \ + connection.slave-type bond \ + connection.autoconnect yes >> "$setup_log" 2>&1 + fi + + nmcli con up "$slave_name" >> "$setup_log" 2>&1 + local ret=$? + [[ $ret -eq 0 ]] || err=$ret + done + + return $err +} + configure_hyper_bridge() { info "Setting up hypervisor bridge" info "Checking $MNIC ipv4.method is auto or manual" @@ -990,7 +1040,7 @@ es_heapsize() { filter_unused_nics() { - if [[ $MNIC ]]; then local grep_string="$MNIC\|bond0"; else local grep_string="bond0"; fi + if [[ $MNIC ]]; then local grep_string="$MNIC\|bond0\|bond1"; else local grep_string="bond0\|bond1"; fi # If we call this function and NICs have already been assigned to the bond interface then add them to the grep search string if [[ $BNICS ]]; then @@ -999,6 +1049,11 @@ filter_unused_nics() { grep_string="$grep_string\|$BONDNIC" done fi + if [[ $MBNICS ]]; then + for BONDNIC in "${MBNICS[@]}"; do + grep_string="$grep_string\|$BONDNIC" + done + fi # Finally, set filtered_nics to any NICs we aren't using (and ignore interfaces that aren't of use) filtered_nics=$(ip link | awk -F: '$0 !~ "lo|vir|veth|br|docker|wl|^[^0-9]"{print $2}' | grep -vwe "$grep_string" | sed 's/ //g' | sed -r 's/(.*)(\.[0-9]+)@\1/\1\2/g') @@ -2084,8 +2139,12 @@ set_initial_firewall_access() { # Set up the management interface on the ISO set_management_interface() { title "Setting up the main interface" + if [[ $MNIC == "bond1" ]]; then + configure_management_bond || fail_setup + fi + if [ "$address_type" = 'DHCP' ]; then - logCmd "nmcli con mod $MNIC connection.autoconnect yes" + logCmd "nmcli con mod $MNIC connection.autoconnect yes ipv4.method auto" logCmd "nmcli con up $MNIC" logCmd "nmcli -p connection show $MNIC" else diff --git a/setup/so-whiptail b/setup/so-whiptail index 9a1d21150..6188d3d30 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -845,18 +845,99 @@ whiptail_management_nic() { [ -n "$TESTING" ] && return filter_unused_nics + local management_nic_options=( "${nic_list_management[@]}" ) + if [[ $is_iso || $is_desktop_iso ]]; then + management_nic_options+=( "BOND" "Configure a bonded management interface" ) + fi - MNIC=$(whiptail --title "$whiptail_title" --menu "Please select the NIC you would like to use for management.\n\nUse the arrow keys to move around and the Enter key to select." 20 75 12 "${nic_list_management[@]}" 3>&1 1>&2 2>&3 ) + MNIC=$(whiptail --title "$whiptail_title" --menu "Please select the NIC you would like to use for management.\n\nUse the arrow keys to move around and the Enter key to select." 20 75 12 "${management_nic_options[@]}" 3>&1 1>&2 2>&3 ) local exitstatus=$? whiptail_check_exitstatus $exitstatus while [ -z "$MNIC" ] do - MNIC=$(whiptail --title "$whiptail_title" --menu "Please select the NIC you would like to use for management.\n\nUse the arrow keys to move around and the Enter key to select." 22 75 12 "${nic_list_management[@]}" 3>&1 1>&2 2>&3 ) + MNIC=$(whiptail --title "$whiptail_title" --menu "Please select the NIC you would like to use for management.\n\nUse the arrow keys to move around and the Enter key to select." 22 75 12 "${management_nic_options[@]}" 3>&1 1>&2 2>&3 ) local exitstatus=$? whiptail_check_exitstatus $exitstatus done + if [[ $MNIC == "BOND" ]]; then + whiptail_management_bond + fi +} + +whiptail_management_bond() { + + [ -n "$TESTING" ] && return + + MBOND_MODE=$(whiptail --title "$whiptail_title" --menu \ + "Choose the bond mode for the management interface.\n\nThe management bond will be created as bond1." 20 75 7 \ + "active-backup" "One active NIC with failover (recommended)" \ + "balance-rr" "Round-robin transmit policy" \ + "balance-xor" "Transmit based on selected hash policy" \ + "broadcast" "Transmit everything on all slave interfaces" \ + "802.3ad" "Dynamic link aggregation (requires switch support)" \ + "balance-tlb" "Adaptive transmit load balancing" \ + "balance-alb" "Adaptive load balancing" 3>&1 1>&2 2>&3) + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + + while [ -z "$MBOND_MODE" ] + do + MBOND_MODE=$(whiptail --title "$whiptail_title" --menu \ + "Choose the bond mode for the management interface.\n\nThe management bond will be created as bond1." 20 75 7 \ + "active-backup" "One active NIC with failover (recommended)" \ + "balance-rr" "Round-robin transmit policy" \ + "balance-xor" "Transmit based on selected hash policy" \ + "broadcast" "Transmit everything on all slave interfaces" \ + "802.3ad" "Dynamic link aggregation (requires switch support)" \ + "balance-tlb" "Adaptive transmit load balancing" \ + "balance-alb" "Adaptive load balancing" 3>&1 1>&2 2>&3) + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + done + + whiptail_management_bond_nics + MNIC="bond1" + + export MBOND_MODE MNIC +} + +whiptail_management_bond_nics() { + + [ -n "$TESTING" ] && return + + MBNICS=() + filter_unused_nics + + MBNICS=$(whiptail --title "$whiptail_title" --checklist "Please add NICs to the Management Interface:" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3) + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + + while [ -z "$MBNICS" ] + do + MBNICS=$(whiptail --title "$whiptail_title" --checklist "Please add NICs to the Management Interface:" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3) + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + done + + MBNICS=$(echo "$MBNICS" | tr -d '"') + + IFS=' ' read -ra MBNICS <<< "$MBNICS" + + for bond_nic in "${MBNICS[@]}"; do + for dev_status in "${nmcli_dev_status_list[@]}"; do + if [[ $dev_status == "${bond_nic}:unmanaged" ]]; then + whiptail \ + --title "$whiptail_title" \ + --msgbox "$bond_nic is unmanaged by Network Manager. Please remove it from other network management tools then re-run setup." \ + 8 75 + exit + fi + done + done + + export MBNICS } whiptail_net_method() { From 3b714db0bfcbfdde05c95e5aa2778338a422ae1b Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 5 May 2026 15:22:40 -0400 Subject: [PATCH 07/20] Show management bond option consistently --- setup/so-functions | 12 +++++++++++- setup/so-setup | 10 ++++++++++ setup/so-whiptail | 4 +--- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 4d60963c5..571270227 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1443,7 +1443,7 @@ network_init() { title "Initializing Network" disable_ipv6 set_hostname - if [[ ( $is_iso || $is_desktop_iso ) ]]; then + if [[ $is_iso || $is_desktop_iso || $MNIC == "bond1" ]]; then set_management_interface fi } @@ -1465,6 +1465,16 @@ network_init_whiptail() { whiptail_network_notice whiptail_dhcp_warn whiptail_management_nic + if [[ $MNIC == "bond1" ]]; then + whiptail_dhcp_or_static + + if [ "$address_type" != 'DHCP' ]; then + collect_int_ip_mask + collect_gateway + collect_dns + collect_dns_domain + fi + fi ;; esac } diff --git a/setup/so-setup b/setup/so-setup index 7875b9c99..27aeef1f6 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -292,6 +292,16 @@ if ! [[ -f $install_opt_file ]]; then # Warn about the dangers of DHCP whiptail_dhcp_warn whiptail_management_nic + if [[ $MNIC == "bond1" ]]; then + whiptail_dhcp_or_static + + if [ "$address_type" != 'DHCP' ]; then + collect_int_ip_mask + collect_gateway + collect_dns + collect_dns_domain + fi + fi fi # Initializing the network based on the previous information network_init diff --git a/setup/so-whiptail b/setup/so-whiptail index 6188d3d30..a0c9d797b 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -846,9 +846,7 @@ whiptail_management_nic() { filter_unused_nics local management_nic_options=( "${nic_list_management[@]}" ) - if [[ $is_iso || $is_desktop_iso ]]; then - management_nic_options+=( "BOND" "Configure a bonded management interface" ) - fi + management_nic_options+=( "BOND" "Configure a bonded management interface" ) MNIC=$(whiptail --title "$whiptail_title" --menu "Please select the NIC you would like to use for management.\n\nUse the arrow keys to move around and the Enter key to select." 20 75 12 "${management_nic_options[@]}" 3>&1 1>&2 2>&3 ) local exitstatus=$? From ecb92d43fcca7ebd614d4b48de3d076d2d8fd029 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 5 May 2026 15:30:09 -0400 Subject: [PATCH 08/20] Limit management bond setup to ISO installs --- setup/so-functions | 14 ++------------ setup/so-setup | 10 ---------- setup/so-whiptail | 4 +++- 3 files changed, 5 insertions(+), 23 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 571270227..4dbbddecc 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1040,7 +1040,7 @@ es_heapsize() { filter_unused_nics() { - if [[ $MNIC ]]; then local grep_string="$MNIC\|bond0\|bond1"; else local grep_string="bond0\|bond1"; fi + if [[ $MNIC ]]; then local grep_string="$MNIC\|bond0"; else local grep_string="bond0"; fi # If we call this function and NICs have already been assigned to the bond interface then add them to the grep search string if [[ $BNICS ]]; then @@ -1443,7 +1443,7 @@ network_init() { title "Initializing Network" disable_ipv6 set_hostname - if [[ $is_iso || $is_desktop_iso || $MNIC == "bond1" ]]; then + if [[ $is_iso || $is_desktop_iso ]]; then set_management_interface fi } @@ -1465,16 +1465,6 @@ network_init_whiptail() { whiptail_network_notice whiptail_dhcp_warn whiptail_management_nic - if [[ $MNIC == "bond1" ]]; then - whiptail_dhcp_or_static - - if [ "$address_type" != 'DHCP' ]; then - collect_int_ip_mask - collect_gateway - collect_dns - collect_dns_domain - fi - fi ;; esac } diff --git a/setup/so-setup b/setup/so-setup index 27aeef1f6..7875b9c99 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -292,16 +292,6 @@ if ! [[ -f $install_opt_file ]]; then # Warn about the dangers of DHCP whiptail_dhcp_warn whiptail_management_nic - if [[ $MNIC == "bond1" ]]; then - whiptail_dhcp_or_static - - if [ "$address_type" != 'DHCP' ]; then - collect_int_ip_mask - collect_gateway - collect_dns - collect_dns_domain - fi - fi fi # Initializing the network based on the previous information network_init diff --git a/setup/so-whiptail b/setup/so-whiptail index a0c9d797b..6188d3d30 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -846,7 +846,9 @@ whiptail_management_nic() { filter_unused_nics local management_nic_options=( "${nic_list_management[@]}" ) - management_nic_options+=( "BOND" "Configure a bonded management interface" ) + if [[ $is_iso || $is_desktop_iso ]]; then + management_nic_options+=( "BOND" "Configure a bonded management interface" ) + fi MNIC=$(whiptail --title "$whiptail_title" --menu "Please select the NIC you would like to use for management.\n\nUse the arrow keys to move around and the Enter key to select." 20 75 12 "${management_nic_options[@]}" 3>&1 1>&2 2>&3 ) local exitstatus=$? From 3e493222200bd45a07f7202cff66a20efb5747c0 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 5 May 2026 15:35:12 -0400 Subject: [PATCH 09/20] Allow preconfigured management bond in requirements --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 4dbbddecc..12dd5160a 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -556,7 +556,7 @@ check_requirements() { local req_cores local req_storage local nic_list - readarray -t nic_list <<< "$(ip link| awk -F: '$0 !~ "lo|vir|veth|br|docker|wl|^[^0-9]"{print $2}' | grep -vwe "bond0\|bond1" | sed 's/ //g' | sed -r 's/(.*)(\.[0-9]+)@\1/\1\2/g')" + readarray -t nic_list <<< "$(ip link| awk -F: '$0 !~ "lo|vir|veth|br|docker|wl|^[^0-9]"{print $2}' | grep -vwe "bond0" | sed 's/ //g' | sed -r 's/(.*)(\.[0-9]+)@\1/\1\2/g')" local num_nics=${#nic_list[@]} if [[ $is_eval ]]; then From 499f7102bd1302fa2efe4030603f9a5211f7b1bf Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 7 May 2026 11:27:49 -0400 Subject: [PATCH 10/20] cleanup status code --- salt/elasticsearch/files/ingest/common | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/ingest/common b/salt/elasticsearch/files/ingest/common index 409bf5af2..5923977c6 100644 --- a/salt/elasticsearch/files/ingest/common +++ b/salt/elasticsearch/files/ingest/common @@ -63,7 +63,8 @@ { "set": { "if": "ctx.event?.dataset != null && !ctx.event.dataset.contains('.')", "field": "event.dataset", "value": "{{event.module}}.{{event.dataset}}" } }, { "split": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "dataset_tag_temp" } }, { "append": { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp.1}}" } }, - { "convert": { "if": "ctx.http?.response?.status_code != null", "field": "http.response.status_code", "type":"long", "ignore_missing": true } }, + { "grok": { "if": "ctx.http?.response?.status_code instanceof String", "field": "http.response.status_code", "patterns": ["%{NUMBER:http.response.status_code:long}(?:\\s+%{GREEDYDATA})?"], "ignore_failure": true } }, + { "convert": { "if": "ctx.http?.response?.status_code != null && !(ctx.http.response.status_code instanceof Number)", "field": "http.response.status_code", "type": "long", "ignore_failure": true } }, { "set": { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } }, { "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "dataset_tag_temp", "event.dataset_temp" ], "ignore_missing": true, "ignore_failure": true } }, { "pipeline": { "name": "global@custom", "ignore_missing_pipeline": true, "description": "[Fleet] Global pipeline for all data streams" } } From e1d830da762ee2058f8651db72b6ef95ca32bdc0 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 8 May 2026 09:11:24 -0400 Subject: [PATCH 11/20] proc_creation per OS type --- salt/soc/files/soc/sigma_so_pipeline.yaml | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/salt/soc/files/soc/sigma_so_pipeline.yaml b/salt/soc/files/soc/sigma_so_pipeline.yaml index 4462bde42..44f1c38e1 100644 --- a/salt/soc/files/soc/sigma_so_pipeline.yaml +++ b/salt/soc/files/soc/sigma_so_pipeline.yaml @@ -126,15 +126,36 @@ transformations: fields: - event.code # Maps process_creation rules to endpoint process creation logs - # This is an OS-agnostic mapping, to account for logs that don't specify source OS - id: endpoint_process_create_windows_add-fields type: add_condition conditions: event.category: 'process' event.type: 'start' + host.os.type: 'windows' rule_conditions: - type: logsource category: process_creation + product: windows + - id: endpoint_process_create_macos_add-fields + type: add_condition + conditions: + event.category: 'process' + event.type: 'start' + host.os.type: 'macos' + rule_conditions: + - type: logsource + category: process_creation + product: macos + - id: endpoint_process_create_linux_add-fields + type: add_condition + conditions: + event.category: 'process' + event.type: 'start' + host.os.type: 'linux' + rule_conditions: + - type: logsource + category: process_creation + product: linux # Maps file_event rules to endpoint file creation logs # This is an OS-agnostic mapping, to account for logs that don't specify source OS - id: endpoint_file_create_add-fields From 4a2177c8278f20e871c1538f481d28b572f17672 Mon Sep 17 00:00:00 2001 From: Jorge Reyes <94730068+reyesj2@users.noreply.github.com> Date: Mon, 11 May 2026 16:15:56 -0500 Subject: [PATCH 12/20] update redis index template missing redis integration component templates --- salt/elasticsearch/defaults.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 6fb795bce..52964b9cf 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -3958,10 +3958,13 @@ elasticsearch: - vulnerability-mappings - common-settings - common-dynamic-mappings + - logs-redis.log@package + - logs-redis.log@custom data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: [] + ignore_missing_component_templates: + - logs-redis.log@custom index_patterns: - logs-redis.log* priority: 501 From 492ae80da7cf52c19751cb607824a6dc0ff90c86 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 11 May 2026 16:51:38 -0500 Subject: [PATCH 13/20] add ingest latency metrics --- salt/elasticsearch/files/ingest/global@custom | 77 ++++++++++++++++++- salt/logstash/defaults.yaml | 5 +- .../so/0012_input_elastic_agent.conf.jinja | 14 +++- .../so/0013_input_lumberjack_fleet.conf | 23 ------ .../so/0013_input_lumberjack_fleet.conf.jinja | 26 +++++++ .../config/so/0800_input_kafka.conf.jinja | 6 ++ .../config/so/0900_input_redis.conf.jinja | 9 ++- .../so/9805_output_elastic_agent.conf.jinja | 8 ++ .../9806_output_lumberjack_fleet.conf.jinja | 19 +++-- .../config/so/9999_output_redis.conf.jinja | 9 ++- salt/logstash/soc_logstash.yaml | 5 ++ 11 files changed, 162 insertions(+), 39 deletions(-) delete mode 100644 salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf create mode 100644 salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf.jinja diff --git a/salt/elasticsearch/files/ingest/global@custom b/salt/elasticsearch/files/ingest/global@custom index bafb783a4..979c5c1b8 100644 --- a/salt/elasticsearch/files/ingest/global@custom +++ b/salt/elasticsearch/files/ingest/global@custom @@ -177,12 +177,84 @@ "description": "Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip" } }, + { + "script": { + "description": "Snapshot event.ingested into _tmp.event_ingested_pre_fleet before .fleet_final_pipeline-1 overwrites it with ES ingest time", + "lang": "painless", + "if": "ctx.event?.ingested != null && ctx.event?.created == null", + "ignore_failure": true, + "source": "ctx.putIfAbsent('_tmp', [:]); ctx._tmp.event_ingested_pre_fleet = ctx.event.ingested;" + } + }, { "pipeline": { "name": ".fleet_final_pipeline-1", "ignore_missing_pipeline": true } }, + { + "script": { + "description": "Calculate time from Elastic Agent to Logstash.", + "lang": "painless", + "if": "ctx._tmp?.logstash_from_agent != null", + "ignore_failure": true, + "source": "ZonedDateTime start = ctx._tmp.event_ingested_pre_fleet != null ? ZonedDateTime.parse(ctx._tmp.event_ingested_pre_fleet) : ZonedDateTime.parse(ctx['@timestamp']); ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_elasticagent_to_logstash = ChronoUnit.SECONDS.between(start, ZonedDateTime.parse(ctx._tmp.logstash_from_agent));" + } + }, + { + "script": { + "description": "Calculate time from Logstash to Redis", + "lang": "painless", + "if": "ctx._tmp?.logstash_from_agent != null && ctx._tmp?.logstash_to_redis != null", + "ignore_failure": true, + "source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_logstash_to_redis = ChronoUnit.SECONDS.between(ZonedDateTime.parse(ctx._tmp.logstash_from_agent), ZonedDateTime.parse(ctx._tmp.logstash_to_redis));" + } + }, + { + "script": { + "description": "Calculate time message spends in redis queue (logstash delay in pulling event).", + "lang": "painless", + "if": "ctx._tmp?.logstash_to_redis != null && ctx._tmp?.logstash_from_redis != null", + "ignore_failure": true, + "source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_redis_to_logstash = ChronoUnit.SECONDS.between(ZonedDateTime.parse(ctx._tmp.logstash_to_redis), ZonedDateTime.parse(ctx._tmp.logstash_from_redis));" + } + }, + { + "script": { + "description": "Calculate time from Logstash to Elasticsearch (after read from Redis).", + "lang": "painless", + "if": "ctx._tmp?.logstash_from_redis != null", + "ignore_failure": true, + "source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_logstash_to_elasticsearch = ChronoUnit.SECONDS.between(ZonedDateTime.parse(ctx._tmp.logstash_from_redis), metadata().now);" + } + }, + { + "script": { + "description": "Calculate time from Elastic Agent to Kafka.", + "lang": "painless", + "if": "ctx._tmp?.logstash_from_kafka != null && ctx._tmp?.logstash_from_agent == null", + "ignore_failure": true, + "source": "ZonedDateTime start = ctx._tmp.event_ingested_pre_fleet != null ? ZonedDateTime.parse(ctx._tmp.event_ingested_pre_fleet) : ZonedDateTime.parse(ctx['@timestamp']); ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_elasticagent_to_kafka = ChronoUnit.SECONDS.between(start, ZonedDateTime.parse(ctx._tmp.logstash_from_kafka));" + } + }, + { + "script": { + "description": "Calculate time message spends in Kafka queue (logstash delay in pulling event).", + "lang": "painless", + "if": "ctx._tmp?.logstash_from_kafka != null && ctx.metadata?.kafka?.timestamp != null && ctx._tmp?.logstash_from_agent == null", + "ignore_failure": true, + "source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_kafka_queue = ChronoUnit.SECONDS.between(ZonedDateTime.ofInstant(Instant.ofEpochMilli(Long.parseLong(ctx.metadata.kafka.timestamp.toString())), ZoneId.of('UTC')), ZonedDateTime.parse(ctx._tmp.logstash_from_kafka));" + } + }, + { + "script": { + "description": "Calculate time from Logstash to Elasticsearch (after read from Kafka).", + "lang": "painless", + "if": "ctx._tmp?.logstash_from_kafka != null && ctx._tmp?.logstash_from_agent == null", + "ignore_failure": true, + "source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_kafka_to_elasticsearch = ChronoUnit.SECONDS.between(ZonedDateTime.parse(ctx._tmp.logstash_from_kafka), metadata().now);" + } + }, { "remove": { "field": "event.agent_id_status", @@ -202,11 +274,12 @@ "event.dataset_temp", "dataset_tag_temp", "module_temp", - "datastream_dataset_temp" + "datastream_dataset_temp", + "_tmp" ], "ignore_missing": true, "ignore_failure": true } } ] -} \ No newline at end of file +} diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml index 520182555..db5e4ee58 100644 --- a/salt/logstash/defaults.yaml +++ b/salt/logstash/defaults.yaml @@ -26,12 +26,12 @@ logstash: manager: - so/0011_input_endgame.conf - so/0012_input_elastic_agent.conf.jinja - - so/0013_input_lumberjack_fleet.conf + - so/0013_input_lumberjack_fleet.conf.jinja - so/9999_output_redis.conf.jinja receiver: - so/0011_input_endgame.conf - so/0012_input_elastic_agent.conf.jinja - - so/0013_input_lumberjack_fleet.conf + - so/0013_input_lumberjack_fleet.conf.jinja - so/9999_output_redis.conf.jinja search: - so/0900_input_redis.conf.jinja @@ -69,4 +69,5 @@ logstash: pipeline_x_batch_x_size: 125 pipeline_x_ecs_compatibility: disabled dmz_nodes: [] + latency_metrics: False diff --git a/salt/logstash/pipelines/config/so/0012_input_elastic_agent.conf.jinja b/salt/logstash/pipelines/config/so/0012_input_elastic_agent.conf.jinja index a4d699aff..32dcac224 100644 --- a/salt/logstash/pipelines/config/so/0012_input_elastic_agent.conf.jinja +++ b/salt/logstash/pipelines/config/so/0012_input_elastic_agent.conf.jinja @@ -1,3 +1,4 @@ +{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %} input { elastic_agent { port => 5055 @@ -11,10 +12,15 @@ input { } } filter { -if ![metadata] { - mutate { - rename => {"@metadata" => "metadata"} + {% if LOGSTASH_MERGED.get('latency_metrics', False) %} + ruby { + code => "event.set('[_tmp][logstash_from_agent]', Time.now().utc.iso8601(3));" + } + {% endif %} + if ![metadata] { + mutate { + rename => {"@metadata" => "metadata"} + } } } -} diff --git a/salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf b/salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf deleted file mode 100644 index b31ffee8d..000000000 --- a/salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf +++ /dev/null @@ -1,23 +0,0 @@ -input { - elastic_agent { - port => 5056 - tags => [ "elastic-agent", "fleet-lumberjack-input" ] - ssl_enabled => true - ssl_certificate => "/usr/share/logstash/elasticfleet-lumberjack.crt" - ssl_key => "/usr/share/logstash/elasticfleet-lumberjack.key" - ecs_compatibility => v8 - id => "fleet-lumberjack-in" - codec => "json" - } -} - - -filter { -if ![metadata] { - mutate { - rename => {"@metadata" => "metadata"} - } -} -} - - diff --git a/salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf.jinja b/salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf.jinja new file mode 100644 index 000000000..a04df5fd1 --- /dev/null +++ b/salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf.jinja @@ -0,0 +1,26 @@ +{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %} +input { + elastic_agent { + port => 5056 + tags => [ "elastic-agent", "fleet-lumberjack-input" ] + ssl_enabled => true + ssl_certificate => "/usr/share/logstash/elasticfleet-lumberjack.crt" + ssl_key => "/usr/share/logstash/elasticfleet-lumberjack.key" + ecs_compatibility => v8 + id => "fleet-lumberjack-in" + codec => "json" + } +} + +filter { + {% if LOGSTASH_MERGED.get('latency_metrics', False) %} + ruby { + code => "event.set('[_tmp][logstash_from_fleet]', Time.now().utc.iso8601(3));" + } + {% endif %} + if ![metadata] { + mutate { + rename => {"@metadata" => "metadata"} + } + } +} diff --git a/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja b/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja index 7478375b0..769f71ea9 100644 --- a/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja +++ b/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja @@ -1,3 +1,4 @@ +{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %} {%- set kafka_password = salt['pillar.get']('kafka:config:password') %} {%- set kafka_trustpass = salt['pillar.get']('kafka:config:trustpass') %} {%- set kafka_brokers = salt['pillar.get']('kafka:nodes', {}) %} @@ -30,6 +31,11 @@ input { } } filter { + {% if LOGSTASH_MERGED.get('latency_metrics', False) %} + ruby { + code => "event.set('[_tmp][logstash_from_kafka]', Time.now().utc.iso8601(3));" + } + {% endif %} if ![metadata] { mutate { rename => { "@metadata" => "metadata" } diff --git a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja index ad9fae5f2..4bf388f4f 100644 --- a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja @@ -1,4 +1,4 @@ -{%- from 'logstash/map.jinja' import LOGSTASH_REDIS_NODES with context %} +{%- from 'logstash/map.jinja' import LOGSTASH_REDIS_NODES, LOGSTASH_MERGED %} {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %} {%- for index in range(LOGSTASH_REDIS_NODES|length) %} @@ -18,3 +18,10 @@ input { } {% endfor %} {% endfor -%} +filter { + {% if LOGSTASH_MERGED.get('latency_metrics', False) %} + ruby { + code => "event.set('[_tmp][logstash_from_redis]', Time.now().utc.iso8601(3));" + } + {% endif %} +} diff --git a/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja b/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja index 4fe138dd8..f973070a5 100644 --- a/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja +++ b/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja @@ -1,3 +1,11 @@ +{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %} +{% if LOGSTASH_MERGED.get('latency_metrics', False) %} +filter { + ruby { + code => "event.set('[_tmp][logstash_to_elasticsearch]', Time.now().utc.iso8601(3));" + } +} +{% endif %} output { if "elastic-agent" in [tags] and "so-ip-mappings" in [tags] { elasticsearch { diff --git a/salt/logstash/pipelines/config/so/9806_output_lumberjack_fleet.conf.jinja b/salt/logstash/pipelines/config/so/9806_output_lumberjack_fleet.conf.jinja index 50328e833..602c5fece 100644 --- a/salt/logstash/pipelines/config/so/9806_output_lumberjack_fleet.conf.jinja +++ b/salt/logstash/pipelines/config/so/9806_output_lumberjack_fleet.conf.jinja @@ -13,13 +13,20 @@ filter { add_tag => "fleet-lumberjack-{{ GLOBALS.hostname }}" } } - -output { - lumberjack { - codec => json +{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %} +{% if LOGSTASH_MERGED.get('latency_metrics', False) %} +filter { + ruby { + code => "event.set('[_tmp][fleet_to_logstash]', Time.now().utc.iso8601(3));" + } +} +{% endif %} +output { + lumberjack { + codec => json hosts => {{ FAILOVER_LOGSTASH_NODES }} ssl_certificate => "/usr/share/filebeat/ca.crt" - port => 5056 + port => 5056 id => "fleet-lumberjack-{{ GLOBALS.hostname }}" - } + } } \ No newline at end of file diff --git a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja index 0d3b3324b..af13915f7 100644 --- a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja @@ -1,10 +1,17 @@ +{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %} {%- if grains.role in ['so-heavynode', 'so-receiver'] %} {%- set HOST = GLOBALS.hostname %} {%- else %} {%- set HOST = GLOBALS.manager %} {%- endif %} {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %} - +{% if LOGSTASH_MERGED.get('latency_metrics', False) %} +filter { + ruby { + code => "event.set('[_tmp][logstash_to_redis]', Time.now().utc.iso8601(3));" + } +} +{% endif %} output { redis { host => '{{ HOST }}' diff --git a/salt/logstash/soc_logstash.yaml b/salt/logstash/soc_logstash.yaml index 5a5816a9e..40794afe4 100644 --- a/salt/logstash/soc_logstash.yaml +++ b/salt/logstash/soc_logstash.yaml @@ -86,3 +86,8 @@ logstash: multiline: True advanced: True forcedType: "[]string" + latency_metrics: + description: Enable latency metrics within events processed by logstash. Useful for pinpointing log ingest delay. + forcedType: bool + global: False + advanced: True From 306b0af4d04c5cd4873037f474779eb3c89f4bfc Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 12 May 2026 09:55:06 -0400 Subject: [PATCH 14/20] Initial commit --- .../tools/sbin/so-detections-overrides-import | 391 +++++++++++++++++ .../so-detections-overrides-import_test.py | 406 ++++++++++++++++++ 2 files changed, 797 insertions(+) create mode 100755 salt/manager/tools/sbin/so-detections-overrides-import create mode 100644 salt/manager/tools/sbin/so-detections-overrides-import_test.py diff --git a/salt/manager/tools/sbin/so-detections-overrides-import b/salt/manager/tools/sbin/so-detections-overrides-import new file mode 100755 index 000000000..1f32bf04a --- /dev/null +++ b/salt/manager/tools/sbin/so-detections-overrides-import @@ -0,0 +1,391 @@ +#!/usr/bin/env python3 + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# Imports detection overrides (e.g. from so-detections-backup) into the so-detection +# index. Reads . files (NDJSON, one override per line) from a source +# directory, looks up the matching detection by publicId+engine, validates each +# override against the same rules SOC enforces, dedupes against existing overrides +# (operational fields only), and appends new ones. + +import argparse +import ipaddress +import json +import os +import re +import sys +from datetime import datetime + +import requests +from requests.auth import HTTPBasicAuth +import urllib3 + +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +DEFAULT_INDEX = "so-detection" +AUTH_FILE = "/opt/so/conf/elasticsearch/curl.config" +ES_URL = "https://localhost:9200" + +# Engines we know how to handle and the file extension the backup script writes. +ENGINES = { + "suricata": "txt", + "sigma": "yaml", +} + +# Standard Suricata variables that ship with Security Onion. Anything else +# referenced in an override is "custom" and the user needs to make sure it +# exists in SOC Config before the override will function. +BUILTIN_SURICATA_VARS = { + "$HOME_NET", "$EXTERNAL_NET", + "$HTTP_SERVERS", "$DNS_SERVERS", "$SQL_SERVERS", "$SMTP_SERVERS", + "$TELNET_SERVERS", "$AIM_SERVERS", "$DC_SERVERS", "$MODBUS_SERVER", + "$MODBUS_CLIENT", "$ENIP_CLIENT", "$ENIP_SERVER", + "$HTTP_PORTS", "$SHELLCODE_PORTS", "$ORACLE_PORTS", "$SSH_PORTS", + "$FTP_PORTS", "$FILE_DATA_PORTS", +} + +VAR_PATTERN = re.compile(r"\$[A-Z_][A-Z0-9_]*") + +# Canonical valid values, per securityonion-soc/model/detection.go. +SURICATA_OVERRIDE_TYPES = {"suppress", "threshold", "modify"} +SUPPRESS_TRACKS = {"by_src", "by_dst", "by_either"} +THRESHOLD_TRACKS = {"by_src", "by_dst", "by_both"} +THRESHOLD_TYPES = {"limit", "threshold", "both"} + +STALE_WARNING = """\ +WARNING: so-detections-backup does not remove backup files when overrides are +deleted via the Security Onion web UI. As a result, files in the source +directory may represent overrides that were intentionally deleted and should +NOT be re-imported. + +Before continuing, verify that the source directory reflects the overrides you +actually want imported. Remove any files corresponding to overrides you previously deleted. +""" + + +def make_session(auth_file): + with open(auth_file, "r") as f: + for line in f: + if line.startswith("user ="): + creds = line.split("=", 1)[1].strip().replace('"', "") + user, _, password = creds.partition(":") + session = requests.Session() + session.auth = HTTPBasicAuth(user, password) + session.headers.update({"Content-Type": "application/json"}) + session.verify = False + return session + raise RuntimeError(f"Could not find 'user =' line in {auth_file}") + + +def find_detection(session, index, public_id, engine): + query = { + "query": {"bool": {"must": [ + {"term": {"so_detection.publicId": public_id}}, + {"term": {"so_detection.engine": engine}}, + ]}}, + "size": 2, + } + r = session.get(f"{ES_URL}/{index}/_search", json=query) + r.raise_for_status() + hits = r.json().get("hits", {}).get("hits", []) + if not hits: + return None, None, None + if len(hits) > 1: + # Shouldn't happen — publicId is unique per engine — but flag it. + print(f" WARN: {len(hits)} detections matched publicId={public_id} engine={engine}; using first") + hit = hits[0] + existing = hit["_source"].get("so_detection", {}).get("overrides") or [] + return hit["_id"], hit["_index"], existing + + +def update_overrides(session, doc_index, doc_id, overrides): + body = {"doc": {"so_detection": {"overrides": overrides}}} + r = session.post(f"{ES_URL}/{doc_index}/_update/{doc_id}", json=body) + r.raise_for_status() + return r.json() + + +def dedupe_key(override): + """Operational fields only, per Override.Equal() in detection.go. + Excludes timestamps and isEnabled so re-imports don't appear unique.""" + t = override.get("type") + if t == "suppress": + return (t, override.get("track"), override.get("ip")) + if t == "threshold": + return (t, override.get("thresholdType"), override.get("track"), + override.get("count"), override.get("seconds")) + if t == "modify": + return (t, override.get("regex"), override.get("value")) + return (t,) + + +def _validate_suricata_ip(ip): + if not ip: + return "ip cannot be empty" + if ip.startswith("$"): + return None + if ip.startswith("[") and ip.endswith("]"): + for part in ip[1:-1].split(","): + err = _validate_single_ip(part.strip()) + if err: + return f"invalid IP in list: {err}" + return None + return _validate_single_ip(ip) + + +def _validate_single_ip(ip): + try: + if "/" in ip: + ipaddress.ip_network(ip, strict=False) + else: + ipaddress.ip_address(ip) + except ValueError: + return f"invalid IP/CIDR {ip!r}" + return None + + +def validate_override(override, engine): + """Mirror Override.Validate() from securityonion-soc/model/detection.go. + Returns None on success, an error string otherwise.""" + if engine != "suricata": + return None # sigma not yet supported; gated earlier in main() + + t = override.get("type") + if not t: + return "override type is required" + if t not in SURICATA_OVERRIDE_TYPES: + return f"invalid type {t!r}: must be one of {sorted(SURICATA_OVERRIDE_TYPES)}" + + has = {k: override.get(k) is not None for k in + ("regex", "value", "thresholdType", "track", "ip", "count", "seconds", "customFilter")} + + if t == "suppress": + if not has["ip"] or not has["track"]: + return "suppress requires 'ip' and 'track'" + if any(has[k] for k in ("regex", "value", "thresholdType", "count", "seconds", "customFilter")): + return "suppress has unnecessary fields" + if override["track"] not in SUPPRESS_TRACKS: + return f"invalid track {override['track']!r}: must be one of {sorted(SUPPRESS_TRACKS)}" + return _validate_suricata_ip(override["ip"]) + + if t == "threshold": + if not all(has[k] for k in ("thresholdType", "track", "count", "seconds")): + return "threshold requires 'thresholdType', 'track', 'count', 'seconds'" + if any(has[k] for k in ("regex", "value", "customFilter")): + return "threshold has unnecessary fields" + if override["thresholdType"] not in THRESHOLD_TYPES: + return f"invalid thresholdType {override['thresholdType']!r}: must be one of {sorted(THRESHOLD_TYPES)}" + if override["track"] not in THRESHOLD_TRACKS: + return f"invalid track {override['track']!r}: must be one of {sorted(THRESHOLD_TRACKS)}" + if not isinstance(override["count"], int) or override["count"] <= 0: + return f"count must be a positive integer, got {override['count']!r}" + if not isinstance(override["seconds"], int) or override["seconds"] <= 0: + return f"seconds must be a positive integer, got {override['seconds']!r}" + return None + + if t == "modify": + if not has["regex"] or not has["value"]: + return "modify requires 'regex' and 'value'" + if any(has[k] for k in ("thresholdType", "track", "count", "seconds", "customFilter")): + return "modify has unnecessary fields" + try: + re.compile(override["regex"]) + except re.error as e: + return f"invalid regex: {e}" + return None + + +def parse_overrides_file(path): + """Parse a file written by so-detections-backup.py: NDJSON, one override + per line. Returns a list of (override_dict, line_number).""" + overrides = [] + with open(path, "r") as f: + for i, line in enumerate(f, start=1): + line = line.strip() + if not line: + continue + overrides.append((json.loads(line), i)) + return overrides + + +def describe(override): + """Human-readable summary of the operational fields for a given override type.""" + t = override.get("type") + if t == "suppress": + return f"type=suppress track={override.get('track')} ip={override.get('ip')}" + if t == "threshold": + return (f"type=threshold track={override.get('track')} " + f"thresholdType={override.get('thresholdType')} " + f"count={override.get('count')} seconds={override.get('seconds')}") + if t == "modify": + return f"type=modify regex={override.get('regex')!r}" + return f"type={t}" + + +def collect_custom_vars(override): + found = set() + for value in override.values(): + if isinstance(value, str): + for match in VAR_PATTERN.findall(value): + if match not in BUILTIN_SURICATA_VARS: + found.add(match) + return found + + +def parse_args(): + p = argparse.ArgumentParser( + description="Import detection overrides into the so-detection index.", + ) + p.add_argument("--source", "-s", required=True, + help="Source directory containing . override files.") + p.add_argument("--engine", "-e", default="suricata", choices=list(ENGINES.keys()), + help="Detection engine (default: suricata). Sigma not yet supported.") + p.add_argument("--dry-run", "-n", action="store_true", + help="Print what would happen without writing to Elasticsearch.") + p.add_argument("--no-import-note", action="store_true", + help="Do not prepend '[Imported YYYY-MM-DD] ' to the override note.") + p.add_argument("--index", "-i", default=DEFAULT_INDEX, + help=f"Elasticsearch index to update (default: {DEFAULT_INDEX}).") + return p.parse_args() + + +def confirm_proceed(args): + """Show the stale-backup warning. Dry-run prints it and continues. Real + runs require the user typing 'yes' at the prompt.""" + print(STALE_WARNING) + if args.dry_run: + print("(dry-run: no acknowledgement required)\n") + return True + answer = input("Type 'yes' to acknowledge and continue: ").strip().lower() + print() + return answer == "yes" + + +def main(): + args = parse_args() + + if args.engine == "sigma": + print("ERROR: sigma overrides are not yet supported.", file=sys.stderr) + sys.exit(2) + + if not os.path.isdir(args.source): + print(f"ERROR: source directory not found: {args.source}", file=sys.stderr) + sys.exit(1) + + extension = ENGINES[args.engine] + files = sorted(f for f in os.listdir(args.source) if f.endswith(f".{extension}")) + if not files: + print(f"No *.{extension} files found in {args.source}") + sys.exit(0) + + if not confirm_proceed(args): + print("Aborted.") + sys.exit(1) + + session = make_session(AUTH_FILE) + today = datetime.now().strftime("%Y-%m-%d") + note_prefix = "" if args.no_import_note else f"[Imported {today}] " + + counts = {"added": 0, "skipped_dedupe": 0, "skipped_not_found": 0, "invalid": 0, "error": 0} + custom_vars = set() + + mode = "DRY-RUN" if args.dry_run else "IMPORT" + print(f"[{mode}] engine={args.engine} source={args.source} index={args.index}\n") + + for filename in files: + public_id = os.path.splitext(filename)[0] + path = os.path.join(args.source, filename) + print(f"{public_id}:") + + try: + new_overrides = parse_overrides_file(path) + except (json.JSONDecodeError, OSError) as e: + print(f" ERROR: could not parse {filename}: {e}") + counts["error"] += 1 + continue + + if not new_overrides: + print(" SKIP: empty file") + continue + + try: + doc_id, doc_index, existing = find_detection(session, args.index, public_id, args.engine) + except requests.HTTPError as e: + print(f" ERROR: search failed: {e}") + counts["error"] += 1 + continue + + if doc_id is None: + print(f" WARN: no detection found for publicId={public_id} engine={args.engine}; skipping") + counts["skipped_not_found"] += len(new_overrides) + continue + + existing_keys = {dedupe_key(o) for o in existing} + merged = list(existing) + added_this_file = 0 + + for override, line_no in new_overrides: + err = validate_override(override, args.engine) + if err: + print(f" INVALID (line {line_no}): {err}") + counts["invalid"] += 1 + continue + + custom_vars.update(collect_custom_vars(override)) + key = dedupe_key(override) + if key in existing_keys: + print(f" SKIP (line {line_no}): duplicate of existing override [{describe(override)}]") + counts["skipped_dedupe"] += 1 + continue + + if note_prefix: + override = dict(override) + override["note"] = note_prefix + (override.get("note") or "") + + merged.append(override) + existing_keys.add(key) + added_this_file += 1 + print(f" ADD (line {line_no}): {describe(override)}") + + if added_this_file == 0: + continue + + if args.dry_run: + print(f" DRY-RUN: would update {doc_index}/{doc_id} " + f"({len(existing)} existing → {len(merged)} total)") + counts["added"] += added_this_file + continue + + try: + update_overrides(session, doc_index, doc_id, merged) + print(f" UPDATED {doc_index}/{doc_id} ({len(existing)} → {len(merged)})") + counts["added"] += added_this_file + except requests.HTTPError as e: + print(f" ERROR: update failed: {e}") + counts["error"] += 1 + + print() + print("=" * 60) + print(f"Summary ({mode}):") + print(f" Overrides added: {counts['added']}") + print(f" Skipped (already present): {counts['skipped_dedupe']}") + print(f" Skipped (no detection): {counts['skipped_not_found']}") + print(f" Invalid (failed checks): {counts['invalid']}") + print(f" Errors: {counts['error']}") + + if custom_vars and args.engine == "suricata": + print() + print("WARNING: detected custom Suricata variables in imported overrides:") + for v in sorted(custom_vars): + print(f" {v}") + print("If any of these are not already defined in SOC Config (Suricata variables),") + print("you must add them manually before the rules will function correctly.") + + sys.exit(0 if counts["error"] == 0 and counts["invalid"] == 0 else 1) + + +if __name__ == "__main__": + main() diff --git a/salt/manager/tools/sbin/so-detections-overrides-import_test.py b/salt/manager/tools/sbin/so-detections-overrides-import_test.py new file mode 100644 index 000000000..ed74e44cb --- /dev/null +++ b/salt/manager/tools/sbin/so-detections-overrides-import_test.py @@ -0,0 +1,406 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +import json +import os +import tempfile +import unittest +from importlib.machinery import SourceFileLoader +from io import StringIO +from unittest.mock import MagicMock, patch + +# The script has no .py extension, so importlib.import_module won't find it by +# name. SourceFileLoader loads source Python regardless of extension. +HERE = os.path.dirname(os.path.abspath(__file__)) +SCRIPT = os.path.join(HERE, "so-detections-overrides-import") +soi = SourceFileLoader("so_overrides_import", SCRIPT).load_module() + + +class TestValidateSuppress(unittest.TestCase): + def test_valid(self): + self.assertIsNone(soi.validate_override( + {"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}, "suricata")) + + def test_valid_var(self): + self.assertIsNone(soi.validate_override( + {"type": "suppress", "track": "by_either", "ip": "$HOME_NET"}, "suricata")) + + def test_valid_cidr(self): + self.assertIsNone(soi.validate_override( + {"type": "suppress", "track": "by_dst", "ip": "10.0.0.0/8"}, "suricata")) + + def test_valid_bracket_list(self): + self.assertIsNone(soi.validate_override( + {"type": "suppress", "track": "by_src", "ip": "[1.2.3.4,10.0.0.0/8]"}, "suricata")) + + def test_missing_ip(self): + err = soi.validate_override({"type": "suppress", "track": "by_src"}, "suricata") + self.assertIn("requires", err) + + def test_missing_track(self): + err = soi.validate_override({"type": "suppress", "ip": "1.2.3.4"}, "suricata") + self.assertIn("requires", err) + + def test_invalid_track(self): + err = soi.validate_override( + {"type": "suppress", "track": "by_both", "ip": "1.2.3.4"}, "suricata") + self.assertIn("invalid track", err) + + def test_invalid_ip(self): + err = soi.validate_override( + {"type": "suppress", "track": "by_src", "ip": "not-an-ip"}, "suricata") + self.assertIn("invalid IP", err) + + def test_unnecessary_field(self): + err = soi.validate_override( + {"type": "suppress", "track": "by_src", "ip": "1.2.3.4", "count": 5}, "suricata") + self.assertIn("unnecessary fields", err) + + +class TestValidateThreshold(unittest.TestCase): + def test_valid(self): + self.assertIsNone(soi.validate_override({ + "type": "threshold", "track": "by_src", + "thresholdType": "limit", "count": 10, "seconds": 60, + }, "suricata")) + + def test_valid_by_both(self): + self.assertIsNone(soi.validate_override({ + "type": "threshold", "track": "by_both", + "thresholdType": "both", "count": 1, "seconds": 1, + }, "suricata")) + + def test_track_by_either_invalid(self): + err = soi.validate_override({ + "type": "threshold", "track": "by_either", + "thresholdType": "limit", "count": 10, "seconds": 60, + }, "suricata") + self.assertIn("invalid track", err) + + def test_invalid_threshold_type(self): + err = soi.validate_override({ + "type": "threshold", "track": "by_src", + "thresholdType": "bogus", "count": 10, "seconds": 60, + }, "suricata") + self.assertIn("invalid thresholdType", err) + + def test_zero_count(self): + err = soi.validate_override({ + "type": "threshold", "track": "by_src", + "thresholdType": "limit", "count": 0, "seconds": 60, + }, "suricata") + self.assertIn("count", err) + + def test_negative_seconds(self): + err = soi.validate_override({ + "type": "threshold", "track": "by_src", + "thresholdType": "limit", "count": 10, "seconds": -1, + }, "suricata") + self.assertIn("seconds", err) + + def test_missing_field(self): + err = soi.validate_override({ + "type": "threshold", "track": "by_src", + "thresholdType": "limit", "count": 10, # missing seconds + }, "suricata") + self.assertIn("requires", err) + + def test_unnecessary_field(self): + err = soi.validate_override({ + "type": "threshold", "track": "by_src", + "thresholdType": "limit", "count": 10, "seconds": 60, + "regex": "foo", + }, "suricata") + self.assertIn("unnecessary fields", err) + + +class TestValidateModify(unittest.TestCase): + def test_valid(self): + self.assertIsNone(soi.validate_override( + {"type": "modify", "regex": r"content:\"foo\"", "value": "content:bar"}, "suricata")) + + def test_invalid_regex(self): + err = soi.validate_override( + {"type": "modify", "regex": "(unbalanced", "value": "x"}, "suricata") + self.assertIn("invalid regex", err) + + def test_missing_value(self): + err = soi.validate_override({"type": "modify", "regex": "x"}, "suricata") + self.assertIn("requires", err) + + def test_unnecessary_field(self): + err = soi.validate_override( + {"type": "modify", "regex": "x", "value": "y", "track": "by_src"}, "suricata") + self.assertIn("unnecessary fields", err) + + +class TestValidateMisc(unittest.TestCase): + def test_unknown_type(self): + err = soi.validate_override({"type": "suppresss", "track": "by_src", "ip": "1.2.3.4"}, "suricata") + self.assertIn("invalid type", err) + + def test_missing_type(self): + err = soi.validate_override({"track": "by_src"}, "suricata") + self.assertIn("type is required", err) + + def test_non_suricata_engine_skipped(self): + # validate_override returns None for non-suricata engines (sigma is gated in main). + self.assertIsNone(soi.validate_override({"type": "anything"}, "sigma")) + + +class TestValidateIP(unittest.TestCase): + def test_plain_ipv4(self): + self.assertIsNone(soi._validate_suricata_ip("1.2.3.4")) + + def test_plain_ipv6(self): + self.assertIsNone(soi._validate_suricata_ip("::1")) + + def test_cidr(self): + self.assertIsNone(soi._validate_suricata_ip("10.0.0.0/8")) + + def test_var(self): + self.assertIsNone(soi._validate_suricata_ip("$CONCOURSEWORKERS")) + + def test_bracket_list(self): + self.assertIsNone(soi._validate_suricata_ip("[1.2.3.4, 10.0.0.0/8]")) + + def test_bracket_list_bad_member(self): + err = soi._validate_suricata_ip("[1.2.3.4,nope]") + self.assertIn("invalid IP in list", err) + + def test_empty(self): + self.assertIn("empty", soi._validate_suricata_ip("")) + + def test_invalid(self): + self.assertIn("invalid", soi._validate_suricata_ip("999.999.999.999")) + + +class TestDedupeKey(unittest.TestCase): + def test_suppress(self): + a = {"type": "suppress", "track": "by_src", "ip": "1.2.3.4", "count": 99} + b = {"type": "suppress", "track": "by_src", "ip": "1.2.3.4"} + # count is irrelevant for suppress dedupe + self.assertEqual(soi.dedupe_key(a), soi.dedupe_key(b)) + + def test_suppress_differs_on_ip(self): + a = {"type": "suppress", "track": "by_src", "ip": "1.2.3.4"} + b = {"type": "suppress", "track": "by_src", "ip": "5.6.7.8"} + self.assertNotEqual(soi.dedupe_key(a), soi.dedupe_key(b)) + + def test_threshold(self): + a = {"type": "threshold", "track": "by_src", "thresholdType": "limit", + "count": 10, "seconds": 60, "ip": "ignored"} + b = {"type": "threshold", "track": "by_src", "thresholdType": "limit", + "count": 10, "seconds": 60} + self.assertEqual(soi.dedupe_key(a), soi.dedupe_key(b)) + + def test_threshold_differs_on_count(self): + a = {"type": "threshold", "track": "by_src", "thresholdType": "limit", + "count": 10, "seconds": 60} + b = {"type": "threshold", "track": "by_src", "thresholdType": "limit", + "count": 20, "seconds": 60} + self.assertNotEqual(soi.dedupe_key(a), soi.dedupe_key(b)) + + def test_modify(self): + a = {"type": "modify", "regex": "x", "value": "y"} + b = {"type": "modify", "regex": "x", "value": "y"} + self.assertEqual(soi.dedupe_key(a), soi.dedupe_key(b)) + + +class TestDescribe(unittest.TestCase): + def test_suppress(self): + s = soi.describe({"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}) + self.assertIn("suppress", s) + self.assertIn("by_src", s) + self.assertIn("1.2.3.4", s) + + def test_threshold_includes_count(self): + s = soi.describe({"type": "threshold", "track": "by_src", + "thresholdType": "limit", "count": 10, "seconds": 60}) + self.assertIn("count=10", s) + self.assertIn("seconds=60", s) + + def test_modify(self): + s = soi.describe({"type": "modify", "regex": "foo"}) + self.assertIn("modify", s) + self.assertIn("foo", s) + + +class TestParseOverridesFile(unittest.TestCase): + def _write(self, content): + fd, path = tempfile.mkstemp(suffix=".txt") + os.close(fd) + with open(path, "w") as f: + f.write(content) + self.addCleanup(os.unlink, path) + return path + + def test_single_line(self): + path = self._write('{"type":"suppress","track":"by_src","ip":"1.2.3.4"}') + result = soi.parse_overrides_file(path) + self.assertEqual(len(result), 1) + self.assertEqual(result[0][0]["type"], "suppress") + self.assertEqual(result[0][1], 1) + + def test_ndjson(self): + path = self._write( + '{"type":"suppress","track":"by_src","ip":"1.2.3.4"}\n' + '{"type":"suppress","track":"by_dst","ip":"5.6.7.8"}\n' + ) + result = soi.parse_overrides_file(path) + self.assertEqual(len(result), 2) + self.assertEqual(result[1][1], 2) + + def test_empty(self): + path = self._write("") + self.assertEqual(soi.parse_overrides_file(path), []) + + def test_blank_lines_skipped(self): + path = self._write('\n{"type":"suppress","track":"by_src","ip":"1.2.3.4"}\n\n') + result = soi.parse_overrides_file(path) + self.assertEqual(len(result), 1) + self.assertEqual(result[0][1], 2) # line number reflects original position + + def test_invalid_raises(self): + path = self._write("not json") + with self.assertRaises(json.JSONDecodeError): + soi.parse_overrides_file(path) + + +class TestCollectCustomVars(unittest.TestCase): + def test_finds_custom(self): + v = soi.collect_custom_vars({"ip": "$CONCOURSEWORKERS"}) + self.assertEqual(v, {"$CONCOURSEWORKERS"}) + + def test_filters_builtins(self): + v = soi.collect_custom_vars({"ip": "$HOME_NET"}) + self.assertEqual(v, set()) + + def test_mixed(self): + v = soi.collect_custom_vars({"ip": "[$HOME_NET,$MYNET]"}) + self.assertEqual(v, {"$MYNET"}) + + def test_non_string_fields_ignored(self): + v = soi.collect_custom_vars({"count": 10, "isEnabled": True}) + self.assertEqual(v, set()) + + +class TestMakeSession(unittest.TestCase): + def _write(self, content): + fd, path = tempfile.mkstemp() + os.close(fd) + with open(path, "w") as f: + f.write(content) + self.addCleanup(os.unlink, path) + return path + + def test_valid_auth_file(self): + path = self._write('user = "admin:secret"\n') + session = soi.make_session(path) + self.assertEqual(session.auth.username, "admin") + self.assertEqual(session.auth.password, "secret") + self.assertFalse(session.verify) + + def test_missing_user_line(self): + path = self._write("# no user line here\n") + with self.assertRaises(RuntimeError): + soi.make_session(path) + + +class TestFindDetection(unittest.TestCase): + def _session_with_response(self, payload): + session = MagicMock() + response = MagicMock() + response.json.return_value = payload + response.raise_for_status.return_value = None + session.get.return_value = response + return session + + def test_found(self): + session = self._session_with_response({"hits": {"hits": [{ + "_id": "abc", "_index": "so-detection", + "_source": {"so_detection": {"overrides": [{"type": "suppress"}]}}, + }]}}) + doc_id, idx, existing = soi.find_detection(session, "so-detection", "2049201", "suricata") + self.assertEqual(doc_id, "abc") + self.assertEqual(idx, "so-detection") + self.assertEqual(len(existing), 1) + + def test_not_found(self): + session = self._session_with_response({"hits": {"hits": []}}) + doc_id, idx, existing = soi.find_detection(session, "so-detection", "x", "suricata") + self.assertIsNone(doc_id) + self.assertIsNone(idx) + self.assertIsNone(existing) + + def test_no_overrides_field(self): + session = self._session_with_response({"hits": {"hits": [{ + "_id": "abc", "_index": "so-detection", + "_source": {"so_detection": {}}, + }]}}) + _, _, existing = soi.find_detection(session, "so-detection", "x", "suricata") + self.assertEqual(existing, []) + + def test_multiple_hits_warns(self): + session = self._session_with_response({"hits": {"hits": [ + {"_id": "a", "_index": "i", "_source": {"so_detection": {"overrides": []}}}, + {"_id": "b", "_index": "i", "_source": {"so_detection": {"overrides": []}}}, + ]}}) + with patch("sys.stdout", new=StringIO()) as out: + doc_id, _, _ = soi.find_detection(session, "i", "x", "suricata") + self.assertEqual(doc_id, "a") + self.assertIn("WARN", out.getvalue()) + + +class TestUpdateOverrides(unittest.TestCase): + def test_posts_to_update_endpoint(self): + session = MagicMock() + response = MagicMock() + response.raise_for_status.return_value = None + response.json.return_value = {"result": "updated"} + session.post.return_value = response + + result = soi.update_overrides(session, "so-detection", "abc", [{"type": "suppress"}]) + + self.assertEqual(result, {"result": "updated"}) + url = session.post.call_args[0][0] + self.assertIn("/_update/abc", url) + body = session.post.call_args[1]["json"] + self.assertEqual(body["doc"]["so_detection"]["overrides"], [{"type": "suppress"}]) + + +class TestConfirmProceed(unittest.TestCase): + def test_dry_run_skips_prompt(self): + args = MagicMock(dry_run=True) + with patch("sys.stdout", new=StringIO()): + self.assertTrue(soi.confirm_proceed(args)) + + def test_yes_input(self): + args = MagicMock(dry_run=False) + with patch("sys.stdout", new=StringIO()): + with patch("builtins.input", return_value="yes"): + self.assertTrue(soi.confirm_proceed(args)) + + def test_yes_input_case_insensitive(self): + args = MagicMock(dry_run=False) + with patch("sys.stdout", new=StringIO()): + with patch("builtins.input", return_value="YES"): + self.assertTrue(soi.confirm_proceed(args)) + + def test_no_input_aborts(self): + args = MagicMock(dry_run=False) + with patch("sys.stdout", new=StringIO()): + with patch("builtins.input", return_value="no"): + self.assertFalse(soi.confirm_proceed(args)) + + def test_empty_input_aborts(self): + args = MagicMock(dry_run=False) + with patch("sys.stdout", new=StringIO()): + with patch("builtins.input", return_value=""): + self.assertFalse(soi.confirm_proceed(args)) + + +if __name__ == "__main__": + unittest.main() From 125610ed42fc9d10090bd11848ced0f6efc69411 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 12 May 2026 10:11:22 -0400 Subject: [PATCH 15/20] Additional test coverage --- .../tools/sbin/so-detections-overrides-import | 14 +- .../so-detections-overrides-import_test.py | 196 +++++++++++++++++- 2 files changed, 191 insertions(+), 19 deletions(-) diff --git a/salt/manager/tools/sbin/so-detections-overrides-import b/salt/manager/tools/sbin/so-detections-overrides-import index 1f32bf04a..e1cad3ac0 100755 --- a/salt/manager/tools/sbin/so-detections-overrides-import +++ b/salt/manager/tools/sbin/so-detections-overrides-import @@ -32,7 +32,6 @@ ES_URL = "https://localhost:9200" # Engines we know how to handle and the file extension the backup script writes. ENGINES = { "suricata": "txt", - "sigma": "yaml", } # Standard Suricata variables that ship with Security Onion. Anything else @@ -119,7 +118,6 @@ def dedupe_key(override): override.get("count"), override.get("seconds")) if t == "modify": return (t, override.get("regex"), override.get("value")) - return (t,) def _validate_suricata_ip(ip): @@ -150,9 +148,6 @@ def _validate_single_ip(ip): def validate_override(override, engine): """Mirror Override.Validate() from securityonion-soc/model/detection.go. Returns None on success, an error string otherwise.""" - if engine != "suricata": - return None # sigma not yet supported; gated earlier in main() - t = override.get("type") if not t: return "override type is required" @@ -222,7 +217,6 @@ def describe(override): f"count={override.get('count')} seconds={override.get('seconds')}") if t == "modify": return f"type=modify regex={override.get('regex')!r}" - return f"type={t}" def collect_custom_vars(override): @@ -242,7 +236,7 @@ def parse_args(): p.add_argument("--source", "-s", required=True, help="Source directory containing . override files.") p.add_argument("--engine", "-e", default="suricata", choices=list(ENGINES.keys()), - help="Detection engine (default: suricata). Sigma not yet supported.") + help="Detection engine (default: suricata).") p.add_argument("--dry-run", "-n", action="store_true", help="Print what would happen without writing to Elasticsearch.") p.add_argument("--no-import-note", action="store_true", @@ -267,10 +261,6 @@ def confirm_proceed(args): def main(): args = parse_args() - if args.engine == "sigma": - print("ERROR: sigma overrides are not yet supported.", file=sys.stderr) - sys.exit(2) - if not os.path.isdir(args.source): print(f"ERROR: source directory not found: {args.source}", file=sys.stderr) sys.exit(1) @@ -376,7 +366,7 @@ def main(): print(f" Invalid (failed checks): {counts['invalid']}") print(f" Errors: {counts['error']}") - if custom_vars and args.engine == "suricata": + if custom_vars: print() print("WARNING: detected custom Suricata variables in imported overrides:") for v in sorted(custom_vars): diff --git a/salt/manager/tools/sbin/so-detections-overrides-import_test.py b/salt/manager/tools/sbin/so-detections-overrides-import_test.py index ed74e44cb..5f5361ea4 100644 --- a/salt/manager/tools/sbin/so-detections-overrides-import_test.py +++ b/salt/manager/tools/sbin/so-detections-overrides-import_test.py @@ -3,19 +3,28 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. +import importlib.util import json import os +import shutil +import sys import tempfile import unittest from importlib.machinery import SourceFileLoader from io import StringIO from unittest.mock import MagicMock, patch -# The script has no .py extension, so importlib.import_module won't find it by -# name. SourceFileLoader loads source Python regardless of extension. +import requests + +# The script has no .py extension; spec_from_file_location can't auto-detect a +# loader, so we hand it a SourceFileLoader explicitly. (load_module() is +# deprecated in 3.14 and slated for removal in 3.15.) HERE = os.path.dirname(os.path.abspath(__file__)) SCRIPT = os.path.join(HERE, "so-detections-overrides-import") -soi = SourceFileLoader("so_overrides_import", SCRIPT).load_module() +_loader = SourceFileLoader("so_overrides_import", SCRIPT) +_spec = importlib.util.spec_from_loader("so_overrides_import", _loader) +soi = importlib.util.module_from_spec(_spec) +_loader.exec_module(soi) class TestValidateSuppress(unittest.TestCase): @@ -145,10 +154,6 @@ class TestValidateMisc(unittest.TestCase): err = soi.validate_override({"track": "by_src"}, "suricata") self.assertIn("type is required", err) - def test_non_suricata_engine_skipped(self): - # validate_override returns None for non-suricata engines (sigma is gated in main). - self.assertIsNone(soi.validate_override({"type": "anything"}, "sigma")) - class TestValidateIP(unittest.TestCase): def test_plain_ipv4(self): @@ -402,5 +407,182 @@ class TestConfirmProceed(unittest.TestCase): self.assertFalse(soi.confirm_proceed(args)) +class TestParseArgs(unittest.TestCase): + def test_defaults(self): + with patch.object(sys, "argv", ["cmd", "--source", "/some/path"]): + args = soi.parse_args() + self.assertEqual(args.source, "/some/path") + self.assertEqual(args.engine, "suricata") + self.assertFalse(args.dry_run) + self.assertFalse(args.no_import_note) + self.assertEqual(args.index, soi.DEFAULT_INDEX) + + def test_all_options(self): + argv = ["cmd", "-s", "/x", "-e", "suricata", "-n", + "--no-import-note", "-i", "alt-index"] + with patch.object(sys, "argv", argv): + args = soi.parse_args() + self.assertEqual(args.source, "/x") + self.assertTrue(args.dry_run) + self.assertTrue(args.no_import_note) + self.assertEqual(args.index, "alt-index") + + +class TestMain(unittest.TestCase): + def setUp(self): + self.tmpdir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmpdir, ignore_errors=True) + # Stub make_session so tests don't need /opt/so/conf/elasticsearch/curl.config. + p = patch.object(soi, "make_session", return_value=MagicMock()) + p.start() + self.addCleanup(p.stop) + + def _write_file(self, public_id, overrides, ext="txt"): + """Write an NDJSON override file. Entries may be dicts or raw strings (for malformed input).""" + path = os.path.join(self.tmpdir, f"{public_id}.{ext}") + with open(path, "w") as f: + for o in overrides: + f.write(o if isinstance(o, str) else json.dumps(o)) + f.write("\n") + return path + + def _run_main(self, *extra_argv, input_response="yes"): + """Run main() with stdout/stderr captured and input mocked. Returns (stdout, stderr, exit_code).""" + argv = ["cmd", "--source", self.tmpdir, *extra_argv] + out, err = StringIO(), StringIO() + with patch.object(sys, "argv", argv), \ + patch("sys.stdout", new=out), \ + patch("sys.stderr", new=err), \ + patch("builtins.input", return_value=input_response): + with self.assertRaises(SystemExit) as cm: + soi.main() + return out.getvalue(), err.getvalue(), cm.exception.code + + def test_source_dir_missing(self): + argv = ["cmd", "--source", "/no/such/path/here"] + err = StringIO() + with patch.object(sys, "argv", argv), patch("sys.stderr", new=err): + with self.assertRaises(SystemExit) as cm: + soi.main() + self.assertEqual(cm.exception.code, 1) + self.assertIn("source directory not found", err.getvalue()) + + def test_no_files_found(self): + out, _, code = self._run_main() + self.assertEqual(code, 0) + self.assertIn("No *.txt files found", out) + + def test_user_aborts(self): + self._write_file("1001", [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}]) + out, _, code = self._run_main(input_response="no") + self.assertEqual(code, 1) + self.assertIn("Aborted", out) + + def test_parse_error_increments_error(self): + # Malformed JSON line — parse_overrides_file raises JSONDecodeError. + self._write_file("1002", ["not json"]) + out, _, code = self._run_main("--dry-run") + self.assertEqual(code, 1) # invalid+error → non-zero + self.assertIn("could not parse", out) + self.assertIn("Errors: 1", out) + + def test_empty_file_skipped(self): + # Blank lines only — parse_overrides_file returns []; main reports "empty file" and continues. + path = os.path.join(self.tmpdir, "1003.txt") + with open(path, "w") as f: + f.write("\n\n") + out, _, code = self._run_main("--dry-run") + self.assertEqual(code, 0) + self.assertIn("empty file", out) + + @patch.object(soi, "find_detection") + def test_search_http_error(self, mock_find): + mock_find.side_effect = requests.HTTPError("boom") + self._write_file("1004", [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}]) + out, _, code = self._run_main("--dry-run") + self.assertEqual(code, 1) + self.assertIn("search failed", out) + + @patch.object(soi, "find_detection") + def test_no_detection_found(self, mock_find): + mock_find.return_value = (None, None, None) + self._write_file("1005", [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}]) + out, _, code = self._run_main("--dry-run") + self.assertEqual(code, 0) + self.assertIn("no detection found", out) + self.assertIn("Skipped (no detection): 1", out) + + @patch.object(soi, "find_detection") + def test_all_duplicates_no_update(self, mock_find): + existing = [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}] + mock_find.return_value = ("doc1", "so-detection", existing) + self._write_file("1006", [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}]) + out, _, code = self._run_main("--dry-run") + self.assertEqual(code, 0) + self.assertIn("SKIP", out) + self.assertNotIn("DRY-RUN: would update", out) # added_this_file == 0 branch + + @patch.object(soi, "update_overrides") + @patch.object(soi, "find_detection") + def test_happy_path_full(self, mock_find, mock_update): + # Exercises: ADD, dedupe SKIP, INVALID, note prefix, UPDATE, custom-vars warning, exit=1 (invalid present) + existing = [{"type": "suppress", "track": "by_src", "ip": "9.9.9.9"}] + mock_find.return_value = ("doc1", "so-detection", existing) + mock_update.return_value = {"result": "updated"} + self._write_file("1007", [ + {"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}, # ADD + {"type": "suppress", "track": "by_src", "ip": "9.9.9.9"}, # SKIP (dupe of existing) + {"type": "suppress", "track": "bogus", "ip": "1.2.3.4"}, # INVALID + {"type": "suppress", "track": "by_src", "ip": "$CONCOURSEWORKERS"}, # ADD + custom var + ]) + out, _, code = self._run_main() + self.assertEqual(code, 1) # one invalid -> non-zero + + mock_update.assert_called_once() + merged = mock_update.call_args[0][3] + self.assertEqual(len(merged), 3) # 1 existing + 2 new + new_notes = [o.get("note", "") for o in merged if o.get("ip") in ("1.2.3.4", "$CONCOURSEWORKERS")] + self.assertTrue(all(n.startswith("[Imported ") for n in new_notes)) + + self.assertIn("ADD", out) + self.assertIn("SKIP", out) + self.assertIn("INVALID", out) + self.assertIn("UPDATED", out) + self.assertIn("$CONCOURSEWORKERS", out) + + @patch.object(soi, "update_overrides") + @patch.object(soi, "find_detection") + def test_no_import_note_preserves_note(self, mock_find, mock_update): + mock_find.return_value = ("doc1", "so-detection", []) + mock_update.return_value = {"result": "updated"} + self._write_file("1008", [ + {"type": "suppress", "track": "by_src", "ip": "1.2.3.4", "note": "original"}, + ]) + _, _, code = self._run_main("--no-import-note") + self.assertEqual(code, 0) + merged = mock_update.call_args[0][3] + self.assertEqual(merged[0]["note"], "original") # no prefix applied + + @patch.object(soi, "find_detection") + def test_dry_run_skips_update(self, mock_find): + mock_find.return_value = ("doc1", "so-detection", []) + self._write_file("1009", [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}]) + with patch.object(soi, "update_overrides") as mock_update: + out, _, code = self._run_main("--dry-run") + self.assertEqual(code, 0) + mock_update.assert_not_called() + self.assertIn("DRY-RUN: would update", out) + + @patch.object(soi, "update_overrides") + @patch.object(soi, "find_detection") + def test_update_http_error(self, mock_find, mock_update): + mock_find.return_value = ("doc1", "so-detection", []) + mock_update.side_effect = requests.HTTPError("nope") + self._write_file("1010", [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}]) + out, _, code = self._run_main() + self.assertEqual(code, 1) + self.assertIn("update failed", out) + + if __name__ == "__main__": unittest.main() From f637dc62d1fa44fe5d367fa8e8826d4eaf16766c Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 12 May 2026 13:29:32 -0500 Subject: [PATCH 16/20] use temp files to prevent jq arg too long --- salt/manager/tools/sbin/soup | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index c0f8b61c1..157f76ee3 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -556,14 +556,23 @@ check_transform_health_and_reauthorize() { # - unhealthy (any non-green health status) # - metadata has run_as_kibana_system: false (this fix is specific to transforms started prior to Kibana 9.3.3) # - are not orphaned (integration is not somehow missing/corrupt/uninstalled) + local tmp_transforms tmp_stats tmp_installed + tmp_transforms=$(mktemp) + tmp_stats=$(mktemp) + tmp_installed=$(mktemp) + + echo "$transforms_doc" > "$tmp_transforms" + echo "$stats_doc" > "$tmp_stats" + echo "$installed_doc" > "$tmp_installed" + local unhealthy_transforms unhealthy_transforms=$(jq -c -n \ - --argjson t "$transforms_doc" \ - --argjson s "$stats_doc" \ - --argjson i "$installed_doc" ' - ($i.items | map({key: .name, value: .version}) | from_entries) as $pkg_ver - | ($s.transforms | map({key: .id, value: .health.status}) | from_entries) as $health - | [ $t.transforms[] + --slurpfile t "$tmp_transforms" \ + --slurpfile s "$tmp_stats" \ + --slurpfile i "$tmp_installed" ' + ($i[0].items | map({key: .name, value: .version}) | from_entries) as $pkg_ver + | ($s[0].transforms | map({key: .id, value: .health.status}) | from_entries) as $health + | [ $t[0].transforms[] | select(._meta.run_as_kibana_system == false) | select(($health[.id] // "unknown") != "green") | {id, pkg: ._meta.package.name, ver: ($pkg_ver[._meta.package.name])} @@ -604,6 +613,8 @@ check_transform_health_and_reauthorize() { (( total_failures += $(jq 'map(select(.success != true)) | length' <<< "$resp" 2>/dev/null) )) done <<< "$unhealthy_transforms" + rm -f "$tmp_transforms" "$tmp_stats" "$tmp_installed" + if [[ "$total_failures" -gt 0 ]]; then echo "Some transform(s) failed to reauthorize." fi From 01fb1aa15665a0382889910edcbc2951050ed80f Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 12 May 2026 15:19:44 -0500 Subject: [PATCH 17/20] check pillars for ScanLNK and rename to ScanLnk --- salt/manager/tools/sbin/soup | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index c0f8b61c1..a5d39e090 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -644,6 +644,27 @@ ensure_postgres_secret() { chown socore:socore "$secrets_file" } +rename_strelka_scan_lnk() { + echo "Renaming strelka pillar ScanLNK to ScanLnk." + local STRELKA_FILE=/opt/so/saltstack/local/pillar/strelka/soc_strelka.sls + local MINIONDIR=/opt/so/saltstack/local/pillar/minions + local OLD_KEY=strelka.backend.config.backend.scanners.ScanLNK + local NEW_KEY=strelka.backend.config.backend.scanners.ScanLnk + local TMP_VALUE_FILE + TMP_VALUE_FILE=$(mktemp) + + for pillar_file in "$STRELKA_FILE" "$MINIONDIR"/*.sls; do + [[ -f "$pillar_file" ]] || continue + # Skip if ScanLNK doesn't exist + so-yaml.py get "$pillar_file" "$OLD_KEY" > "$TMP_VALUE_FILE" 2>/dev/null || continue + echo "Found 'ScanLNK' key in $pillar_file. Renaming to 'ScanLnk'." + so-yaml.py add "$pillar_file" "$NEW_KEY" "file:$TMP_VALUE_FILE" + so-yaml.py remove "$pillar_file" "$OLD_KEY" + done + + rm -f "$TMP_VALUE_FILE" +} + up_to_3.1.0() { ensure_postgres_local_pillar ensure_postgres_secret @@ -651,7 +672,7 @@ up_to_3.1.0() { elasticsearch_backup_index_templates # Clear existing component template state file. rm -f /opt/so/state/esfleet_component_templates.json - + rename_strelka_scan_lnk INSTALLEDVERSION=3.1.0 } From b068ad2b3572ec00f40f181e2606e2034243da4d Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 13 May 2026 10:53:11 -0400 Subject: [PATCH 18/20] remove stig from hypervisor and managerhype --- salt/top.sls | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/salt/top.sls b/salt/top.sls index ff789e89d..cf743edd1 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -119,7 +119,7 @@ base: - kafka - pcap.cleanup - '*_manager or *_managerhype and G@saltversion:{{saltversion}} and not I@node_data:False': + '*_manager and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - salt.master - registry @@ -146,6 +146,32 @@ base: - stig - kafka + '*_managerhype and G@saltversion:{{saltversion}} and not I@node_data:False': + - match: compound + - salt.master + - registry + - nginx + - influxdb + - postgres + - strelka.manager + - soc + - kratos + - hydra + - firewall + - manager + - sensoroni + - telegraf + - backup.config_backup + - elasticsearch + - logstash + - redis + - elastic-fleet-package-registry + - kibana + - elastalert + - utility + - elasticfleet + - kafka + '*_managerhype and I@features:vrt and G@saltversion:{{saltversion}}': - match: compound - manager.hypervisor @@ -286,7 +312,6 @@ base: - libvirt - libvirt.images - elasticfleet.install_agent_grid - - stig '*_desktop and G@saltversion:{{saltversion}}': - sensoroni From 72327285b202755b0bded4f3119e0debe8d6037c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 13 May 2026 11:58:21 -0400 Subject: [PATCH 19/20] Change Telegraf output from BOTH to INFLUXDB --- salt/telegraf/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/telegraf/defaults.yaml b/salt/telegraf/defaults.yaml index ead122b0a..24f58b157 100644 --- a/salt/telegraf/defaults.yaml +++ b/salt/telegraf/defaults.yaml @@ -1,6 +1,6 @@ telegraf: enabled: False - output: BOTH + output: INFLUXDB config: interval: '30s' metric_batch_size: 1000 From d56bf0182334c79161ad9cb809e8f92cc73b22c9 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 13 May 2026 12:32:54 -0500 Subject: [PATCH 20/20] add zeek.ja4d ingest pipeline --- salt/elasticsearch/files/ingest/zeek.ja4d | 71 +++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 salt/elasticsearch/files/ingest/zeek.ja4d diff --git a/salt/elasticsearch/files/ingest/zeek.ja4d b/salt/elasticsearch/files/ingest/zeek.ja4d new file mode 100644 index 000000000..206622c49 --- /dev/null +++ b/salt/elasticsearch/files/ingest/zeek.ja4d @@ -0,0 +1,71 @@ +{ + "description": "zeek.ja4d", + "processors": [ + { + "set": { + "field": "event.dataset", + "value": "ja4d" + } + }, + { + "remove": { + "field": [ + "host" + ], + "ignore_failure": true + } + }, + { + "json": { + "field": "message", + "target_field": "message2", + "ignore_failure": true + } + }, + { + "rename": { + "field": "message2.ja4d", + "target_field": "hash.ja4d", + "ignore_missing": true, + "if": "ctx?.message2?.ja4d != null && ctx.message2.ja4d.length() > 0" + } + }, + { + "rename": { + "field": "message2.client_mac", + "target_field": "host.mac", + "ignore_missing": true, + "if": "ctx?.message2?.client_mac != null && ctx.message2.client_mac.length() > 0" + } + }, + { + "rename": { + "field": "message2.hostname", + "target_field": "host.hostname", + "ignore_missing": true, + "if": "ctx?.message2?.hostname != null && ctx.message2.hostname.length() > 0" + } + }, + { + "rename": { + "field": "message2.requested_ip", + "target_field": "dhcp.requested_address", + "ignore_missing": true, + "if": "ctx?.message2?.requested_ip != null && ctx.message2.requested_ip.length() > 0" + } + }, + { + "rename": { + "field": "message2.vendor_class_id", + "target_field": "zeek.ja4d.vendor_class_id", + "ignore_missing": true, + "if": "ctx?.message2?.vendor_class_id != null && ctx.message2.vendor_class_id.length() > 0" + } + }, + { + "pipeline": { + "name": "zeek.common" + } + } + ] +} \ No newline at end of file