idstools removal refactor

salt/manager/tools/sbin/soup

@@ -426,6 +426,7 @@ preupgrade_changes() {
     [[ "$INSTALLEDVERSION" == 2.4.160 ]] && up_to_2.4.170
     [[ "$INSTALLEDVERSION" == 2.4.170 ]] && up_to_2.4.180
     [[ "$INSTALLEDVERSION" == 2.4.180 ]] && up_to_2.4.190
+    [[ "$INSTALLEDVERSION" == 2.4.190 ]] && up_to_2.4.200
     true
 }
 
@@ -457,6 +458,7 @@ postupgrade_changes() {
     [[ "$POSTVERSION"  == 2.4.160 ]] && post_to_2.4.170
     [[ "$POSTVERSION"  == 2.4.170 ]] && post_to_2.4.180
     [[ "$POSTVERSION"  == 2.4.180 ]] && post_to_2.4.190
+    [[ "$POSTVERSION"  == 2.4.190 ]] && post_to_2.4.200
     true
 }
 
@@ -636,6 +638,11 @@ post_to_2.4.190() {
   POSTVERSION=2.4.190
 }
 
+post_to_2.4.200() {
+  echo "Nothing to apply"
+  POSTVERSION=2.4.200
+}
+
 repo_sync() {
   echo "Sync the local repo."
   su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync."
@@ -903,6 +910,13 @@ up_to_2.4.190() {
   INSTALLEDVERSION=2.4.190
 }
 
+up_to_2.4.200() {
+  echo "Migrating idstools config"
+  suricata_idstools_removal
+
+  INSTALLEDVERSION=2.4.200
+}
+
 add_hydra_pillars() {
   mkdir -p /opt/so/saltstack/local/pillar/hydra
   touch /opt/so/saltstack/local/pillar/hydra/soc_hydra.sls
@@ -986,6 +1000,8 @@ rollover_index() {
 }
 
 suricata_idstools_migration() {
+  # For 2.4.70
+
   #Backup the pillars for idstools
   mkdir -p /nsm/backup/detections-migration/idstools
   rsync -av /opt/so/saltstack/local/pillar/idstools/* /nsm/backup/detections-migration/idstools
@@ -1086,6 +1102,113 @@ playbook_migration() {
   echo "Playbook Migration is complete...."
 }
 
+suricata_idstools_removal() {
+# For SOUPs beginning with 2.4.200
+
+# Create syncBlock file
+cat > /opt/so/conf/soc/fingerprints/suricataengine.syncBlock << EOF
+Suricata rulset sync is blocked until this file is removed. Make sure that you have manually added any custom Suricata rulesets via SOC config - review the documentation for more details: securityonion.net/docs
+EOF
+
+# Track if we have custom configs
+CUSTOM_CONFIGS_FOUND=0
+
+# ETPRO Check
+ETPRO=$(grep "--etpro" /usr/sbin/so-rule-update)
+if [[ -n "$ETPRO" ]]; then
+    echo "Grid is using ETPRO."
+    # Add ETPRO yaml to SOC pillar file
+    if [[ $is_airgap -eq 0 ]]; then
+        #TODO /opt/so/saltstack/local/pillar/soc/soc_soc.sls
+    else
+        #TODO /opt/so/saltstack/local/pillar/soc/soc_soc.sls
+    fi
+fi
+
+#idstools conf parse
+RULECAT_CONF="/opt/so/conf/idstools/etc/rulecat.conf"
+
+# Parse RULECAT_CONF and check for custom configs
+if [[ ! -f "$RULECAT_CONF" ]]; then
+    echo "Warning: $RULECAT_CONF not found - leaving syncBlock."
+    return 0
+fi
+
+echo "Parsing $RULECAT_CONF for custom configurations..."
+
+# Default values to check against
+DEFAULT_URL="--url=http://MANAGER:7788/suricata/emerging-all.rules"
+DEFAULT_DISABLE="--disable=/opt/so/idstools/etc/disable.conf"
+DEFAULT_ENABLE="--enable=/opt/so/idstools/etc/enable.conf"
+DEFAULT_MODIFY="--modify=/opt/so/idstools/etc/modify.conf"
+
+# Valid --local patterns
+VALID_LOCAL_PATTERNS=(
+    "/opt/so/rules/nids/suri/local.rules" # 2/24
+    "/opt/so/rules/nids/suri/extraction.rules" # 2/24
+    "/opt/so/rules/nids/suri/filters.rules" # 2/24
+    "/opt/so/rules/nids/extraction.rules" # 9/23
+    "/opt/so/rules/nids/filters.rules" # 9/23
+    "/opt/so/rules/nids/local.rules" # 8/23
+    "/opt/so/rules/nids/sorules/extraction.rules" # 8/23
+    "/opt/so/rules/nids/sorules/filters.rules" # 8/23
+  )
+
+# Parse each line in the config file
+while IFS= read -r line; do
+    # Skip empty lines and comments
+    [[ -z "$line" || "$line" =~ ^[[:space:]]*# ]] && continue
+    
+    # Check for non-default --local parameter
+    if [[ "$line" =~ ^--local= ]]; then
+        local_path="${line#--local=}"
+        is_valid=0
+        for pattern in "${VALID_LOCAL_PATTERNS[@]}"; do
+            if [[ "$local_path" == "$pattern" ]]; then
+                is_valid=1
+                break
+            fi
+        done
+        if [[ $is_valid -eq 0 ]]; then
+            echo "Custom --local parameter detected: $line"
+            echo "Custom configuration found: $line" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock
+            return 0
+        fi
+    fi
+    
+    # Check for non-default --url parameter (default contains 7788)
+    if [[ "$line" =~ ^--url= ]] && [[ ! "$line" =~ 7788 ]]; then
+        echo "Custom --url parameter detected: $line"
+        echo "Custom configuration found: $line" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock
+        return 0
+    fi
+    
+    # Sanity checks for other parameters
+    if [[ "$line" =~ ^--disable= ]] && [[ "$line" != "$DEFAULT_DISABLE" ]]; then
+        echo "Custom --disable parameter detected: $line"
+        echo "Custom configuration found: $line" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock
+        return 0
+    fi
+    
+    if [[ "$line" =~ ^--enable= ]] && [[ "$line" != "$DEFAULT_ENABLE" ]]; then
+        echo "Custom --enable parameter detected: $line"
+        echo "Custom configuration found: $line" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock
+        return 0
+    fi
+    
+    if [[ "$line" =~ ^--modify= ]] && [[ "$line" != "$DEFAULT_MODIFY" ]]; then
+        echo "Custom --modify parameter detected: $line"
+        echo "Custom configuration found: $line" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock
+        return 0
+    fi
+    
+done < "$RULECAT_CONF"
+
+# If we reach here, no custom configs were found
+echo "idstools migration completed successfully - removing Suricata engine syncBlock"
+rm -f /opt/so/conf/soc/fingerprints/suricataengine.syncBlock
+}
+
 determine_elastic_agent_upgrade() {
   if [[ $is_airgap -eq 0 ]]; then
     update_elastic_agent_airgap

salt/soc/enabled.sls

@@ -26,6 +26,7 @@ so-soc:
       - /nsm/rules:/nsm/rules:rw
       - /opt/so/conf/strelka:/opt/sensoroni/yara:rw
       - /opt/so/conf/sigma:/opt/sensoroni/sigma:rw
+      - /opt/so/conf/suricata:/opt/sensoroni/suricata:rw
       - /opt/so/rules/elastalert/rules:/opt/sensoroni/elastalert:rw
       - /opt/so/rules/nids/suri:/opt/sensoroni/nids:rw
       - /opt/so/conf/soc/fingerprints:/opt/sensoroni/fingerprints:rw

salt/soc/soc_soc.yaml

@@ -590,11 +590,11 @@ soc:
                 label: Ruleset License
                 required: True
               - field: readOnly
-                label: Read Only
+                label: Read Only (Prevents changes to the rule itself - can still be enabled/disabled/tuned)
                 forcedType: bool
                 required: False
               - field: deleteUnreferenced
-                label: Delete Unreferenced
+                label: Delete Unreferenced (Deletes rules that are no longer referenced by ruleset source)
                 forcedType: bool
                 required: False
             airgap: *serulesetSources

salt/suricata/config.sls

@@ -53,13 +53,15 @@ suridir:
   file.directory:
     - name: /opt/so/conf/suricata
     - user: 940
-    - group: 940
+    - group: 939
+    - mode: 775
 
 suriruledir:
   file.directory:
     - name: /opt/so/conf/suricata/rules
     - user: 940
-    - group: 940
+    - group: 939
+    - mode: 775
     - makedirs: True
 
 surilogdir:
@@ -124,10 +126,7 @@ suriconfig:
 surithresholding:
   file.managed:
     - name: /opt/so/conf/suricata/threshold.conf
-    - source: salt://suricata/files/threshold.conf.jinja
-    - user: 940
-    - group: 940
-    - template: jinja
+    - replace: False
 
 suriclassifications:
   file.managed:

salt/suricata/files/threshold.conf.jinja

@@ -1,35 +0,0 @@
-{% import_yaml 'suricata/thresholding/sids.yaml' as THRESHOLDING %}
-{% if THRESHOLDING -%}
-  
-  {% for EACH_SID in THRESHOLDING -%}
-    {% for ACTIONS_LIST in THRESHOLDING[EACH_SID] -%}
-      {% for EACH_ACTION in ACTIONS_LIST -%}
-        
-        {%- if EACH_ACTION == 'threshold' %}
-{{ EACH_ACTION }} gen_id {{ ACTIONS_LIST[EACH_ACTION].gen_id }}, sig_id {{ EACH_SID }}, type {{ ACTIONS_LIST[EACH_ACTION].type }}, track {{ ACTIONS_LIST[EACH_ACTION].track }}, count {{ ACTIONS_LIST[EACH_ACTION].count }}, seconds {{ ACTIONS_LIST[EACH_ACTION].seconds }}
-        
-        {%- elif EACH_ACTION == 'rate_filter' %}
-          {%- if ACTIONS_LIST[EACH_ACTION].new_action not in ['drop','reject'] %}
-{{ EACH_ACTION }} gen_id {{ ACTIONS_LIST[EACH_ACTION].gen_id }}, sig_id {{ EACH_SID }}, track {{ ACTIONS_LIST[EACH_ACTION].track }}, count {{ ACTIONS_LIST[EACH_ACTION].count }}, seconds {{ ACTIONS_LIST[EACH_ACTION].seconds }}, new_action {{ ACTIONS_LIST[EACH_ACTION].new_action }}, timeout {{ ACTIONS_LIST[EACH_ACTION].timeout }}
-          {%- else %}
-##### Security Onion does not support drop or reject actions for rate_filter
-##### {{ EACH_ACTION }} gen_id {{ ACTIONS_LIST[EACH_ACTION].gen_id }}, sig_id {{ EACH_SID }}, track {{ ACTIONS_LIST[EACH_ACTION].track }}, count {{ ACTIONS_LIST[EACH_ACTION].count }}, seconds {{ ACTIONS_LIST[EACH_ACTION].seconds }}, new_action {{ ACTIONS_LIST[EACH_ACTION].new_action }}, timeout {{ ACTIONS_LIST[EACH_ACTION].timeout }}
-          {%- endif %}
-        
-        {%- elif EACH_ACTION == 'suppress' %}
-          {%- if ACTIONS_LIST[EACH_ACTION].track is defined %}
-{{ EACH_ACTION }} gen_id {{ ACTIONS_LIST[EACH_ACTION].gen_id }}, sig_id {{ EACH_SID }}, track {{ ACTIONS_LIST[EACH_ACTION].track }}, ip {{ ACTIONS_LIST[EACH_ACTION].ip }}
-          {%- else %}
-{{ EACH_ACTION }} gen_id {{ ACTIONS_LIST[EACH_ACTION].gen_id }}, sig_id {{ EACH_SID }}
-          {%- endif %}
-        
-        {%- endif %}
-      
-      {%- endfor %}
-    {%- endfor %}
-  {%- endfor %}
-
-{%- else %}
-##### Navigate to suricata > thresholding > SIDS in SOC to define thresholding
-
-{%- endif %}