From a77157391cc23e2318cb41edaac6fe81fe62f518 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Wed, 17 Sep 2025 10:42:05 -0400 Subject: [PATCH 01/34] remove idstools --- pillar/top.sls | 6 -- salt/allowed_states.map.jinja | 1 - salt/common/tools/sbin/so-image-common | 3 - salt/docker/defaults.yaml | 5 - salt/docker/soc_docker.yaml | 3 +- salt/idstools/config.sls | 65 ------------- salt/idstools/defaults.yaml | 10 -- salt/idstools/disabled.sls | 31 ------- salt/idstools/enabled.sls | 91 ------------------- salt/idstools/etc/disable.conf | 16 ---- salt/idstools/etc/enable.conf | 16 ---- salt/idstools/etc/modify.conf | 12 --- salt/idstools/etc/rulecat.conf | 23 ----- salt/idstools/init.sls | 13 --- salt/idstools/map.jinja | 7 -- salt/idstools/rules/extraction.rules | 26 ------ salt/idstools/rules/filters.rules | 11 --- salt/idstools/rules/local.rules | 1 - salt/idstools/soc_idstools.yaml | 72 --------------- salt/idstools/sostatus.sls | 21 ----- salt/idstools/sync_files.sls | 37 -------- salt/idstools/tools/sbin/so-idstools-restart | 12 --- salt/idstools/tools/sbin/so-idstools-start | 12 --- salt/idstools/tools/sbin/so-idstools-stop | 12 --- salt/idstools/tools/sbin_jinja/so-rule-update | 40 -------- salt/logrotate/defaults.yaml | 10 -- salt/logrotate/soc_logrotate.yaml | 7 -- salt/manager/tools/sbin/so-minion | 16 ---- salt/salt/files/engines.conf | 24 ----- salt/soc/defaults.yaml | 62 ++++++++++++- salt/soc/enabled.sls | 2 +- salt/soc/merged.map.jinja | 59 +++++++++++- salt/soc/soc_soc.yaml | 46 ++++++++++ salt/suricata/defaults.yaml | 2 +- salt/top.sls | 5 - setup/so-functions | 8 +- setup/so-variables | 6 -- 37 files changed, 169 insertions(+), 624 deletions(-) delete mode 100644 salt/idstools/config.sls delete mode 100644 salt/idstools/defaults.yaml delete mode 100644 salt/idstools/disabled.sls delete mode 100644 salt/idstools/enabled.sls delete mode 100644 salt/idstools/etc/disable.conf delete mode 100644 salt/idstools/etc/enable.conf delete mode 100644 salt/idstools/etc/modify.conf delete mode 100644 salt/idstools/etc/rulecat.conf delete mode 100644 salt/idstools/init.sls delete mode 100644 salt/idstools/map.jinja delete mode 100644 salt/idstools/rules/extraction.rules delete mode 100644 salt/idstools/rules/filters.rules delete mode 100644 salt/idstools/rules/local.rules delete mode 100644 salt/idstools/soc_idstools.yaml delete mode 100644 salt/idstools/sostatus.sls delete mode 100644 salt/idstools/sync_files.sls delete mode 100755 salt/idstools/tools/sbin/so-idstools-restart delete mode 100755 salt/idstools/tools/sbin/so-idstools-start delete mode 100755 salt/idstools/tools/sbin/so-idstools-stop delete mode 100755 salt/idstools/tools/sbin_jinja/so-rule-update diff --git a/pillar/top.sls b/pillar/top.sls index 1fdb59deb..a1114b80c 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -43,8 +43,6 @@ base: - secrets - manager.soc_manager - manager.adv_manager - - idstools.soc_idstools - - idstools.adv_idstools - logstash.nodes - logstash.soc_logstash - logstash.adv_logstash @@ -117,8 +115,6 @@ base: - elastalert.adv_elastalert - manager.soc_manager - manager.adv_manager - - idstools.soc_idstools - - idstools.adv_idstools - soc.soc_soc - soc.adv_soc - kibana.soc_kibana @@ -158,8 +154,6 @@ base: {% endif %} - secrets - healthcheck.standalone - - idstools.soc_idstools - - idstools.adv_idstools - kratos.soc_kratos - kratos.adv_kratos - hydra.soc_hydra diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 068722ca2..346ed7f12 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -38,7 +38,6 @@ 'hydra', 'elasticfleet', 'elastic-fleet-package-registry', - 'idstools', 'suricata.manager', 'utility' ] %} diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 7fd35d5ac..e2fe4f715 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -25,7 +25,6 @@ container_list() { if [ $MANAGERCHECK == 'so-import' ]; then TRUSTED_CONTAINERS=( "so-elasticsearch" - "so-idstools" "so-influxdb" "so-kibana" "so-kratos" @@ -49,7 +48,6 @@ container_list() { "so-elastic-fleet-package-registry" "so-elasticsearch" "so-idh" - "so-idstools" "so-influxdb" "so-kafka" "so-kibana" @@ -71,7 +69,6 @@ container_list() { ) else TRUSTED_CONTAINERS=( - "so-idstools" "so-elasticsearch" "so-logstash" "so-nginx" diff --git a/salt/docker/defaults.yaml b/salt/docker/defaults.yaml index 2d7ad4e1c..456a187d6 100644 --- a/salt/docker/defaults.yaml +++ b/salt/docker/defaults.yaml @@ -24,11 +24,6 @@ docker: custom_bind_mounts: [] extra_hosts: [] extra_env: [] - 'so-idstools': - final_octet: 25 - custom_bind_mounts: [] - extra_hosts: [] - extra_env: [] 'so-influxdb': final_octet: 26 port_bindings: diff --git a/salt/docker/soc_docker.yaml b/salt/docker/soc_docker.yaml index dacbf2302..3c4475236 100644 --- a/salt/docker/soc_docker.yaml +++ b/salt/docker/soc_docker.yaml @@ -41,7 +41,6 @@ docker: forcedType: "[]string" so-elastic-fleet: *dockerOptions so-elasticsearch: *dockerOptions - so-idstools: *dockerOptions so-influxdb: *dockerOptions so-kibana: *dockerOptions so-kratos: *dockerOptions @@ -102,4 +101,4 @@ docker: multiline: True forcedType: "[]string" so-zeek: *dockerOptions - so-kafka: *dockerOptions \ No newline at end of file + so-kafka: *dockerOptions diff --git a/salt/idstools/config.sls b/salt/idstools/config.sls deleted file mode 100644 index cea75ab9a..000000000 --- a/salt/idstools/config.sls +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} - -include: - - idstools.sync_files - -idstoolslogdir: - file.directory: - - name: /opt/so/log/idstools - - user: 939 - - group: 939 - - makedirs: True - -idstools_sbin: - file.recurse: - - name: /usr/sbin - - source: salt://idstools/tools/sbin - - user: 939 - - group: 939 - - file_mode: 755 - -# If this is used, exclude so-rule-update -#idstools_sbin_jinja: -# file.recurse: -# - name: /usr/sbin -# - source: salt://idstools/tools/sbin_jinja -# - user: 939 -# - group: 939 -# - file_mode: 755 -# - template: jinja - -idstools_so-rule-update: - file.managed: - - name: /usr/sbin/so-rule-update - - source: salt://idstools/tools/sbin_jinja/so-rule-update - - user: 939 - - group: 939 - - mode: 755 - - template: jinja - -suricatacustomdirsfile: - file.directory: - - name: /nsm/rules/detect-suricata/custom_file - - user: 939 - - group: 939 - - makedirs: True - -suricatacustomdirsurl: - file.directory: - - name: /nsm/rules/detect-suricata/custom_temp - - user: 939 - - group: 939 - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/salt/idstools/defaults.yaml b/salt/idstools/defaults.yaml deleted file mode 100644 index 1be100cec..000000000 --- a/salt/idstools/defaults.yaml +++ /dev/null @@ -1,10 +0,0 @@ -idstools: - enabled: False - config: - urls: [] - ruleset: ETOPEN - oinkcode: "" - sids: - enabled: [] - disabled: [] - modify: [] diff --git a/salt/idstools/disabled.sls b/salt/idstools/disabled.sls deleted file mode 100644 index ab0e10d7a..000000000 --- a/salt/idstools/disabled.sls +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} - -include: - - idstools.sostatus - -so-idstools: - docker_container.absent: - - force: True - -so-idstools_so-status.disabled: - file.comment: - - name: /opt/so/conf/so-status/so-status.conf - - regex: ^so-idstools$ - -so-rule-update: - cron.absent: - - identifier: so-rule-update - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/salt/idstools/enabled.sls b/salt/idstools/enabled.sls deleted file mode 100644 index 365b38772..000000000 --- a/salt/idstools/enabled.sls +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} -{% from 'docker/docker.map.jinja' import DOCKER %} -{% from 'vars/globals.map.jinja' import GLOBALS %} -{% set proxy = salt['pillar.get']('manager:proxy') %} - -include: - - idstools.config - - idstools.sostatus - -so-idstools: - docker_container.running: - - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-idstools:{{ GLOBALS.so_version }} - - hostname: so-idstools - - user: socore - - networks: - - sobridge: - - ipv4_address: {{ DOCKER.containers['so-idstools'].ip }} - {% if proxy %} - - environment: - - http_proxy={{ proxy }} - - https_proxy={{ proxy }} - - no_proxy={{ salt['pillar.get']('manager:no_proxy') }} - {% if DOCKER.containers['so-idstools'].extra_env %} - {% for XTRAENV in DOCKER.containers['so-idstools'].extra_env %} - - {{ XTRAENV }} - {% endfor %} - {% endif %} - {% elif DOCKER.containers['so-idstools'].extra_env %} - - environment: - {% for XTRAENV in DOCKER.containers['so-idstools'].extra_env %} - - {{ XTRAENV }} - {% endfor %} - {% endif %} - - binds: - - /opt/so/conf/idstools/etc:/opt/so/idstools/etc:ro - - /opt/so/rules/nids/suri:/opt/so/rules/nids/suri:rw - - /nsm/rules/:/nsm/rules/:rw - {% if DOCKER.containers['so-idstools'].custom_bind_mounts %} - {% for BIND in DOCKER.containers['so-idstools'].custom_bind_mounts %} - - {{ BIND }} - {% endfor %} - {% endif %} - - extra_hosts: - - {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }} - {% if DOCKER.containers['so-idstools'].extra_hosts %} - {% for XTRAHOST in DOCKER.containers['so-idstools'].extra_hosts %} - - {{ XTRAHOST }} - {% endfor %} - {% endif %} - - watch: - - file: idstoolsetcsync - - file: idstools_so-rule-update - -delete_so-idstools_so-status.disabled: - file.uncomment: - - name: /opt/so/conf/so-status/so-status.conf - - regex: ^so-idstools$ - -so-rule-update: - cron.present: - - name: /usr/sbin/so-rule-update > /opt/so/log/idstools/download_cron.log 2>&1 - - identifier: so-rule-update - - user: root - - minute: '1' - - hour: '7' - -# order this last to give so-idstools container time to be ready -run_so-rule-update: - cmd.run: - - name: '/usr/sbin/so-rule-update > /opt/so/log/idstools/download_idstools_state.log 2>&1' - - require: - - docker_container: so-idstools - - onchanges: - - file: idstools_so-rule-update - - file: idstoolsetcsync - - file: synclocalnidsrules - - order: last - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/salt/idstools/etc/disable.conf b/salt/idstools/etc/disable.conf deleted file mode 100644 index 84144a495..000000000 --- a/salt/idstools/etc/disable.conf +++ /dev/null @@ -1,16 +0,0 @@ -{%- set disabled_sids = salt['pillar.get']('idstools:sids:disabled', {}) -%} -# idstools - disable.conf - -# Example of disabling a rule by signature ID (gid is optional). -# 1:2019401 -# 2019401 - -# Example of disabling a rule by regular expression. -# - All regular expression matches are case insensitive. -# re:hearbleed -# re:MS(0[7-9]|10)-\d+ -{%- if disabled_sids != None %} -{%- for sid in disabled_sids %} -{{ sid }} -{%- endfor %} -{%- endif %} \ No newline at end of file diff --git a/salt/idstools/etc/enable.conf b/salt/idstools/etc/enable.conf deleted file mode 100644 index 5da0bfc61..000000000 --- a/salt/idstools/etc/enable.conf +++ /dev/null @@ -1,16 +0,0 @@ -{%- set enabled_sids = salt['pillar.get']('idstools:sids:enabled', {}) -%} -# idstools-rulecat - enable.conf - -# Example of enabling a rule by signature ID (gid is optional). -# 1:2019401 -# 2019401 - -# Example of enabling a rule by regular expression. -# - All regular expression matches are case insensitive. -# re:hearbleed -# re:MS(0[7-9]|10)-\d+ -{%- if enabled_sids != None %} -{%- for sid in enabled_sids %} -{{ sid }} -{%- endfor %} -{%- endif %} \ No newline at end of file diff --git a/salt/idstools/etc/modify.conf b/salt/idstools/etc/modify.conf deleted file mode 100644 index 4ea75ada2..000000000 --- a/salt/idstools/etc/modify.conf +++ /dev/null @@ -1,12 +0,0 @@ -{%- set modify_sids = salt['pillar.get']('idstools:sids:modify', {}) -%} -# idstools-rulecat - modify.conf - -# Format: "" "" - -# Example changing the seconds for rule 2019401 to 3600. -#2019401 "seconds \d+" "seconds 3600" -{%- if modify_sids != None %} -{%- for sid in modify_sids %} -{{ sid }} -{%- endfor %} -{%- endif %} \ No newline at end of file diff --git a/salt/idstools/etc/rulecat.conf b/salt/idstools/etc/rulecat.conf deleted file mode 100644 index e4ec611db..000000000 --- a/salt/idstools/etc/rulecat.conf +++ /dev/null @@ -1,23 +0,0 @@ -{%- from 'vars/globals.map.jinja' import GLOBALS -%} -{%- from 'soc/merged.map.jinja' import SOCMERGED -%} ---suricata-version=7.0.3 ---merged=/opt/so/rules/nids/suri/all.rules ---output=/nsm/rules/detect-suricata/custom_temp ---local=/opt/so/rules/nids/suri/local.rules -{%- if GLOBALS.md_engine == "SURICATA" %} ---local=/opt/so/rules/nids/suri/extraction.rules ---local=/opt/so/rules/nids/suri/filters.rules -{%- endif %} ---url=http://{{ GLOBALS.manager }}:7788/suricata/emerging-all.rules ---disable=/opt/so/idstools/etc/disable.conf ---enable=/opt/so/idstools/etc/enable.conf ---modify=/opt/so/idstools/etc/modify.conf -{%- if SOCMERGED.config.server.modules.suricataengine.customRulesets %} - {%- for ruleset in SOCMERGED.config.server.modules.suricataengine.customRulesets %} - {%- if 'url' in ruleset %} ---url={{ ruleset.url }} - {%- elif 'file' in ruleset %} ---local={{ ruleset.file }} - {%- endif %} - {%- endfor %} -{%- endif %} diff --git a/salt/idstools/init.sls b/salt/idstools/init.sls deleted file mode 100644 index ac1d51717..000000000 --- a/salt/idstools/init.sls +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'idstools/map.jinja' import IDSTOOLSMERGED %} - -include: -{% if IDSTOOLSMERGED.enabled %} - - idstools.enabled -{% else %} - - idstools.disabled -{% endif %} diff --git a/salt/idstools/map.jinja b/salt/idstools/map.jinja deleted file mode 100644 index 97d12279b..000000000 --- a/salt/idstools/map.jinja +++ /dev/null @@ -1,7 +0,0 @@ -{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one - or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at - https://securityonion.net/license; you may not use this file except in compliance with the - Elastic License 2.0. #} - -{% import_yaml 'idstools/defaults.yaml' as IDSTOOLSDEFAULTS with context %} -{% set IDSTOOLSMERGED = salt['pillar.get']('idstools', IDSTOOLSDEFAULTS.idstools, merge=True) %} diff --git a/salt/idstools/rules/extraction.rules b/salt/idstools/rules/extraction.rules deleted file mode 100644 index 3ebbd41b1..000000000 --- a/salt/idstools/rules/extraction.rules +++ /dev/null @@ -1,26 +0,0 @@ -# Extract all PDF mime type -alert http any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100000; rev:1;) -alert smtp any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100001; rev:1;) -alert nfs any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100002; rev:1;) -alert smb any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100003; rev:1;) -# Extract EXE/DLL file types -alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100004; rev:1;) -alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100005; rev:1;) -alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100006; rev:1;) -alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100007; rev:1;) -alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100008; rev:1;) -alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100009; rev:1;) -alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100010; rev:1;) -alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100011; rev:1;) - -# Extract all Zip files -alert http any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100012; rev:1;) -alert smtp any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100013; rev:1;) -alert nfs any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100014; rev:1;) -alert smb any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100015; rev:1;) - -# Extract Word Docs -alert http any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100016; rev:1;) -alert smtp any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100017; rev:1;) -alert nfs any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100018; rev:1;) -alert smb any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100019; rev:1;) \ No newline at end of file diff --git a/salt/idstools/rules/filters.rules b/salt/idstools/rules/filters.rules deleted file mode 100644 index 051d1913f..000000000 --- a/salt/idstools/rules/filters.rules +++ /dev/null @@ -1,11 +0,0 @@ -# Start the filters at sid 1200000 -# Example of filtering out *google.com from being in the dns log. -#config dns any any -> any any (dns.query; content:"google.com"; config: logging disable, type tx, scope tx; sid:1200000;) -# Example of filtering out *google.com from being in the http log. -#config http any any -> any any (http.host; content:"google.com"; config: logging disable, type tx, scope tx; sid:1200001;) -# Example of filtering out someuseragent from being in the http log. -#config http any any -> any any (http.user_agent; content:"someuseragent"; config: logging disable, type tx, scope tx; sid:1200002;) -# Example of filtering out Google's certificate from being in the ssl log. -#config tls any any -> any any (tls.fingerprint; content:"4f:a4:5e:58:7e:d9:db:20:09:d7:b6:c7:ff:58:c4:7b:dc:3f:55:b4"; config: logging disable, type tx, scope tx; sid:1200003;) -# Example of filtering out a md5 of a file from being in the files log. -#config fileinfo any any -> any any (fileinfo.filemd5; content:"7a125dc69c82d5caf94d3913eecde4b5"; config: logging disable, type tx, scope tx; sid:1200004;) diff --git a/salt/idstools/rules/local.rules b/salt/idstools/rules/local.rules deleted file mode 100644 index ac11dfa58..000000000 --- a/salt/idstools/rules/local.rules +++ /dev/null @@ -1 +0,0 @@ -# Add your custom Suricata rules in this file. \ No newline at end of file diff --git a/salt/idstools/soc_idstools.yaml b/salt/idstools/soc_idstools.yaml deleted file mode 100644 index 4f7a53e91..000000000 --- a/salt/idstools/soc_idstools.yaml +++ /dev/null @@ -1,72 +0,0 @@ -idstools: - enabled: - description: Enables or disables the IDStools process which is used by the Detection system. - config: - oinkcode: - description: Enter your registration code or oinkcode for paid NIDS rulesets. - title: Registration Code - global: True - forcedType: string - helpLink: rules.html - ruleset: - description: 'Defines the ruleset you want to run. Options are ETOPEN or ETPRO. Once you have changed the ruleset here, you will need to wait for the rule update to take place (every 24 hours), or you can force the update by nagivating to Detections --> Options dropdown menu --> Suricata --> Full Update. WARNING! Changing the ruleset will remove all existing non-overlapping Suricata rules of the previous ruleset and their associated overrides. This removal cannot be undone.' - global: True - regex: ETPRO\b|ETOPEN\b - helpLink: rules.html - urls: - description: This is a list of additional rule download locations. This feature is currently disabled. - global: True - multiline: True - forcedType: "[]string" - readonly: True - helpLink: rules.html - sids: - disabled: - description: Contains the list of NIDS rules (or regex patterns) disabled across the grid. This setting is readonly; Use the Detections screen to disable rules. - global: True - multiline: True - forcedType: "[]string" - regex: \d*|re:.* - helpLink: managing-alerts.html - readonlyUi: True - advanced: true - enabled: - description: Contains the list of NIDS rules (or regex patterns) enabled across the grid. This setting is readonly; Use the Detections screen to enable rules. - global: True - multiline: True - forcedType: "[]string" - regex: \d*|re:.* - helpLink: managing-alerts.html - readonlyUi: True - advanced: true - modify: - description: Contains the list of NIDS rules (SID "REGEX_SEARCH_TERM" "REGEX_REPLACE_TERM"). This setting is readonly; Use the Detections screen to modify rules. - global: True - multiline: True - forcedType: "[]string" - helpLink: managing-alerts.html - readonlyUi: True - advanced: true - rules: - local__rules: - description: Contains the list of custom NIDS rules applied to the grid. This setting is readonly; Use the Detections screen to adjust rules. - file: True - global: True - advanced: True - title: Local Rules - helpLink: local-rules.html - readonlyUi: True - filters__rules: - description: If you are using Suricata for metadata, then you can set custom filters for that metadata here. - file: True - global: True - advanced: True - title: Filter Rules - helpLink: suricata.html - extraction__rules: - description: If you are using Suricata for metadata, then you can set a list of MIME types for file extraction here. - file: True - global: True - advanced: True - title: Extraction Rules - helpLink: suricata.html diff --git a/salt/idstools/sostatus.sls b/salt/idstools/sostatus.sls deleted file mode 100644 index 408b10742..000000000 --- a/salt/idstools/sostatus.sls +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} - -append_so-idstools_so-status.conf: - file.append: - - name: /opt/so/conf/so-status/so-status.conf - - text: so-idstools - - unless: grep -q so-idstools /opt/so/conf/so-status/so-status.conf - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/salt/idstools/sync_files.sls b/salt/idstools/sync_files.sls deleted file mode 100644 index cdacfaa74..000000000 --- a/salt/idstools/sync_files.sls +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - -idstoolsdir: - file.directory: - - name: /opt/so/conf/idstools/etc - - user: 939 - - group: 939 - - makedirs: True - -idstoolsetcsync: - file.recurse: - - name: /opt/so/conf/idstools/etc - - source: salt://idstools/etc - - user: 939 - - group: 939 - - template: jinja - -rulesdir: - file.directory: - - name: /opt/so/rules/nids/suri - - user: 939 - - group: 939 - - makedirs: True - -# Don't show changes because all.rules can be large -synclocalnidsrules: - file.recurse: - - name: /opt/so/rules/nids/suri/ - - source: salt://idstools/rules/ - - user: 939 - - group: 939 - - show_changes: False - - include_pat: 'E@.rules' diff --git a/salt/idstools/tools/sbin/so-idstools-restart b/salt/idstools/tools/sbin/so-idstools-restart deleted file mode 100755 index f2abbd0a5..000000000 --- a/salt/idstools/tools/sbin/so-idstools-restart +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - -/usr/sbin/so-restart idstools $1 diff --git a/salt/idstools/tools/sbin/so-idstools-start b/salt/idstools/tools/sbin/so-idstools-start deleted file mode 100755 index e17b5e521..000000000 --- a/salt/idstools/tools/sbin/so-idstools-start +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - -/usr/sbin/so-start idstools $1 diff --git a/salt/idstools/tools/sbin/so-idstools-stop b/salt/idstools/tools/sbin/so-idstools-stop deleted file mode 100755 index f2d188d06..000000000 --- a/salt/idstools/tools/sbin/so-idstools-stop +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - -/usr/sbin/so-stop idstools $1 diff --git a/salt/idstools/tools/sbin_jinja/so-rule-update b/salt/idstools/tools/sbin_jinja/so-rule-update deleted file mode 100755 index 9ac09ed15..000000000 --- a/salt/idstools/tools/sbin_jinja/so-rule-update +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -# if this script isn't already running -if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then - - . /usr/sbin/so-common - -{%- from 'vars/globals.map.jinja' import GLOBALS %} -{%- from 'idstools/map.jinja' import IDSTOOLSMERGED %} - -{%- set proxy = salt['pillar.get']('manager:proxy') %} -{%- set noproxy = salt['pillar.get']('manager:no_proxy', '') %} - -{%- if proxy %} -# Download the rules from the internet - export http_proxy={{ proxy }} - export https_proxy={{ proxy }} - export no_proxy="{{ noproxy }}" -{%- endif %} - - mkdir -p /nsm/rules/suricata - chown -R socore:socore /nsm/rules/suricata -{%- if not GLOBALS.airgap %} -# Download the rules from the internet -{%- if IDSTOOLSMERGED.config.ruleset == 'ETOPEN' %} - docker exec so-idstools idstools-rulecat -v --suricata-version 7.0.3 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force -{%- elif IDSTOOLSMERGED.config.ruleset == 'ETPRO' %} - docker exec so-idstools idstools-rulecat -v --suricata-version 7.0.3 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force --etpro={{ IDSTOOLSMERGED.config.oinkcode }} -{%- endif %} -{%- endif %} - - - argstr="" - for arg in "$@"; do - argstr="${argstr} \"${arg}\"" - done - - docker exec so-idstools /bin/bash -c "cd /opt/so/idstools/etc && idstools-rulecat --force ${argstr}" - -fi diff --git a/salt/logrotate/defaults.yaml b/salt/logrotate/defaults.yaml index 2f7247ff2..4df6713ef 100644 --- a/salt/logrotate/defaults.yaml +++ b/salt/logrotate/defaults.yaml @@ -1,15 +1,5 @@ logrotate: config: - /opt/so/log/idstools/*_x_log: - - daily - - rotate 14 - - missingok - - copytruncate - - compress - - create - - extension .log - - dateext - - dateyesterday /opt/so/log/nginx/*_x_log: - daily - rotate 14 diff --git a/salt/logrotate/soc_logrotate.yaml b/salt/logrotate/soc_logrotate.yaml index 56f879e4f..21b54755e 100644 --- a/salt/logrotate/soc_logrotate.yaml +++ b/salt/logrotate/soc_logrotate.yaml @@ -1,12 +1,5 @@ logrotate: config: - "/opt/so/log/idstools/*_x_log": - description: List of logrotate options for this file. - title: /opt/so/log/idstools/*.log - advanced: True - multiline: True - global: True - forcedType: "[]string" "/opt/so/log/nginx/*_x_log": description: List of logrotate options for this file. title: /opt/so/log/nginx/*.log diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 34ebdaeec..c91a7a793 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -603,16 +603,6 @@ function add_kratos_to_minion() { fi } -function add_idstools_to_minion() { - printf '%s\n'\ - "idstools:"\ - " enabled: True"\ - " " >> $PILLARFILE - if [ $? -ne 0 ]; then - log "ERROR" "Failed to add idstools configuration to $PILLARFILE" - return 1 - fi -} function add_elastic_fleet_package_registry_to_minion() { printf '%s\n'\ @@ -740,7 +730,6 @@ function createEVAL() { add_soc_to_minion || return 1 add_registry_to_minion || return 1 add_kratos_to_minion || return 1 - add_idstools_to_minion || return 1 add_elastic_fleet_package_registry_to_minion || return 1 } @@ -761,7 +750,6 @@ function createSTANDALONE() { add_soc_to_minion || return 1 add_registry_to_minion || return 1 add_kratos_to_minion || return 1 - add_idstools_to_minion || return 1 add_elastic_fleet_package_registry_to_minion || return 1 } @@ -778,7 +766,6 @@ function createMANAGER() { add_soc_to_minion || return 1 add_registry_to_minion || return 1 add_kratos_to_minion || return 1 - add_idstools_to_minion || return 1 add_elastic_fleet_package_registry_to_minion || return 1 } @@ -795,7 +782,6 @@ function createMANAGERSEARCH() { add_soc_to_minion || return 1 add_registry_to_minion || return 1 add_kratos_to_minion || return 1 - add_idstools_to_minion || return 1 add_elastic_fleet_package_registry_to_minion || return 1 } @@ -810,7 +796,6 @@ function createIMPORT() { add_soc_to_minion || return 1 add_registry_to_minion || return 1 add_kratos_to_minion || return 1 - add_idstools_to_minion || return 1 add_elastic_fleet_package_registry_to_minion || return 1 } @@ -895,7 +880,6 @@ function createMANAGERHYPE() { add_soc_to_minion || return 1 add_registry_to_minion || return 1 add_kratos_to_minion || return 1 - add_idstools_to_minion || return 1 add_elastic_fleet_package_registry_to_minion || return 1 } diff --git a/salt/salt/files/engines.conf b/salt/salt/files/engines.conf index 15d55e18f..8192ee201 100644 --- a/salt/salt/files/engines.conf +++ b/salt/salt/files/engines.conf @@ -6,30 +6,6 @@ engines: interval: 60 - pillarWatch: fpa: - - files: - - /opt/so/saltstack/local/pillar/idstools/soc_idstools.sls - - /opt/so/saltstack/local/pillar/idstools/adv_idstools.sls - pillar: idstools.config.ruleset - default: ETOPEN - actions: - from: - '*': - to: - '*': - - cmd.run: - cmd: /usr/sbin/so-rule-update - - files: - - /opt/so/saltstack/local/pillar/idstools/soc_idstools.sls - - /opt/so/saltstack/local/pillar/idstools/adv_idstools.sls - pillar: idstools.config.oinkcode - default: '' - actions: - from: - '*': - to: - '*': - - cmd.run: - cmd: /usr/sbin/so-rule-update - files: - /opt/so/saltstack/local/pillar/global/soc_global.sls - /opt/so/saltstack/local/pillar/global/adv_global.sls diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 0e3e50240..8b4708d38 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1554,12 +1554,72 @@ soc: disableRegex: [] enableRegex: [] failAfterConsecutiveErrorCount: 10 - communityRulesFile: /nsm/rules/suricata/emerging-all.rules rulesFingerprintFile: /opt/sensoroni/fingerprints/emerging-all.fingerprint stateFilePath: /opt/sensoroni/fingerprints/suricataengine.state integrityCheckFrequencySeconds: 1200 ignoredSidRanges: - '1100000-1101000' + rulesetSources: + default: + - name: Emerging-Threats + description: "Emerging Threats ruleset - To enable ET Pro, enter your license key below. Leave empty for ET Open (free) rules." + licenseKey: "" + enabled: true + sourceType: url + sourcePath: 'https://rules.emergingthreats.net/open/suricata/emerging.rules.tar.gz' + urlHash: "https://rules.emergingthreats.net/open/suricata/emerging.rules.tar.gz.md5" + license: "BSD" + excludeFiles: + - "*deleted*" + - "*retired*" + proxyURL: "" + proxyUsername: "" + proxyPassword: "" + proxyCACert: "" + insecureSkipVerify: false + readOnly: true + deleteUnreferenced: true + - name: local-rules + id: local-rules + description: "Local custom rules from files (*.rules) in a directory on the filesystem" + license: "custom" + sourceType: directory + sourcePath: /nsm/rules/local/ + readOnly: false + deleteUnreferenced: false + enabled: false + excludeFiles: + - "*backup*" + airgap: + - name: Emerging-Threats + description: "Emerging Threats ruleset - To enable ET Pro, enter your license key below. Leave empty for ET Open (free) rules." + licenseKey: "" + enabled: true + sourceType: url + sourcePath: 'https://rules.emergingthreats.net/open/suricata/emerging.rules.tar.gz' + urlHash: "https://rules.emergingthreats.net/open/suricata/emerging.rules.tar.gz.md5" + license: "BSD" + excludeFiles: + - "*deleted*" + - "*retired*" + proxyURL: "" + proxyUsername: "" + proxyPassword: "" + proxyCACert: "" + insecureSkipVerify: false + readOnly: true + deleteUnreferenced: true + - name: local-rules + id: local-rules + description: "Local custom rules from files (*.rules) in a directory on the filesystem" + license: "custom" + sourceType: directory + sourcePath: /nsm/rules/local/ + readOnly: false + deleteUnreferenced: false + enabled: false + excludeFiles: + - "*backup*" navigator: intervalMinutes: 30 outputPath: /opt/sensoroni/navigator diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 09e2c16a8..6dd7b71ae 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -27,7 +27,7 @@ so-soc: - /opt/so/conf/strelka:/opt/sensoroni/yara:rw - /opt/so/conf/sigma:/opt/sensoroni/sigma:rw - /opt/so/rules/elastalert/rules:/opt/sensoroni/elastalert:rw - - /opt/so/rules/nids/suri:/opt/sensoroni/nids:ro + - /opt/so/rules/nids/suri:/opt/sensoroni/nids:rw - /opt/so/conf/soc/fingerprints:/opt/sensoroni/fingerprints:rw - /nsm/soc/jobs:/opt/sensoroni/jobs:rw - /nsm/soc/uploads:/nsm/soc/uploads:rw diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index e053ce63f..b43ccaf1b 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -50,17 +50,74 @@ {% do SOCMERGED.config.server.modules.elastalertengine.update({'enabledSigmaRules': SOCMERGED.config.server.modules.elastalertengine.enabledSigmaRules.default}) %} {% endif %} -{# set elastalertengine.rulesRepos and strelkaengine.rulesRepos based on airgap or not #} +{# set elastalertengine.rulesRepos, strelkaengine.rulesRepos, and suricataengine.rulesetSources based on airgap or not #} {% if GLOBALS.airgap %} {% do SOCMERGED.config.server.modules.elastalertengine.update({'rulesRepos': SOCMERGED.config.server.modules.elastalertengine.rulesRepos.airgap}) %} {% do SOCMERGED.config.server.modules.strelkaengine.update({'rulesRepos': SOCMERGED.config.server.modules.strelkaengine.rulesRepos.airgap}) %} +{#% if SOCMERGED.config.server.modules.suricataengine.rulesetSources is mapping %#} +{% do SOCMERGED.config.server.modules.suricataengine.update({'rulesetSources': SOCMERGED.config.server.modules.suricataengine.rulesetSources.airgap}) %} +{#% endif %#} {% do SOCMERGED.config.server.update({'airgapEnabled': true}) %} {% else %} {% do SOCMERGED.config.server.modules.elastalertengine.update({'rulesRepos': SOCMERGED.config.server.modules.elastalertengine.rulesRepos.default}) %} {% do SOCMERGED.config.server.modules.strelkaengine.update({'rulesRepos': SOCMERGED.config.server.modules.strelkaengine.rulesRepos.default}) %} +{#% if SOCMERGED.config.server.modules.suricataengine.rulesetSources is mapping %#} +{% do SOCMERGED.config.server.modules.suricataengine.update({'rulesetSources': SOCMERGED.config.server.modules.suricataengine.rulesetSources.default}) %} +{#% endif %#} {% do SOCMERGED.config.server.update({'airgapEnabled': false}) %} {% endif %} + +{# Define the Detections custom ruleset that should always be present #} +{% set CUSTOM_RULESET = { + 'name': 'custom', + 'description': 'User-created custom rules created via the Detections module in the SOC UI', + 'sourceType': 'elasticsearch', + 'sourcePath': 'so_detection.ruleset:__custom__', + 'readOnly': false, + 'deleteUnreferenced': false, + 'license': 'Custom', + 'enabled': true +} %} + +{# Always append the custom ruleset to suricataengine.rulesetSources if not already present #} +{% if SOCMERGED.config.server.modules.suricataengine is defined and SOCMERGED.config.server.modules.suricataengine.rulesetSources is defined %} +{% if SOCMERGED.config.server.modules.suricataengine.rulesetSources is not mapping %} +{% set custom_names = SOCMERGED.config.server.modules.suricataengine.rulesetSources | selectattr('name', 'equalto', 'custom') | list %} +{% if custom_names | length == 0 %} +{% do SOCMERGED.config.server.modules.suricataengine.rulesetSources.append(CUSTOM_RULESET) %} +{% endif %} +{% endif %} +{% endif %} + +{# Transform Emerging-Threats ruleset based on license key #} +{% if SOCMERGED.config.server.modules.suricataengine is defined and SOCMERGED.config.server.modules.suricataengine.rulesetSources is defined %} +{% if SOCMERGED.config.server.modules.suricataengine.rulesetSources is not mapping %} +{% for ruleset in SOCMERGED.config.server.modules.suricataengine.rulesetSources %} +{% if ruleset.name == 'Emerging-Threats' %} +{% if ruleset.licenseKey and ruleset.licenseKey != '' %} +{# License key is defined - transform to ETPRO #} +{% do ruleset.update({ + 'name': 'ETPRO', + 'sourcePath': 'https://rules.emergingthreatspro.com/' ~ ruleset.licenseKey ~ '/suricata-7.0.3/etpro.rules.tar.gz', + 'urlHash': 'https://rules.emergingthreatspro.com/' ~ ruleset.licenseKey ~ '/suricata-7.0.3/etpro.rules.tar.gz.md5', + 'license': 'Commercial' + }) %} +{% else %} +{# No license key - explicitly set to ETOPEN #} +{% do ruleset.update({ + 'name': 'ETOPEN', + 'sourcePath': 'https://rules.emergingthreats.net/open/suricata-7.0.3/emerging.rules.tar.gz', + 'urlHash': 'https://rules.emergingthreats.net/open/suricata-7.0.3/emerging.rules.tar.gz.md5', + 'license': 'BSD' + }) %} +{% endif %} +{% endif %} +{% endfor %} +{% endif %} +{% endif %} + + {# set playbookRepos based on airgap or not #} {% if GLOBALS.airgap %} {% do SOCMERGED.config.server.modules.playbook.update({'playbookRepos': SOCMERGED.config.server.modules.playbook.playbookRepos.airgap}) %} diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index b292d1460..623df4ea3 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -543,6 +543,52 @@ soc: advanced: True forcedType: "[]string" helpLink: detections.html#rule-engine-status + rulesetSources: + default: &serulesetSources + description: "Ruleset sources for Suricata rules. Supports URL downloads and local directories. Refer to the linked documentation for details on how to configure this setting." + global: True + advanced: False + forcedType: "[]{}" + helpLink: suricata.html + syntax: json + uiElements: + - field: name + label: Ruleset Name (This will be the name of the ruleset in the UI) + required: True + readonly: True + - field: description + label: Description + - field: enabled + label: Enabled (If false, existing rules & overrides will be removed) + forcedType: bool + required: True + - field: licenseKey + label: License Key + required: False + - field: sourceType + label: Source Type + required: True + options: + - url + - directory + - field: sourcePath + label: Source Path (full url or directory path) + required: True + - field: excludeFiles + label: Exclude Files (list of file names to exclude, separated by commas) + required: False + - field: license + label: Ruleset License + required: True + - field: readOnly + label: Read Only + forcedType: bool + required: False + - field: deleteUnreferenced + label: Delete Unreferenced + forcedType: bool + required: False + airgap: *serulesetSources navigator: intervalMinutes: description: How often to generate the Navigator Layers. (minutes) diff --git a/salt/suricata/defaults.yaml b/salt/suricata/defaults.yaml index d819d1cf9..b4c615157 100644 --- a/salt/suricata/defaults.yaml +++ b/salt/suricata/defaults.yaml @@ -459,7 +459,7 @@ suricata: append: "yes" default-rule-path: /etc/suricata/rules rule-files: - - all.rules + - all-rulesets.rules classification-file: /etc/suricata/classification.config reference-config-file: /etc/suricata/reference.config threshold-file: /etc/suricata/threshold.conf diff --git a/salt/top.sls b/salt/top.sls index a75346462..c465307c0 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -74,7 +74,6 @@ base: - sensoroni - telegraf - firewall - - idstools - suricata.manager - healthcheck - elasticsearch @@ -106,7 +105,6 @@ base: - firewall - sensoroni - telegraf - - idstools - suricata.manager - healthcheck - elasticsearch @@ -142,7 +140,6 @@ base: - sensoroni - telegraf - backup.config_backup - - idstools - suricata.manager - elasticsearch - logstash @@ -177,7 +174,6 @@ base: - sensoroni - telegraf - backup.config_backup - - idstools - suricata.manager - elasticsearch - logstash @@ -208,7 +204,6 @@ base: - sensoroni - telegraf - firewall - - idstools - suricata.manager - pcap - elasticsearch diff --git a/setup/so-functions b/setup/so-functions index 522446be4..334dc4a0d 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -829,7 +829,6 @@ create_manager_pillars() { backup_pillar docker_pillar redis_pillar - idstools_pillar kratos_pillar hydra_pillar soc_pillar @@ -1295,11 +1294,6 @@ ls_heapsize() { } -idstools_pillar() { - title "Ading IDSTOOLS pillar options" - touch $adv_idstools_pillar_file -} - nginx_pillar() { title "Creating the NGINX pillar" [[ -z "$TESTING" ]] && return @@ -1475,7 +1469,7 @@ make_some_dirs() { mkdir -p $local_salt_dir/salt/firewall/portgroups mkdir -p $local_salt_dir/salt/firewall/ports - for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni soc docker zeek suricata nginx telegraf logstash soc manager kratos hydra idstools idh elastalert stig global kafka versionlock hypervisor vm; do + for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni soc docker zeek suricata nginx telegraf logstash soc manager kratos hydra idh elastalert stig global kafka versionlock hypervisor vm; do mkdir -p $local_salt_dir/pillar/$THEDIR touch $local_salt_dir/pillar/$THEDIR/adv_$THEDIR.sls touch $local_salt_dir/pillar/$THEDIR/soc_$THEDIR.sls diff --git a/setup/so-variables b/setup/so-variables index fc253df0a..a0d7aadc1 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -166,12 +166,6 @@ export hydra_pillar_file adv_hydra_pillar_file="$local_salt_dir/pillar/hydra/adv_hydra.sls" export adv_hydra_pillar_file -idstools_pillar_file="$local_salt_dir/pillar/idstools/soc_idstools.sls" -export idstools_pillar_file - -adv_idstools_pillar_file="$local_salt_dir/pillar/idstools/adv_idstools.sls" -export adv_idstools_pillar_file - nginx_pillar_file="$local_salt_dir/pillar/nginx/soc_nginx.sls" export nginx_pillar_file From 11518f6eea8b78753da9f0cc93c48ac9f70f94b7 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Tue, 11 Nov 2025 13:41:32 -0500 Subject: [PATCH 02/34] idstools removal refactor --- salt/manager/tools/sbin/soup | 123 +++++++++++++++++++++++ salt/soc/enabled.sls | 1 + salt/soc/soc_soc.yaml | 4 +- salt/suricata/config.sls | 11 +- salt/suricata/files/threshold.conf.jinja | 35 ------- 5 files changed, 131 insertions(+), 43 deletions(-) delete mode 100644 salt/suricata/files/threshold.conf.jinja diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index f32b6edf8..d6ad73d63 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -426,6 +426,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" == 2.4.160 ]] && up_to_2.4.170 [[ "$INSTALLEDVERSION" == 2.4.170 ]] && up_to_2.4.180 [[ "$INSTALLEDVERSION" == 2.4.180 ]] && up_to_2.4.190 + [[ "$INSTALLEDVERSION" == 2.4.190 ]] && up_to_2.4.200 true } @@ -457,6 +458,7 @@ postupgrade_changes() { [[ "$POSTVERSION" == 2.4.160 ]] && post_to_2.4.170 [[ "$POSTVERSION" == 2.4.170 ]] && post_to_2.4.180 [[ "$POSTVERSION" == 2.4.180 ]] && post_to_2.4.190 + [[ "$POSTVERSION" == 2.4.190 ]] && post_to_2.4.200 true } @@ -636,6 +638,11 @@ post_to_2.4.190() { POSTVERSION=2.4.190 } +post_to_2.4.200() { + echo "Nothing to apply" + POSTVERSION=2.4.200 +} + repo_sync() { echo "Sync the local repo." su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync." @@ -903,6 +910,13 @@ up_to_2.4.190() { INSTALLEDVERSION=2.4.190 } +up_to_2.4.200() { + echo "Migrating idstools config" + suricata_idstools_removal + + INSTALLEDVERSION=2.4.200 +} + add_hydra_pillars() { mkdir -p /opt/so/saltstack/local/pillar/hydra touch /opt/so/saltstack/local/pillar/hydra/soc_hydra.sls @@ -986,6 +1000,8 @@ rollover_index() { } suricata_idstools_migration() { + # For 2.4.70 + #Backup the pillars for idstools mkdir -p /nsm/backup/detections-migration/idstools rsync -av /opt/so/saltstack/local/pillar/idstools/* /nsm/backup/detections-migration/idstools @@ -1086,6 +1102,113 @@ playbook_migration() { echo "Playbook Migration is complete...." } +suricata_idstools_removal() { +# For SOUPs beginning with 2.4.200 + +# Create syncBlock file +cat > /opt/so/conf/soc/fingerprints/suricataengine.syncBlock << EOF +Suricata rulset sync is blocked until this file is removed. Make sure that you have manually added any custom Suricata rulesets via SOC config - review the documentation for more details: securityonion.net/docs +EOF + +# Track if we have custom configs +CUSTOM_CONFIGS_FOUND=0 + +# ETPRO Check +ETPRO=$(grep "--etpro" /usr/sbin/so-rule-update) +if [[ -n "$ETPRO" ]]; then + echo "Grid is using ETPRO." + # Add ETPRO yaml to SOC pillar file + if [[ $is_airgap -eq 0 ]]; then + #TODO /opt/so/saltstack/local/pillar/soc/soc_soc.sls + else + #TODO /opt/so/saltstack/local/pillar/soc/soc_soc.sls + fi +fi + +#idstools conf parse +RULECAT_CONF="/opt/so/conf/idstools/etc/rulecat.conf" + +# Parse RULECAT_CONF and check for custom configs +if [[ ! -f "$RULECAT_CONF" ]]; then + echo "Warning: $RULECAT_CONF not found - leaving syncBlock." + return 0 +fi + +echo "Parsing $RULECAT_CONF for custom configurations..." + +# Default values to check against +DEFAULT_URL="--url=http://MANAGER:7788/suricata/emerging-all.rules" +DEFAULT_DISABLE="--disable=/opt/so/idstools/etc/disable.conf" +DEFAULT_ENABLE="--enable=/opt/so/idstools/etc/enable.conf" +DEFAULT_MODIFY="--modify=/opt/so/idstools/etc/modify.conf" + +# Valid --local patterns +VALID_LOCAL_PATTERNS=( + "/opt/so/rules/nids/suri/local.rules" # 2/24 + "/opt/so/rules/nids/suri/extraction.rules" # 2/24 + "/opt/so/rules/nids/suri/filters.rules" # 2/24 + "/opt/so/rules/nids/extraction.rules" # 9/23 + "/opt/so/rules/nids/filters.rules" # 9/23 + "/opt/so/rules/nids/local.rules" # 8/23 + "/opt/so/rules/nids/sorules/extraction.rules" # 8/23 + "/opt/so/rules/nids/sorules/filters.rules" # 8/23 + ) + +# Parse each line in the config file +while IFS= read -r line; do + # Skip empty lines and comments + [[ -z "$line" || "$line" =~ ^[[:space:]]*# ]] && continue + + # Check for non-default --local parameter + if [[ "$line" =~ ^--local= ]]; then + local_path="${line#--local=}" + is_valid=0 + for pattern in "${VALID_LOCAL_PATTERNS[@]}"; do + if [[ "$local_path" == "$pattern" ]]; then + is_valid=1 + break + fi + done + if [[ $is_valid -eq 0 ]]; then + echo "Custom --local parameter detected: $line" + echo "Custom configuration found: $line" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock + return 0 + fi + fi + + # Check for non-default --url parameter (default contains 7788) + if [[ "$line" =~ ^--url= ]] && [[ ! "$line" =~ 7788 ]]; then + echo "Custom --url parameter detected: $line" + echo "Custom configuration found: $line" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock + return 0 + fi + + # Sanity checks for other parameters + if [[ "$line" =~ ^--disable= ]] && [[ "$line" != "$DEFAULT_DISABLE" ]]; then + echo "Custom --disable parameter detected: $line" + echo "Custom configuration found: $line" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock + return 0 + fi + + if [[ "$line" =~ ^--enable= ]] && [[ "$line" != "$DEFAULT_ENABLE" ]]; then + echo "Custom --enable parameter detected: $line" + echo "Custom configuration found: $line" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock + return 0 + fi + + if [[ "$line" =~ ^--modify= ]] && [[ "$line" != "$DEFAULT_MODIFY" ]]; then + echo "Custom --modify parameter detected: $line" + echo "Custom configuration found: $line" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock + return 0 + fi + +done < "$RULECAT_CONF" + +# If we reach here, no custom configs were found +echo "idstools migration completed successfully - removing Suricata engine syncBlock" +rm -f /opt/so/conf/soc/fingerprints/suricataengine.syncBlock +} + determine_elastic_agent_upgrade() { if [[ $is_airgap -eq 0 ]]; then update_elastic_agent_airgap diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 62f3f85ba..62873ebdd 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -26,6 +26,7 @@ so-soc: - /nsm/rules:/nsm/rules:rw - /opt/so/conf/strelka:/opt/sensoroni/yara:rw - /opt/so/conf/sigma:/opt/sensoroni/sigma:rw + - /opt/so/conf/suricata:/opt/sensoroni/suricata:rw - /opt/so/rules/elastalert/rules:/opt/sensoroni/elastalert:rw - /opt/so/rules/nids/suri:/opt/sensoroni/nids:rw - /opt/so/conf/soc/fingerprints:/opt/sensoroni/fingerprints:rw diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index bc67a5295..85e49def6 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -590,11 +590,11 @@ soc: label: Ruleset License required: True - field: readOnly - label: Read Only + label: Read Only (Prevents changes to the rule itself - can still be enabled/disabled/tuned) forcedType: bool required: False - field: deleteUnreferenced - label: Delete Unreferenced + label: Delete Unreferenced (Deletes rules that are no longer referenced by ruleset source) forcedType: bool required: False airgap: *serulesetSources diff --git a/salt/suricata/config.sls b/salt/suricata/config.sls index 00364f384..a43dd95a3 100644 --- a/salt/suricata/config.sls +++ b/salt/suricata/config.sls @@ -53,13 +53,15 @@ suridir: file.directory: - name: /opt/so/conf/suricata - user: 940 - - group: 940 + - group: 939 + - mode: 775 suriruledir: file.directory: - name: /opt/so/conf/suricata/rules - user: 940 - - group: 940 + - group: 939 + - mode: 775 - makedirs: True surilogdir: @@ -124,10 +126,7 @@ suriconfig: surithresholding: file.managed: - name: /opt/so/conf/suricata/threshold.conf - - source: salt://suricata/files/threshold.conf.jinja - - user: 940 - - group: 940 - - template: jinja + - replace: False suriclassifications: file.managed: diff --git a/salt/suricata/files/threshold.conf.jinja b/salt/suricata/files/threshold.conf.jinja deleted file mode 100644 index a439dad96..000000000 --- a/salt/suricata/files/threshold.conf.jinja +++ /dev/null @@ -1,35 +0,0 @@ -{% import_yaml 'suricata/thresholding/sids.yaml' as THRESHOLDING %} -{% if THRESHOLDING -%} - - {% for EACH_SID in THRESHOLDING -%} - {% for ACTIONS_LIST in THRESHOLDING[EACH_SID] -%} - {% for EACH_ACTION in ACTIONS_LIST -%} - - {%- if EACH_ACTION == 'threshold' %} -{{ EACH_ACTION }} gen_id {{ ACTIONS_LIST[EACH_ACTION].gen_id }}, sig_id {{ EACH_SID }}, type {{ ACTIONS_LIST[EACH_ACTION].type }}, track {{ ACTIONS_LIST[EACH_ACTION].track }}, count {{ ACTIONS_LIST[EACH_ACTION].count }}, seconds {{ ACTIONS_LIST[EACH_ACTION].seconds }} - - {%- elif EACH_ACTION == 'rate_filter' %} - {%- if ACTIONS_LIST[EACH_ACTION].new_action not in ['drop','reject'] %} -{{ EACH_ACTION }} gen_id {{ ACTIONS_LIST[EACH_ACTION].gen_id }}, sig_id {{ EACH_SID }}, track {{ ACTIONS_LIST[EACH_ACTION].track }}, count {{ ACTIONS_LIST[EACH_ACTION].count }}, seconds {{ ACTIONS_LIST[EACH_ACTION].seconds }}, new_action {{ ACTIONS_LIST[EACH_ACTION].new_action }}, timeout {{ ACTIONS_LIST[EACH_ACTION].timeout }} - {%- else %} -##### Security Onion does not support drop or reject actions for rate_filter -##### {{ EACH_ACTION }} gen_id {{ ACTIONS_LIST[EACH_ACTION].gen_id }}, sig_id {{ EACH_SID }}, track {{ ACTIONS_LIST[EACH_ACTION].track }}, count {{ ACTIONS_LIST[EACH_ACTION].count }}, seconds {{ ACTIONS_LIST[EACH_ACTION].seconds }}, new_action {{ ACTIONS_LIST[EACH_ACTION].new_action }}, timeout {{ ACTIONS_LIST[EACH_ACTION].timeout }} - {%- endif %} - - {%- elif EACH_ACTION == 'suppress' %} - {%- if ACTIONS_LIST[EACH_ACTION].track is defined %} -{{ EACH_ACTION }} gen_id {{ ACTIONS_LIST[EACH_ACTION].gen_id }}, sig_id {{ EACH_SID }}, track {{ ACTIONS_LIST[EACH_ACTION].track }}, ip {{ ACTIONS_LIST[EACH_ACTION].ip }} - {%- else %} -{{ EACH_ACTION }} gen_id {{ ACTIONS_LIST[EACH_ACTION].gen_id }}, sig_id {{ EACH_SID }} - {%- endif %} - - {%- endif %} - - {%- endfor %} - {%- endfor %} - {%- endfor %} - -{%- else %} -##### Navigate to suricata > thresholding > SIDS in SOC to define thresholding - -{%- endif %} From 13789bc56f7660763190f8f94be65253b205dcdd Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Tue, 11 Nov 2025 13:45:37 -0500 Subject: [PATCH 03/34] idstools removal refactor --- salt/manager/tools/sbin/soup | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index d6ad73d63..ed33481ee 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1120,8 +1120,10 @@ if [[ -n "$ETPRO" ]]; then # Add ETPRO yaml to SOC pillar file if [[ $is_airgap -eq 0 ]]; then #TODO /opt/so/saltstack/local/pillar/soc/soc_soc.sls + echo "TODO" else #TODO /opt/so/saltstack/local/pillar/soc/soc_soc.sls + echo "TODO" fi fi From 3a8a6bf5ff9eaa9c3d075539e14f0182c146e6b0 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Tue, 11 Nov 2025 14:12:51 -0500 Subject: [PATCH 04/34] idstools removal refactor --- salt/manager/tools/sbin/soup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index ed33481ee..f063314f9 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1114,8 +1114,9 @@ EOF CUSTOM_CONFIGS_FOUND=0 # ETPRO Check -ETPRO=$(grep "--etpro" /usr/sbin/so-rule-update) +ETPRO=$(grep "\--etpro" /usr/sbin/so-rule-update) if [[ -n "$ETPRO" ]]; then + ETPRO_KEY=$(echo "$ETPRO" | awk -F'--etpro=' '{print $2}' | awk '{print $1}') echo "Grid is using ETPRO." # Add ETPRO yaml to SOC pillar file if [[ $is_airgap -eq 0 ]]; then From 55bbbdb58d27cfbe15c233c3c71daa925d4bc723 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Tue, 11 Nov 2025 14:34:28 -0500 Subject: [PATCH 05/34] idstools removal refactor --- salt/manager/tools/sbin/soup | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index f063314f9..774ca0430 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1131,6 +1131,8 @@ fi #idstools conf parse RULECAT_CONF="/opt/so/conf/idstools/etc/rulecat.conf" +echo "Checking $RULECAT_CONF for custom configurations..." + # Parse RULECAT_CONF and check for custom configs if [[ ! -f "$RULECAT_CONF" ]]; then echo "Warning: $RULECAT_CONF not found - leaving syncBlock." From 1f24796eba263c238875d71e4e037aa980f02f5a Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Wed, 12 Nov 2025 08:48:47 -0500 Subject: [PATCH 06/34] Fix ETPRO check --- salt/manager/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 774ca0430..3fa8b8c56 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1114,7 +1114,7 @@ EOF CUSTOM_CONFIGS_FOUND=0 # ETPRO Check -ETPRO=$(grep "\--etpro" /usr/sbin/so-rule-update) +ETPRO=$(grep "\--etpro" /usr/sbin/so-rule-update || true) if [[ -n "$ETPRO" ]]; then ETPRO_KEY=$(echo "$ETPRO" | awk -F'--etpro=' '{print $2}' | awk '{print $1}') echo "Grid is using ETPRO." From 81d7c313af01ee750e1e25d51778e9ac4c65f08b Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Wed, 12 Nov 2025 11:11:01 -0500 Subject: [PATCH 07/34] remove dupe --- salt/suricata/config.sls | 6 ------ 1 file changed, 6 deletions(-) diff --git a/salt/suricata/config.sls b/salt/suricata/config.sls index beda362d3..3379697bc 100644 --- a/salt/suricata/config.sls +++ b/salt/suricata/config.sls @@ -10,12 +10,6 @@ {% from 'suricata/map.jinja' import SURICATAMERGED %} {% from 'bpf/suricata.map.jinja' import SURICATABPF, SURICATA_BPF_STATUS, SURICATA_BPF_CALC %} -suridir: - file.directory: - - name: /opt/so/conf/suricata - - user: 940 - - group: 940 - {% if GLOBALS.pcap_engine in ["SURICATA", "TRANSITION"] %} {% from 'bpf/pcap.map.jinja' import PCAPBPF, PCAP_BPF_STATUS, PCAP_BPF_CALC %} # BPF compilation and configuration From 573dded921696cf1d765e98d2a9332c303ab2828 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Thu, 13 Nov 2025 09:25:20 -0500 Subject: [PATCH 08/34] refactor to hash --- salt/manager/tools/sbin/soup | 207 +++++++++++++++++++---------------- 1 file changed, 110 insertions(+), 97 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 5e7d9fbc7..0fb83d73f 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -639,7 +639,9 @@ post_to_2.4.190() { } post_to_2.4.200() { - echo "Nothing to apply" + echo "Initiating Suricata idstools migration..." + suricata_idstools_removal_post + POSTVERSION=2.4.200 } @@ -911,8 +913,8 @@ up_to_2.4.190() { } up_to_2.4.200() { - echo "Migrating idstools config" - suricata_idstools_removal + echo "Backing up idstools config..." + suricata_idstools_removal_pre INSTALLEDVERSION=2.4.200 } @@ -1102,116 +1104,127 @@ playbook_migration() { echo "Playbook Migration is complete...." } -suricata_idstools_removal() { -# For SOUPs beginning with 2.4.200 +suricata_idstools_removal_pre() { +# For SOUPs beginning with 2.4.200 - pre SOUP checks # Create syncBlock file cat > /opt/so/conf/soc/fingerprints/suricataengine.syncBlock << EOF -Suricata rulset sync is blocked until this file is removed. Make sure that you have manually added any custom Suricata rulesets via SOC config - review the documentation for more details: securityonion.net/docs +Suricata ruleset sync is blocked until this file is removed. Make sure that you have manually added any custom Suricata rulesets via SOC config - review the documentation for more details: securityonion.net/docs EOF -# Track if we have custom configs -CUSTOM_CONFIGS_FOUND=0 +# TODO - backup custom rules & overrides +mkdir -p /nsm/backup/detections-migration/2-4-200 +cp /usr/sbin/so-rule-update /nsm/backup/detections-migration/2-4-200 +cp /opt/so/conf/idstools/etc/rulecat.conf /nsm/backup/detections-migration/2-4-200 -# ETPRO Check -ETPRO=$(grep "\--etpro" /usr/sbin/so-rule-update || true) -if [[ -n "$ETPRO" ]]; then - ETPRO_KEY=$(echo "$ETPRO" | awk -F'--etpro=' '{print $2}' | awk '{print $1}') - echo "Grid is using ETPRO." - # Add ETPRO yaml to SOC pillar file - if [[ $is_airgap -eq 0 ]]; then - #TODO /opt/so/saltstack/local/pillar/soc/soc_soc.sls - echo "TODO" - else - #TODO /opt/so/saltstack/local/pillar/soc/soc_soc.sls - echo "TODO" +} + +suricata_idstools_removal_post() { +# For SOUPs beginning with 2.4.200 - post SOUP checks + +echo "Checking idstools configuration for custom modifications..." + +# Normalize file content for consistent hashing +# Args: $1 - file path +normalize_file() { + local file="$1" + + if [[ ! -f "$file" ]]; then + echo "FILE_NOT_FOUND" + return 1 fi -fi -#idstools conf parse -RULECAT_CONF="/opt/so/conf/idstools/etc/rulecat.conf" + # Strip whitespace, normalize hostname, remove blank lines + sed -E \ + -e 's/^[[:space:]]+//; s/[[:space:]]+$//' \ + -e '/^$/d' \ + -e 's|--url=http://[^:]+:7788|--url=http://MANAGER:7788|' \ + "$file" +} -echo "Checking $RULECAT_CONF for custom configurations..." +# Hash normalized content +hash_file() { + local file="$1" -# Parse RULECAT_CONF and check for custom configs -if [[ ! -f "$RULECAT_CONF" ]]; then - echo "Warning: $RULECAT_CONF not found - leaving syncBlock." - return 0 -fi + local normalized=$(normalize_file "$file") -echo "Parsing $RULECAT_CONF for custom configurations..." + if [[ "$normalized" == "FILE_NOT_FOUND" ]]; then + echo "FILE_NOT_FOUND" + return 1 + fi -# Default values to check against -DEFAULT_URL="--url=http://MANAGER:7788/suricata/emerging-all.rules" -DEFAULT_DISABLE="--disable=/opt/so/idstools/etc/disable.conf" -DEFAULT_ENABLE="--enable=/opt/so/idstools/etc/enable.conf" -DEFAULT_MODIFY="--modify=/opt/so/idstools/etc/modify.conf" + echo -n "$normalized" | sha256sum | awk '{print $1}' +} -# Valid --local patterns -VALID_LOCAL_PATTERNS=( - "/opt/so/rules/nids/suri/local.rules" # 2/24 - "/opt/so/rules/nids/suri/extraction.rules" # 2/24 - "/opt/so/rules/nids/suri/filters.rules" # 2/24 - "/opt/so/rules/nids/extraction.rules" # 9/23 - "/opt/so/rules/nids/filters.rules" # 9/23 - "/opt/so/rules/nids/local.rules" # 8/23 - "/opt/so/rules/nids/sorules/extraction.rules" # 8/23 - "/opt/so/rules/nids/sorules/filters.rules" # 8/23 - ) +# Known-default hashes +KNOWN_SO_RULE_UPDATE_HASHES=( + "8f1fe1cb65c08aab78830315b952785c7ccdcc108c5c0474f427e29d4e39ee5f" # non-Airgap + "d23ac5a962c709dcb888103effb71444df72b46009b6c426e280dbfbc7d74d40" # Airgap +) -# Parse each line in the config file -while IFS= read -r line; do - # Skip empty lines and comments - [[ -z "$line" || "$line" =~ ^[[:space:]]*# ]] && continue - - # Check for non-default --local parameter - if [[ "$line" =~ ^--local= ]]; then - local_path="${line#--local=}" - is_valid=0 - for pattern in "${VALID_LOCAL_PATTERNS[@]}"; do - if [[ "$local_path" == "$pattern" ]]; then - is_valid=1 - break - fi - done - if [[ $is_valid -eq 0 ]]; then - echo "Custom --local parameter detected: $line" - echo "Custom configuration found: $line" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock - return 0 +KNOWN_RULECAT_CONF_HASHES=( + "17fc663a83b30d4ba43ac6643666b0c96343c5ea6ea833fe6a8362fe415b666b" # default +) + +# Check a config file against known hashes +# Args: $1 - file path, $2 - array name of known hashes +check_config_file() { + local file="$1" + local known_hashes_array="$2" + local file_display_name=$(basename "$file") + + if [[ ! -f "$file" ]]; then + echo "Warning: $file not found" + return 1 + fi + + echo "Hashing $file..." + local file_hash=$(hash_file "$file") + + if [[ "$file_hash" == "FILE_NOT_FOUND" ]]; then + echo "Warning: Could not read $file" + return 1 + fi + + echo " Hash: $file_hash" + + # Check if hash matches any known default + local match_found=0 + local -n known_hashes=$known_hashes_array + for known_hash in "${known_hashes[@]}"; do + if [[ "$file_hash" == "$known_hash" ]]; then + match_found=1 + echo " Matches known default configuration" + break fi - fi - - # Check for non-default --url parameter (default contains 7788) - if [[ "$line" =~ ^--url= ]] && [[ ! "$line" =~ 7788 ]]; then - echo "Custom --url parameter detected: $line" - echo "Custom configuration found: $line" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock - return 0 - fi - - # Sanity checks for other parameters - if [[ "$line" =~ ^--disable= ]] && [[ "$line" != "$DEFAULT_DISABLE" ]]; then - echo "Custom --disable parameter detected: $line" - echo "Custom configuration found: $line" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock - return 0 - fi - - if [[ "$line" =~ ^--enable= ]] && [[ "$line" != "$DEFAULT_ENABLE" ]]; then - echo "Custom --enable parameter detected: $line" - echo "Custom configuration found: $line" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock - return 0 - fi - - if [[ "$line" =~ ^--modify= ]] && [[ "$line" != "$DEFAULT_MODIFY" ]]; then - echo "Custom --modify parameter detected: $line" - echo "Custom configuration found: $line" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock - return 0 - fi - -done < "$RULECAT_CONF" + done -# If we reach here, no custom configs were found -echo "idstools migration completed successfully - removing Suricata engine syncBlock" -rm -f /opt/so/conf/soc/fingerprints/suricataengine.syncBlock + if [[ $match_found -eq 0 ]]; then + echo "Does not match known default - custom configuration detected" + echo "Custom $file_display_name detected (hash: $file_hash)" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock + return 1 + fi + + return 0 +} + +# Check so-rule-update and rulecat.conf +SO_RULE_UPDATE="/nsm/backup/detections-migration/2-4-200/so-rule-update" +RULECAT_CONF="/nsm/backup/detections-migration/2-4-200/rulecat.conf" + +custom_found=0 + +check_config_file "$SO_RULE_UPDATE" "KNOWN_SO_RULE_UPDATE_HASHES" || custom_found=1 +check_config_file "$RULECAT_CONF" "KNOWN_RULECAT_CONF_HASHES" || custom_found=1 + +# If no custom configs found, remove syncBlock +if [[ $custom_found -eq 0 ]]; then + echo "idstools migration completed successfully - removing Suricata engine syncBlock" + rm -f /opt/so/conf/soc/fingerprints/suricataengine.syncBlock +else + echo "Custom idstools configuration detected - syncBlock remains in place" + echo "Review /opt/so/conf/soc/fingerprints/suricataengine.syncBlock for details" +fi } determine_elastic_agent_upgrade() { From 37b3fd9b7b26a4e4267a0b2cbe07d8a7fd8bc8ed Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Thu, 13 Nov 2025 10:41:12 -0500 Subject: [PATCH 09/34] add detections backup --- salt/manager/tools/sbin/soup | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 0fb83d73f..cb5ec65d4 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1112,11 +1112,35 @@ cat > /opt/so/conf/soc/fingerprints/suricataengine.syncBlock << EOF Suricata ruleset sync is blocked until this file is removed. Make sure that you have manually added any custom Suricata rulesets via SOC config - review the documentation for more details: securityonion.net/docs EOF -# TODO - backup custom rules & overrides +# Backup custom rules & overrides mkdir -p /nsm/backup/detections-migration/2-4-200 cp /usr/sbin/so-rule-update /nsm/backup/detections-migration/2-4-200 cp /opt/so/conf/idstools/etc/rulecat.conf /nsm/backup/detections-migration/2-4-200 +if [[ -f /opt/so/conf/soc/so-detections-backup.py ]]; then + python3 /opt/so/conf/soc/so-detections-backup.py + + # Verify backup by comparing counts + echo "Verifying detection overrides backup..." + es_override_count=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -k -L \ + "https://localhost:9200/so-detection/_count" \ + -H "Content-Type: application/json" \ + -d '{"query": {"bool": {"must": [{"exists": {"field": "so_detection.overrides"}}]}}}' | jq -r '.count') + + backup_override_count=$(find /nsm/backup/detections/repo/*/overrides -type f 2>/dev/null | wc -l) + + echo " Elasticsearch overrides: $es_override_count" + echo " Backed up overrides: $backup_override_count" + + if [[ "$es_override_count" -eq "$backup_override_count" ]]; then + echo " Override backup verified successfully" + else + echo " Warning: Override counts do not match" + fi +else + echo "SOC Detections backup script not found, skipping detection backup" +fi + } suricata_idstools_removal_post() { From b2606b6094c2c387db32502b6d55931b65035468 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Thu, 13 Nov 2025 14:10:51 -0500 Subject: [PATCH 10/34] fix perms --- salt/manager/tools/sbin/soup | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index cb5ec65d4..fae574cae 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1108,6 +1108,8 @@ suricata_idstools_removal_pre() { # For SOUPs beginning with 2.4.200 - pre SOUP checks # Create syncBlock file +install -d -o 939 -g 939 -m 755 /opt/so/conf/soc/fingerprints +install -o 939 -g 939 -m 644 /dev/null /opt/so/conf/soc/fingerprints/suricataengine.syncBlock cat > /opt/so/conf/soc/fingerprints/suricataengine.syncBlock << EOF Suricata ruleset sync is blocked until this file is removed. Make sure that you have manually added any custom Suricata rulesets via SOC config - review the documentation for more details: securityonion.net/docs EOF From f047677d8a2c4df46fc67ea87f93e766d34a332c Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Fri, 14 Nov 2025 09:03:08 -0500 Subject: [PATCH 11/34] Check correct files --- salt/manager/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index fae574cae..64f1880d7 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1235,8 +1235,8 @@ check_config_file() { } # Check so-rule-update and rulecat.conf -SO_RULE_UPDATE="/nsm/backup/detections-migration/2-4-200/so-rule-update" -RULECAT_CONF="/nsm/backup/detections-migration/2-4-200/rulecat.conf" +SO_RULE_UPDATE="/usr/sbin/so-rule-update" +RULECAT_CONF="/opt/so/conf/idstools/etc/rulecat.conf" custom_found=0 From 431e5abf89a609247815d2384f1cdacf1601b5cf Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Fri, 14 Nov 2025 09:39:33 -0500 Subject: [PATCH 12/34] Extract ETPRO key if found --- salt/manager/tools/sbin/soup | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 64f1880d7..84b4c7903 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1134,10 +1134,15 @@ if [[ -f /opt/so/conf/soc/so-detections-backup.py ]]; then echo " Elasticsearch overrides: $es_override_count" echo " Backed up overrides: $backup_override_count" - if [[ "$es_override_count" -eq "$backup_override_count" ]]; then - echo " Override backup verified successfully" + if [[ "$es_override_count" -gt 0 ]]; then + if [[ "$backup_override_count" -gt 0 ]]; then + echo " Override backup verified successfully" + else + echo " Error: Elasticsearch has $es_override_count overrides but backup has 0 files" + exit 1 + fi else - echo " Warning: Override counts do not match" + echo " No overrides to backup" fi else echo "SOC Detections backup script not found, skipping detection backup" @@ -1228,6 +1233,15 @@ check_config_file() { if [[ $match_found -eq 0 ]]; then echo "Does not match known default - custom configuration detected" echo "Custom $file_display_name detected (hash: $file_hash)" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock + + # If this is so-rule-update, check for ETPRO key + if [[ "$file_display_name" == "so-rule-update" ]]; then + etpro_key=$(grep -oP '\-\-etpro=\K[0-9a-fA-F]+' "$file" 2>/dev/null || true) + if [[ -n "$etpro_key" ]]; then + echo "ETPRO key found: $etpro_key" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock + fi + fi + return 1 fi From af7f7d0728eb7e537c08f45836d79682e69d8f39 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Mon, 17 Nov 2025 12:00:08 -0500 Subject: [PATCH 13/34] Fix file paths --- salt/soc/enabled.sls | 3 ++- salt/suricata/config.sls | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 62873ebdd..0319c6c81 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -28,7 +28,8 @@ so-soc: - /opt/so/conf/sigma:/opt/sensoroni/sigma:rw - /opt/so/conf/suricata:/opt/sensoroni/suricata:rw - /opt/so/rules/elastalert/rules:/opt/sensoroni/elastalert:rw - - /opt/so/rules/nids/suri:/opt/sensoroni/nids:rw + - /opt/so/saltstack/local/salt/suricata/rules:/opt/sensoroni/suricata/rules:rw + - /opt/so/saltstack/local/salt/suricata/files:/opt/sensoroni/suricata/threshold:rw - /opt/so/conf/soc/fingerprints:/opt/sensoroni/fingerprints:rw - /nsm/soc/jobs:/opt/sensoroni/jobs:rw - /nsm/soc/uploads:/nsm/soc/uploads:rw diff --git a/salt/suricata/config.sls b/salt/suricata/config.sls index 3379697bc..685aa66e7 100644 --- a/salt/suricata/config.sls +++ b/salt/suricata/config.sls @@ -158,7 +158,10 @@ suriconfig: surithresholding: file.managed: - name: /opt/so/conf/suricata/threshold.conf - - replace: False + - source: salt://suricata/files/threshold.conf + - user: 940 + - group: 940 + - contents: 'This file is managed by Security Onion. Do not modify by hand.' suriclassifications: file.managed: From 1b55642c868287bb2bb49d9adbb022bcb9c586a5 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Tue, 18 Nov 2025 09:58:14 -0500 Subject: [PATCH 14/34] Refactor rules location --- salt/allowed_states.map.jinja | 1 - salt/suricata/config.sls | 8 +++----- salt/suricata/enabled.sls | 2 +- salt/suricata/manager.sls | 30 ------------------------------ salt/suricata/rules/PLACEHOLDER | 0 salt/top.sls | 5 ----- 6 files changed, 4 insertions(+), 42 deletions(-) delete mode 100644 salt/suricata/manager.sls create mode 100644 salt/suricata/rules/PLACEHOLDER diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index c41573522..2393f92d7 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -38,7 +38,6 @@ 'hydra', 'elasticfleet', 'elastic-fleet-package-registry', - 'suricata.manager', 'utility' ] %} diff --git a/salt/suricata/config.sls b/salt/suricata/config.sls index 685aa66e7..c7c687bae 100644 --- a/salt/suricata/config.sls +++ b/salt/suricata/config.sls @@ -90,7 +90,7 @@ suridir: suriruledir: file.directory: - - name: /opt/so/conf/suricata/rules + - name: /opt/so/rules/suricata - user: 940 - group: 939 - mode: 775 @@ -118,12 +118,10 @@ suridatadir: - mode: 770 - makedirs: True -# salt:// would resolve to /opt/so/rules/nids because of the defined file_roots and -# not existing under /opt/so/saltstack/local/salt or /opt/so/saltstack/default/salt surirulesync: file.recurse: - - name: /opt/so/conf/suricata/rules/ - - source: salt://suri/ + - name: /opt/so/rules/suricata/ + - source: salt://suricata/rules/ - user: 940 - group: 940 - show_changes: False diff --git a/salt/suricata/enabled.sls b/salt/suricata/enabled.sls index 34e9f2e4c..1576a0629 100644 --- a/salt/suricata/enabled.sls +++ b/salt/suricata/enabled.sls @@ -36,7 +36,7 @@ so-suricata: - /opt/so/conf/suricata/suricata.yaml:/etc/suricata/suricata.yaml:ro - /opt/so/conf/suricata/threshold.conf:/etc/suricata/threshold.conf:ro - /opt/so/conf/suricata/classification.config:/etc/suricata/classification.config:ro - - /opt/so/conf/suricata/rules:/etc/suricata/rules:ro + - /opt/so/rules/suricata:/etc/suricata/rules:ro - /opt/so/log/suricata/:/var/log/suricata/:rw - /nsm/suricata/:/nsm/:rw - /nsm/suricata/extracted:/var/log/suricata//filestore:rw diff --git a/salt/suricata/manager.sls b/salt/suricata/manager.sls deleted file mode 100644 index 3d5183556..000000000 --- a/salt/suricata/manager.sls +++ /dev/null @@ -1,30 +0,0 @@ -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls in allowed_states %} - -surilocaldir: - file.directory: - - name: /opt/so/saltstack/local/salt/suricata - - user: socore - - group: socore - - makedirs: True - -ruleslink: - file.symlink: - - name: /opt/so/saltstack/local/salt/suricata/rules - - user: socore - - group: socore - - target: /opt/so/rules/nids/suri - -refresh_salt_master_fileserver_suricata_ruleslink: - salt.runner: - - name: fileserver.update - - onchanges: - - file: ruleslink - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/salt/suricata/rules/PLACEHOLDER b/salt/suricata/rules/PLACEHOLDER new file mode 100644 index 000000000..e69de29bb diff --git a/salt/top.sls b/salt/top.sls index 613878860..d80806564 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -74,7 +74,6 @@ base: - sensoroni - telegraf - firewall - - suricata.manager - healthcheck - elasticsearch - elastic-fleet-package-registry @@ -105,7 +104,6 @@ base: - firewall - sensoroni - telegraf - - suricata.manager - healthcheck - elasticsearch - logstash @@ -140,7 +138,6 @@ base: - sensoroni - telegraf - backup.config_backup - - suricata.manager - elasticsearch - logstash - redis @@ -174,7 +171,6 @@ base: - sensoroni - telegraf - backup.config_backup - - suricata.manager - elasticsearch - logstash - redis @@ -204,7 +200,6 @@ base: - sensoroni - telegraf - firewall - - suricata.manager - pcap - elasticsearch - elastic-fleet-package-registry From 148ef7ef21dd40a6008d9a5e59afd4fa4f318f2b Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Tue, 18 Nov 2025 11:57:30 -0500 Subject: [PATCH 15/34] add default ruleset --- salt/soc/defaults.yaml | 8 ++++++++ salt/suricata/config.sls | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 65cdd385d..53cbb10e1 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1586,6 +1586,14 @@ soc: insecureSkipVerify: false readOnly: true deleteUnreferenced: true + - name: ABUSECH-SSLBL + deleteUnreferenced: true + description: 'Abuse.ch SSL Blacklist' + enabled: false + license: CC0-1.0 + readOnly: true + sourcePath: https://sslbl.abuse.ch/blacklist/sslblacklist_tls_cert.tar.gz + sourceType: url - name: local-rules id: local-rules description: "Local custom rules from files (*.rules) in a directory on the filesystem" diff --git a/salt/suricata/config.sls b/salt/suricata/config.sls index c7c687bae..7ce605e0b 100644 --- a/salt/suricata/config.sls +++ b/salt/suricata/config.sls @@ -159,7 +159,7 @@ surithresholding: - source: salt://suricata/files/threshold.conf - user: 940 - group: 940 - - contents: 'This file is managed by Security Onion. Do not modify by hand.' + - onlyif: salt://suricata/files/threshold.conf suriclassifications: file.managed: From ced3af818c8b18c1a6961b4b1c4b57680c2e26a4 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Tue, 25 Nov 2025 13:51:50 -0500 Subject: [PATCH 16/34] Refactor for Airgap --- salt/manager/init.sls | 1 + salt/soc/config.sls | 34 +++++++++++- salt/soc/defaults.yaml | 46 ++++++++++++---- ...tections_custom_repo_template_readme.jinja | 55 +++++++++++++++++++ salt/soc/merged.map.jinja | 11 ++++ salt/suricata/config.sls | 14 +++++ salt/suricata/files/so_extraction.rules | 26 +++++++++ salt/suricata/files/so_filters.rules | 12 ++++ 8 files changed, 188 insertions(+), 11 deletions(-) create mode 100644 salt/suricata/files/so_extraction.rules create mode 100644 salt/suricata/files/so_filters.rules diff --git a/salt/manager/init.sls b/salt/manager/init.sls index f59c33652..da829c1ce 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -206,6 +206,7 @@ git_config_set_safe_dirs: - multivar: - /nsm/rules/custom-local-repos/local-sigma - /nsm/rules/custom-local-repos/local-yara + - /nsm/rules/custom-local-repos/local-suricata - /nsm/securityonion-resources - /opt/so/conf/soc/ai_summary_repos/securityonion-resources - /nsm/airgap-resources/playbooks diff --git a/salt/soc/config.sls b/salt/soc/config.sls index 78a495e0a..7e2beefa0 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -215,7 +215,6 @@ socsensoronirepos: - mode: 775 - makedirs: True - create_custom_local_yara_repo_template: git.present: - name: /nsm/rules/custom-local-repos/local-yara @@ -249,6 +248,39 @@ add_readme_custom_local_sigma_repo_template: - context: repo_type: "sigma" +create_custom_local_suricata_repo_template: + git.present: + - name: /nsm/rules/custom-local-repos/local-suricata + - bare: False + - force: True + +add_readme_custom_local_suricata_repo_template: + file.managed: + - name: /nsm/rules/custom-local-repos/local-suricata/README + - source: salt://soc/files/soc/detections_custom_repo_template_readme.jinja + - user: 939 + - group: 939 + - template: jinja + - context: + repo_type: "suricata" + +etpro_airgap_folder: + file.directory: + - name: /nsm/rules/custom-local-repos/local-etpro-suricata + - user: 939 + - group: 939 + - makedirs: True + +add_readme_etpro_airgap_template: + file.managed: + - name: /nsm/rules/custom-local-repos/local-etpro-suricata/README + - source: salt://soc/files/soc/detections_custom_repo_template_readme.jinja + - user: 939 + - group: 939 + - template: jinja + - context: + repo_type: "suricata-etpro" + socore_own_custom_repos: file.directory: - name: /nsm/rules/custom-local-repos/ diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 53cbb10e1..685d0744c 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1595,16 +1595,29 @@ soc: sourcePath: https://sslbl.abuse.ch/blacklist/sslblacklist_tls_cert.tar.gz sourceType: url - name: local-rules - id: local-rules - description: "Local custom rules from files (*.rules) in a directory on the filesystem" + description: "Local rules from files (*.rules) in a directory on the filesystem" license: "custom" sourceType: directory - sourcePath: /nsm/rules/local/ + sourcePath: /nsm/rules/custom-local-repos/local-suricata readOnly: false deleteUnreferenced: false + enabled: true + - name: SO_FILTERS + deleteUnreferenced: true + description: Filter rules for when Suricata is set as the metadata engine enabled: false - excludeFiles: - - "*backup*" + license: Elastic-2.0 + readOnly: true + sourcePath: /nsm/rules/suricata/so_filters.rules + sourceType: directory + - name: SO_EXTRACTIONS + description: Extraction rules for when Suricata is set as the metadata engine + deleteUnreferenced: true + enabled: false + license: Elastic-2.0 + readOnly: true + sourcePath: /nsm/rules/suricata/so_extraction.rules + sourceType: directory airgap: - name: Emerging-Threats description: "Emerging Threats ruleset - To enable ET Pro, enter your license key below. Leave empty for ET Open (free) rules." @@ -1625,16 +1638,29 @@ soc: readOnly: true deleteUnreferenced: true - name: local-rules - id: local-rules - description: "Local custom rules from files (*.rules) in a directory on the filesystem" + description: "Local rules from files (*.rules) in a directory on the filesystem" license: "custom" sourceType: directory - sourcePath: /nsm/rules/local/ + sourcePath: /nsm/rules/custom-local-repos/local-suricata readOnly: false deleteUnreferenced: false + enabled: true + - name: SO_FILTERS + deleteUnreferenced: true + description: Filter rules for when Suricata is set as the metadata engine enabled: false - excludeFiles: - - "*backup*" + license: Elastic-2.0 + readOnly: true + sourcePath: /nsm/rules/suricata/so_filters.rules + sourceType: directory + - name: SO_EXTRACTIONS + description: Extraction rules for when Suricata is set as the metadata engine + deleteUnreferenced: true + enabled: false + license: Elastic-2.0 + readOnly: true + sourcePath: /nsm/rules/suricata/so_extraction.rules + sourceType: directory navigator: intervalMinutes: 30 outputPath: /opt/sensoroni/navigator diff --git a/salt/soc/files/soc/detections_custom_repo_template_readme.jinja b/salt/soc/files/soc/detections_custom_repo_template_readme.jinja index 228a467bf..060b8ec6e 100644 --- a/salt/soc/files/soc/detections_custom_repo_template_readme.jinja +++ b/salt/soc/files/soc/detections_custom_repo_template_readme.jinja @@ -45,6 +45,61 @@ Finally, commit it: The next time the Strelka / YARA engine syncs, the new rule should be imported If there are errors, review the sync log to troubleshoot further. +{% elif repo_type == 'suricata' %} +# Suricata Local Custom Rules Repository + +This folder has already been initialized as a git repo +and your Security Onion grid is configured to import any Suricata rule files found here. + +Just add your rule file and commit it. + +For example: + +** Note: If this is your first time making changes to this repo, you may run into the following error: + +fatal: detected dubious ownership in repository at '/nsm/rules/custom-local-repos/local-suricata' +To add an exception for this directory, call: + git config --global --add safe.directory /nsm/rules/custom-local-repos/local-suricata + +This means that the user you are running commands as does not match the user that is used for this git repo (socore). +You will need to make sure your rule files are accessible to the socore user, so either su to socore +or add the exception and then chown the rule files later. + +Also, you will be asked to set some configuration: +``` +Author identity unknown +*** Please tell me who you are. +Run + git config --global user.email "you@example.com" + git config --global user.name "Your Name" +to set your account's default identity. +Omit --global to set the identity only in this repository. +``` + +Run these commands, ommitting the `--global`. + +With that out of the way: + +First, create the rule file with a .rules extension: +`vi my_custom_rules.rules` + +Next, use git to stage the new rule to be committed: +`git add my_custom_rules.rules` + +Finally, commit it: +`git commit -m "Initial commit of my_custom_rule.rules"` + +The next time the Suricata engine syncs, the new rule/s should be imported +If there are errors, review the sync log to troubleshoot further. + +{% elif repo_type == 'suricata-etpro' %} +# Suricata ETPRO - Airgap + +This folder has been initialized for use with ETPRO during Airgap deployment. + +Just add your ETPRO rule/s file to this folder and the Suricata engine will import them. + +If there are errors, review the sync log to troubleshoot further. {% elif repo_type == 'sigma' %} # Sigma Local Custom Rules Repository diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index b43ccaf1b..cd23a17f3 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -90,6 +90,17 @@ {% endif %} {% endif %} +{# Enable SO_FILTERS and SO_EXTRACTIONS when Suricata is the metadata engine #} +{% if SOCMERGED.config.server.modules.suricataengine is defined and SOCMERGED.config.server.modules.suricataengine.rulesetSources is defined %} +{% if SOCMERGED.config.server.modules.suricataengine.rulesetSources is not mapping %} +{% for ruleset in SOCMERGED.config.server.modules.suricataengine.rulesetSources %} +{% if ruleset.name in ['SO_FILTERS', 'SO_EXTRACTIONS'] and GLOBALS.md_engine == 'SURICATA' %} +{% do ruleset.update({'enabled': true}) %} +{% endif %} +{% endfor %} +{% endif %} +{% endif %} + {# Transform Emerging-Threats ruleset based on license key #} {% if SOCMERGED.config.server.modules.suricataengine is defined and SOCMERGED.config.server.modules.suricataengine.rulesetSources is defined %} {% if SOCMERGED.config.server.modules.suricataengine.rulesetSources is not mapping %} diff --git a/salt/suricata/config.sls b/salt/suricata/config.sls index 7ce605e0b..46f5b1d7e 100644 --- a/salt/suricata/config.sls +++ b/salt/suricata/config.sls @@ -126,6 +126,20 @@ surirulesync: - group: 940 - show_changes: False +suriextractionrules: + file.managed: + - name: /nsm/rules/suricata/so_extraction.rules + - source: salt://suricata/files/so_extraction.rules + - user: 939 + - group: 939 + +surifiltersrules: + file.managed: + - name: /nsm/rules/suricata/so_filters.rules + - source: salt://suricata/files/so_filters.rules + - user: 939 + - group: 939 + surilogscript: file.managed: - name: /usr/local/bin/surilogcompress diff --git a/salt/suricata/files/so_extraction.rules b/salt/suricata/files/so_extraction.rules new file mode 100644 index 000000000..d43812144 --- /dev/null +++ b/salt/suricata/files/so_extraction.rules @@ -0,0 +1,26 @@ +# Extract all PDF mime type +alert http any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100000; rev:1;) +alert smtp any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100001; rev:1;) +alert nfs any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100002; rev:1;) +alert smb any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100003; rev:1;) +# Extract EXE/DLL file types +alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100004; rev:1;) +alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100005; rev:1;) +alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100006; rev:1;) +alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100007; rev:1;) +alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100008; rev:1;) +alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100009; rev:1;) +alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100010; rev:1;) +alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100011; rev:1;) + +# Extract all Zip files +alert http any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100012; rev:1;) +alert smtp any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100013; rev:1;) +alert nfs any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100014; rev:1;) +alert smb any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100015; rev:1;) + +# Extract Word Docs +alert http any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100016; rev:1;) +alert smtp any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100017; rev:1;) +alert nfs any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100018; rev:1;) +alert smb any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100019; rev:1;) diff --git a/salt/suricata/files/so_filters.rules b/salt/suricata/files/so_filters.rules new file mode 100644 index 000000000..c49eaec26 --- /dev/null +++ b/salt/suricata/files/so_filters.rules @@ -0,0 +1,12 @@ +# Start the filters at sid 1200000 +# Example of filtering out *google.com from being in the dns log. +#config dns any any -> any any (dns.query; content:"google.com"; config: logging disable, type tx, scope tx; sid:1200000;) +# Example of filtering out *google.com from being in the http log. +#config http any any -> any any (http.host; content:"google.com"; config: logging disable, type tx, scope tx; sid:1200001;) +# Example of filtering out someuseragent from being in the http log. +#config http any any -> any any (http.user_agent; content:"someuseragent"; config: logging disable, type tx, scope tx; sid:1200002;) +# Example of filtering out Google's certificate from being in the ssl log. +#config tls any any -> any any (tls.fingerprint; content:"4f:a4:5e:58:7e:d9:db:20:09:d7:b6:c7:ff:58:c4:7b:dc:3f:55:b4"; config: logging disable, type tx, scope tx; sid:1200003;) +# Example of filtering out a md5 of a file from being in the files log. +#config fileinfo any any -> any any (fileinfo.filemd5; content:"7a125dc69c82d5caf94d3913eecde4b5"; config: logging disable, type tx, scope tx; sid:1200004;) + From 1284150382791b919c0ebd415a6da4f55d546563 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Thu, 27 Nov 2025 08:39:19 -0500 Subject: [PATCH 17/34] Move to manager init --- salt/manager/init.sls | 22 ++++++++++++++++++++++ salt/suricata/config.sls | 14 -------------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/salt/manager/init.sls b/salt/manager/init.sls index da829c1ce..cf97a6f0b 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -211,6 +211,28 @@ git_config_set_safe_dirs: - /opt/so/conf/soc/ai_summary_repos/securityonion-resources - /nsm/airgap-resources/playbooks - /opt/so/conf/soc/playbooks + +surinsmrulesdir: + file.directory: + - name: /nsm/rules/suricata + - user: 939 + - group: 939 + - makedirs: True + +suriextractionrules: + file.managed: + - name: /nsm/rules/suricata/so_extraction.rules + - source: salt://suricata/files/so_extraction.rules + - user: 939 + - group: 939 + +surifiltersrules: + file.managed: + - name: /nsm/rules/suricata/so_filters.rules + - source: salt://suricata/files/so_filters.rules + - user: 939 + - group: 939 + {% else %} {{sls}}_state_not_allowed: diff --git a/salt/suricata/config.sls b/salt/suricata/config.sls index 46f5b1d7e..7ce605e0b 100644 --- a/salt/suricata/config.sls +++ b/salt/suricata/config.sls @@ -126,20 +126,6 @@ surirulesync: - group: 940 - show_changes: False -suriextractionrules: - file.managed: - - name: /nsm/rules/suricata/so_extraction.rules - - source: salt://suricata/files/so_extraction.rules - - user: 939 - - group: 939 - -surifiltersrules: - file.managed: - - name: /nsm/rules/suricata/so_filters.rules - - source: salt://suricata/files/so_filters.rules - - user: 939 - - group: 939 - surilogscript: file.managed: - name: /usr/local/bin/surilogcompress From 89a9106d79e002747e9e526d9046f61ea5d7107b Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Sat, 29 Nov 2025 15:17:28 -0500 Subject: [PATCH 18/34] Add context --- salt/manager/tools/sbin/soup | 9 +++++---- salt/soc/merged.map.jinja | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index f55395691..ba954ca6f 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1234,11 +1234,12 @@ check_config_file() { echo "Does not match known default - custom configuration detected" echo "Custom $file_display_name detected (hash: $file_hash)" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock - # If this is so-rule-update, check for ETPRO key + # If this is so-rule-update, check for ETPRO license code and write out to the syncBlock file + # If ETPRO is enabled, the license code already exists in the so-rule-update script, this is just making it easier to migrate if [[ "$file_display_name" == "so-rule-update" ]]; then - etpro_key=$(grep -oP '\-\-etpro=\K[0-9a-fA-F]+' "$file" 2>/dev/null || true) - if [[ -n "$etpro_key" ]]; then - echo "ETPRO key found: $etpro_key" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock + etpro_code=$(grep -oP '\-\-etpro=\K[0-9a-fA-F]+' "$file" 2>/dev/null || true) + if [[ -n "$etpro_code" ]]; then + echo "ETPRO code found: $etpro_code" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock fi fi diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index cd23a17f3..e1532462c 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -108,6 +108,7 @@ {% if ruleset.name == 'Emerging-Threats' %} {% if ruleset.licenseKey and ruleset.licenseKey != '' %} {# License key is defined - transform to ETPRO #} +{# Engine Version is hardcoded in the URL - this does not change often: https://community.emergingthreats.net/t/supported-engines/71 #} {% do ruleset.update({ 'name': 'ETPRO', 'sourcePath': 'https://rules.emergingthreatspro.com/' ~ ruleset.licenseKey ~ '/suricata-7.0.3/etpro.rules.tar.gz', From 87477ae4f66903d0fcf08a744bd74365b0b2af47 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Sat, 29 Nov 2025 15:40:10 -0500 Subject: [PATCH 19/34] Removed uneeded bind --- salt/soc/enabled.sls | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 0319c6c81..62e673ffc 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -26,7 +26,6 @@ so-soc: - /nsm/rules:/nsm/rules:rw - /opt/so/conf/strelka:/opt/sensoroni/yara:rw - /opt/so/conf/sigma:/opt/sensoroni/sigma:rw - - /opt/so/conf/suricata:/opt/sensoroni/suricata:rw - /opt/so/rules/elastalert/rules:/opt/sensoroni/elastalert:rw - /opt/so/saltstack/local/salt/suricata/rules:/opt/sensoroni/suricata/rules:rw - /opt/so/saltstack/local/salt/suricata/files:/opt/sensoroni/suricata/threshold:rw From 65c96b2edf07d05a43317786e5afd6e45b2c1c26 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Sat, 29 Nov 2025 16:27:22 -0500 Subject: [PATCH 20/34] Add error handling --- salt/manager/tools/sbin/soup | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index ba954ca6f..af0222414 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1124,10 +1124,17 @@ if [[ -f /opt/so/conf/soc/so-detections-backup.py ]]; then # Verify backup by comparing counts echo "Verifying detection overrides backup..." - es_override_count=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -k -L \ - "https://localhost:9200/so-detection/_count" \ - -H "Content-Type: application/json" \ - -d '{"query": {"bool": {"must": [{"exists": {"field": "so_detection.overrides"}}]}}}' | jq -r '.count') + es_override_count=$(/sbin/so-elasticsearch-query 'so-detection/_count' \ + -H 'Content-Type: application/json' \ + -d '{"query": {"bool": {"must": [{"exists": {"field": "so_detection.overrides"}}]}}}' | jq -r '.count') || { + echo " Error: Failed to query Elasticsearch for override count" + exit 1 + } + + if [[ ! "$es_override_count" =~ ^[0-9]+$ ]]; then + echo " Error: Invalid override count from Elasticsearch: '$es_override_count'" + exit 1 + fi backup_override_count=$(find /nsm/backup/detections/repo/*/overrides -type f 2>/dev/null | wc -l) From e96cfd35f70b115435f494933f45d1f4fbe25408 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Sat, 29 Nov 2025 17:00:51 -0500 Subject: [PATCH 21/34] Refactor for simplicity --- salt/manager/tools/sbin/soup | 61 +++++++++++++----------------------- 1 file changed, 22 insertions(+), 39 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index af0222414..f2e584bf6 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1162,36 +1162,22 @@ suricata_idstools_removal_post() { echo "Checking idstools configuration for custom modifications..." -# Normalize file content for consistent hashing +# Normalize and hash file content for consistent comparison # Args: $1 - file path -normalize_file() { +# Outputs: SHA256 hash to stdout +# Returns: 0 on success, 1 on failure +hash_normalized_file() { local file="$1" - if [[ ! -f "$file" ]]; then - echo "FILE_NOT_FOUND" + if [[ ! -r "$file" ]]; then return 1 fi - # Strip whitespace, normalize hostname, remove blank lines sed -E \ -e 's/^[[:space:]]+//; s/[[:space:]]+$//' \ -e '/^$/d' \ -e 's|--url=http://[^:]+:7788|--url=http://MANAGER:7788|' \ - "$file" -} - -# Hash normalized content -hash_file() { - local file="$1" - - local normalized=$(normalize_file "$file") - - if [[ "$normalized" == "FILE_NOT_FOUND" ]]; then - echo "FILE_NOT_FOUND" - return 1 - fi - - echo -n "$normalized" | sha256sum | awk '{print $1}' + "$file" | sha256sum | awk '{print $1}' } # Known-default hashes @@ -1213,47 +1199,44 @@ check_config_file() { if [[ ! -f "$file" ]]; then echo "Warning: $file not found" + echo "$file_display_name not found - manual verification required" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock return 1 fi echo "Hashing $file..." - local file_hash=$(hash_file "$file") - - if [[ "$file_hash" == "FILE_NOT_FOUND" ]]; then + local file_hash + if ! file_hash=$(hash_normalized_file "$file"); then echo "Warning: Could not read $file" + echo "$file_display_name not readable - manual verification required" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock return 1 fi echo " Hash: $file_hash" # Check if hash matches any known default - local match_found=0 local -n known_hashes=$known_hashes_array for known_hash in "${known_hashes[@]}"; do if [[ "$file_hash" == "$known_hash" ]]; then - match_found=1 echo " Matches known default configuration" - break + return 0 fi done - if [[ $match_found -eq 0 ]]; then - echo "Does not match known default - custom configuration detected" - echo "Custom $file_display_name detected (hash: $file_hash)" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock + # No match - custom configuration detected + echo "Does not match known default - custom configuration detected" + echo "Custom $file_display_name detected (hash: $file_hash)" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock - # If this is so-rule-update, check for ETPRO license code and write out to the syncBlock file - # If ETPRO is enabled, the license code already exists in the so-rule-update script, this is just making it easier to migrate - if [[ "$file_display_name" == "so-rule-update" ]]; then - etpro_code=$(grep -oP '\-\-etpro=\K[0-9a-fA-F]+' "$file" 2>/dev/null || true) - if [[ -n "$etpro_code" ]]; then - echo "ETPRO code found: $etpro_code" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock - fi + # If this is so-rule-update, check for ETPRO license code and write out to the syncBlock file + # If ETPRO is enabled, the license code already exists in the so-rule-update script, this is just making it easier to migrate + if [[ "$file_display_name" == "so-rule-update" ]]; then + local etpro_code + etpro_code=$(grep -oP '\-\-etpro=\K[0-9a-fA-F]+' "$file" 2>/dev/null) || true + if [[ -n "$etpro_code" ]]; then + echo "ETPRO code found: $etpro_code" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock fi - - return 1 fi - return 0 + return 1 } # Check so-rule-update and rulecat.conf From bda83a47a2f2f45af528e5da823b76aef34603a4 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Sat, 29 Nov 2025 17:45:22 -0500 Subject: [PATCH 22/34] Remove header --- salt/manager/tools/sbin/soup | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index f2e584bf6..59690c0bf 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1125,7 +1125,6 @@ if [[ -f /opt/so/conf/soc/so-detections-backup.py ]]; then # Verify backup by comparing counts echo "Verifying detection overrides backup..." es_override_count=$(/sbin/so-elasticsearch-query 'so-detection/_count' \ - -H 'Content-Type: application/json' \ -d '{"query": {"bool": {"must": [{"exists": {"field": "so_detection.overrides"}}]}}}' | jq -r '.count') || { echo " Error: Failed to query Elasticsearch for override count" exit 1 From 999f83ce57cac79bb8b4a0dbb4daed4cfd83c6c1 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Mon, 1 Dec 2025 14:21:58 -0500 Subject: [PATCH 23/34] Create dir earlier --- salt/suricata/config.sls | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/salt/suricata/config.sls b/salt/suricata/config.sls index 7ce605e0b..59ae376dc 100644 --- a/salt/suricata/config.sls +++ b/salt/suricata/config.sls @@ -22,6 +22,14 @@ suriPCAPbpfcompilationfailure: {% endif %} {% endif %} +suridir: + file.directory: + - name: /opt/so/conf/suricata + - user: 940 + - group: 939 + - mode: 775 + - makedirs: True + # BPF applied to all of Suricata - alerts/metadata/pcap suribpf: file.managed: @@ -81,13 +89,6 @@ suricata_sbin_jinja: - file_mode: 755 - template: jinja -suridir: - file.directory: - - name: /opt/so/conf/suricata - - user: 940 - - group: 939 - - mode: 775 - suriruledir: file.directory: - name: /opt/so/rules/suricata From 8abd4c9c78a8caaa258c7837cfc89244e0f974f3 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Tue, 2 Dec 2025 12:42:15 -0500 Subject: [PATCH 24/34] Remove idstools files --- salt/manager/tools/sbin/soup | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 59690c0bf..eb424ba72 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1255,6 +1255,24 @@ else echo "Custom idstools configuration detected - syncBlock remains in place" echo "Review /opt/so/conf/soc/fingerprints/suricataengine.syncBlock for details" fi + +echo "Cleaning up idstools" +echo "Stopping and removing the idstools container..." +if [ -n "$(docker ps -q -f name=^so-idstools$)" ]; then + image_name=$(docker ps -a --filter name=^so-idstools$ --format '{{.Image}}' 2>/dev/null || true) + docker stop so-idstools || echo "Warning: failed to stop so-idstools container" + docker rm so-idstools || echo "Warning: failed to remove so-idstools container" + + if [[ -n "$image_name" ]]; then + echo "Removing idstools image: $image_name" + docker rmi "$image_name" || echo "Warning: failed to remove image $image_name" + fi +fi + +echo "Removing idstools symlink and scripts..." +rm /opt/so/saltstack/local/salt/suricata/rules +rm -rf /usr/sbin/so-idstools* + } determine_elastic_agent_upgrade() { From 52f70dc49ae87b3f1f064c1091fd132c4afbc1ba Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Tue, 2 Dec 2025 17:40:30 -0500 Subject: [PATCH 25/34] Cleanup idstools --- salt/manager/tools/sbin/soup | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index eb424ba72..2efc1603a 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1272,6 +1272,8 @@ fi echo "Removing idstools symlink and scripts..." rm /opt/so/saltstack/local/salt/suricata/rules rm -rf /usr/sbin/so-idstools* +sed -i '/^so-idstools$/d' /opt/so/conf/so-status/so-status.conf +so-yaml.py removelistitem /etc/salt/master file_roots.base /opt/so/rules/nids } From 23575fdf6c4173ae29c00ee5480d5bb10583f0d2 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Tue, 2 Dec 2025 19:19:57 -0500 Subject: [PATCH 26/34] edit actual file --- salt/manager/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 2efc1603a..67262dcb0 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1272,7 +1272,7 @@ fi echo "Removing idstools symlink and scripts..." rm /opt/so/saltstack/local/salt/suricata/rules rm -rf /usr/sbin/so-idstools* -sed -i '/^so-idstools$/d' /opt/so/conf/so-status/so-status.conf +sed '/^so-idstools$/d' /opt/so/conf/so-status/so-status.conf so-yaml.py removelistitem /etc/salt/master file_roots.base /opt/so/rules/nids } From 41b3ac75540e59169effa32cc278e8504a19a2ba Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Tue, 2 Dec 2025 19:58:56 -0500 Subject: [PATCH 27/34] Backup salt master config --- salt/manager/tools/sbin/soup | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 67262dcb0..6cf5921b5 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1272,7 +1272,10 @@ fi echo "Removing idstools symlink and scripts..." rm /opt/so/saltstack/local/salt/suricata/rules rm -rf /usr/sbin/so-idstools* -sed '/^so-idstools$/d' /opt/so/conf/so-status/so-status.conf +sed -i '/^#\?so-idstools$/d' /opt/so/conf/so-status/so-status.conf + +# Backup the salt master config before editing it +cp /etc/salt/master /nsm/backup/detections-migration/2-4-200 so-yaml.py removelistitem /etc/salt/master file_roots.base /opt/so/rules/nids } From 822c411e83730aec9939c4fd45ec6455bc15aff2 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 2 Dec 2025 21:24:24 -0500 Subject: [PATCH 28/34] Update version to 2.4.0-delta --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 86df31761..09e15369f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.200 +2.4.0-delta From f15a39c1535463c1ae2c5ca2ef45b1fbef9c7911 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Wed, 3 Dec 2025 11:24:04 -0500 Subject: [PATCH 29/34] Add historical hashes --- salt/common/tools/sbin_jinja/so-import-pcap | 2 +- salt/manager/tools/sbin/soup | 33 ++++++++++++++++++--- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/salt/common/tools/sbin_jinja/so-import-pcap b/salt/common/tools/sbin_jinja/so-import-pcap index b630df015..9171c4bc6 100755 --- a/salt/common/tools/sbin_jinja/so-import-pcap +++ b/salt/common/tools/sbin_jinja/so-import-pcap @@ -85,7 +85,7 @@ function suricata() { docker run --rm \ -v /opt/so/conf/suricata/suricata.yaml:/etc/suricata/suricata.yaml:ro \ -v /opt/so/conf/suricata/threshold.conf:/etc/suricata/threshold.conf:ro \ - -v /opt/so/conf/suricata/rules:/etc/suricata/rules:ro \ + -v /opt/so/rules/suricata/:/etc/suricata/rules:ro \ -v ${LOG_PATH}:/var/log/suricata/:rw \ -v ${NSM_PATH}/:/nsm/:rw \ -v "$PCAP:/input.pcap:ro" \ diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 6cf5921b5..2ffed6af3 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1179,14 +1179,39 @@ hash_normalized_file() { "$file" | sha256sum | awk '{print $1}' } -# Known-default hashes +# Known-default hashes for so-rule-update (ETOPEN ruleset) KNOWN_SO_RULE_UPDATE_HASHES=( - "8f1fe1cb65c08aab78830315b952785c7ccdcc108c5c0474f427e29d4e39ee5f" # non-Airgap - "d23ac5a962c709dcb888103effb71444df72b46009b6c426e280dbfbc7d74d40" # Airgap + # 2.4.100+ (suricata 7.0.3, non-airgap) + "5fbd067ced86c8ec72ffb7e1798aa624123b536fb9d78f4b3ad8d3b45db1eae7" # 2.4.100-2.4.190 non-Airgap + # 2.4.90+ airgap (same for 2.4.90 and 2.4.100+) + "61f632c55791338c438c071040f1490066769bcce808b595b5cc7974a90e653a" # 2.4.90+ Airgap + # 2.4.90 (suricata 6.0, non-airgap, comment inside proxy block) + "0380ec52a05933244ab0f0bc506576e1d838483647b40612d5fe4b378e47aedd" # 2.4.90 non-Airgap + # 2.4.10-2.4.80 (suricata 6.0, non-airgap, comment outside proxy block) + "b6e4d1b5a78d57880ad038a9cd2cc6978aeb2dd27d48ea1a44dd866a2aee7ff4" # 2.4.10-2.4.80 non-Airgap + # 2.4.10-2.4.80 airgap + "b20146526ace2b142fde4664f1386a9a1defa319b3a1d113600ad33a1b037dad" # 2.4.10-2.4.80 Airgap + # 2.4.5 and earlier (no pidof check, non-airgap) + "d04f5e4015c348133d28a7840839e82d60009781eaaa1c66f7f67747703590dc" # 2.4.5 non-Airgap ) +# Known-default hashes for rulecat.conf KNOWN_RULECAT_CONF_HASHES=( - "17fc663a83b30d4ba43ac6643666b0c96343c5ea6ea833fe6a8362fe415b666b" # default + # 2.4.100+ (suricata 7.0.3) + "302e75dca9110807f09ade2eec3be1fcfc8b2bf6cf2252b0269bb72efeefe67e" # 2.4.100-2.4.190 without SURICATA md_engine + "8029b7718c324a9afa06a5cf180afde703da1277af4bdd30310a6cfa3d6398cb" # 2.4.100-2.4.190 with SURICATA md_engine + # 2.4.80-2.4.90 (suricata 6.0, with --suricata-version and --output) + "4d8b318e6950a6f60b02f307cf27c929efd39652990c1bd0c8820aa8a307e1e7" # 2.4.80-2.4.90 without SURICATA md_engine + "a1ddf264c86c4e91c81c5a317f745a19466d4311e4533ec3a3c91fed04c11678" # 2.4.80-2.4.90 with SURICATA md_engine + # 2.4.50-2.4.70 (/suri/ path, no --suricata-version) + "86e3afb8d0f00c62337195602636864c98580a13ca9cc85029661a539deae6ae" # 2.4.50-2.4.70 without SURICATA md_engine + "5a97604ca5b820a10273a2d6546bb5e00c5122ca5a7dfe0ba0bfbce5fc026f4b" # 2.4.50-2.4.70 with SURICATA md_engine + # 2.4.20-2.4.40 (/nids/ path without /suri/) + "d098ea9ecd94b5cca35bf33543f8ea8f48066a0785221fabda7fef43d2462c29" # 2.4.20-2.4.40 without SURICATA md_engine + "9dbc60df22ae20d65738ba42e620392577857038ba92278e23ec182081d191cd" # 2.4.20-2.4.40 with SURICATA md_engine + # 2.4.5-2.4.10 (/sorules/ path for extraction/filters) + "490f6843d9fca759ee74db3ada9c702e2440b8393f2cfaf07bbe41aaa6d955c3" # 2.4.5-2.4.10 with SURICATA md_engine + # Note: 2.4.5-2.4.10 without SURICATA md_engine has same hash as 2.4.20-2.4.40 without SURICATA md_engine ) # Check a config file against known hashes From 9304513ce82d7afb85ca5b156ef35f3b338bbf47 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Thu, 4 Dec 2025 12:26:13 -0500 Subject: [PATCH 30/34] Add support for suricata rules load status --- salt/suricata/config.sls | 8 +++++++ salt/suricata/cron/so-suricata-rulestats | 30 ++++++++++++++++++++++++ salt/suricata/enabled.sls | 12 ++++++++++ salt/telegraf/defaults.yaml | 4 ++++ salt/telegraf/scripts/surirules.sh | 30 ++++++++++++++++++++++++ 5 files changed, 84 insertions(+) create mode 100644 salt/suricata/cron/so-suricata-rulestats create mode 100644 salt/telegraf/scripts/surirules.sh diff --git a/salt/suricata/config.sls b/salt/suricata/config.sls index 59ae376dc..2a4a051cf 100644 --- a/salt/suricata/config.sls +++ b/salt/suricata/config.sls @@ -178,6 +178,14 @@ so-suricata-eve-clean: - template: jinja - source: salt://suricata/cron/so-suricata-eve-clean +so-suricata-rulestats: + file.managed: + - name: /usr/sbin/so-suricata-rulestats + - user: root + - group: root + - mode: 755 + - source: salt://suricata/cron/so-suricata-rulestats + {% else %} {{sls}}_state_not_allowed: diff --git a/salt/suricata/cron/so-suricata-rulestats b/salt/suricata/cron/so-suricata-rulestats new file mode 100644 index 000000000..95b51c58a --- /dev/null +++ b/salt/suricata/cron/so-suricata-rulestats @@ -0,0 +1,30 @@ +#!/bin/bash +# +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# Query Suricata for ruleset stats and reload time, write to JSON file for Telegraf to consume + +OUTFILE="/opt/so/log/suricata/rulestats.json" +SURICATASC="docker exec so-suricata /opt/suricata/bin/suricatasc" +SOCKET="/var/run/suricata/suricata-command.socket" + +query() { + timeout 10 $SURICATASC -c "$1" "$SOCKET" 2>/dev/null +} + +STATS=$(query "ruleset-stats") +RELOAD=$(query "ruleset-reload-time") + +if echo "$STATS" | jq -e '.return == "OK"' > /dev/null 2>&1; then + LOADED=$(echo "$STATS" | jq -r '.message[0].rules_loaded') + FAILED=$(echo "$STATS" | jq -r '.message[0].rules_failed') + LAST_RELOAD=$(echo "$RELOAD" | jq -r '.message[0].last_reload') + + jq -n --argjson loaded "$LOADED" --argjson failed "$FAILED" --arg reload "$LAST_RELOAD" \ + '{rules_loaded: $loaded, rules_failed: $failed, last_reload: $reload, return: "OK"}' > "$OUTFILE" +else + echo '{"return":"FAIL"}' > "$OUTFILE" +fi diff --git a/salt/suricata/enabled.sls b/salt/suricata/enabled.sls index 1576a0629..ec521abb3 100644 --- a/salt/suricata/enabled.sls +++ b/salt/suricata/enabled.sls @@ -90,6 +90,18 @@ clean_suricata_eve_files: - month: '*' - dayweek: '*' +# Add rulestats cron - runs every minute to query Suricata for rule load status +suricata_rulestats: + cron.present: + - name: /usr/sbin/so-suricata-rulestats > /dev/null 2>&1 + - identifier: suricata_rulestats + - user: root + - minute: '*' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' + {% else %} {{sls}}_state_not_allowed: diff --git a/salt/telegraf/defaults.yaml b/salt/telegraf/defaults.yaml index 79ad9008d..c0a67b0ca 100644 --- a/salt/telegraf/defaults.yaml +++ b/salt/telegraf/defaults.yaml @@ -21,6 +21,7 @@ telegraf: - sostatus.sh - stenoloss.sh - suriloss.sh + - surirules.sh - zeekcaptureloss.sh - zeekloss.sh standalone: @@ -36,6 +37,7 @@ telegraf: - sostatus.sh - stenoloss.sh - suriloss.sh + - surirules.sh - zeekcaptureloss.sh - zeekloss.sh - features.sh @@ -81,6 +83,7 @@ telegraf: - sostatus.sh - stenoloss.sh - suriloss.sh + - surirules.sh - zeekcaptureloss.sh - zeekloss.sh - features.sh @@ -95,6 +98,7 @@ telegraf: - sostatus.sh - stenoloss.sh - suriloss.sh + - surirules.sh - zeekcaptureloss.sh - zeekloss.sh idh: diff --git a/salt/telegraf/scripts/surirules.sh b/salt/telegraf/scripts/surirules.sh new file mode 100644 index 000000000..b38d5df26 --- /dev/null +++ b/salt/telegraf/scripts/surirules.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# Read Suricata ruleset stats from JSON file written by so-suricata-rulestats cron job +# JSON format: {"rules_loaded":45879,"rules_failed":1,"last_reload":"2025-12-04T14:10:57+0000","return":"OK"} +# or on failure: {"return":"FAIL"} + +# if this script isn't already running +if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then + + STATSFILE="/var/log/suricata/rulestats.json" + + # Check file exists, is less than 90 seconds old, and has valid data + if [ -f "$STATSFILE" ] && [ $(($(date +%s) - $(stat -c %Y "$STATSFILE"))) -lt 90 ] && jq -e '.return == "OK" and .rules_loaded != null and .rules_failed != null' "$STATSFILE" > /dev/null 2>&1; then + LOADED=$(jq -r '.rules_loaded' "$STATSFILE") + FAILED=$(jq -r '.rules_failed' "$STATSFILE") + RELOAD_TIME=$(jq -r '.last_reload // ""' "$STATSFILE") + + echo "surirules loaded=${LOADED}i,failed=${FAILED}i,reload_time=\"${RELOAD_TIME}\",status=\"ok\"" + else + echo "surirules loaded=0i,failed=0i,reload_time=\"\",status=\"unknown\"" + fi + +fi + +exit 0 From dba087ae256db4b3e58e164809d02faf31c553e9 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 5 Dec 2025 09:43:31 -0500 Subject: [PATCH 31/34] Update version from 2.4.0-delta to 2.4.200 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 09e15369f..86df31761 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.0-delta +2.4.200 From b7ad985c7a2bd7c671b9b9372f449ff5f9cb81b7 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Fri, 5 Dec 2025 09:48:46 -0500 Subject: [PATCH 32/34] Add cron.abset --- salt/suricata/config.sls | 2 +- salt/suricata/disabled.sls | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/suricata/config.sls b/salt/suricata/config.sls index 2a4a051cf..b6796031f 100644 --- a/salt/suricata/config.sls +++ b/salt/suricata/config.sls @@ -124,7 +124,7 @@ surirulesync: - name: /opt/so/rules/suricata/ - source: salt://suricata/rules/ - user: 940 - - group: 940 + - group: 939 - show_changes: False surilogscript: diff --git a/salt/suricata/disabled.sls b/salt/suricata/disabled.sls index 49f8f93bf..e7a75867f 100644 --- a/salt/suricata/disabled.sls +++ b/salt/suricata/disabled.sls @@ -23,6 +23,11 @@ clean_suricata_eve_files: cron.absent: - identifier: clean_suricata_eve_files +# Remove rulestats cron +rulestats: + cron.absent: + - identifier: suricata_rulestats + {% else %} {{sls}}_state_not_allowed: From 3f9a9b7019bd1fb456d9c39efdbc16652214253c Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Fri, 5 Dec 2025 10:23:24 -0500 Subject: [PATCH 33/34] tweak threshold --- salt/suricata/config.sls | 1 - salt/suricata/files/threshold.conf | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 salt/suricata/files/threshold.conf diff --git a/salt/suricata/config.sls b/salt/suricata/config.sls index b6796031f..e0b85b7e7 100644 --- a/salt/suricata/config.sls +++ b/salt/suricata/config.sls @@ -160,7 +160,6 @@ surithresholding: - source: salt://suricata/files/threshold.conf - user: 940 - group: 940 - - onlyif: salt://suricata/files/threshold.conf suriclassifications: file.managed: diff --git a/salt/suricata/files/threshold.conf b/salt/suricata/files/threshold.conf new file mode 100644 index 000000000..a03ac31a3 --- /dev/null +++ b/salt/suricata/files/threshold.conf @@ -0,0 +1,2 @@ +# Threshold configuration generated by Security Onion +# This file is automatically generated - do not edit manually \ No newline at end of file From a6b19c4a6c5110860144108a3482c624b359dcb2 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Fri, 5 Dec 2025 12:13:05 -0500 Subject: [PATCH 34/34] Remove idstools config from manager pillar file --- salt/manager/tools/sbin/soup | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 2ffed6af3..c439d2b89 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1299,8 +1299,10 @@ rm /opt/so/saltstack/local/salt/suricata/rules rm -rf /usr/sbin/so-idstools* sed -i '/^#\?so-idstools$/d' /opt/so/conf/so-status/so-status.conf -# Backup the salt master config before editing it -cp /etc/salt/master /nsm/backup/detections-migration/2-4-200 +# Backup the salt master config & manager pillar before editing it +cp /opt/so/saltstack/local/pillar/minions/$MINIONID.sls /nsm/backup/detections-migration/2-4-200/ +cp /etc/salt/master /nsm/backup/detections-migration/2-4-200/ +so-yaml.py remove /opt/so/saltstack/local/pillar/minions/$MINIONID.sls idstools so-yaml.py removelistitem /etc/salt/master file_roots.base /opt/so/rules/nids }