Merge remote-tracking branch 'origin/2.4/dev' into idstools-refactor

add force & certs flag to update fleet certs as needed

pillar/top.sls

@@ -43,8 +43,6 @@ base:
     - secrets
     - manager.soc_manager
     - manager.adv_manager
-    - idstools.soc_idstools
-    - idstools.adv_idstools
     - logstash.nodes
     - logstash.soc_logstash
     - logstash.adv_logstash
@@ -117,8 +115,6 @@ base:
     - elastalert.adv_elastalert
     - manager.soc_manager
     - manager.adv_manager
-    - idstools.soc_idstools
-    - idstools.adv_idstools
     - soc.soc_soc
     - soc.adv_soc
     - kibana.soc_kibana
@@ -158,8 +154,6 @@ base:
     {% endif %}
     - secrets
     - healthcheck.standalone
-    - idstools.soc_idstools
-    - idstools.adv_idstools
     - kratos.soc_kratos
     - kratos.adv_kratos
     - hydra.soc_hydra

salt/allowed_states.map.jinja

@@ -38,8 +38,6 @@
     'hydra',
     'elasticfleet',
     'elastic-fleet-package-registry',
-    'idstools',
-    'suricata.manager',
     'utility'
 ] %}
 

salt/common/tools/sbin/so-image-common

@@ -25,7 +25,6 @@ container_list() {
   if [ $MANAGERCHECK == 'so-import' ]; then
     TRUSTED_CONTAINERS=(
       "so-elasticsearch"
-      "so-idstools"
       "so-influxdb"
       "so-kibana"
       "so-kratos"
@@ -49,7 +48,6 @@ container_list() {
       "so-elastic-fleet-package-registry"
       "so-elasticsearch"
       "so-idh"
-      "so-idstools"
       "so-influxdb"
       "so-kafka"
       "so-kibana"
@@ -69,7 +67,6 @@ container_list() {
     )
   else
     TRUSTED_CONTAINERS=(
-      "so-idstools"
       "so-elasticsearch"
       "so-logstash"
       "so-nginx"

salt/docker/defaults.yaml

@@ -24,11 +24,6 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
-    'so-idstools':
-      final_octet: 25
-      custom_bind_mounts: []
-      extra_hosts: []
-      extra_env: []
     'so-influxdb':
       final_octet: 26
       port_bindings:

salt/docker/soc_docker.yaml

@@ -41,7 +41,6 @@ docker:
         forcedType: "[]string"
     so-elastic-fleet: *dockerOptions
     so-elasticsearch: *dockerOptions
-    so-idstools: *dockerOptions
     so-influxdb: *dockerOptions
     so-kibana: *dockerOptions
     so-kratos: *dockerOptions
@@ -102,4 +101,4 @@ docker:
         multiline: True
         forcedType: "[]string"
     so-zeek: *dockerOptions
-    so-kafka: *dockerOptions
+    so-kafka: *dockerOptions

salt/elasticfleet/enabled.sls

@@ -32,6 +32,16 @@ so-elastic-fleet-auto-configure-logstash-outputs:
     - retry:
         attempts: 4
         interval: 30
+
+{# Separate from above in order to catch elasticfleet-logstash.crt changes and force update to fleet output policy #}
+so-elastic-fleet-auto-configure-logstash-outputs-force:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-outputs-update --force --certs
+    - retry:
+        attempts: 4
+        interval: 30
+    - onchanges:
+        - x509: etc_elasticfleet_logstash_crt
 {% endif %}
 
 # If enabled, automatically update Fleet Server URLs & ES Connection

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update

@@ -8,6 +8,27 @@
 
 . /usr/sbin/so-common
 
+FORCE_UPDATE=false
+UPDATE_CERTS=false
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -f|--force)
+      FORCE_UPDATE=true
+      shift
+      ;;
+    -c| --certs)
+      UPDATE_CERTS=true
+      shift
+      ;;
+    *)
+      echo "Unknown option $1"
+      echo "Usage: $0 [-f|--force] [-c|--certs]"
+      exit 1
+      ;;
+  esac
+done
+
 # Only run on Managers
 if ! is_manager_node; then
     printf "Not a Manager Node... Exiting"
@@ -17,17 +38,42 @@ fi
 function update_logstash_outputs() {
     if logstash_policy=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs/so-manager_logstash" --retry 3 --retry-delay 10 --fail 2>/dev/null); then
         SSL_CONFIG=$(echo "$logstash_policy" | jq -r '.item.ssl')
+        LOGSTASHKEY=$(openssl rsa -in  /etc/pki/elasticfleet-logstash.key)
+        LOGSTASHCRT=$(openssl x509 -in /etc/pki/elasticfleet-logstash.crt)
+        LOGSTASHCA=$(openssl x509 -in  /etc/pki/tls/certs/intca.crt)
         if SECRETS=$(echo "$logstash_policy" | jq -er '.item.secrets' 2>/dev/null); then
-            JSON_STRING=$(jq -n \
-                --arg UPDATEDLIST "$NEW_LIST_JSON" \
-                --argjson SECRETS "$SECRETS" \
-                --argjson SSL_CONFIG "$SSL_CONFIG" \
-                '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl": $SSL_CONFIG,"secrets": $SECRETS}')
+            if [[ "$UPDATE_CERTS" != "true"  ]]; then
+                # Reuse existing secret
+                JSON_STRING=$(jq -n \
+                    --arg UPDATEDLIST "$NEW_LIST_JSON" \
+                    --argjson SECRETS "$SECRETS" \
+                    --argjson SSL_CONFIG "$SSL_CONFIG" \
+                    '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl": $SSL_CONFIG,"secrets": $SECRETS}')
+            else
+                # Update certs, creating new secret
+                JSON_STRING=$(jq -n \
+                    --arg UPDATEDLIST "$NEW_LIST_JSON" \
+                    --arg LOGSTASHKEY "$LOGSTASHKEY" \
+                    --arg LOGSTASHCRT "$LOGSTASHCRT" \
+                    --arg LOGSTASHCA "$LOGSTASHCA" \
+                    '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl": {"certificate": $LOGSTASHCRT,"certificate_authorities":[ $LOGSTASHCA ]},"secrets": {"ssl":{"key": $LOGSTASHKEY }}}')
+            fi
         else
-            JSON_STRING=$(jq -n \
-                --arg UPDATEDLIST "$NEW_LIST_JSON" \
-                --argjson SSL_CONFIG "$SSL_CONFIG" \
-                '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl": $SSL_CONFIG}')
+            if [[ "$UPDATE_CERTS" != "true" ]]; then
+                # Reuse existing ssl config
+                JSON_STRING=$(jq -n \
+                    --arg UPDATEDLIST "$NEW_LIST_JSON" \
+                    --argjson SSL_CONFIG "$SSL_CONFIG" \
+                    '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl": $SSL_CONFIG}')
+            else
+                # Update ssl config
+                JSON_STRING=$(jq -n \
+                    --arg UPDATEDLIST "$NEW_LIST_JSON" \
+                    --arg LOGSTASHKEY "$LOGSTASHKEY" \
+                    --arg LOGSTASHCRT "$LOGSTASHCRT" \
+                    --arg LOGSTASHCA "$LOGSTASHCA" \
+                    '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl": {"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]}}')
+            fi
         fi
     fi
 
@@ -151,7 +197,7 @@ NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "$
 NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}')
 
 # Compare the current & new list of outputs - if different, update the Logstash outputs
-if [ "$NEW_HASH" = "$CURRENT_HASH" ]; then
+if [[ "$NEW_HASH" = "$CURRENT_HASH" ]] && [[ "$FORCE_UPDATE" != "true" ]]; then
     printf "\nHashes match - no update needed.\n"
     printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n"
 

salt/idstools/config.sls

@@ -1,65 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls.split('.')[0] in allowed_states %}
-
-include:
-  - idstools.sync_files
-
-idstoolslogdir:
-  file.directory:
-    - name: /opt/so/log/idstools
-    - user: 939
-    - group: 939
-    - makedirs: True
-
-idstools_sbin:
-  file.recurse:
-    - name: /usr/sbin
-    - source: salt://idstools/tools/sbin
-    - user: 939
-    - group: 939
-    - file_mode: 755
-
-# If this is used, exclude so-rule-update
-#idstools_sbin_jinja:
-#  file.recurse:
-#    - name: /usr/sbin
-#    - source: salt://idstools/tools/sbin_jinja
-#    - user: 939
-#    - group: 939 
-#    - file_mode: 755
-#    - template: jinja
-
-idstools_so-rule-update:
-  file.managed:
-    - name: /usr/sbin/so-rule-update
-    - source: salt://idstools/tools/sbin_jinja/so-rule-update
-    - user: 939
-    - group: 939 
-    - mode: 755
-    - template: jinja
-
-suricatacustomdirsfile:
-  file.directory:
-    - name: /nsm/rules/detect-suricata/custom_file
-    - user: 939
-    - group: 939
-    - makedirs: True
-
-suricatacustomdirsurl:
-  file.directory:
-    - name: /nsm/rules/detect-suricata/custom_temp
-    - user: 939
-    - group: 939
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

salt/idstools/defaults.yaml

@@ -1,10 +0,0 @@
-idstools:
-  enabled: False
-  config:
-    urls: []
-    ruleset: ETOPEN
-    oinkcode: ""
-  sids:
-    enabled: []
-    disabled: []
-    modify: []

salt/idstools/disabled.sls

@@ -1,31 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls.split('.')[0] in allowed_states %}
-
-include:
-  - idstools.sostatus
-  
-so-idstools:
-  docker_container.absent:
-    - force: True
-
-so-idstools_so-status.disabled:
-  file.comment:
-    - name: /opt/so/conf/so-status/so-status.conf
-    - regex: ^so-idstools$
-
-so-rule-update:
-  cron.absent:
-    - identifier: so-rule-update
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

salt/idstools/enabled.sls

@@ -1,91 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls.split('.')[0] in allowed_states %}
-{%   from 'docker/docker.map.jinja' import DOCKER %}
-{%   from 'vars/globals.map.jinja' import GLOBALS %}
-{%   set proxy = salt['pillar.get']('manager:proxy') %}
-
-include:
-  - idstools.config
-  - idstools.sostatus
-
-so-idstools:
-  docker_container.running:
-    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-idstools:{{ GLOBALS.so_version }}
-    - hostname: so-idstools
-    - user: socore
-    - networks:
-      - sobridge:
-        - ipv4_address: {{ DOCKER.containers['so-idstools'].ip }}
-    {% if proxy %}
-    - environment:
-      - http_proxy={{ proxy }}
-      - https_proxy={{ proxy }}
-      - no_proxy={{ salt['pillar.get']('manager:no_proxy') }}
-      {% if DOCKER.containers['so-idstools'].extra_env %}
-        {% for XTRAENV in DOCKER.containers['so-idstools'].extra_env %}
-      - {{ XTRAENV }}
-        {% endfor %}
-      {% endif %}
-    {% elif DOCKER.containers['so-idstools'].extra_env %}
-    - environment:
-      {% for XTRAENV in DOCKER.containers['so-idstools'].extra_env %}
-      - {{ XTRAENV }}
-      {% endfor %}
-    {% endif %}
-    - binds:
-      - /opt/so/conf/idstools/etc:/opt/so/idstools/etc:ro
-      - /opt/so/rules/nids/suri:/opt/so/rules/nids/suri:rw
-      - /nsm/rules/:/nsm/rules/:rw
-    {% if DOCKER.containers['so-idstools'].custom_bind_mounts %}
-      {% for BIND in DOCKER.containers['so-idstools'].custom_bind_mounts %}
-      - {{ BIND }}
-      {% endfor %}
-    {% endif %}
-    - extra_hosts:
-      - {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }}
-    {% if DOCKER.containers['so-idstools'].extra_hosts %}
-      {% for XTRAHOST in DOCKER.containers['so-idstools'].extra_hosts %}
-      - {{ XTRAHOST }}
-      {% endfor %}
-    {% endif %}
-    - watch:
-      - file: idstoolsetcsync
-      - file: idstools_so-rule-update
-
-delete_so-idstools_so-status.disabled:
-  file.uncomment:
-    - name: /opt/so/conf/so-status/so-status.conf
-    - regex: ^so-idstools$
-
-so-rule-update:
-  cron.present:
-    - name: /usr/sbin/so-rule-update > /opt/so/log/idstools/download_cron.log 2>&1
-    - identifier: so-rule-update
-    - user: root
-    - minute: '1'
-    - hour: '7'
-
-# order this last to give so-idstools container time to be ready
-run_so-rule-update:
-  cmd.run:
-    - name: '/usr/sbin/so-rule-update > /opt/so/log/idstools/download_idstools_state.log 2>&1'
-    - require:
-      - docker_container: so-idstools
-    - onchanges:
-      - file: idstools_so-rule-update
-      - file: idstoolsetcsync
-      - file: synclocalnidsrules
-    - order: last
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

salt/idstools/etc/disable.conf

@@ -1,16 +0,0 @@
-{%- set disabled_sids = salt['pillar.get']('idstools:sids:disabled', {}) -%}
-# idstools - disable.conf
-
-# Example of disabling a rule by signature ID (gid is optional).
-# 1:2019401
-# 2019401
-
-# Example of disabling a rule by regular expression.
-# - All regular expression matches are case insensitive.
-# re:hearbleed
-# re:MS(0[7-9]|10)-\d+
-{%- if disabled_sids != None %}
-{%- for sid in disabled_sids %}
-{{ sid }}
-{%- endfor %}
-{%- endif %}

salt/idstools/etc/enable.conf

@@ -1,16 +0,0 @@
-{%- set enabled_sids = salt['pillar.get']('idstools:sids:enabled', {}) -%}
-# idstools-rulecat - enable.conf
-
-# Example of enabling a rule by signature ID (gid is optional).
-# 1:2019401
-# 2019401
-
-# Example of enabling a rule by regular expression.
-# - All regular expression matches are case insensitive.
-# re:hearbleed
-# re:MS(0[7-9]|10)-\d+
-{%- if enabled_sids != None %}
-{%- for sid in enabled_sids %}
-{{ sid }}
-{%- endfor %}
-{%- endif %}

salt/idstools/etc/modify.conf

@@ -1,12 +0,0 @@
-{%- set modify_sids = salt['pillar.get']('idstools:sids:modify', {}) -%}
-# idstools-rulecat - modify.conf
-
-# Format: <sid> "<from>" "<to>"
-
-# Example changing the seconds for rule 2019401 to 3600.
-#2019401 "seconds \d+" "seconds 3600"
-{%- if modify_sids != None %}
-{%- for sid in modify_sids %}
-{{ sid }}
-{%- endfor %}
-{%- endif %}

salt/idstools/etc/rulecat.conf

@@ -1,23 +0,0 @@
-{%- from 'vars/globals.map.jinja' import GLOBALS -%}
-{%- from 'soc/merged.map.jinja' import SOCMERGED -%}
---suricata-version=7.0.3
---merged=/opt/so/rules/nids/suri/all.rules
---output=/nsm/rules/detect-suricata/custom_temp
---local=/opt/so/rules/nids/suri/local.rules
-{%-   if GLOBALS.md_engine == "SURICATA" %}
---local=/opt/so/rules/nids/suri/extraction.rules
---local=/opt/so/rules/nids/suri/filters.rules
-{%-   endif %}
---url=http://{{ GLOBALS.manager }}:7788/suricata/emerging-all.rules
---disable=/opt/so/idstools/etc/disable.conf
---enable=/opt/so/idstools/etc/enable.conf
---modify=/opt/so/idstools/etc/modify.conf
-{%- if SOCMERGED.config.server.modules.suricataengine.customRulesets %}
-    {%- for ruleset in SOCMERGED.config.server.modules.suricataengine.customRulesets %}
-        {%- if 'url' in ruleset %}
---url={{ ruleset.url }}
-        {%- elif 'file' in ruleset %}
---local={{ ruleset.file }}
-        {%- endif %}
-    {%- endfor %}
-{%- endif %}

salt/idstools/init.sls

@@ -1,13 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'idstools/map.jinja' import IDSTOOLSMERGED %}
-
-include:
-{% if IDSTOOLSMERGED.enabled %}
-  - idstools.enabled
-{% else %}
-  - idstools.disabled
-{% endif %}

salt/idstools/map.jinja

@@ -1,7 +0,0 @@
-{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-   https://securityonion.net/license; you may not use this file except in compliance with the
-   Elastic License 2.0. #}
-   
-{% import_yaml 'idstools/defaults.yaml' as IDSTOOLSDEFAULTS with context %}
-{% set IDSTOOLSMERGED = salt['pillar.get']('idstools', IDSTOOLSDEFAULTS.idstools, merge=True) %}

salt/idstools/rules/local.rules

@@ -1 +0,0 @@
-# Add your custom Suricata rules in this file.

salt/idstools/soc_idstools.yaml

@@ -1,72 +0,0 @@
-idstools:
-  enabled:
-    description: Enables or disables the IDStools process which is used by the Detection system.
-  config:
-    oinkcode:
-      description: Enter your registration code or oinkcode for paid NIDS rulesets.
-      title: Registration Code
-      global: True
-      forcedType: string
-      helpLink: rules.html
-    ruleset:
-      description: 'Defines the ruleset you want to run. Options are ETOPEN or ETPRO. Once you have changed the ruleset here, you will need to wait for the rule update to take place (every 24 hours), or you can force the update by nagivating to Detections -->  Options dropdown menu --> Suricata --> Full Update. WARNING! Changing the ruleset will remove all existing non-overlapping Suricata rules of the previous ruleset and their associated overrides. This removal cannot be undone.'
-      global: True
-      regex:  ETPRO\b|ETOPEN\b
-      helpLink: rules.html
-    urls:
-      description: This is a list of additional rule download locations. This feature is currently disabled. 
-      global: True
-      multiline: True
-      forcedType: "[]string"
-      readonly: True
-      helpLink: rules.html
-  sids:
-    disabled:
-      description: Contains the list of NIDS rules (or regex patterns) disabled across the grid. This setting is readonly; Use the Detections screen to disable rules.
-      global: True
-      multiline: True
-      forcedType: "[]string"
-      regex: \d*|re:.*
-      helpLink: managing-alerts.html
-      readonlyUi: True
-      advanced: true
-    enabled:
-      description: Contains the list of NIDS rules (or regex patterns) enabled across the grid. This setting is readonly; Use the Detections screen to enable rules.
-      global: True
-      multiline: True
-      forcedType: "[]string"
-      regex: \d*|re:.*
-      helpLink: managing-alerts.html
-      readonlyUi: True
-      advanced: true
-    modify:
-      description: Contains the list of NIDS rules (SID "REGEX_SEARCH_TERM" "REGEX_REPLACE_TERM"). This setting is readonly; Use the Detections screen to modify rules.
-      global: True
-      multiline: True
-      forcedType: "[]string"
-      helpLink: managing-alerts.html
-      readonlyUi: True
-      advanced: true
-  rules:
-    local__rules:
-      description: Contains the list of custom NIDS rules applied to the grid. This setting is readonly; Use the Detections screen to adjust rules.
-      file: True
-      global: True
-      advanced: True
-      title: Local Rules
-      helpLink: local-rules.html
-      readonlyUi: True
-    filters__rules:
-      description: If you are using Suricata for metadata, then you can set custom filters for that metadata here.
-      file: True
-      global: True
-      advanced: True
-      title: Filter Rules
-      helpLink: suricata.html
-    extraction__rules:
-      description: If you are using Suricata for metadata, then you can set a list of MIME types for file extraction here.
-      file: True
-      global: True
-      advanced: True
-      title: Extraction Rules
-      helpLink: suricata.html

salt/idstools/sostatus.sls

@@ -1,21 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls.split('.')[0] in allowed_states %}
-
-append_so-idstools_so-status.conf:
-  file.append:
-    - name: /opt/so/conf/so-status/so-status.conf
-    - text: so-idstools
-    - unless: grep -q so-idstools /opt/so/conf/so-status/so-status.conf
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

salt/idstools/sync_files.sls

@@ -1,37 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-idstoolsdir:
-  file.directory:
-    - name: /opt/so/conf/idstools/etc
-    - user: 939
-    - group: 939
-    - makedirs: True
-
-idstoolsetcsync:
-  file.recurse:
-    - name: /opt/so/conf/idstools/etc
-    - source: salt://idstools/etc
-    - user: 939
-    - group: 939
-    - template: jinja
-
-rulesdir:
-  file.directory:
-    - name: /opt/so/rules/nids/suri
-    - user: 939
-    - group: 939
-    - makedirs: True
-
-# Don't show changes because all.rules can be large
-synclocalnidsrules:
-  file.recurse:
-    - name: /opt/so/rules/nids/suri/
-    - source: salt://idstools/rules/
-    - user: 939
-    - group: 939
-    - show_changes: False
-    - include_pat: 'E@.rules'

salt/idstools/tools/sbin/so-idstools-restart

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-/usr/sbin/so-restart idstools $1

salt/idstools/tools/sbin/so-idstools-start

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-/usr/sbin/so-start idstools $1

salt/idstools/tools/sbin/so-idstools-stop

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-/usr/sbin/so-stop idstools $1

salt/idstools/tools/sbin_jinja/so-rule-update

@@ -1,40 +0,0 @@
-#!/bin/bash
-
-# if this script isn't already running
-if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
-
-    . /usr/sbin/so-common
-
-{%- from 'vars/globals.map.jinja' import GLOBALS %}
-{%- from 'idstools/map.jinja' import IDSTOOLSMERGED %}
-
-{%- set proxy = salt['pillar.get']('manager:proxy') %}
-{%- set noproxy = salt['pillar.get']('manager:no_proxy', '') %}
-
-{%- if proxy %}
-# Download the rules from the internet
-    export http_proxy={{ proxy }}
-    export https_proxy={{ proxy }}
-    export no_proxy="{{ noproxy }}"
-{%- endif %}
-
-    mkdir -p /nsm/rules/suricata
-    chown -R socore:socore /nsm/rules/suricata
-{%- if not GLOBALS.airgap %}
-# Download the rules from the internet
-{%-   if IDSTOOLSMERGED.config.ruleset == 'ETOPEN' %}
-    docker exec so-idstools idstools-rulecat -v --suricata-version 7.0.3 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force
-{%-   elif IDSTOOLSMERGED.config.ruleset == 'ETPRO' %}
-    docker exec so-idstools idstools-rulecat -v --suricata-version 7.0.3 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force --etpro={{ IDSTOOLSMERGED.config.oinkcode }}
-{%-   endif %}
-{%- endif %}
-
-
-    argstr=""
-    for arg in "$@"; do
-        argstr="${argstr} \"${arg}\""
-    done
-
-    docker exec so-idstools /bin/bash -c "cd /opt/so/idstools/etc && idstools-rulecat --force ${argstr}"
-
-fi

salt/logrotate/defaults.yaml

@@ -1,15 +1,5 @@
 logrotate:
   config:
-    /opt/so/log/idstools/*_x_log:
-      - daily
-      - rotate 14
-      - missingok
-      - copytruncate
-      - compress
-      - create
-      - extension .log
-      - dateext
-      - dateyesterday
     /opt/so/log/nginx/*_x_log:
       - daily
       - rotate 14

salt/logrotate/soc_logrotate.yaml

@@ -1,12 +1,5 @@
 logrotate:
   config:
-    "/opt/so/log/idstools/*_x_log":
-      description: List of logrotate options for this file.
-      title: /opt/so/log/idstools/*.log
-      advanced: True
-      multiline: True
-      global: True
-      forcedType: "[]string"
     "/opt/so/log/nginx/*_x_log":
       description: List of logrotate options for this file.
       title: /opt/so/log/nginx/*.log

salt/manager/init.sls

@@ -206,10 +206,33 @@ git_config_set_safe_dirs:
     - multivar:
       - /nsm/rules/custom-local-repos/local-sigma
       - /nsm/rules/custom-local-repos/local-yara
+      - /nsm/rules/custom-local-repos/local-suricata
       - /nsm/securityonion-resources
       - /opt/so/conf/soc/ai_summary_repos/securityonion-resources
       - /nsm/airgap-resources/playbooks
       - /opt/so/conf/soc/playbooks
+
+surinsmrulesdir:
+  file.directory:
+    - name: /nsm/rules/suricata
+    - user: 939
+    - group: 939
+    - makedirs: True
+
+suriextractionrules:
+  file.managed:
+    - name: /nsm/rules/suricata/so_extraction.rules
+    - source: salt://suricata/files/so_extraction.rules
+    - user: 939
+    - group: 939
+
+surifiltersrules:
+  file.managed:
+    - name: /nsm/rules/suricata/so_filters.rules
+    - source: salt://suricata/files/so_filters.rules
+    - user: 939
+    - group: 939
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/manager/tools/sbin/so-minion

@@ -604,16 +604,6 @@ function add_kratos_to_minion() {
     fi
 }
 
-function add_idstools_to_minion() {
-    printf '%s\n'\
-    "idstools:"\
-    "  enabled: True"\
-	"  " >> $PILLARFILE
-    if [ $? -ne 0 ]; then
-        log "ERROR" "Failed to add idstools configuration to $PILLARFILE"
-        return 1
-    fi
-}
 
 function add_elastic_fleet_package_registry_to_minion() {
     printf '%s\n'\
@@ -741,7 +731,6 @@ function createEVAL() {
 	add_soc_to_minion || return 1
 	add_registry_to_minion || return 1
 	add_kratos_to_minion || return 1
-	add_idstools_to_minion || return 1
 	add_elastic_fleet_package_registry_to_minion || return 1
 }
 
@@ -762,7 +751,6 @@ function createSTANDALONE() {
 	add_soc_to_minion || return 1
 	add_registry_to_minion || return 1
 	add_kratos_to_minion || return 1
-	add_idstools_to_minion || return 1
 	add_elastic_fleet_package_registry_to_minion || return 1
 }
 
@@ -779,7 +767,6 @@ function createMANAGER() {
 	add_soc_to_minion || return 1
 	add_registry_to_minion || return 1
 	add_kratos_to_minion || return 1
-	add_idstools_to_minion || return 1
 	add_elastic_fleet_package_registry_to_minion || return 1
 }
 
@@ -796,7 +783,6 @@ function createMANAGERSEARCH() {
 	add_soc_to_minion || return 1
 	add_registry_to_minion || return 1
 	add_kratos_to_minion || return 1
-	add_idstools_to_minion || return 1
 	add_elastic_fleet_package_registry_to_minion || return 1
 }
 
@@ -811,7 +797,6 @@ function createIMPORT() {
 	add_soc_to_minion || return 1
 	add_registry_to_minion || return 1
 	add_kratos_to_minion || return 1
-	add_idstools_to_minion || return 1
 	add_elastic_fleet_package_registry_to_minion || return 1
 }
 
@@ -896,7 +881,6 @@ function createMANAGERHYPE() {
 	add_soc_to_minion || return 1
 	add_registry_to_minion || return 1
 	add_kratos_to_minion || return 1
-	add_idstools_to_minion || return 1
 	add_elastic_fleet_package_registry_to_minion || return 1
 }
 

salt/manager/tools/sbin/soup

@@ -426,6 +426,7 @@ preupgrade_changes() {
     [[ "$INSTALLEDVERSION" == 2.4.160 ]] && up_to_2.4.170
     [[ "$INSTALLEDVERSION" == 2.4.170 ]] && up_to_2.4.180
     [[ "$INSTALLEDVERSION" == 2.4.180 ]] && up_to_2.4.190
+    [[ "$INSTALLEDVERSION" == 2.4.190 ]] && up_to_2.4.200
     true
 }
 
@@ -457,6 +458,7 @@ postupgrade_changes() {
     [[ "$POSTVERSION"  == 2.4.160 ]] && post_to_2.4.170
     [[ "$POSTVERSION"  == 2.4.170 ]] && post_to_2.4.180
     [[ "$POSTVERSION"  == 2.4.180 ]] && post_to_2.4.190
+    [[ "$POSTVERSION"  == 2.4.190 ]] && post_to_2.4.200
     true
 }
 
@@ -636,6 +638,13 @@ post_to_2.4.190() {
   POSTVERSION=2.4.190
 }
 
+post_to_2.4.200() {
+  echo "Initiating Suricata idstools migration..."
+  suricata_idstools_removal_post
+
+  POSTVERSION=2.4.200
+}
+
 repo_sync() {
   echo "Sync the local repo."
   su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync."
@@ -903,6 +912,13 @@ up_to_2.4.190() {
   INSTALLEDVERSION=2.4.190
 }
 
+up_to_2.4.200() {
+  echo "Backing up idstools config..."
+  suricata_idstools_removal_pre
+
+  INSTALLEDVERSION=2.4.200
+}
+
 add_hydra_pillars() {
   mkdir -p /opt/so/saltstack/local/pillar/hydra
   touch /opt/so/saltstack/local/pillar/hydra/soc_hydra.sls
@@ -986,6 +1002,8 @@ rollover_index() {
 }
 
 suricata_idstools_migration() {
+  # For 2.4.70
+
   #Backup the pillars for idstools
   mkdir -p /nsm/backup/detections-migration/idstools
   rsync -av /opt/so/saltstack/local/pillar/idstools/* /nsm/backup/detections-migration/idstools
@@ -1086,6 +1104,159 @@ playbook_migration() {
   echo "Playbook Migration is complete...."
 }
 
+suricata_idstools_removal_pre() {
+# For SOUPs beginning with 2.4.200 - pre SOUP checks
+
+# Create syncBlock file
+install -d -o 939 -g 939 -m 755 /opt/so/conf/soc/fingerprints
+install -o 939 -g 939 -m 644 /dev/null /opt/so/conf/soc/fingerprints/suricataengine.syncBlock
+cat > /opt/so/conf/soc/fingerprints/suricataengine.syncBlock << EOF
+Suricata ruleset sync is blocked until this file is removed. Make sure that you have manually added any custom Suricata rulesets via SOC config - review the documentation for more details: securityonion.net/docs
+EOF
+
+# Backup custom rules & overrides
+mkdir -p /nsm/backup/detections-migration/2-4-200
+cp /usr/sbin/so-rule-update /nsm/backup/detections-migration/2-4-200
+cp /opt/so/conf/idstools/etc/rulecat.conf /nsm/backup/detections-migration/2-4-200
+
+if [[ -f /opt/so/conf/soc/so-detections-backup.py ]]; then
+    python3 /opt/so/conf/soc/so-detections-backup.py
+
+    # Verify backup by comparing counts
+    echo "Verifying detection overrides backup..."
+    es_override_count=$(/sbin/so-elasticsearch-query 'so-detection/_count' \
+       -d '{"query": {"bool": {"must": [{"exists": {"field": "so_detection.overrides"}}]}}}' | jq -r '.count') || {
+        echo "  Error: Failed to query Elasticsearch for override count"
+        exit 1
+    }
+
+    if [[ ! "$es_override_count" =~ ^[0-9]+$ ]]; then
+        echo "  Error: Invalid override count from Elasticsearch: '$es_override_count'"
+        exit 1
+    fi
+
+    backup_override_count=$(find /nsm/backup/detections/repo/*/overrides -type f 2>/dev/null | wc -l)
+
+    echo "  Elasticsearch overrides: $es_override_count"
+    echo "  Backed up overrides: $backup_override_count"
+
+    if [[ "$es_override_count" -gt 0 ]]; then
+        if [[ "$backup_override_count" -gt 0 ]]; then
+            echo "  Override backup verified successfully"
+        else
+            echo "  Error: Elasticsearch has $es_override_count overrides but backup has 0 files"
+            exit 1
+        fi
+    else
+        echo "  No overrides to backup"
+    fi
+else
+    echo "SOC Detections backup script not found, skipping detection backup"
+fi
+
+}
+
+suricata_idstools_removal_post() {
+# For SOUPs beginning with 2.4.200 - post SOUP checks
+
+echo "Checking idstools configuration for custom modifications..."
+
+# Normalize and hash file content for consistent comparison
+# Args: $1 - file path
+# Outputs: SHA256 hash to stdout
+# Returns: 0 on success, 1 on failure
+hash_normalized_file() {
+    local file="$1"
+
+    if [[ ! -r "$file" ]]; then
+        return 1
+    fi
+
+    sed -E \
+        -e 's/^[[:space:]]+//; s/[[:space:]]+$//' \
+        -e '/^$/d' \
+        -e 's|--url=http://[^:]+:7788|--url=http://MANAGER:7788|' \
+        "$file" | sha256sum | awk '{print $1}'
+}
+
+# Known-default hashes
+KNOWN_SO_RULE_UPDATE_HASHES=(
+    "8f1fe1cb65c08aab78830315b952785c7ccdcc108c5c0474f427e29d4e39ee5f"  # non-Airgap
+    "d23ac5a962c709dcb888103effb71444df72b46009b6c426e280dbfbc7d74d40"  # Airgap
+)
+
+KNOWN_RULECAT_CONF_HASHES=(
+    "17fc663a83b30d4ba43ac6643666b0c96343c5ea6ea833fe6a8362fe415b666b"  # default
+)
+
+# Check a config file against known hashes
+# Args: $1 - file path, $2 - array name of known hashes
+check_config_file() {
+    local file="$1"
+    local known_hashes_array="$2"
+    local file_display_name=$(basename "$file")
+
+    if [[ ! -f "$file" ]]; then
+        echo "Warning: $file not found"
+        echo "$file_display_name not found - manual verification required" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock
+        return 1
+    fi
+
+    echo "Hashing $file..."
+    local file_hash
+    if ! file_hash=$(hash_normalized_file "$file"); then
+        echo "Warning: Could not read $file"
+        echo "$file_display_name not readable - manual verification required" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock
+        return 1
+    fi
+
+    echo "  Hash: $file_hash"
+
+    # Check if hash matches any known default
+    local -n known_hashes=$known_hashes_array
+    for known_hash in "${known_hashes[@]}"; do
+        if [[ "$file_hash" == "$known_hash" ]]; then
+            echo "  Matches known default configuration"
+            return 0
+        fi
+    done
+
+    # No match - custom configuration detected
+    echo "Does not match known default - custom configuration detected"
+    echo "Custom $file_display_name detected (hash: $file_hash)" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock
+
+    # If this is so-rule-update, check for ETPRO license code and write out to the syncBlock file
+    # If ETPRO is enabled, the license code already exists in the so-rule-update script, this is just making it easier to migrate
+    if [[ "$file_display_name" == "so-rule-update" ]]; then
+        local etpro_code
+        etpro_code=$(grep -oP '\-\-etpro=\K[0-9a-fA-F]+' "$file" 2>/dev/null) || true
+        if [[ -n "$etpro_code" ]]; then
+            echo "ETPRO code found: $etpro_code" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock
+        fi
+    fi
+
+    return 1
+}
+
+# Check so-rule-update and rulecat.conf
+SO_RULE_UPDATE="/usr/sbin/so-rule-update"
+RULECAT_CONF="/opt/so/conf/idstools/etc/rulecat.conf"
+
+custom_found=0
+
+check_config_file "$SO_RULE_UPDATE" "KNOWN_SO_RULE_UPDATE_HASHES" || custom_found=1
+check_config_file "$RULECAT_CONF" "KNOWN_RULECAT_CONF_HASHES" || custom_found=1
+
+# If no custom configs found, remove syncBlock
+if [[ $custom_found -eq 0 ]]; then
+    echo "idstools migration completed successfully - removing Suricata engine syncBlock"
+    rm -f /opt/so/conf/soc/fingerprints/suricataengine.syncBlock
+else
+    echo "Custom idstools configuration detected - syncBlock remains in place"
+    echo "Review /opt/so/conf/soc/fingerprints/suricataengine.syncBlock for details"
+fi
+}
+
 determine_elastic_agent_upgrade() {
   if [[ $is_airgap -eq 0 ]]; then
     update_elastic_agent_airgap

salt/salt/files/engines.conf

@@ -6,30 +6,6 @@ engines:
       interval: 60
   - pillarWatch:
       fpa:
-        - files:
-            - /opt/so/saltstack/local/pillar/idstools/soc_idstools.sls
-            - /opt/so/saltstack/local/pillar/idstools/adv_idstools.sls
-          pillar: idstools.config.ruleset
-          default: ETOPEN
-          actions:
-            from:
-              '*':
-                to:
-                  '*':
-                  - cmd.run:
-                      cmd: /usr/sbin/so-rule-update
-        - files:
-            - /opt/so/saltstack/local/pillar/idstools/soc_idstools.sls
-            - /opt/so/saltstack/local/pillar/idstools/adv_idstools.sls
-          pillar: idstools.config.oinkcode
-          default: ''
-          actions:
-            from:
-              '*':
-                to:
-                  '*':
-                  - cmd.run:
-                      cmd: /usr/sbin/so-rule-update
         - files:
             - /opt/so/saltstack/local/pillar/global/soc_global.sls
             - /opt/so/saltstack/local/pillar/global/adv_global.sls

salt/soc/config.sls

@@ -215,7 +215,6 @@ socsensoronirepos:
     - mode: 775
     - makedirs: True
 
-
 create_custom_local_yara_repo_template:
   git.present:
     - name: /nsm/rules/custom-local-repos/local-yara
@@ -249,6 +248,39 @@ add_readme_custom_local_sigma_repo_template:
     - context:
         repo_type: "sigma"
 
+create_custom_local_suricata_repo_template:
+  git.present:
+    - name: /nsm/rules/custom-local-repos/local-suricata
+    - bare: False
+    - force: True
+
+add_readme_custom_local_suricata_repo_template:
+  file.managed:
+    - name: /nsm/rules/custom-local-repos/local-suricata/README
+    - source: salt://soc/files/soc/detections_custom_repo_template_readme.jinja
+    - user: 939
+    - group: 939
+    - template: jinja
+    - context:
+        repo_type: "suricata"
+
+etpro_airgap_folder:
+  file.directory:
+    - name: /nsm/rules/custom-local-repos/local-etpro-suricata
+    - user: 939
+    - group: 939
+    - makedirs: True
+
+add_readme_etpro_airgap_template:
+  file.managed:
+    - name: /nsm/rules/custom-local-repos/local-etpro-suricata/README
+    - source: salt://soc/files/soc/detections_custom_repo_template_readme.jinja
+    - user: 939
+    - group: 939
+    - template: jinja
+    - context:
+        repo_type: "suricata-etpro"
+
 socore_own_custom_repos:
   file.directory:
     - name: /nsm/rules/custom-local-repos/

salt/soc/defaults.yaml

@@ -1563,12 +1563,106 @@ soc:
           disableRegex: []
           enableRegex: []
           failAfterConsecutiveErrorCount: 10
-          communityRulesFile: /nsm/rules/suricata/emerging-all.rules
           rulesFingerprintFile: /opt/sensoroni/fingerprints/emerging-all.fingerprint
           stateFilePath: /opt/sensoroni/fingerprints/suricataengine.state
           integrityCheckFrequencySeconds: 1200
           ignoredSidRanges:
             - '1100000-1101000'
+          rulesetSources:
+            default:
+              - name: Emerging-Threats
+                description: "Emerging Threats ruleset - To enable ET Pro, enter your license key below. Leave empty for ET Open (free) rules."
+                licenseKey: ""
+                enabled: true
+                sourceType: url
+                sourcePath: 'https://rules.emergingthreats.net/open/suricata/emerging.rules.tar.gz'
+                urlHash: "https://rules.emergingthreats.net/open/suricata/emerging.rules.tar.gz.md5"
+                license: "BSD"
+                excludeFiles:
+                  - "*deleted*"
+                  - "*retired*"
+                proxyURL: ""
+                proxyUsername: ""
+                proxyPassword: ""
+                proxyCACert: ""
+                insecureSkipVerify: false
+                readOnly: true
+                deleteUnreferenced: true
+              - name: ABUSECH-SSLBL
+                deleteUnreferenced: true
+                description: 'Abuse.ch SSL Blacklist'
+                enabled: false
+                license: CC0-1.0
+                readOnly: true
+                sourcePath: https://sslbl.abuse.ch/blacklist/sslblacklist_tls_cert.tar.gz
+                sourceType: url
+              - name: local-rules
+                description: "Local rules from files (*.rules) in a directory on the filesystem"
+                license: "custom"
+                sourceType: directory
+                sourcePath: /nsm/rules/custom-local-repos/local-suricata
+                readOnly: false
+                deleteUnreferenced: false
+                enabled: true
+              - name: SO_FILTERS
+                deleteUnreferenced: true
+                description: Filter rules for when Suricata is set as the metadata engine
+                enabled: false
+                license: Elastic-2.0
+                readOnly: true
+                sourcePath: /nsm/rules/suricata/so_filters.rules
+                sourceType: directory
+              - name: SO_EXTRACTIONS
+                description: Extraction rules for when Suricata is set as the metadata engine
+                deleteUnreferenced: true
+                enabled: false
+                license: Elastic-2.0
+                readOnly: true
+                sourcePath: /nsm/rules/suricata/so_extraction.rules
+                sourceType: directory
+            airgap:
+              - name: Emerging-Threats
+                description: "Emerging Threats ruleset - To enable ET Pro, enter your license key below. Leave empty for ET Open (free) rules."
+                licenseKey: ""
+                enabled: true
+                sourceType: url
+                sourcePath: 'https://rules.emergingthreats.net/open/suricata/emerging.rules.tar.gz'
+                urlHash: "https://rules.emergingthreats.net/open/suricata/emerging.rules.tar.gz.md5"
+                license: "BSD"
+                excludeFiles:
+                  - "*deleted*"
+                  - "*retired*"
+                proxyURL: ""
+                proxyUsername: ""
+                proxyPassword: ""
+                proxyCACert: ""
+                insecureSkipVerify: false
+                readOnly: true
+                deleteUnreferenced: true
+              - name: local-rules
+                description: "Local rules from files (*.rules) in a directory on the filesystem"
+                license: "custom"
+                sourceType: directory
+                sourcePath: /nsm/rules/custom-local-repos/local-suricata
+                readOnly: false
+                deleteUnreferenced: false
+                enabled: true
+              - name: SO_FILTERS
+                deleteUnreferenced: true
+                description: Filter rules for when Suricata is set as the metadata engine
+                enabled: false
+                license: Elastic-2.0
+                readOnly: true
+                sourcePath: /nsm/rules/suricata/so_filters.rules
+                sourceType: directory
+              - name: SO_EXTRACTIONS
+                description: Extraction rules for when Suricata is set as the metadata engine
+                deleteUnreferenced: true
+                enabled: false
+                license: Elastic-2.0
+                readOnly: true
+                sourcePath: /nsm/rules/suricata/so_extraction.rules
+                sourceType: directory
         navigator:
           intervalMinutes: 30
           outputPath: /opt/sensoroni/navigator

salt/soc/enabled.sls

@@ -27,7 +27,8 @@ so-soc:
       - /opt/so/conf/strelka:/opt/sensoroni/yara:rw
       - /opt/so/conf/sigma:/opt/sensoroni/sigma:rw
       - /opt/so/rules/elastalert/rules:/opt/sensoroni/elastalert:rw
-      - /opt/so/rules/nids/suri:/opt/sensoroni/nids:ro
+      - /opt/so/saltstack/local/salt/suricata/rules:/opt/sensoroni/suricata/rules:rw
+      - /opt/so/saltstack/local/salt/suricata/files:/opt/sensoroni/suricata/threshold:rw
       - /opt/so/conf/soc/fingerprints:/opt/sensoroni/fingerprints:rw
       - /nsm/soc/jobs:/opt/sensoroni/jobs:rw
       - /nsm/soc/uploads:/nsm/soc/uploads:rw

salt/soc/files/soc/detections_custom_repo_template_readme.jinja

@@ -45,6 +45,61 @@ Finally, commit it:
 The next time the Strelka / YARA engine syncs, the new rule should be imported
 If there are errors, review the sync log to troubleshoot further.
 
+{% elif repo_type == 'suricata' %}
+# Suricata Local Custom Rules Repository
+
+This folder has already been initialized as a git repo 
+and your Security Onion grid is configured to import any Suricata rule files found here. 
+
+Just add your rule file and commit it.
+
+For example:
+
+** Note: If this is your first time making changes to this repo, you may run into the following error:
+
+fatal: detected dubious ownership in repository at '/nsm/rules/custom-local-repos/local-suricata'
+To add an exception for this directory, call:
+	git config --global --add safe.directory /nsm/rules/custom-local-repos/local-suricata
+
+This means that the user you are running commands as does not match the user that is used for this git repo (socore).
+You will need to make sure your rule files are accessible to the socore user, so either su to socore 
+or add the exception and then chown the rule files later.
+
+Also, you will be asked to set some configuration:
+```
+Author identity unknown
+*** Please tell me who you are.
+Run
+  git config --global user.email "you@example.com"
+  git config --global user.name "Your Name"
+to set your account's default identity.
+Omit --global to set the identity only in this repository.
+```
+
+Run these commands, ommitting the `--global`.
+
+With that out of the way:
+
+First, create the rule file with a .rules extension:
+`vi my_custom_rules.rules`
+
+Next, use git to stage the new rule to be committed:
+`git add my_custom_rules.rules`
+
+Finally, commit it:
+`git commit -m "Initial commit of my_custom_rule.rules"`
+
+The next time the Suricata engine syncs, the new rule/s should be imported
+If there are errors, review the sync log to troubleshoot further.
+
+{% elif repo_type == 'suricata-etpro' %}
+# Suricata ETPRO - Airgap
+
+This folder has been initialized for use with ETPRO during Airgap deployment.
+
+Just add your ETPRO rule/s file to this folder and the Suricata engine will import them.
+
+If there are errors, review the sync log to troubleshoot further.
 {% elif repo_type == 'sigma' %}
 # Sigma Local Custom Rules Repository
 

salt/soc/merged.map.jinja

@@ -50,17 +50,86 @@
 {%   do SOCMERGED.config.server.modules.elastalertengine.update({'enabledSigmaRules': SOCMERGED.config.server.modules.elastalertengine.enabledSigmaRules.default}) %}
 {% endif %}
 
-{# set elastalertengine.rulesRepos and strelkaengine.rulesRepos based on airgap or not #}
+{# set elastalertengine.rulesRepos, strelkaengine.rulesRepos, and suricataengine.rulesetSources based on airgap or not #}
 {% if GLOBALS.airgap %}
 {%   do SOCMERGED.config.server.modules.elastalertengine.update({'rulesRepos': SOCMERGED.config.server.modules.elastalertengine.rulesRepos.airgap}) %}
 {%   do SOCMERGED.config.server.modules.strelkaengine.update({'rulesRepos': SOCMERGED.config.server.modules.strelkaengine.rulesRepos.airgap}) %}
+{#%   if SOCMERGED.config.server.modules.suricataengine.rulesetSources is mapping %#}
+{%     do SOCMERGED.config.server.modules.suricataengine.update({'rulesetSources': SOCMERGED.config.server.modules.suricataengine.rulesetSources.airgap}) %}
+{#%   endif %#}
 {%   do SOCMERGED.config.server.update({'airgapEnabled': true}) %}
 {% else %}
 {%   do SOCMERGED.config.server.modules.elastalertengine.update({'rulesRepos': SOCMERGED.config.server.modules.elastalertengine.rulesRepos.default}) %}
 {%   do SOCMERGED.config.server.modules.strelkaengine.update({'rulesRepos': SOCMERGED.config.server.modules.strelkaengine.rulesRepos.default}) %}
+{#%   if SOCMERGED.config.server.modules.suricataengine.rulesetSources is mapping %#}
+{%     do SOCMERGED.config.server.modules.suricataengine.update({'rulesetSources': SOCMERGED.config.server.modules.suricataengine.rulesetSources.default}) %}
+{#%   endif %#}
 {%   do SOCMERGED.config.server.update({'airgapEnabled': false}) %}
 {% endif %}
 
+
+{# Define the Detections custom ruleset that should always be present #}
+{% set CUSTOM_RULESET = {
+  'name': 'custom',
+  'description': 'User-created custom rules created via the Detections module in the SOC UI',
+  'sourceType': 'elasticsearch',
+  'sourcePath': 'so_detection.ruleset:__custom__',
+  'readOnly': false,
+  'deleteUnreferenced': false,
+  'license': 'Custom',
+  'enabled': true
+} %}
+
+{# Always append the custom ruleset to suricataengine.rulesetSources if not already present #}
+{% if SOCMERGED.config.server.modules.suricataengine is defined and SOCMERGED.config.server.modules.suricataengine.rulesetSources is defined %}
+{%   if SOCMERGED.config.server.modules.suricataengine.rulesetSources is not mapping %}
+{%     set custom_names = SOCMERGED.config.server.modules.suricataengine.rulesetSources | selectattr('name', 'equalto', 'custom') | list %}
+{%     if custom_names | length == 0 %}
+{%       do SOCMERGED.config.server.modules.suricataengine.rulesetSources.append(CUSTOM_RULESET) %}
+{%     endif %}
+{%   endif %}
+{% endif %}
+
+{# Enable SO_FILTERS and SO_EXTRACTIONS when Suricata is the metadata engine #}
+{% if SOCMERGED.config.server.modules.suricataengine is defined and SOCMERGED.config.server.modules.suricataengine.rulesetSources is defined %}
+{%   if SOCMERGED.config.server.modules.suricataengine.rulesetSources is not mapping %}
+{%     for ruleset in SOCMERGED.config.server.modules.suricataengine.rulesetSources %}
+{%       if ruleset.name in ['SO_FILTERS', 'SO_EXTRACTIONS'] and GLOBALS.md_engine == 'SURICATA' %}
+{%         do ruleset.update({'enabled': true}) %}
+{%       endif %}
+{%     endfor %}
+{%   endif %}
+{% endif %}
+
+{# Transform Emerging-Threats ruleset based on license key #}
+{% if SOCMERGED.config.server.modules.suricataengine is defined and SOCMERGED.config.server.modules.suricataengine.rulesetSources is defined %}
+{%   if SOCMERGED.config.server.modules.suricataengine.rulesetSources is not mapping %}
+{%     for ruleset in SOCMERGED.config.server.modules.suricataengine.rulesetSources %}
+{%     if ruleset.name == 'Emerging-Threats' %}
+{%       if ruleset.licenseKey and ruleset.licenseKey != '' %}
+{#         License key is defined - transform to ETPRO #}
+{#         Engine Version is hardcoded in the URL - this does not change often: https://community.emergingthreats.net/t/supported-engines/71 #}
+{%         do ruleset.update({
+             'name': 'ETPRO',
+             'sourcePath': 'https://rules.emergingthreatspro.com/' ~ ruleset.licenseKey ~ '/suricata-7.0.3/etpro.rules.tar.gz',
+             'urlHash': 'https://rules.emergingthreatspro.com/' ~ ruleset.licenseKey ~ '/suricata-7.0.3/etpro.rules.tar.gz.md5',
+             'license': 'Commercial'
+           }) %}
+{%       else %}
+{#         No license key - explicitly set to ETOPEN #}
+{%         do ruleset.update({
+             'name': 'ETOPEN',
+             'sourcePath': 'https://rules.emergingthreats.net/open/suricata-7.0.3/emerging.rules.tar.gz',
+             'urlHash': 'https://rules.emergingthreats.net/open/suricata-7.0.3/emerging.rules.tar.gz.md5',
+             'license': 'BSD'
+           }) %}
+{%       endif %}
+{%     endif %}
+{%     endfor %}
+{%   endif %}
+{% endif %}
+
+
 {# set playbookRepos based on airgap or not #}
 {% if GLOBALS.airgap %}
 {%   do SOCMERGED.config.server.modules.playbook.update({'playbookRepos': SOCMERGED.config.server.modules.playbook.playbookRepos.airgap}) %}

salt/soc/soc_soc.yaml

@@ -563,6 +563,52 @@ soc:
             advanced: True
             forcedType: "[]string"
             helpLink: detections.html#rule-engine-status
+          rulesetSources:
+            default: &serulesetSources
+              description: "Ruleset sources for Suricata rules. Supports URL downloads and local directories. Refer to the linked documentation for details on how to configure this setting."
+              global: True
+              advanced: False
+              forcedType: "[]{}"
+              helpLink: suricata.html
+              syntax: json
+              uiElements:
+              - field: name
+                label: Ruleset Name (This will be the name of the ruleset in the UI)
+                required: True
+                readonly: True
+              - field: description
+                label: Description
+              - field: enabled
+                label: Enabled (If false, existing rules & overrides will be removed)
+                forcedType: bool
+                required: True
+              - field: licenseKey
+                label: License Key
+                required: False
+              - field: sourceType
+                label: Source Type
+                required: True
+                options:
+                  - url
+                  - directory
+              - field: sourcePath
+                label: Source Path (full url or directory path)
+                required: True
+              - field: excludeFiles
+                label: Exclude Files (list of file names to exclude, separated by commas)
+                required: False
+              - field: license
+                label: Ruleset License
+                required: True
+              - field: readOnly
+                label: Read Only (Prevents changes to the rule itself - can still be enabled/disabled/tuned)
+                forcedType: bool
+                required: False
+              - field: deleteUnreferenced
+                label: Delete Unreferenced (Deletes rules that are no longer referenced by ruleset source)
+                forcedType: bool
+                required: False
+            airgap: *serulesetSources
         navigator:
           intervalMinutes:
             description: How often to generate the Navigator Layers. (minutes)

salt/suricata/config.sls

@@ -10,12 +10,6 @@
 {%   from 'suricata/map.jinja' import SURICATAMERGED %}
 {%   from 'bpf/suricata.map.jinja' import SURICATABPF, SURICATA_BPF_STATUS, SURICATA_BPF_CALC %}
 
-suridir:
-  file.directory:
-    - name: /opt/so/conf/suricata
-    - user: 940
-    - group: 940
-
 {%   if GLOBALS.pcap_engine in ["SURICATA", "TRANSITION"] %}
 {%     from 'bpf/pcap.map.jinja' import PCAPBPF, PCAP_BPF_STATUS, PCAP_BPF_CALC %}
 # BPF compilation and configuration
@@ -28,6 +22,14 @@ suriPCAPbpfcompilationfailure:
 {%     endif %}
 {%   endif %}
 
+suridir:
+  file.directory:
+    - name: /opt/so/conf/suricata
+    - user: 940
+    - group: 939
+    - mode: 775
+    - makedirs: True
+
 # BPF applied to all of Suricata - alerts/metadata/pcap
 suribpf:
   file.managed:
@@ -89,9 +91,11 @@ suricata_sbin_jinja:
 
 suriruledir:
   file.directory:
-    - name: /opt/so/conf/suricata/rules
+    - name: /opt/so/rules/suricata
     - user: 940
-    - group: 940
+    - group: 939
+    - mode: 775
+    - makedirs: True
 
 surilogdir:
   file.directory:
@@ -115,12 +119,10 @@ suridatadir:
     - mode: 770
     - makedirs: True
 
-# salt:// would resolve to /opt/so/rules/nids because of the defined file_roots and
-#  not existing under /opt/so/saltstack/local/salt or /opt/so/saltstack/default/salt
 surirulesync:
   file.recurse:
-    - name: /opt/so/conf/suricata/rules/
-    - source: salt://suri/
+    - name: /opt/so/rules/suricata/
+    - source: salt://suricata/rules/
     - user: 940
     - group: 940
     - show_changes: False
@@ -155,10 +157,10 @@ suriconfig:
 surithresholding:
   file.managed:
     - name: /opt/so/conf/suricata/threshold.conf
-    - source: salt://suricata/files/threshold.conf.jinja
+    - source: salt://suricata/files/threshold.conf
     - user: 940
     - group: 940
-    - template: jinja
+    - onlyif: salt://suricata/files/threshold.conf
 
 suriclassifications:
   file.managed:

salt/suricata/defaults.yaml

@@ -467,7 +467,7 @@ suricata:
         append: "yes"
     default-rule-path: /etc/suricata/rules
     rule-files:
-      - all.rules
+      - all-rulesets.rules
     classification-file: /etc/suricata/classification.config
     reference-config-file: /etc/suricata/reference.config
     threshold-file: /etc/suricata/threshold.conf

salt/suricata/enabled.sls

@@ -36,7 +36,7 @@ so-suricata:
       - /opt/so/conf/suricata/suricata.yaml:/etc/suricata/suricata.yaml:ro
       - /opt/so/conf/suricata/threshold.conf:/etc/suricata/threshold.conf:ro
       - /opt/so/conf/suricata/classification.config:/etc/suricata/classification.config:ro
-      - /opt/so/conf/suricata/rules:/etc/suricata/rules:ro
+      - /opt/so/rules/suricata:/etc/suricata/rules:ro
       - /opt/so/log/suricata/:/var/log/suricata/:rw
       - /nsm/suricata/:/nsm/:rw
       - /nsm/suricata/extracted:/var/log/suricata//filestore:rw

salt/suricata/files/so_extraction.rules

@@ -23,4 +23,4 @@ alert smb any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestor
 alert http any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100016; rev:1;)
 alert smtp any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100017; rev:1;)
 alert nfs any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100018; rev:1;)
-alert smb any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100019; rev:1;)
+alert smb any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100019; rev:1;)

salt/suricata/files/so_filters.rules

@@ -9,3 +9,4 @@
 #config tls any any -> any any (tls.fingerprint; content:"4f:a4:5e:58:7e:d9:db:20:09:d7:b6:c7:ff:58:c4:7b:dc:3f:55:b4"; config: logging disable, type tx, scope tx; sid:1200003;)
 # Example of filtering out a md5 of a file from being in the files log.
 #config fileinfo any any -> any any (fileinfo.filemd5; content:"7a125dc69c82d5caf94d3913eecde4b5"; config: logging disable, type tx, scope tx; sid:1200004;)
+

salt/suricata/files/threshold.conf.jinja

@@ -1,35 +0,0 @@
-{% import_yaml 'suricata/thresholding/sids.yaml' as THRESHOLDING %}
-{% if THRESHOLDING -%}
-  
-  {% for EACH_SID in THRESHOLDING -%}
-    {% for ACTIONS_LIST in THRESHOLDING[EACH_SID] -%}
-      {% for EACH_ACTION in ACTIONS_LIST -%}
-        
-        {%- if EACH_ACTION == 'threshold' %}
-{{ EACH_ACTION }} gen_id {{ ACTIONS_LIST[EACH_ACTION].gen_id }}, sig_id {{ EACH_SID }}, type {{ ACTIONS_LIST[EACH_ACTION].type }}, track {{ ACTIONS_LIST[EACH_ACTION].track }}, count {{ ACTIONS_LIST[EACH_ACTION].count }}, seconds {{ ACTIONS_LIST[EACH_ACTION].seconds }}
-        
-        {%- elif EACH_ACTION == 'rate_filter' %}
-          {%- if ACTIONS_LIST[EACH_ACTION].new_action not in ['drop','reject'] %}
-{{ EACH_ACTION }} gen_id {{ ACTIONS_LIST[EACH_ACTION].gen_id }}, sig_id {{ EACH_SID }}, track {{ ACTIONS_LIST[EACH_ACTION].track }}, count {{ ACTIONS_LIST[EACH_ACTION].count }}, seconds {{ ACTIONS_LIST[EACH_ACTION].seconds }}, new_action {{ ACTIONS_LIST[EACH_ACTION].new_action }}, timeout {{ ACTIONS_LIST[EACH_ACTION].timeout }}
-          {%- else %}
-##### Security Onion does not support drop or reject actions for rate_filter
-##### {{ EACH_ACTION }} gen_id {{ ACTIONS_LIST[EACH_ACTION].gen_id }}, sig_id {{ EACH_SID }}, track {{ ACTIONS_LIST[EACH_ACTION].track }}, count {{ ACTIONS_LIST[EACH_ACTION].count }}, seconds {{ ACTIONS_LIST[EACH_ACTION].seconds }}, new_action {{ ACTIONS_LIST[EACH_ACTION].new_action }}, timeout {{ ACTIONS_LIST[EACH_ACTION].timeout }}
-          {%- endif %}
-        
-        {%- elif EACH_ACTION == 'suppress' %}
-          {%- if ACTIONS_LIST[EACH_ACTION].track is defined %}
-{{ EACH_ACTION }} gen_id {{ ACTIONS_LIST[EACH_ACTION].gen_id }}, sig_id {{ EACH_SID }}, track {{ ACTIONS_LIST[EACH_ACTION].track }}, ip {{ ACTIONS_LIST[EACH_ACTION].ip }}
-          {%- else %}
-{{ EACH_ACTION }} gen_id {{ ACTIONS_LIST[EACH_ACTION].gen_id }}, sig_id {{ EACH_SID }}
-          {%- endif %}
-        
-        {%- endif %}
-      
-      {%- endfor %}
-    {%- endfor %}
-  {%- endfor %}
-
-{%- else %}
-##### Navigate to suricata > thresholding > SIDS in SOC to define thresholding
-
-{%- endif %}

salt/suricata/manager.sls

@@ -1,30 +0,0 @@
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls in allowed_states %}
-
-surilocaldir:
-  file.directory:
-    - name: /opt/so/saltstack/local/salt/suricata
-    - user: socore
-    - group: socore
-    - makedirs: True
-
-ruleslink:
-  file.symlink:
-    - name: /opt/so/saltstack/local/salt/suricata/rules
-    - user: socore
-    - group: socore
-    - target: /opt/so/rules/nids/suri
-
-refresh_salt_master_fileserver_suricata_ruleslink:
-  salt.runner:
-    - name: fileserver.update
-    - onchanges:
-      - file: ruleslink
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

salt/top.sls

@@ -74,8 +74,6 @@ base:
     - sensoroni
     - telegraf
     - firewall
-    - idstools
-    - suricata.manager
     - healthcheck
     - elasticsearch
     - elastic-fleet-package-registry
@@ -106,8 +104,6 @@ base:
     - firewall
     - sensoroni
     - telegraf
-    - idstools
-    - suricata.manager
     - healthcheck
     - elasticsearch
     - logstash
@@ -142,8 +138,6 @@ base:
     - sensoroni
     - telegraf
     - backup.config_backup
-    - idstools
-    - suricata.manager
     - elasticsearch
     - logstash
     - redis
@@ -177,8 +171,6 @@ base:
     - sensoroni
     - telegraf
     - backup.config_backup
-    - idstools
-    - suricata.manager
     - elasticsearch
     - logstash
     - redis
@@ -208,8 +200,6 @@ base:
     - sensoroni
     - telegraf
     - firewall
-    - idstools
-    - suricata.manager
     - pcap
     - elasticsearch
     - elastic-fleet-package-registry

setup/so-functions

@@ -816,7 +816,6 @@ create_manager_pillars() {
 	backup_pillar
 	docker_pillar
 	redis_pillar
-	idstools_pillar
 	kratos_pillar
 	hydra_pillar
 	soc_pillar
@@ -1282,11 +1281,6 @@ ls_heapsize() {
 
 }
 
-idstools_pillar() {
-	title "Ading IDSTOOLS pillar options"
-	touch $adv_idstools_pillar_file
-}
-
 nginx_pillar() {
 	title "Creating the NGINX pillar"
 	[[ -z "$TESTING" ]] && return
@@ -1462,7 +1456,7 @@ make_some_dirs() {
 	mkdir -p $local_salt_dir/salt/firewall/portgroups
 	mkdir -p $local_salt_dir/salt/firewall/ports
 
-	for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni soc docker zeek suricata nginx telegraf logstash soc manager kratos hydra idstools idh elastalert stig global kafka versionlock hypervisor vm; do
+	for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni soc docker zeek suricata nginx telegraf logstash soc manager kratos hydra idh elastalert stig global kafka versionlock hypervisor vm; do
 	mkdir -p $local_salt_dir/pillar/$THEDIR
 	touch $local_salt_dir/pillar/$THEDIR/adv_$THEDIR.sls
 	touch $local_salt_dir/pillar/$THEDIR/soc_$THEDIR.sls

setup/so-variables

@@ -166,12 +166,6 @@ export hydra_pillar_file
 adv_hydra_pillar_file="$local_salt_dir/pillar/hydra/adv_hydra.sls"
 export adv_hydra_pillar_file
 
-idstools_pillar_file="$local_salt_dir/pillar/idstools/soc_idstools.sls"
-export idstools_pillar_file
-
-adv_idstools_pillar_file="$local_salt_dir/pillar/idstools/adv_idstools.sls"
-export adv_idstools_pillar_file
-
 nginx_pillar_file="$local_salt_dir/pillar/nginx/soc_nginx.sls"
 export nginx_pillar_file