Update defaults.yaml

pillar/top.sls

@@ -43,6 +43,8 @@ base:
     - secrets
     - manager.soc_manager
     - manager.adv_manager
+    - idstools.soc_idstools
+    - idstools.adv_idstools
     - logstash.nodes
     - logstash.soc_logstash
     - logstash.adv_logstash
@@ -115,6 +117,8 @@ base:
     - elastalert.adv_elastalert
     - manager.soc_manager
     - manager.adv_manager
+    - idstools.soc_idstools
+    - idstools.adv_idstools
     - soc.soc_soc
     - soc.adv_soc
     - kibana.soc_kibana
@@ -154,6 +158,8 @@ base:
     {% endif %}
     - secrets
     - healthcheck.standalone
+    - idstools.soc_idstools
+    - idstools.adv_idstools
     - kratos.soc_kratos
     - kratos.adv_kratos
     - hydra.soc_hydra

salt/allowed_states.map.jinja

@@ -38,6 +38,7 @@
     'hydra',
     'elasticfleet',
     'elastic-fleet-package-registry',
+    'idstools',
     'suricata.manager',
     'utility'
 ] %}

salt/common/tools/sbin/so-common

@@ -395,7 +395,7 @@ is_manager_node() {
 }
 
 is_sensor_node() {
-	# Check to see if this is a sensor (forward) node
+	# Check to see if this is a sensor node
 	is_single_node_grid && return 0
 	grep "role: so-" /etc/salt/grains | grep -E "sensor|heavynode" &> /dev/null
 }

salt/common/tools/sbin/so-image-common

@@ -25,6 +25,7 @@ container_list() {
   if [ $MANAGERCHECK == 'so-import' ]; then
     TRUSTED_CONTAINERS=(
       "so-elasticsearch"
+      "so-idstools"
       "so-influxdb"
       "so-kibana"
       "so-kratos"
@@ -48,6 +49,7 @@ container_list() {
       "so-elastic-fleet-package-registry"
       "so-elasticsearch"
       "so-idh"
+      "so-idstools"
       "so-influxdb"
       "so-kafka"
       "so-kibana"
@@ -67,6 +69,7 @@ container_list() {
     )
   else
     TRUSTED_CONTAINERS=(
+      "so-idstools"
       "so-elasticsearch"
       "so-logstash"
       "so-nginx"

salt/docker/defaults.yaml

@@ -24,6 +24,11 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+    'so-idstools':
+      final_octet: 25
+      custom_bind_mounts: []
+      extra_hosts: []
+      extra_env: []
     'so-influxdb':
       final_octet: 26
       port_bindings:

salt/docker/soc_docker.yaml

@@ -41,6 +41,7 @@ docker:
         forcedType: "[]string"
     so-elastic-fleet: *dockerOptions
     so-elasticsearch: *dockerOptions
+    so-idstools: *dockerOptions
     so-influxdb: *dockerOptions
     so-kibana: *dockerOptions
     so-kratos: *dockerOptions

salt/idstools/config.sls

@@ -0,0 +1,65 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+
+include:
+  - idstools.sync_files
+
+idstoolslogdir:
+  file.directory:
+    - name: /opt/so/log/idstools
+    - user: 939
+    - group: 939
+    - makedirs: True
+
+idstools_sbin:
+  file.recurse:
+    - name: /usr/sbin
+    - source: salt://idstools/tools/sbin
+    - user: 939
+    - group: 939
+    - file_mode: 755
+
+# If this is used, exclude so-rule-update
+#idstools_sbin_jinja:
+#  file.recurse:
+#    - name: /usr/sbin
+#    - source: salt://idstools/tools/sbin_jinja
+#    - user: 939
+#    - group: 939 
+#    - file_mode: 755
+#    - template: jinja
+
+idstools_so-rule-update:
+  file.managed:
+    - name: /usr/sbin/so-rule-update
+    - source: salt://idstools/tools/sbin_jinja/so-rule-update
+    - user: 939
+    - group: 939 
+    - mode: 755
+    - template: jinja
+
+suricatacustomdirsfile:
+  file.directory:
+    - name: /nsm/rules/detect-suricata/custom_file
+    - user: 939
+    - group: 939
+    - makedirs: True
+
+suricatacustomdirsurl:
+  file.directory:
+    - name: /nsm/rules/detect-suricata/custom_temp
+    - user: 939
+    - group: 939
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/idstools/defaults.yaml

@@ -0,0 +1,10 @@
+idstools:
+  enabled: False
+  config:
+    urls: []
+    ruleset: ETOPEN
+    oinkcode: ""
+  sids:
+    enabled: []
+    disabled: []
+    modify: []

salt/idstools/disabled.sls

@@ -0,0 +1,31 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+
+include:
+  - idstools.sostatus
+  
+so-idstools:
+  docker_container.absent:
+    - force: True
+
+so-idstools_so-status.disabled:
+  file.comment:
+    - name: /opt/so/conf/so-status/so-status.conf
+    - regex: ^so-idstools$
+
+so-rule-update:
+  cron.absent:
+    - identifier: so-rule-update
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/idstools/enabled.sls

@@ -0,0 +1,91 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+{%   from 'docker/docker.map.jinja' import DOCKER %}
+{%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   set proxy = salt['pillar.get']('manager:proxy') %}
+
+include:
+  - idstools.config
+  - idstools.sostatus
+
+so-idstools:
+  docker_container.running:
+    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-idstools:{{ GLOBALS.so_version }}
+    - hostname: so-idstools
+    - user: socore
+    - networks:
+      - sobridge:
+        - ipv4_address: {{ DOCKER.containers['so-idstools'].ip }}
+    {% if proxy %}
+    - environment:
+      - http_proxy={{ proxy }}
+      - https_proxy={{ proxy }}
+      - no_proxy={{ salt['pillar.get']('manager:no_proxy') }}
+      {% if DOCKER.containers['so-idstools'].extra_env %}
+        {% for XTRAENV in DOCKER.containers['so-idstools'].extra_env %}
+      - {{ XTRAENV }}
+        {% endfor %}
+      {% endif %}
+    {% elif DOCKER.containers['so-idstools'].extra_env %}
+    - environment:
+      {% for XTRAENV in DOCKER.containers['so-idstools'].extra_env %}
+      - {{ XTRAENV }}
+      {% endfor %}
+    {% endif %}
+    - binds:
+      - /opt/so/conf/idstools/etc:/opt/so/idstools/etc:ro
+      - /opt/so/rules/nids/suri:/opt/so/rules/nids/suri:rw
+      - /nsm/rules/:/nsm/rules/:rw
+    {% if DOCKER.containers['so-idstools'].custom_bind_mounts %}
+      {% for BIND in DOCKER.containers['so-idstools'].custom_bind_mounts %}
+      - {{ BIND }}
+      {% endfor %}
+    {% endif %}
+    - extra_hosts:
+      - {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }}
+    {% if DOCKER.containers['so-idstools'].extra_hosts %}
+      {% for XTRAHOST in DOCKER.containers['so-idstools'].extra_hosts %}
+      - {{ XTRAHOST }}
+      {% endfor %}
+    {% endif %}
+    - watch:
+      - file: idstoolsetcsync
+      - file: idstools_so-rule-update
+
+delete_so-idstools_so-status.disabled:
+  file.uncomment:
+    - name: /opt/so/conf/so-status/so-status.conf
+    - regex: ^so-idstools$
+
+so-rule-update:
+  cron.present:
+    - name: /usr/sbin/so-rule-update > /opt/so/log/idstools/download_cron.log 2>&1
+    - identifier: so-rule-update
+    - user: root
+    - minute: '1'
+    - hour: '7'
+
+# order this last to give so-idstools container time to be ready
+run_so-rule-update:
+  cmd.run:
+    - name: '/usr/sbin/so-rule-update > /opt/so/log/idstools/download_idstools_state.log 2>&1'
+    - require:
+      - docker_container: so-idstools
+    - onchanges:
+      - file: idstools_so-rule-update
+      - file: idstoolsetcsync
+      - file: synclocalnidsrules
+    - order: last
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/idstools/etc/disable.conf

@@ -0,0 +1,16 @@
+{%- set disabled_sids = salt['pillar.get']('idstools:sids:disabled', {}) -%}
+# idstools - disable.conf
+
+# Example of disabling a rule by signature ID (gid is optional).
+# 1:2019401
+# 2019401
+
+# Example of disabling a rule by regular expression.
+# - All regular expression matches are case insensitive.
+# re:hearbleed
+# re:MS(0[7-9]|10)-\d+
+{%- if disabled_sids != None %}
+{%- for sid in disabled_sids %}
+{{ sid }}
+{%- endfor %}
+{%- endif %}

salt/idstools/etc/enable.conf

@@ -0,0 +1,16 @@
+{%- set enabled_sids = salt['pillar.get']('idstools:sids:enabled', {}) -%}
+# idstools-rulecat - enable.conf
+
+# Example of enabling a rule by signature ID (gid is optional).
+# 1:2019401
+# 2019401
+
+# Example of enabling a rule by regular expression.
+# - All regular expression matches are case insensitive.
+# re:hearbleed
+# re:MS(0[7-9]|10)-\d+
+{%- if enabled_sids != None %}
+{%- for sid in enabled_sids %}
+{{ sid }}
+{%- endfor %}
+{%- endif %}

salt/idstools/etc/modify.conf

@@ -0,0 +1,12 @@
+{%- set modify_sids = salt['pillar.get']('idstools:sids:modify', {}) -%}
+# idstools-rulecat - modify.conf
+
+# Format: <sid> "<from>" "<to>"
+
+# Example changing the seconds for rule 2019401 to 3600.
+#2019401 "seconds \d+" "seconds 3600"
+{%- if modify_sids != None %}
+{%- for sid in modify_sids %}
+{{ sid }}
+{%- endfor %}
+{%- endif %}

salt/idstools/etc/rulecat.conf

@@ -0,0 +1,23 @@
+{%- from 'vars/globals.map.jinja' import GLOBALS -%}
+{%- from 'soc/merged.map.jinja' import SOCMERGED -%}
+--suricata-version=7.0.3
+--merged=/opt/so/rules/nids/suri/all.rules
+--output=/nsm/rules/detect-suricata/custom_temp
+--local=/opt/so/rules/nids/suri/local.rules
+{%-   if GLOBALS.md_engine == "SURICATA" %}
+--local=/opt/so/rules/nids/suri/extraction.rules
+--local=/opt/so/rules/nids/suri/filters.rules
+{%-   endif %}
+--url=http://{{ GLOBALS.manager }}:7788/suricata/emerging-all.rules
+--disable=/opt/so/idstools/etc/disable.conf
+--enable=/opt/so/idstools/etc/enable.conf
+--modify=/opt/so/idstools/etc/modify.conf
+{%- if SOCMERGED.config.server.modules.suricataengine.customRulesets %}
+    {%- for ruleset in SOCMERGED.config.server.modules.suricataengine.customRulesets %}
+        {%- if 'url' in ruleset %}
+--url={{ ruleset.url }}
+        {%- elif 'file' in ruleset %}
+--local={{ ruleset.file }}
+        {%- endif %}
+    {%- endfor %}
+{%- endif %}

salt/idstools/init.sls

@@ -0,0 +1,13 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'idstools/map.jinja' import IDSTOOLSMERGED %}
+
+include:
+{% if IDSTOOLSMERGED.enabled %}
+  - idstools.enabled
+{% else %}
+  - idstools.disabled
+{% endif %}

salt/idstools/map.jinja

@@ -0,0 +1,7 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+   
+{% import_yaml 'idstools/defaults.yaml' as IDSTOOLSDEFAULTS with context %}
+{% set IDSTOOLSMERGED = salt['pillar.get']('idstools', IDSTOOLSDEFAULTS.idstools, merge=True) %}

salt/idstools/rules/extraction.rules

@@ -0,0 +1,26 @@
+# Extract all PDF mime type
+alert http any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100000; rev:1;)
+alert smtp any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100001; rev:1;)
+alert nfs any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100002; rev:1;)
+alert smb any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100003; rev:1;)
+# Extract EXE/DLL file types
+alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100004; rev:1;)
+alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100005; rev:1;)
+alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100006; rev:1;)
+alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100007; rev:1;)
+alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100008; rev:1;)
+alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100009; rev:1;)
+alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100010; rev:1;)
+alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100011; rev:1;)
+
+# Extract all Zip files
+alert http any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100012; rev:1;)
+alert smtp any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100013; rev:1;)
+alert nfs any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100014; rev:1;)
+alert smb any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100015; rev:1;)
+
+# Extract Word Docs
+alert http any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100016; rev:1;)
+alert smtp any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100017; rev:1;)
+alert nfs any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100018; rev:1;)
+alert smb any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100019; rev:1;)

salt/idstools/rules/filters.rules

@@ -0,0 +1,11 @@
+# Start the filters at sid 1200000
+# Example of filtering out *google.com from being in the dns log.
+#config dns any any -> any any (dns.query; content:"google.com"; config: logging disable, type tx, scope tx; sid:1200000;)
+# Example of filtering out  *google.com from being in the http log.
+#config http any any -> any any (http.host; content:"google.com"; config: logging disable, type tx, scope tx; sid:1200001;)
+# Example of filtering out  someuseragent from being in the http log.
+#config http any any -> any any (http.user_agent; content:"someuseragent"; config: logging disable, type tx, scope tx; sid:1200002;)
+# Example of filtering out  Google's certificate from being in the ssl log.
+#config tls any any -> any any (tls.fingerprint; content:"4f:a4:5e:58:7e:d9:db:20:09:d7:b6:c7:ff:58:c4:7b:dc:3f:55:b4"; config: logging disable, type tx, scope tx; sid:1200003;)
+# Example of filtering out a md5 of a file from being in the files log.
+#config fileinfo any any -> any any (fileinfo.filemd5; content:"7a125dc69c82d5caf94d3913eecde4b5"; config: logging disable, type tx, scope tx; sid:1200004;)

salt/idstools/rules/local.rules

@@ -0,0 +1 @@
+# Add your custom Suricata rules in this file.

salt/idstools/soc_idstools.yaml

@@ -0,0 +1,72 @@
+idstools:
+  enabled:
+    description: Enables or disables the IDStools process which is used by the Detection system.
+  config:
+    oinkcode:
+      description: Enter your registration code or oinkcode for paid NIDS rulesets.
+      title: Registration Code
+      global: True
+      forcedType: string
+      helpLink: rules.html
+    ruleset:
+      description: 'Defines the ruleset you want to run. Options are ETOPEN or ETPRO. Once you have changed the ruleset here, you will need to wait for the rule update to take place (every 24 hours), or you can force the update by nagivating to Detections -->  Options dropdown menu --> Suricata --> Full Update. WARNING! Changing the ruleset will remove all existing non-overlapping Suricata rules of the previous ruleset and their associated overrides. This removal cannot be undone.'
+      global: True
+      regex:  ETPRO\b|ETOPEN\b
+      helpLink: rules.html
+    urls:
+      description: This is a list of additional rule download locations. This feature is currently disabled. 
+      global: True
+      multiline: True
+      forcedType: "[]string"
+      readonly: True
+      helpLink: rules.html
+  sids:
+    disabled:
+      description: Contains the list of NIDS rules (or regex patterns) disabled across the grid. This setting is readonly; Use the Detections screen to disable rules.
+      global: True
+      multiline: True
+      forcedType: "[]string"
+      regex: \d*|re:.*
+      helpLink: managing-alerts.html
+      readonlyUi: True
+      advanced: true
+    enabled:
+      description: Contains the list of NIDS rules (or regex patterns) enabled across the grid. This setting is readonly; Use the Detections screen to enable rules.
+      global: True
+      multiline: True
+      forcedType: "[]string"
+      regex: \d*|re:.*
+      helpLink: managing-alerts.html
+      readonlyUi: True
+      advanced: true
+    modify:
+      description: Contains the list of NIDS rules (SID "REGEX_SEARCH_TERM" "REGEX_REPLACE_TERM"). This setting is readonly; Use the Detections screen to modify rules.
+      global: True
+      multiline: True
+      forcedType: "[]string"
+      helpLink: managing-alerts.html
+      readonlyUi: True
+      advanced: true
+  rules:
+    local__rules:
+      description: Contains the list of custom NIDS rules applied to the grid. This setting is readonly; Use the Detections screen to adjust rules.
+      file: True
+      global: True
+      advanced: True
+      title: Local Rules
+      helpLink: local-rules.html
+      readonlyUi: True
+    filters__rules:
+      description: If you are using Suricata for metadata, then you can set custom filters for that metadata here.
+      file: True
+      global: True
+      advanced: True
+      title: Filter Rules
+      helpLink: suricata.html
+    extraction__rules:
+      description: If you are using Suricata for metadata, then you can set a list of MIME types for file extraction here.
+      file: True
+      global: True
+      advanced: True
+      title: Extraction Rules
+      helpLink: suricata.html

salt/idstools/sostatus.sls

@@ -0,0 +1,21 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+
+append_so-idstools_so-status.conf:
+  file.append:
+    - name: /opt/so/conf/so-status/so-status.conf
+    - text: so-idstools
+    - unless: grep -q so-idstools /opt/so/conf/so-status/so-status.conf
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/idstools/sync_files.sls

@@ -0,0 +1,37 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
+idstoolsdir:
+  file.directory:
+    - name: /opt/so/conf/idstools/etc
+    - user: 939
+    - group: 939
+    - makedirs: True
+
+idstoolsetcsync:
+  file.recurse:
+    - name: /opt/so/conf/idstools/etc
+    - source: salt://idstools/etc
+    - user: 939
+    - group: 939
+    - template: jinja
+
+rulesdir:
+  file.directory:
+    - name: /opt/so/rules/nids/suri
+    - user: 939
+    - group: 939
+    - makedirs: True
+
+# Don't show changes because all.rules can be large
+synclocalnidsrules:
+  file.recurse:
+    - name: /opt/so/rules/nids/suri/
+    - source: salt://idstools/rules/
+    - user: 939
+    - group: 939
+    - show_changes: False
+    - include_pat: 'E@.rules'

salt/idstools/tools/sbin/so-idstools-restart

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
+
+. /usr/sbin/so-common
+
+/usr/sbin/so-restart idstools $1

salt/idstools/tools/sbin/so-idstools-start

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
+
+. /usr/sbin/so-common
+
+/usr/sbin/so-start idstools $1

salt/idstools/tools/sbin/so-idstools-stop

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
+
+. /usr/sbin/so-common
+
+/usr/sbin/so-stop idstools $1

salt/idstools/tools/sbin_jinja/so-rule-update

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# if this script isn't already running
+if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
+
+    . /usr/sbin/so-common
+
+{%- from 'vars/globals.map.jinja' import GLOBALS %}
+{%- from 'idstools/map.jinja' import IDSTOOLSMERGED %}
+
+{%- set proxy = salt['pillar.get']('manager:proxy') %}
+{%- set noproxy = salt['pillar.get']('manager:no_proxy', '') %}
+
+{%- if proxy %}
+# Download the rules from the internet
+    export http_proxy={{ proxy }}
+    export https_proxy={{ proxy }}
+    export no_proxy="{{ noproxy }}"
+{%- endif %}
+
+    mkdir -p /nsm/rules/suricata
+    chown -R socore:socore /nsm/rules/suricata
+{%- if not GLOBALS.airgap %}
+# Download the rules from the internet
+{%-   if IDSTOOLSMERGED.config.ruleset == 'ETOPEN' %}
+    docker exec so-idstools idstools-rulecat -v --suricata-version 7.0.3 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force
+{%-   elif IDSTOOLSMERGED.config.ruleset == 'ETPRO' %}
+    docker exec so-idstools idstools-rulecat -v --suricata-version 7.0.3 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force --etpro={{ IDSTOOLSMERGED.config.oinkcode }}
+{%-   endif %}
+{%- endif %}
+
+
+    argstr=""
+    for arg in "$@"; do
+        argstr="${argstr} \"${arg}\""
+    done
+
+    docker exec so-idstools /bin/bash -c "cd /opt/so/idstools/etc && idstools-rulecat --force ${argstr}"
+
+fi

salt/logrotate/defaults.yaml

@@ -1,5 +1,15 @@
 logrotate:
   config:
+    /opt/so/log/idstools/*_x_log:
+      - daily
+      - rotate 14
+      - missingok
+      - copytruncate
+      - compress
+      - create
+      - extension .log
+      - dateext
+      - dateyesterday
     /opt/so/log/nginx/*_x_log:
       - daily
       - rotate 14

salt/logrotate/soc_logrotate.yaml

@@ -1,5 +1,12 @@
 logrotate:
   config:
+    "/opt/so/log/idstools/*_x_log":
+      description: List of logrotate options for this file.
+      title: /opt/so/log/idstools/*.log
+      advanced: True
+      multiline: True
+      global: True
+      forcedType: "[]string"
     "/opt/so/log/nginx/*_x_log":
       description: List of logrotate options for this file.
       title: /opt/so/log/nginx/*.log

salt/manager/tools/sbin/so-minion

@@ -604,6 +604,16 @@ function add_kratos_to_minion() {
     fi
 }
 
+function add_idstools_to_minion() {
+    printf '%s\n'\
+    "idstools:"\
+    "  enabled: True"\
+	"  " >> $PILLARFILE
+    if [ $? -ne 0 ]; then
+        log "ERROR" "Failed to add idstools configuration to $PILLARFILE"
+        return 1
+    fi
+}
 
 function add_elastic_fleet_package_registry_to_minion() {
     printf '%s\n'\
@@ -731,6 +741,7 @@ function createEVAL() {
 	add_soc_to_minion || return 1
 	add_registry_to_minion || return 1
 	add_kratos_to_minion || return 1
+	add_idstools_to_minion || return 1
 	add_elastic_fleet_package_registry_to_minion || return 1
 }
 
@@ -751,6 +762,7 @@ function createSTANDALONE() {
 	add_soc_to_minion || return 1
 	add_registry_to_minion || return 1
 	add_kratos_to_minion || return 1
+	add_idstools_to_minion || return 1
 	add_elastic_fleet_package_registry_to_minion || return 1
 }
 
@@ -767,6 +779,7 @@ function createMANAGER() {
 	add_soc_to_minion || return 1
 	add_registry_to_minion || return 1
 	add_kratos_to_minion || return 1
+	add_idstools_to_minion || return 1
 	add_elastic_fleet_package_registry_to_minion || return 1
 }
 
@@ -783,6 +796,7 @@ function createMANAGERSEARCH() {
 	add_soc_to_minion || return 1
 	add_registry_to_minion || return 1
 	add_kratos_to_minion || return 1
+	add_idstools_to_minion || return 1
 	add_elastic_fleet_package_registry_to_minion || return 1
 }
 
@@ -797,6 +811,7 @@ function createIMPORT() {
 	add_soc_to_minion || return 1
 	add_registry_to_minion || return 1
 	add_kratos_to_minion || return 1
+	add_idstools_to_minion || return 1
 	add_elastic_fleet_package_registry_to_minion || return 1
 }
 
@@ -881,6 +896,7 @@ function createMANAGERHYPE() {
 	add_soc_to_minion || return 1
 	add_registry_to_minion || return 1
 	add_kratos_to_minion || return 1
+	add_idstools_to_minion || return 1
 	add_elastic_fleet_package_registry_to_minion || return 1
 }
 

salt/manager/tools/sbin/soup

@@ -1679,7 +1679,7 @@ This appears to be a distributed deployment. Other nodes should update themselve
 
 Each minion is on a random 15 minute check-in period and things like network bandwidth can be a factor in how long the actual upgrade takes. If you have a heavy node on a slow link, it is going to take a while to get the containers to it. Depending on what changes happened between the versions, Elasticsearch might not be able to talk to said heavy node until the update is complete.
 
-If it looks like you’re missing data after the upgrade, please avoid restarting services and instead make sure at least one search node has completed its upgrade. The best way to do this is to run 'sudo salt-call state.highstate' from a search node and make sure there are no errors. Typically if it works on one node it will work on the rest. Forward nodes are less complex and will update as they check in so you can monitor those from the Grid section of SOC.
+If it looks like you’re missing data after the upgrade, please avoid restarting services and instead make sure at least one search node has completed its upgrade. The best way to do this is to run 'sudo salt-call state.highstate' from a search node and make sure there are no errors. Typically if it works on one node it will work on the rest. Sensor nodes are less complex and will update as they check in so you can monitor those from the Grid section of SOC.
 
 For more information, please see $DOC_BASE_URL/soup.html#distributed-deployments.
 

salt/pcap/soc_pcap.yaml

@@ -7,7 +7,7 @@ pcap:
       description: By default, Stenographer limits the number of files in the pcap directory to 30000 to avoid limitations with the ext3 filesystem. However, if you're using the ext4 or xfs filesystems, then it is safe to increase this value. So if you have a large amount of storage and find that you only have 3 weeks worth of PCAP on disk while still having plenty of free space, then you may want to increase this default setting.
       helpLink: stenographer.html
     diskfreepercentage:
-      description: Stenographer will purge old PCAP on a regular basis to keep the disk free percentage at this level. If you have a distributed deployment with dedicated forward nodes, then the default value of 10 should be reasonable since Stenographer should be the main consumer of disk space in the /nsm partition. However, if you have systems that run both Stenographer and Elasticsearch at the same time (like eval and standalone installations), then you’ll want to make sure that this value is no lower than 21 so that you avoid Elasticsearch hitting its watermark setting at 80% disk usage. If you have an older standalone installation, then you may need to manually change this value to 21.
+      description: Stenographer will purge old PCAP on a regular basis to keep the disk free percentage at this level. If you have a distributed deployment with dedicated Sensor nodes, then the default value of 10 should be reasonable since Stenographer should be the main consumer of disk space in the /nsm partition. However, if you have systems that run both Stenographer and Elasticsearch at the same time (like eval and standalone installations), then you’ll want to make sure that this value is no lower than 21 so that you avoid Elasticsearch hitting its watermark setting at 80% disk usage. If you have an older standalone installation, then you may need to manually change this value to 21.
       helpLink: stenographer.html
     blocks:
       description: The number of 1MB packet blocks used by Stenographer and AF_PACKET to store packets in memory, per thread. You shouldn't need to change this. 

salt/salt/files/engines.conf

@@ -6,6 +6,30 @@ engines:
       interval: 60
   - pillarWatch:
       fpa:
+        - files:
+            - /opt/so/saltstack/local/pillar/idstools/soc_idstools.sls
+            - /opt/so/saltstack/local/pillar/idstools/adv_idstools.sls
+          pillar: idstools.config.ruleset
+          default: ETOPEN
+          actions:
+            from:
+              '*':
+                to:
+                  '*':
+                  - cmd.run:
+                      cmd: /usr/sbin/so-rule-update
+        - files:
+            - /opt/so/saltstack/local/pillar/idstools/soc_idstools.sls
+            - /opt/so/saltstack/local/pillar/idstools/adv_idstools.sls
+          pillar: idstools.config.oinkcode
+          default: ''
+          actions:
+            from:
+              '*':
+                to:
+                  '*':
+                  - cmd.run:
+                      cmd: /usr/sbin/so-rule-update
         - files:
             - /opt/so/saltstack/local/pillar/global/soc_global.sls
             - /opt/so/saltstack/local/pillar/global/adv_global.sls

salt/soc/defaults.yaml

@@ -1561,72 +1561,12 @@ soc:
           disableRegex: []
           enableRegex: []
           failAfterConsecutiveErrorCount: 10
+          communityRulesFile: /nsm/rules/suricata/emerging-all.rules
           rulesFingerprintFile: /opt/sensoroni/fingerprints/emerging-all.fingerprint
           stateFilePath: /opt/sensoroni/fingerprints/suricataengine.state
           integrityCheckFrequencySeconds: 1200
           ignoredSidRanges:
             - '1100000-1101000'
-          rulesetSources:
-            default:
-              - name: Emerging-Threats
-                description: "Emerging Threats ruleset - To enable ET Pro, enter your license key below. Leave empty for ET Open (free) rules."
-                licenseKey: ""
-                enabled: true
-                sourceType: url
-                sourcePath: 'https://rules.emergingthreats.net/open/suricata/emerging.rules.tar.gz'
-                urlHash: "https://rules.emergingthreats.net/open/suricata/emerging.rules.tar.gz.md5"
-                license: "BSD"
-                excludeFiles:
-                  - "*deleted*"
-                  - "*retired*"
-                proxyURL: ""
-                proxyUsername: ""
-                proxyPassword: ""
-                proxyCACert: ""
-                insecureSkipVerify: false
-                readOnly: true
-                deleteUnreferenced: true
-              - name: local-rules
-                id: local-rules
-                description: "Local custom rules from files (*.rules) in a directory on the filesystem"
-                license: "custom"
-                sourceType: directory
-                sourcePath: /nsm/rules/local/
-                readOnly: false
-                deleteUnreferenced: false
-                enabled: false
-                excludeFiles:
-                  - "*backup*"
-            airgap:
-              - name: Emerging-Threats
-                description: "Emerging Threats ruleset - To enable ET Pro, enter your license key below. Leave empty for ET Open (free) rules."
-                licenseKey: ""
-                enabled: true
-                sourceType: url
-                sourcePath: 'https://rules.emergingthreats.net/open/suricata/emerging.rules.tar.gz'
-                urlHash: "https://rules.emergingthreats.net/open/suricata/emerging.rules.tar.gz.md5"
-                license: "BSD"
-                excludeFiles:
-                  - "*deleted*"
-                  - "*retired*"
-                proxyURL: ""
-                proxyUsername: ""
-                proxyPassword: ""
-                proxyCACert: ""
-                insecureSkipVerify: false
-                readOnly: true
-                deleteUnreferenced: true
-              - name: local-rules
-                id: local-rules
-                description: "Local custom rules from files (*.rules) in a directory on the filesystem"
-                license: "custom"
-                sourceType: directory
-                sourcePath: /nsm/rules/local/
-                readOnly: false
-                deleteUnreferenced: false
-                enabled: false
-                excludeFiles:
-                  - "*backup*"
         navigator:
           intervalMinutes: 30
           outputPath: /opt/sensoroni/navigator
@@ -2631,3 +2571,8 @@ soc:
               contextLimitSmall: 128000
               contextLimitLarge: 128000
               lowBalanceColorAlert: 500000
+            - id: qwen-235b
+              displayName: QWEN 235B
+              contextLimitSmall: 256000
+              contextLimitLarge: 256000
+              lowBalanceColorAlert: 500000

salt/soc/enabled.sls

@@ -27,7 +27,7 @@ so-soc:
       - /opt/so/conf/strelka:/opt/sensoroni/yara:rw
       - /opt/so/conf/sigma:/opt/sensoroni/sigma:rw
       - /opt/so/rules/elastalert/rules:/opt/sensoroni/elastalert:rw
-      - /opt/so/rules/nids/suri:/opt/sensoroni/nids:rw
+      - /opt/so/rules/nids/suri:/opt/sensoroni/nids:ro
       - /opt/so/conf/soc/fingerprints:/opt/sensoroni/fingerprints:rw
       - /nsm/soc/jobs:/opt/sensoroni/jobs:rw
       - /nsm/soc/uploads:/nsm/soc/uploads:rw

salt/soc/merged.map.jinja

@@ -50,74 +50,17 @@
 {%   do SOCMERGED.config.server.modules.elastalertengine.update({'enabledSigmaRules': SOCMERGED.config.server.modules.elastalertengine.enabledSigmaRules.default}) %}
 {% endif %}
 
-{# set elastalertengine.rulesRepos, strelkaengine.rulesRepos, and suricataengine.rulesetSources based on airgap or not #}
+{# set elastalertengine.rulesRepos and strelkaengine.rulesRepos based on airgap or not #}
 {% if GLOBALS.airgap %}
 {%   do SOCMERGED.config.server.modules.elastalertengine.update({'rulesRepos': SOCMERGED.config.server.modules.elastalertengine.rulesRepos.airgap}) %}
 {%   do SOCMERGED.config.server.modules.strelkaengine.update({'rulesRepos': SOCMERGED.config.server.modules.strelkaengine.rulesRepos.airgap}) %}
-{#%   if SOCMERGED.config.server.modules.suricataengine.rulesetSources is mapping %#}
-{%     do SOCMERGED.config.server.modules.suricataengine.update({'rulesetSources': SOCMERGED.config.server.modules.suricataengine.rulesetSources.airgap}) %}
-{#%   endif %#}
 {%   do SOCMERGED.config.server.update({'airgapEnabled': true}) %}
 {% else %}
 {%   do SOCMERGED.config.server.modules.elastalertengine.update({'rulesRepos': SOCMERGED.config.server.modules.elastalertengine.rulesRepos.default}) %}
 {%   do SOCMERGED.config.server.modules.strelkaengine.update({'rulesRepos': SOCMERGED.config.server.modules.strelkaengine.rulesRepos.default}) %}
-{#%   if SOCMERGED.config.server.modules.suricataengine.rulesetSources is mapping %#}
-{%     do SOCMERGED.config.server.modules.suricataengine.update({'rulesetSources': SOCMERGED.config.server.modules.suricataengine.rulesetSources.default}) %}
-{#%   endif %#}
 {%   do SOCMERGED.config.server.update({'airgapEnabled': false}) %}
 {% endif %}
 
-
-{# Define the Detections custom ruleset that should always be present #}
-{% set CUSTOM_RULESET = {
-  'name': 'custom',
-  'description': 'User-created custom rules created via the Detections module in the SOC UI',
-  'sourceType': 'elasticsearch',
-  'sourcePath': 'so_detection.ruleset:__custom__',
-  'readOnly': false,
-  'deleteUnreferenced': false,
-  'license': 'Custom',
-  'enabled': true
-} %}
-
-{# Always append the custom ruleset to suricataengine.rulesetSources if not already present #}
-{% if SOCMERGED.config.server.modules.suricataengine is defined and SOCMERGED.config.server.modules.suricataengine.rulesetSources is defined %}
-{%   if SOCMERGED.config.server.modules.suricataengine.rulesetSources is not mapping %}
-{%     set custom_names = SOCMERGED.config.server.modules.suricataengine.rulesetSources | selectattr('name', 'equalto', 'custom') | list %}
-{%     if custom_names | length == 0 %}
-{%       do SOCMERGED.config.server.modules.suricataengine.rulesetSources.append(CUSTOM_RULESET) %}
-{%     endif %}
-{%   endif %}
-{% endif %}
-
-{# Transform Emerging-Threats ruleset based on license key #}
-{% if SOCMERGED.config.server.modules.suricataengine is defined and SOCMERGED.config.server.modules.suricataengine.rulesetSources is defined %}
-{%   if SOCMERGED.config.server.modules.suricataengine.rulesetSources is not mapping %}
-{%     for ruleset in SOCMERGED.config.server.modules.suricataengine.rulesetSources %}
-{%     if ruleset.name == 'Emerging-Threats' %}
-{%       if ruleset.licenseKey and ruleset.licenseKey != '' %}
-{#         License key is defined - transform to ETPRO #}
-{%         do ruleset.update({
-             'name': 'ETPRO',
-             'sourcePath': 'https://rules.emergingthreatspro.com/' ~ ruleset.licenseKey ~ '/suricata-7.0.3/etpro.rules.tar.gz',
-             'urlHash': 'https://rules.emergingthreatspro.com/' ~ ruleset.licenseKey ~ '/suricata-7.0.3/etpro.rules.tar.gz.md5',
-             'license': 'Commercial'
-           }) %}
-{%       else %}
-{#         No license key - explicitly set to ETOPEN #}
-{%         do ruleset.update({
-             'name': 'ETOPEN',
-             'sourcePath': 'https://rules.emergingthreats.net/open/suricata-7.0.3/emerging.rules.tar.gz',
-             'urlHash': 'https://rules.emergingthreats.net/open/suricata-7.0.3/emerging.rules.tar.gz.md5',
-             'license': 'BSD'
-           }) %}
-{%       endif %}
-{%     endif %}
-{%     endfor %}
-{%   endif %}
-{% endif %}
-
-
 {# set playbookRepos based on airgap or not #}
 {% if GLOBALS.airgap %}
 {%   do SOCMERGED.config.server.modules.playbook.update({'playbookRepos': SOCMERGED.config.server.modules.playbook.playbookRepos.airgap}) %}

salt/soc/soc_soc.yaml

@@ -552,52 +552,6 @@ soc:
             advanced: True
             forcedType: "[]string"
             helpLink: detections.html#rule-engine-status
-          rulesetSources:
-            default: &serulesetSources
-              description: "Ruleset sources for Suricata rules. Supports URL downloads and local directories. Refer to the linked documentation for details on how to configure this setting."
-              global: True
-              advanced: False
-              forcedType: "[]{}"
-              helpLink: suricata.html
-              syntax: json
-              uiElements:
-              - field: name
-                label: Ruleset Name (This will be the name of the ruleset in the UI)
-                required: True
-                readonly: True
-              - field: description
-                label: Description
-              - field: enabled
-                label: Enabled (If false, existing rules & overrides will be removed)
-                forcedType: bool
-                required: True
-              - field: licenseKey
-                label: License Key
-                required: False
-              - field: sourceType
-                label: Source Type
-                required: True
-                options:
-                  - url
-                  - directory
-              - field: sourcePath
-                label: Source Path (full url or directory path)
-                required: True
-              - field: excludeFiles
-                label: Exclude Files (list of file names to exclude, separated by commas)
-                required: False
-              - field: license
-                label: Ruleset License
-                required: True
-              - field: readOnly
-                label: Read Only
-                forcedType: bool
-                required: False
-              - field: deleteUnreferenced
-                label: Delete Unreferenced
-                forcedType: bool
-                required: False
-            airgap: *serulesetSources
         navigator:
           intervalMinutes:
             description: How often to generate the Navigator Layers. (minutes)

salt/suricata/defaults.yaml

@@ -459,7 +459,7 @@ suricata:
         append: "yes"
     default-rule-path: /etc/suricata/rules
     rule-files:
-      - all-rulesets.rules
+      - all.rules
     classification-file: /etc/suricata/classification.config
     reference-config-file: /etc/suricata/reference.config
     threshold-file: /etc/suricata/threshold.conf

salt/top.sls

@@ -74,6 +74,7 @@ base:
     - sensoroni
     - telegraf
     - firewall
+    - idstools
     - suricata.manager
     - healthcheck
     - elasticsearch
@@ -105,6 +106,7 @@ base:
     - firewall
     - sensoroni
     - telegraf
+    - idstools
     - suricata.manager
     - healthcheck
     - elasticsearch
@@ -140,6 +142,7 @@ base:
     - sensoroni
     - telegraf
     - backup.config_backup
+    - idstools
     - suricata.manager
     - elasticsearch
     - logstash
@@ -174,6 +177,7 @@ base:
     - sensoroni
     - telegraf
     - backup.config_backup
+    - idstools
     - suricata.manager
     - elasticsearch
     - logstash
@@ -204,6 +208,7 @@ base:
     - sensoroni
     - telegraf
     - firewall
+    - idstools
     - suricata.manager
     - pcap
     - elasticsearch

setup/so-functions

@@ -836,6 +836,7 @@ create_manager_pillars() {
 	backup_pillar
 	docker_pillar
 	redis_pillar
+	idstools_pillar
 	kratos_pillar
 	hydra_pillar
 	soc_pillar
@@ -1301,6 +1302,11 @@ ls_heapsize() {
 
 }
 
+idstools_pillar() {
+	title "Ading IDSTOOLS pillar options"
+	touch $adv_idstools_pillar_file
+}
+
 nginx_pillar() {
 	title "Creating the NGINX pillar"
 	[[ -z "$TESTING" ]] && return
@@ -1476,7 +1482,7 @@ make_some_dirs() {
 	mkdir -p $local_salt_dir/salt/firewall/portgroups
 	mkdir -p $local_salt_dir/salt/firewall/ports
 
-	for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni soc docker zeek suricata nginx telegraf logstash soc manager kratos hydra idh elastalert stig global kafka versionlock hypervisor vm; do
+	for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni soc docker zeek suricata nginx telegraf logstash soc manager kratos hydra idstools idh elastalert stig global kafka versionlock hypervisor vm; do
 	mkdir -p $local_salt_dir/pillar/$THEDIR
 	touch $local_salt_dir/pillar/$THEDIR/adv_$THEDIR.sls
 	touch $local_salt_dir/pillar/$THEDIR/soc_$THEDIR.sls

setup/so-variables

@@ -166,6 +166,12 @@ export hydra_pillar_file
 adv_hydra_pillar_file="$local_salt_dir/pillar/hydra/adv_hydra.sls"
 export adv_hydra_pillar_file
 
+idstools_pillar_file="$local_salt_dir/pillar/idstools/soc_idstools.sls"
+export idstools_pillar_file
+
+adv_idstools_pillar_file="$local_salt_dir/pillar/idstools/adv_idstools.sls"
+export adv_idstools_pillar_file
+
 nginx_pillar_file="$local_salt_dir/pillar/nginx/soc_nginx.sls"
 export nginx_pillar_file
 

setup/so-whiptail

@@ -676,8 +676,8 @@ whiptail_install_type_dist_existing() {
 	EOM
 	
 	install_type=$(whiptail --title "$whiptail_title" --menu "$node_msg" 19 75 7 \
-		"SENSOR" "Create a forward only sensor " \
+		"SENSOR" "Add a Sensor Node for monitoring network traffic " \
-		"SEARCHNODE" "Add a search node with parsing " \
+		"SEARCHNODE" "Add a Search Node with parsing " \
 		"FLEET" "Dedicated Elastic Fleet Node " \
 		"HEAVYNODE" "Sensor + Search Node " \
 		"IDH" "Intrusion Detection Honeypot Node " \