diff --git a/pillar/top.sls b/pillar/top.sls index 1fdb59deb..a1114b80c 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -43,8 +43,6 @@ base: - secrets - manager.soc_manager - manager.adv_manager - - idstools.soc_idstools - - idstools.adv_idstools - logstash.nodes - logstash.soc_logstash - logstash.adv_logstash @@ -117,8 +115,6 @@ base: - elastalert.adv_elastalert - manager.soc_manager - manager.adv_manager - - idstools.soc_idstools - - idstools.adv_idstools - soc.soc_soc - soc.adv_soc - kibana.soc_kibana @@ -158,8 +154,6 @@ base: {% endif %} - secrets - healthcheck.standalone - - idstools.soc_idstools - - idstools.adv_idstools - kratos.soc_kratos - kratos.adv_kratos - hydra.soc_hydra diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 068722ca2..346ed7f12 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -38,7 +38,6 @@ 'hydra', 'elasticfleet', 'elastic-fleet-package-registry', - 'idstools', 'suricata.manager', 'utility' ] %} diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 7fd35d5ac..e2fe4f715 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -25,7 +25,6 @@ container_list() { if [ $MANAGERCHECK == 'so-import' ]; then TRUSTED_CONTAINERS=( "so-elasticsearch" - "so-idstools" "so-influxdb" "so-kibana" "so-kratos" @@ -49,7 +48,6 @@ container_list() { "so-elastic-fleet-package-registry" "so-elasticsearch" "so-idh" - "so-idstools" "so-influxdb" "so-kafka" "so-kibana" @@ -71,7 +69,6 @@ container_list() { ) else TRUSTED_CONTAINERS=( - "so-idstools" "so-elasticsearch" "so-logstash" "so-nginx" diff --git a/salt/docker/defaults.yaml b/salt/docker/defaults.yaml index 2d7ad4e1c..456a187d6 100644 --- a/salt/docker/defaults.yaml +++ b/salt/docker/defaults.yaml @@ -24,11 +24,6 @@ docker: custom_bind_mounts: [] extra_hosts: [] extra_env: [] - 'so-idstools': - final_octet: 25 - custom_bind_mounts: [] - extra_hosts: [] - extra_env: [] 'so-influxdb': final_octet: 26 port_bindings: diff --git a/salt/docker/soc_docker.yaml b/salt/docker/soc_docker.yaml index dacbf2302..3c4475236 100644 --- a/salt/docker/soc_docker.yaml +++ b/salt/docker/soc_docker.yaml @@ -41,7 +41,6 @@ docker: forcedType: "[]string" so-elastic-fleet: *dockerOptions so-elasticsearch: *dockerOptions - so-idstools: *dockerOptions so-influxdb: *dockerOptions so-kibana: *dockerOptions so-kratos: *dockerOptions @@ -102,4 +101,4 @@ docker: multiline: True forcedType: "[]string" so-zeek: *dockerOptions - so-kafka: *dockerOptions \ No newline at end of file + so-kafka: *dockerOptions diff --git a/salt/idstools/config.sls b/salt/idstools/config.sls deleted file mode 100644 index cea75ab9a..000000000 --- a/salt/idstools/config.sls +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} - -include: - - idstools.sync_files - -idstoolslogdir: - file.directory: - - name: /opt/so/log/idstools - - user: 939 - - group: 939 - - makedirs: True - -idstools_sbin: - file.recurse: - - name: /usr/sbin - - source: salt://idstools/tools/sbin - - user: 939 - - group: 939 - - file_mode: 755 - -# If this is used, exclude so-rule-update -#idstools_sbin_jinja: -# file.recurse: -# - name: /usr/sbin -# - source: salt://idstools/tools/sbin_jinja -# - user: 939 -# - group: 939 -# - file_mode: 755 -# - template: jinja - -idstools_so-rule-update: - file.managed: - - name: /usr/sbin/so-rule-update - - source: salt://idstools/tools/sbin_jinja/so-rule-update - - user: 939 - - group: 939 - - mode: 755 - - template: jinja - -suricatacustomdirsfile: - file.directory: - - name: /nsm/rules/detect-suricata/custom_file - - user: 939 - - group: 939 - - makedirs: True - -suricatacustomdirsurl: - file.directory: - - name: /nsm/rules/detect-suricata/custom_temp - - user: 939 - - group: 939 - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/salt/idstools/defaults.yaml b/salt/idstools/defaults.yaml deleted file mode 100644 index 1be100cec..000000000 --- a/salt/idstools/defaults.yaml +++ /dev/null @@ -1,10 +0,0 @@ -idstools: - enabled: False - config: - urls: [] - ruleset: ETOPEN - oinkcode: "" - sids: - enabled: [] - disabled: [] - modify: [] diff --git a/salt/idstools/disabled.sls b/salt/idstools/disabled.sls deleted file mode 100644 index ab0e10d7a..000000000 --- a/salt/idstools/disabled.sls +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} - -include: - - idstools.sostatus - -so-idstools: - docker_container.absent: - - force: True - -so-idstools_so-status.disabled: - file.comment: - - name: /opt/so/conf/so-status/so-status.conf - - regex: ^so-idstools$ - -so-rule-update: - cron.absent: - - identifier: so-rule-update - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/salt/idstools/enabled.sls b/salt/idstools/enabled.sls deleted file mode 100644 index 365b38772..000000000 --- a/salt/idstools/enabled.sls +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} -{% from 'docker/docker.map.jinja' import DOCKER %} -{% from 'vars/globals.map.jinja' import GLOBALS %} -{% set proxy = salt['pillar.get']('manager:proxy') %} - -include: - - idstools.config - - idstools.sostatus - -so-idstools: - docker_container.running: - - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-idstools:{{ GLOBALS.so_version }} - - hostname: so-idstools - - user: socore - - networks: - - sobridge: - - ipv4_address: {{ DOCKER.containers['so-idstools'].ip }} - {% if proxy %} - - environment: - - http_proxy={{ proxy }} - - https_proxy={{ proxy }} - - no_proxy={{ salt['pillar.get']('manager:no_proxy') }} - {% if DOCKER.containers['so-idstools'].extra_env %} - {% for XTRAENV in DOCKER.containers['so-idstools'].extra_env %} - - {{ XTRAENV }} - {% endfor %} - {% endif %} - {% elif DOCKER.containers['so-idstools'].extra_env %} - - environment: - {% for XTRAENV in DOCKER.containers['so-idstools'].extra_env %} - - {{ XTRAENV }} - {% endfor %} - {% endif %} - - binds: - - /opt/so/conf/idstools/etc:/opt/so/idstools/etc:ro - - /opt/so/rules/nids/suri:/opt/so/rules/nids/suri:rw - - /nsm/rules/:/nsm/rules/:rw - {% if DOCKER.containers['so-idstools'].custom_bind_mounts %} - {% for BIND in DOCKER.containers['so-idstools'].custom_bind_mounts %} - - {{ BIND }} - {% endfor %} - {% endif %} - - extra_hosts: - - {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }} - {% if DOCKER.containers['so-idstools'].extra_hosts %} - {% for XTRAHOST in DOCKER.containers['so-idstools'].extra_hosts %} - - {{ XTRAHOST }} - {% endfor %} - {% endif %} - - watch: - - file: idstoolsetcsync - - file: idstools_so-rule-update - -delete_so-idstools_so-status.disabled: - file.uncomment: - - name: /opt/so/conf/so-status/so-status.conf - - regex: ^so-idstools$ - -so-rule-update: - cron.present: - - name: /usr/sbin/so-rule-update > /opt/so/log/idstools/download_cron.log 2>&1 - - identifier: so-rule-update - - user: root - - minute: '1' - - hour: '7' - -# order this last to give so-idstools container time to be ready -run_so-rule-update: - cmd.run: - - name: '/usr/sbin/so-rule-update > /opt/so/log/idstools/download_idstools_state.log 2>&1' - - require: - - docker_container: so-idstools - - onchanges: - - file: idstools_so-rule-update - - file: idstoolsetcsync - - file: synclocalnidsrules - - order: last - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/salt/idstools/etc/disable.conf b/salt/idstools/etc/disable.conf deleted file mode 100644 index 84144a495..000000000 --- a/salt/idstools/etc/disable.conf +++ /dev/null @@ -1,16 +0,0 @@ -{%- set disabled_sids = salt['pillar.get']('idstools:sids:disabled', {}) -%} -# idstools - disable.conf - -# Example of disabling a rule by signature ID (gid is optional). -# 1:2019401 -# 2019401 - -# Example of disabling a rule by regular expression. -# - All regular expression matches are case insensitive. -# re:hearbleed -# re:MS(0[7-9]|10)-\d+ -{%- if disabled_sids != None %} -{%- for sid in disabled_sids %} -{{ sid }} -{%- endfor %} -{%- endif %} \ No newline at end of file diff --git a/salt/idstools/etc/enable.conf b/salt/idstools/etc/enable.conf deleted file mode 100644 index 5da0bfc61..000000000 --- a/salt/idstools/etc/enable.conf +++ /dev/null @@ -1,16 +0,0 @@ -{%- set enabled_sids = salt['pillar.get']('idstools:sids:enabled', {}) -%} -# idstools-rulecat - enable.conf - -# Example of enabling a rule by signature ID (gid is optional). -# 1:2019401 -# 2019401 - -# Example of enabling a rule by regular expression. -# - All regular expression matches are case insensitive. -# re:hearbleed -# re:MS(0[7-9]|10)-\d+ -{%- if enabled_sids != None %} -{%- for sid in enabled_sids %} -{{ sid }} -{%- endfor %} -{%- endif %} \ No newline at end of file diff --git a/salt/idstools/etc/modify.conf b/salt/idstools/etc/modify.conf deleted file mode 100644 index 4ea75ada2..000000000 --- a/salt/idstools/etc/modify.conf +++ /dev/null @@ -1,12 +0,0 @@ -{%- set modify_sids = salt['pillar.get']('idstools:sids:modify', {}) -%} -# idstools-rulecat - modify.conf - -# Format: "" "" - -# Example changing the seconds for rule 2019401 to 3600. -#2019401 "seconds \d+" "seconds 3600" -{%- if modify_sids != None %} -{%- for sid in modify_sids %} -{{ sid }} -{%- endfor %} -{%- endif %} \ No newline at end of file diff --git a/salt/idstools/etc/rulecat.conf b/salt/idstools/etc/rulecat.conf deleted file mode 100644 index e4ec611db..000000000 --- a/salt/idstools/etc/rulecat.conf +++ /dev/null @@ -1,23 +0,0 @@ -{%- from 'vars/globals.map.jinja' import GLOBALS -%} -{%- from 'soc/merged.map.jinja' import SOCMERGED -%} ---suricata-version=7.0.3 ---merged=/opt/so/rules/nids/suri/all.rules ---output=/nsm/rules/detect-suricata/custom_temp ---local=/opt/so/rules/nids/suri/local.rules -{%- if GLOBALS.md_engine == "SURICATA" %} ---local=/opt/so/rules/nids/suri/extraction.rules ---local=/opt/so/rules/nids/suri/filters.rules -{%- endif %} ---url=http://{{ GLOBALS.manager }}:7788/suricata/emerging-all.rules ---disable=/opt/so/idstools/etc/disable.conf ---enable=/opt/so/idstools/etc/enable.conf ---modify=/opt/so/idstools/etc/modify.conf -{%- if SOCMERGED.config.server.modules.suricataengine.customRulesets %} - {%- for ruleset in SOCMERGED.config.server.modules.suricataengine.customRulesets %} - {%- if 'url' in ruleset %} ---url={{ ruleset.url }} - {%- elif 'file' in ruleset %} ---local={{ ruleset.file }} - {%- endif %} - {%- endfor %} -{%- endif %} diff --git a/salt/idstools/init.sls b/salt/idstools/init.sls deleted file mode 100644 index ac1d51717..000000000 --- a/salt/idstools/init.sls +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'idstools/map.jinja' import IDSTOOLSMERGED %} - -include: -{% if IDSTOOLSMERGED.enabled %} - - idstools.enabled -{% else %} - - idstools.disabled -{% endif %} diff --git a/salt/idstools/map.jinja b/salt/idstools/map.jinja deleted file mode 100644 index 97d12279b..000000000 --- a/salt/idstools/map.jinja +++ /dev/null @@ -1,7 +0,0 @@ -{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one - or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at - https://securityonion.net/license; you may not use this file except in compliance with the - Elastic License 2.0. #} - -{% import_yaml 'idstools/defaults.yaml' as IDSTOOLSDEFAULTS with context %} -{% set IDSTOOLSMERGED = salt['pillar.get']('idstools', IDSTOOLSDEFAULTS.idstools, merge=True) %} diff --git a/salt/idstools/rules/extraction.rules b/salt/idstools/rules/extraction.rules deleted file mode 100644 index 3ebbd41b1..000000000 --- a/salt/idstools/rules/extraction.rules +++ /dev/null @@ -1,26 +0,0 @@ -# Extract all PDF mime type -alert http any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100000; rev:1;) -alert smtp any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100001; rev:1;) -alert nfs any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100002; rev:1;) -alert smb any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100003; rev:1;) -# Extract EXE/DLL file types -alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100004; rev:1;) -alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100005; rev:1;) -alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100006; rev:1;) -alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100007; rev:1;) -alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100008; rev:1;) -alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100009; rev:1;) -alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100010; rev:1;) -alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100011; rev:1;) - -# Extract all Zip files -alert http any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100012; rev:1;) -alert smtp any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100013; rev:1;) -alert nfs any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100014; rev:1;) -alert smb any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100015; rev:1;) - -# Extract Word Docs -alert http any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100016; rev:1;) -alert smtp any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100017; rev:1;) -alert nfs any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100018; rev:1;) -alert smb any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100019; rev:1;) \ No newline at end of file diff --git a/salt/idstools/rules/filters.rules b/salt/idstools/rules/filters.rules deleted file mode 100644 index 051d1913f..000000000 --- a/salt/idstools/rules/filters.rules +++ /dev/null @@ -1,11 +0,0 @@ -# Start the filters at sid 1200000 -# Example of filtering out *google.com from being in the dns log. -#config dns any any -> any any (dns.query; content:"google.com"; config: logging disable, type tx, scope tx; sid:1200000;) -# Example of filtering out *google.com from being in the http log. -#config http any any -> any any (http.host; content:"google.com"; config: logging disable, type tx, scope tx; sid:1200001;) -# Example of filtering out someuseragent from being in the http log. -#config http any any -> any any (http.user_agent; content:"someuseragent"; config: logging disable, type tx, scope tx; sid:1200002;) -# Example of filtering out Google's certificate from being in the ssl log. -#config tls any any -> any any (tls.fingerprint; content:"4f:a4:5e:58:7e:d9:db:20:09:d7:b6:c7:ff:58:c4:7b:dc:3f:55:b4"; config: logging disable, type tx, scope tx; sid:1200003;) -# Example of filtering out a md5 of a file from being in the files log. -#config fileinfo any any -> any any (fileinfo.filemd5; content:"7a125dc69c82d5caf94d3913eecde4b5"; config: logging disable, type tx, scope tx; sid:1200004;) diff --git a/salt/idstools/rules/local.rules b/salt/idstools/rules/local.rules deleted file mode 100644 index ac11dfa58..000000000 --- a/salt/idstools/rules/local.rules +++ /dev/null @@ -1 +0,0 @@ -# Add your custom Suricata rules in this file. \ No newline at end of file diff --git a/salt/idstools/soc_idstools.yaml b/salt/idstools/soc_idstools.yaml deleted file mode 100644 index 4f7a53e91..000000000 --- a/salt/idstools/soc_idstools.yaml +++ /dev/null @@ -1,72 +0,0 @@ -idstools: - enabled: - description: Enables or disables the IDStools process which is used by the Detection system. - config: - oinkcode: - description: Enter your registration code or oinkcode for paid NIDS rulesets. - title: Registration Code - global: True - forcedType: string - helpLink: rules.html - ruleset: - description: 'Defines the ruleset you want to run. Options are ETOPEN or ETPRO. Once you have changed the ruleset here, you will need to wait for the rule update to take place (every 24 hours), or you can force the update by nagivating to Detections --> Options dropdown menu --> Suricata --> Full Update. WARNING! Changing the ruleset will remove all existing non-overlapping Suricata rules of the previous ruleset and their associated overrides. This removal cannot be undone.' - global: True - regex: ETPRO\b|ETOPEN\b - helpLink: rules.html - urls: - description: This is a list of additional rule download locations. This feature is currently disabled. - global: True - multiline: True - forcedType: "[]string" - readonly: True - helpLink: rules.html - sids: - disabled: - description: Contains the list of NIDS rules (or regex patterns) disabled across the grid. This setting is readonly; Use the Detections screen to disable rules. - global: True - multiline: True - forcedType: "[]string" - regex: \d*|re:.* - helpLink: managing-alerts.html - readonlyUi: True - advanced: true - enabled: - description: Contains the list of NIDS rules (or regex patterns) enabled across the grid. This setting is readonly; Use the Detections screen to enable rules. - global: True - multiline: True - forcedType: "[]string" - regex: \d*|re:.* - helpLink: managing-alerts.html - readonlyUi: True - advanced: true - modify: - description: Contains the list of NIDS rules (SID "REGEX_SEARCH_TERM" "REGEX_REPLACE_TERM"). This setting is readonly; Use the Detections screen to modify rules. - global: True - multiline: True - forcedType: "[]string" - helpLink: managing-alerts.html - readonlyUi: True - advanced: true - rules: - local__rules: - description: Contains the list of custom NIDS rules applied to the grid. This setting is readonly; Use the Detections screen to adjust rules. - file: True - global: True - advanced: True - title: Local Rules - helpLink: local-rules.html - readonlyUi: True - filters__rules: - description: If you are using Suricata for metadata, then you can set custom filters for that metadata here. - file: True - global: True - advanced: True - title: Filter Rules - helpLink: suricata.html - extraction__rules: - description: If you are using Suricata for metadata, then you can set a list of MIME types for file extraction here. - file: True - global: True - advanced: True - title: Extraction Rules - helpLink: suricata.html diff --git a/salt/idstools/sostatus.sls b/salt/idstools/sostatus.sls deleted file mode 100644 index 408b10742..000000000 --- a/salt/idstools/sostatus.sls +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} - -append_so-idstools_so-status.conf: - file.append: - - name: /opt/so/conf/so-status/so-status.conf - - text: so-idstools - - unless: grep -q so-idstools /opt/so/conf/so-status/so-status.conf - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/salt/idstools/sync_files.sls b/salt/idstools/sync_files.sls deleted file mode 100644 index cdacfaa74..000000000 --- a/salt/idstools/sync_files.sls +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - -idstoolsdir: - file.directory: - - name: /opt/so/conf/idstools/etc - - user: 939 - - group: 939 - - makedirs: True - -idstoolsetcsync: - file.recurse: - - name: /opt/so/conf/idstools/etc - - source: salt://idstools/etc - - user: 939 - - group: 939 - - template: jinja - -rulesdir: - file.directory: - - name: /opt/so/rules/nids/suri - - user: 939 - - group: 939 - - makedirs: True - -# Don't show changes because all.rules can be large -synclocalnidsrules: - file.recurse: - - name: /opt/so/rules/nids/suri/ - - source: salt://idstools/rules/ - - user: 939 - - group: 939 - - show_changes: False - - include_pat: 'E@.rules' diff --git a/salt/idstools/tools/sbin/so-idstools-restart b/salt/idstools/tools/sbin/so-idstools-restart deleted file mode 100755 index f2abbd0a5..000000000 --- a/salt/idstools/tools/sbin/so-idstools-restart +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - -/usr/sbin/so-restart idstools $1 diff --git a/salt/idstools/tools/sbin/so-idstools-start b/salt/idstools/tools/sbin/so-idstools-start deleted file mode 100755 index e17b5e521..000000000 --- a/salt/idstools/tools/sbin/so-idstools-start +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - -/usr/sbin/so-start idstools $1 diff --git a/salt/idstools/tools/sbin/so-idstools-stop b/salt/idstools/tools/sbin/so-idstools-stop deleted file mode 100755 index f2d188d06..000000000 --- a/salt/idstools/tools/sbin/so-idstools-stop +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - -/usr/sbin/so-stop idstools $1 diff --git a/salt/idstools/tools/sbin_jinja/so-rule-update b/salt/idstools/tools/sbin_jinja/so-rule-update deleted file mode 100755 index 9ac09ed15..000000000 --- a/salt/idstools/tools/sbin_jinja/so-rule-update +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -# if this script isn't already running -if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then - - . /usr/sbin/so-common - -{%- from 'vars/globals.map.jinja' import GLOBALS %} -{%- from 'idstools/map.jinja' import IDSTOOLSMERGED %} - -{%- set proxy = salt['pillar.get']('manager:proxy') %} -{%- set noproxy = salt['pillar.get']('manager:no_proxy', '') %} - -{%- if proxy %} -# Download the rules from the internet - export http_proxy={{ proxy }} - export https_proxy={{ proxy }} - export no_proxy="{{ noproxy }}" -{%- endif %} - - mkdir -p /nsm/rules/suricata - chown -R socore:socore /nsm/rules/suricata -{%- if not GLOBALS.airgap %} -# Download the rules from the internet -{%- if IDSTOOLSMERGED.config.ruleset == 'ETOPEN' %} - docker exec so-idstools idstools-rulecat -v --suricata-version 7.0.3 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force -{%- elif IDSTOOLSMERGED.config.ruleset == 'ETPRO' %} - docker exec so-idstools idstools-rulecat -v --suricata-version 7.0.3 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force --etpro={{ IDSTOOLSMERGED.config.oinkcode }} -{%- endif %} -{%- endif %} - - - argstr="" - for arg in "$@"; do - argstr="${argstr} \"${arg}\"" - done - - docker exec so-idstools /bin/bash -c "cd /opt/so/idstools/etc && idstools-rulecat --force ${argstr}" - -fi diff --git a/salt/logrotate/defaults.yaml b/salt/logrotate/defaults.yaml index 2f7247ff2..4df6713ef 100644 --- a/salt/logrotate/defaults.yaml +++ b/salt/logrotate/defaults.yaml @@ -1,15 +1,5 @@ logrotate: config: - /opt/so/log/idstools/*_x_log: - - daily - - rotate 14 - - missingok - - copytruncate - - compress - - create - - extension .log - - dateext - - dateyesterday /opt/so/log/nginx/*_x_log: - daily - rotate 14 diff --git a/salt/logrotate/soc_logrotate.yaml b/salt/logrotate/soc_logrotate.yaml index 56f879e4f..21b54755e 100644 --- a/salt/logrotate/soc_logrotate.yaml +++ b/salt/logrotate/soc_logrotate.yaml @@ -1,12 +1,5 @@ logrotate: config: - "/opt/so/log/idstools/*_x_log": - description: List of logrotate options for this file. - title: /opt/so/log/idstools/*.log - advanced: True - multiline: True - global: True - forcedType: "[]string" "/opt/so/log/nginx/*_x_log": description: List of logrotate options for this file. title: /opt/so/log/nginx/*.log diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 34ebdaeec..c91a7a793 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -603,16 +603,6 @@ function add_kratos_to_minion() { fi } -function add_idstools_to_minion() { - printf '%s\n'\ - "idstools:"\ - " enabled: True"\ - " " >> $PILLARFILE - if [ $? -ne 0 ]; then - log "ERROR" "Failed to add idstools configuration to $PILLARFILE" - return 1 - fi -} function add_elastic_fleet_package_registry_to_minion() { printf '%s\n'\ @@ -740,7 +730,6 @@ function createEVAL() { add_soc_to_minion || return 1 add_registry_to_minion || return 1 add_kratos_to_minion || return 1 - add_idstools_to_minion || return 1 add_elastic_fleet_package_registry_to_minion || return 1 } @@ -761,7 +750,6 @@ function createSTANDALONE() { add_soc_to_minion || return 1 add_registry_to_minion || return 1 add_kratos_to_minion || return 1 - add_idstools_to_minion || return 1 add_elastic_fleet_package_registry_to_minion || return 1 } @@ -778,7 +766,6 @@ function createMANAGER() { add_soc_to_minion || return 1 add_registry_to_minion || return 1 add_kratos_to_minion || return 1 - add_idstools_to_minion || return 1 add_elastic_fleet_package_registry_to_minion || return 1 } @@ -795,7 +782,6 @@ function createMANAGERSEARCH() { add_soc_to_minion || return 1 add_registry_to_minion || return 1 add_kratos_to_minion || return 1 - add_idstools_to_minion || return 1 add_elastic_fleet_package_registry_to_minion || return 1 } @@ -810,7 +796,6 @@ function createIMPORT() { add_soc_to_minion || return 1 add_registry_to_minion || return 1 add_kratos_to_minion || return 1 - add_idstools_to_minion || return 1 add_elastic_fleet_package_registry_to_minion || return 1 } @@ -895,7 +880,6 @@ function createMANAGERHYPE() { add_soc_to_minion || return 1 add_registry_to_minion || return 1 add_kratos_to_minion || return 1 - add_idstools_to_minion || return 1 add_elastic_fleet_package_registry_to_minion || return 1 } diff --git a/salt/salt/files/engines.conf b/salt/salt/files/engines.conf index 15d55e18f..8192ee201 100644 --- a/salt/salt/files/engines.conf +++ b/salt/salt/files/engines.conf @@ -6,30 +6,6 @@ engines: interval: 60 - pillarWatch: fpa: - - files: - - /opt/so/saltstack/local/pillar/idstools/soc_idstools.sls - - /opt/so/saltstack/local/pillar/idstools/adv_idstools.sls - pillar: idstools.config.ruleset - default: ETOPEN - actions: - from: - '*': - to: - '*': - - cmd.run: - cmd: /usr/sbin/so-rule-update - - files: - - /opt/so/saltstack/local/pillar/idstools/soc_idstools.sls - - /opt/so/saltstack/local/pillar/idstools/adv_idstools.sls - pillar: idstools.config.oinkcode - default: '' - actions: - from: - '*': - to: - '*': - - cmd.run: - cmd: /usr/sbin/so-rule-update - files: - /opt/so/saltstack/local/pillar/global/soc_global.sls - /opt/so/saltstack/local/pillar/global/adv_global.sls diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 0e3e50240..8b4708d38 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1554,12 +1554,72 @@ soc: disableRegex: [] enableRegex: [] failAfterConsecutiveErrorCount: 10 - communityRulesFile: /nsm/rules/suricata/emerging-all.rules rulesFingerprintFile: /opt/sensoroni/fingerprints/emerging-all.fingerprint stateFilePath: /opt/sensoroni/fingerprints/suricataengine.state integrityCheckFrequencySeconds: 1200 ignoredSidRanges: - '1100000-1101000' + rulesetSources: + default: + - name: Emerging-Threats + description: "Emerging Threats ruleset - To enable ET Pro, enter your license key below. Leave empty for ET Open (free) rules." + licenseKey: "" + enabled: true + sourceType: url + sourcePath: 'https://rules.emergingthreats.net/open/suricata/emerging.rules.tar.gz' + urlHash: "https://rules.emergingthreats.net/open/suricata/emerging.rules.tar.gz.md5" + license: "BSD" + excludeFiles: + - "*deleted*" + - "*retired*" + proxyURL: "" + proxyUsername: "" + proxyPassword: "" + proxyCACert: "" + insecureSkipVerify: false + readOnly: true + deleteUnreferenced: true + - name: local-rules + id: local-rules + description: "Local custom rules from files (*.rules) in a directory on the filesystem" + license: "custom" + sourceType: directory + sourcePath: /nsm/rules/local/ + readOnly: false + deleteUnreferenced: false + enabled: false + excludeFiles: + - "*backup*" + airgap: + - name: Emerging-Threats + description: "Emerging Threats ruleset - To enable ET Pro, enter your license key below. Leave empty for ET Open (free) rules." + licenseKey: "" + enabled: true + sourceType: url + sourcePath: 'https://rules.emergingthreats.net/open/suricata/emerging.rules.tar.gz' + urlHash: "https://rules.emergingthreats.net/open/suricata/emerging.rules.tar.gz.md5" + license: "BSD" + excludeFiles: + - "*deleted*" + - "*retired*" + proxyURL: "" + proxyUsername: "" + proxyPassword: "" + proxyCACert: "" + insecureSkipVerify: false + readOnly: true + deleteUnreferenced: true + - name: local-rules + id: local-rules + description: "Local custom rules from files (*.rules) in a directory on the filesystem" + license: "custom" + sourceType: directory + sourcePath: /nsm/rules/local/ + readOnly: false + deleteUnreferenced: false + enabled: false + excludeFiles: + - "*backup*" navigator: intervalMinutes: 30 outputPath: /opt/sensoroni/navigator diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 09e2c16a8..6dd7b71ae 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -27,7 +27,7 @@ so-soc: - /opt/so/conf/strelka:/opt/sensoroni/yara:rw - /opt/so/conf/sigma:/opt/sensoroni/sigma:rw - /opt/so/rules/elastalert/rules:/opt/sensoroni/elastalert:rw - - /opt/so/rules/nids/suri:/opt/sensoroni/nids:ro + - /opt/so/rules/nids/suri:/opt/sensoroni/nids:rw - /opt/so/conf/soc/fingerprints:/opt/sensoroni/fingerprints:rw - /nsm/soc/jobs:/opt/sensoroni/jobs:rw - /nsm/soc/uploads:/nsm/soc/uploads:rw diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index e053ce63f..b43ccaf1b 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -50,17 +50,74 @@ {% do SOCMERGED.config.server.modules.elastalertengine.update({'enabledSigmaRules': SOCMERGED.config.server.modules.elastalertengine.enabledSigmaRules.default}) %} {% endif %} -{# set elastalertengine.rulesRepos and strelkaengine.rulesRepos based on airgap or not #} +{# set elastalertengine.rulesRepos, strelkaengine.rulesRepos, and suricataengine.rulesetSources based on airgap or not #} {% if GLOBALS.airgap %} {% do SOCMERGED.config.server.modules.elastalertengine.update({'rulesRepos': SOCMERGED.config.server.modules.elastalertengine.rulesRepos.airgap}) %} {% do SOCMERGED.config.server.modules.strelkaengine.update({'rulesRepos': SOCMERGED.config.server.modules.strelkaengine.rulesRepos.airgap}) %} +{#% if SOCMERGED.config.server.modules.suricataengine.rulesetSources is mapping %#} +{% do SOCMERGED.config.server.modules.suricataengine.update({'rulesetSources': SOCMERGED.config.server.modules.suricataengine.rulesetSources.airgap}) %} +{#% endif %#} {% do SOCMERGED.config.server.update({'airgapEnabled': true}) %} {% else %} {% do SOCMERGED.config.server.modules.elastalertengine.update({'rulesRepos': SOCMERGED.config.server.modules.elastalertengine.rulesRepos.default}) %} {% do SOCMERGED.config.server.modules.strelkaengine.update({'rulesRepos': SOCMERGED.config.server.modules.strelkaengine.rulesRepos.default}) %} +{#% if SOCMERGED.config.server.modules.suricataengine.rulesetSources is mapping %#} +{% do SOCMERGED.config.server.modules.suricataengine.update({'rulesetSources': SOCMERGED.config.server.modules.suricataengine.rulesetSources.default}) %} +{#% endif %#} {% do SOCMERGED.config.server.update({'airgapEnabled': false}) %} {% endif %} + +{# Define the Detections custom ruleset that should always be present #} +{% set CUSTOM_RULESET = { + 'name': 'custom', + 'description': 'User-created custom rules created via the Detections module in the SOC UI', + 'sourceType': 'elasticsearch', + 'sourcePath': 'so_detection.ruleset:__custom__', + 'readOnly': false, + 'deleteUnreferenced': false, + 'license': 'Custom', + 'enabled': true +} %} + +{# Always append the custom ruleset to suricataengine.rulesetSources if not already present #} +{% if SOCMERGED.config.server.modules.suricataengine is defined and SOCMERGED.config.server.modules.suricataengine.rulesetSources is defined %} +{% if SOCMERGED.config.server.modules.suricataengine.rulesetSources is not mapping %} +{% set custom_names = SOCMERGED.config.server.modules.suricataengine.rulesetSources | selectattr('name', 'equalto', 'custom') | list %} +{% if custom_names | length == 0 %} +{% do SOCMERGED.config.server.modules.suricataengine.rulesetSources.append(CUSTOM_RULESET) %} +{% endif %} +{% endif %} +{% endif %} + +{# Transform Emerging-Threats ruleset based on license key #} +{% if SOCMERGED.config.server.modules.suricataengine is defined and SOCMERGED.config.server.modules.suricataengine.rulesetSources is defined %} +{% if SOCMERGED.config.server.modules.suricataengine.rulesetSources is not mapping %} +{% for ruleset in SOCMERGED.config.server.modules.suricataengine.rulesetSources %} +{% if ruleset.name == 'Emerging-Threats' %} +{% if ruleset.licenseKey and ruleset.licenseKey != '' %} +{# License key is defined - transform to ETPRO #} +{% do ruleset.update({ + 'name': 'ETPRO', + 'sourcePath': 'https://rules.emergingthreatspro.com/' ~ ruleset.licenseKey ~ '/suricata-7.0.3/etpro.rules.tar.gz', + 'urlHash': 'https://rules.emergingthreatspro.com/' ~ ruleset.licenseKey ~ '/suricata-7.0.3/etpro.rules.tar.gz.md5', + 'license': 'Commercial' + }) %} +{% else %} +{# No license key - explicitly set to ETOPEN #} +{% do ruleset.update({ + 'name': 'ETOPEN', + 'sourcePath': 'https://rules.emergingthreats.net/open/suricata-7.0.3/emerging.rules.tar.gz', + 'urlHash': 'https://rules.emergingthreats.net/open/suricata-7.0.3/emerging.rules.tar.gz.md5', + 'license': 'BSD' + }) %} +{% endif %} +{% endif %} +{% endfor %} +{% endif %} +{% endif %} + + {# set playbookRepos based on airgap or not #} {% if GLOBALS.airgap %} {% do SOCMERGED.config.server.modules.playbook.update({'playbookRepos': SOCMERGED.config.server.modules.playbook.playbookRepos.airgap}) %} diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index b292d1460..623df4ea3 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -543,6 +543,52 @@ soc: advanced: True forcedType: "[]string" helpLink: detections.html#rule-engine-status + rulesetSources: + default: &serulesetSources + description: "Ruleset sources for Suricata rules. Supports URL downloads and local directories. Refer to the linked documentation for details on how to configure this setting." + global: True + advanced: False + forcedType: "[]{}" + helpLink: suricata.html + syntax: json + uiElements: + - field: name + label: Ruleset Name (This will be the name of the ruleset in the UI) + required: True + readonly: True + - field: description + label: Description + - field: enabled + label: Enabled (If false, existing rules & overrides will be removed) + forcedType: bool + required: True + - field: licenseKey + label: License Key + required: False + - field: sourceType + label: Source Type + required: True + options: + - url + - directory + - field: sourcePath + label: Source Path (full url or directory path) + required: True + - field: excludeFiles + label: Exclude Files (list of file names to exclude, separated by commas) + required: False + - field: license + label: Ruleset License + required: True + - field: readOnly + label: Read Only + forcedType: bool + required: False + - field: deleteUnreferenced + label: Delete Unreferenced + forcedType: bool + required: False + airgap: *serulesetSources navigator: intervalMinutes: description: How often to generate the Navigator Layers. (minutes) diff --git a/salt/suricata/defaults.yaml b/salt/suricata/defaults.yaml index d819d1cf9..b4c615157 100644 --- a/salt/suricata/defaults.yaml +++ b/salt/suricata/defaults.yaml @@ -459,7 +459,7 @@ suricata: append: "yes" default-rule-path: /etc/suricata/rules rule-files: - - all.rules + - all-rulesets.rules classification-file: /etc/suricata/classification.config reference-config-file: /etc/suricata/reference.config threshold-file: /etc/suricata/threshold.conf diff --git a/salt/top.sls b/salt/top.sls index a75346462..c465307c0 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -74,7 +74,6 @@ base: - sensoroni - telegraf - firewall - - idstools - suricata.manager - healthcheck - elasticsearch @@ -106,7 +105,6 @@ base: - firewall - sensoroni - telegraf - - idstools - suricata.manager - healthcheck - elasticsearch @@ -142,7 +140,6 @@ base: - sensoroni - telegraf - backup.config_backup - - idstools - suricata.manager - elasticsearch - logstash @@ -177,7 +174,6 @@ base: - sensoroni - telegraf - backup.config_backup - - idstools - suricata.manager - elasticsearch - logstash @@ -208,7 +204,6 @@ base: - sensoroni - telegraf - firewall - - idstools - suricata.manager - pcap - elasticsearch diff --git a/setup/so-functions b/setup/so-functions index 522446be4..334dc4a0d 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -829,7 +829,6 @@ create_manager_pillars() { backup_pillar docker_pillar redis_pillar - idstools_pillar kratos_pillar hydra_pillar soc_pillar @@ -1295,11 +1294,6 @@ ls_heapsize() { } -idstools_pillar() { - title "Ading IDSTOOLS pillar options" - touch $adv_idstools_pillar_file -} - nginx_pillar() { title "Creating the NGINX pillar" [[ -z "$TESTING" ]] && return @@ -1475,7 +1469,7 @@ make_some_dirs() { mkdir -p $local_salt_dir/salt/firewall/portgroups mkdir -p $local_salt_dir/salt/firewall/ports - for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni soc docker zeek suricata nginx telegraf logstash soc manager kratos hydra idstools idh elastalert stig global kafka versionlock hypervisor vm; do + for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni soc docker zeek suricata nginx telegraf logstash soc manager kratos hydra idh elastalert stig global kafka versionlock hypervisor vm; do mkdir -p $local_salt_dir/pillar/$THEDIR touch $local_salt_dir/pillar/$THEDIR/adv_$THEDIR.sls touch $local_salt_dir/pillar/$THEDIR/soc_$THEDIR.sls diff --git a/setup/so-variables b/setup/so-variables index fc253df0a..a0d7aadc1 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -166,12 +166,6 @@ export hydra_pillar_file adv_hydra_pillar_file="$local_salt_dir/pillar/hydra/adv_hydra.sls" export adv_hydra_pillar_file -idstools_pillar_file="$local_salt_dir/pillar/idstools/soc_idstools.sls" -export idstools_pillar_file - -adv_idstools_pillar_file="$local_salt_dir/pillar/idstools/adv_idstools.sls" -export adv_idstools_pillar_file - nginx_pillar_file="$local_salt_dir/pillar/nginx/soc_nginx.sls" export nginx_pillar_file