From b2e75e77e8a079ee7066f0ab867691ea9cfb496f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 13:50:19 -0400 Subject: [PATCH 01/12] add local.rules and filter.rules to suricata defaults. add extraction.rules, local.rules and filter.rules for suricata metadata --- salt/suricata/defaults.yaml | 3 ++- salt/suricata/suricata_mdengine.yaml | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/salt/suricata/defaults.yaml b/salt/suricata/defaults.yaml index 4651b7268..fd1b00929 100644 --- a/salt/suricata/defaults.yaml +++ b/salt/suricata/defaults.yaml @@ -416,7 +416,6 @@ suricata: enabled: "yes" filename: keyword_perf.log append: "yes" - prefilter: enabled: "yes" filename: prefilter_perf.log @@ -443,6 +442,8 @@ suricata: default-rule-path: /etc/suricata/rules rule-files: - all.rules + - local.rules + - filter.rules classification-file: /etc/suricata/classification.config reference-config-file: /etc/suricata/reference.config threshold-file: /etc/suricata/threshold.conf diff --git a/salt/suricata/suricata_mdengine.yaml b/salt/suricata/suricata_mdengine.yaml index 1c3855501..c6844541f 100644 --- a/salt/suricata/suricata_mdengine.yaml +++ b/salt/suricata/suricata_mdengine.yaml @@ -70,3 +70,9 @@ suricata: - flow #- netflow #- metadata + profiling: + rule-files: + - all.rules + - extraction.rules + - local.rules + - filter.rules From aab89d2483822359cc235827f3f4486024d3b288 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 13:54:58 -0400 Subject: [PATCH 02/12] rule-files does not go under profiling --- salt/suricata/suricata_mdengine.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/salt/suricata/suricata_mdengine.yaml b/salt/suricata/suricata_mdengine.yaml index c6844541f..80299dc5b 100644 --- a/salt/suricata/suricata_mdengine.yaml +++ b/salt/suricata/suricata_mdengine.yaml @@ -70,9 +70,8 @@ suricata: - flow #- netflow #- metadata - profiling: - rule-files: - - all.rules - - extraction.rules - - local.rules - - filter.rules + rule-files: + - all.rules + - extraction.rules + - local.rules + - filter.rules From 9118ac2b569e9fd3e3c994b24a8f6a4502d4331c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 13:59:43 -0400 Subject: [PATCH 03/12] filter.rules to filters.rules --- salt/suricata/defaults.yaml | 2 +- salt/suricata/suricata_mdengine.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/suricata/defaults.yaml b/salt/suricata/defaults.yaml index fd1b00929..4253794a8 100644 --- a/salt/suricata/defaults.yaml +++ b/salt/suricata/defaults.yaml @@ -443,7 +443,7 @@ suricata: rule-files: - all.rules - local.rules - - filter.rules + - filters.rules classification-file: /etc/suricata/classification.config reference-config-file: /etc/suricata/reference.config threshold-file: /etc/suricata/threshold.conf diff --git a/salt/suricata/suricata_mdengine.yaml b/salt/suricata/suricata_mdengine.yaml index 80299dc5b..d1fb7c2c3 100644 --- a/salt/suricata/suricata_mdengine.yaml +++ b/salt/suricata/suricata_mdengine.yaml @@ -74,4 +74,4 @@ suricata: - all.rules - extraction.rules - local.rules - - filter.rules + - filters.rules From 20dedab4b283cf3d7345170abd7ee24dac9bfbc3 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 15:03:06 -0400 Subject: [PATCH 04/12] remove previously add rules files --- salt/suricata/defaults.yaml | 2 -- salt/suricata/suricata_mdengine.yaml | 5 ----- 2 files changed, 7 deletions(-) diff --git a/salt/suricata/defaults.yaml b/salt/suricata/defaults.yaml index 4253794a8..050efa8f8 100644 --- a/salt/suricata/defaults.yaml +++ b/salt/suricata/defaults.yaml @@ -442,8 +442,6 @@ suricata: default-rule-path: /etc/suricata/rules rule-files: - all.rules - - local.rules - - filters.rules classification-file: /etc/suricata/classification.config reference-config-file: /etc/suricata/reference.config threshold-file: /etc/suricata/threshold.conf diff --git a/salt/suricata/suricata_mdengine.yaml b/salt/suricata/suricata_mdengine.yaml index d1fb7c2c3..1c3855501 100644 --- a/salt/suricata/suricata_mdengine.yaml +++ b/salt/suricata/suricata_mdengine.yaml @@ -70,8 +70,3 @@ suricata: - flow #- netflow #- metadata - rule-files: - - all.rules - - extraction.rules - - local.rules - - filters.rules From 230f5868f9ab59ae235d970d5319dd89276bdaab Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 15:14:27 -0400 Subject: [PATCH 05/12] sync sorules --- salt/idstools/sync_files.sls | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/salt/idstools/sync_files.sls b/salt/idstools/sync_files.sls index 64479e937..e8d5edda6 100644 --- a/salt/idstools/sync_files.sls +++ b/salt/idstools/sync_files.sls @@ -26,6 +26,13 @@ rulesdir: - group: 939 - makedirs: True +SOrulesdir: + file.directory: + - name: /opt/so/rules/nids/sorules + - user: 939 + - group: 939 + - makedirs: True + # Don't show changes because all.rules can be large synclocalnidsrules: file.recurse: @@ -35,3 +42,13 @@ synclocalnidsrules: - group: 939 - show_changes: False - include_pat: 'E@.rules' + +# Don't show changes because all.rules can be large +syncnidsSOrules: + file.recurse: + - name: /opt/so/rules/nids/sorules + - source: salt://idstools/sorules/ + - user: 939 + - group: 939 + - show_changes: False + - include_pat: 'E@.rules' From 5c704d7e5864dc16c37224d000c019b257315a5c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 15:20:44 -0400 Subject: [PATCH 06/12] run so-rule-update if idstools configs change --- salt/idstools/sync_files.sls | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/idstools/sync_files.sls b/salt/idstools/sync_files.sls index e8d5edda6..cc6c45baa 100644 --- a/salt/idstools/sync_files.sls +++ b/salt/idstools/sync_files.sls @@ -19,6 +19,12 @@ idstoolsetcsync: - group: 939 - template: jinja +run_so-rule-update: + cmd.run: + - name: '/usr/sbin/so-rule-update > /opt/so/log/idstools/download.log 2>&1' + - onchanges: + - file: idstoolsetcsync + rulesdir: file.directory: - name: /opt/so/rules/nids From 21c80e4953a4de0b0cfa58464808fc44e8665704 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 15:27:23 -0400 Subject: [PATCH 07/12] run so-rule-update after idstools container restart --- salt/idstools/enabled.sls | 8 ++++++++ salt/idstools/sync_files.sls | 6 ------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/salt/idstools/enabled.sls b/salt/idstools/enabled.sls index bf5650773..966cb6786 100644 --- a/salt/idstools/enabled.sls +++ b/salt/idstools/enabled.sls @@ -69,6 +69,14 @@ so-rule-update: - minute: '1' - hour: '7' +run_so-rule-update: + cmd.run: + - name: '/usr/sbin/so-rule-update > /opt/so/log/idstools/download.log 2>&1' + - require: + - docker_container: so-idstools + - onchanges: + - file: idstoolsetcsync + {% else %} {{sls}}_state_not_allowed: diff --git a/salt/idstools/sync_files.sls b/salt/idstools/sync_files.sls index cc6c45baa..e8d5edda6 100644 --- a/salt/idstools/sync_files.sls +++ b/salt/idstools/sync_files.sls @@ -19,12 +19,6 @@ idstoolsetcsync: - group: 939 - template: jinja -run_so-rule-update: - cmd.run: - - name: '/usr/sbin/so-rule-update > /opt/so/log/idstools/download.log 2>&1' - - onchanges: - - file: idstoolsetcsync - rulesdir: file.directory: - name: /opt/so/rules/nids From 2dbe6798498d3086298b79ca355940ed98da96ee Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 17:05:03 -0400 Subject: [PATCH 08/12] force restart of filecheck if the config changes --- salt/strelka/filestream/config.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/strelka/filestream/config.sls b/salt/strelka/filestream/config.sls index a215967ee..d4615b174 100644 --- a/salt/strelka/filestream/config.sls +++ b/salt/strelka/filestream/config.sls @@ -85,6 +85,7 @@ filecheck_restart: - success_retcodes: [0,1] - onchanges: - file: filecheck_script + - file: filecheck_conf filecheck_run: cron.present: From 6da2f117f215ee856fe6800ff91c94ff11cea168 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 17:25:08 -0400 Subject: [PATCH 09/12] change which user runs filecheck cron based on md engine --- salt/strelka/filestream/config.sls | 30 ++++++++++++++++++++++++++---- salt/strelka/map.jinja | 2 -- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/salt/strelka/filestream/config.sls b/salt/strelka/filestream/config.sls index d4615b174..9c0ef1357 100644 --- a/salt/strelka/filestream/config.sls +++ b/salt/strelka/filestream/config.sls @@ -6,7 +6,7 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls.split('.')[0] in allowed_states %} {% from 'strelka/map.jinja' import STRELKAMERGED %} -{% from 'strelka/map.jinja' import filecheck_runas %} +{% from 'vars/globals.map.jinja' import GLOBALS %} include: - strelka.config @@ -87,11 +87,33 @@ filecheck_restart: - file: filecheck_script - file: filecheck_conf -filecheck_run: +{% if GLOBALS.md_engine == 'ZEEK' %} + +filecheck_run_socore: cron.present: - name: 'ps -ef | grep filecheck | grep -v grep > /dev/null 2>&1 || python3 /opt/so/conf/strelka/filecheck >> /opt/so/log/strelka/filecheck_stdout.log 2>&1 &' - - identifier: filecheck_run - - user: {{ filecheck_runas }} + - identifier: filecheck_run_socore + - user: socore + +remove_filecheck_run_suricata: + cron.absent: + - identifier: filecheck_run_suricata + - user: suricata + +{% elif GLOBALS.md_engine == 'SURICATA'%} + +filecheck_run_suricata: + cron.present: + - name: 'ps -ef | grep filecheck | grep -v grep > /dev/null 2>&1 || python3 /opt/so/conf/strelka/filecheck >> /opt/so/log/strelka/filecheck_stdout.log 2>&1 &' + - identifier: filecheck_run_suricata + - user: suricata + +remove_filecheck_run_socore: + cron.absent: + - identifier: filecheck_run_socore + - user: socore + +{% endif %} filcheck_history_clean: cron.present: diff --git a/salt/strelka/map.jinja b/salt/strelka/map.jinja index 646f7a746..387036248 100644 --- a/salt/strelka/map.jinja +++ b/salt/strelka/map.jinja @@ -24,10 +24,8 @@ {% if GLOBALS.md_engine == "SURICATA" %} {% set extract_path = '/nsm/suricata/extracted' %} -{% set filecheck_runas = 'suricata' %} {% else %} {% set extract_path = '/nsm/zeek/extracted/complete' %} -{% set filecheck_runas = 'socore' %} {% endif %} {% do STRELKADEFAULTS.strelka.filecheck.update({'extract_path': extract_path}) %} From 553b758c61e87909f704b9159c13d30022ea3ac4 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 17:28:14 -0400 Subject: [PATCH 10/12] update cronjobs first, the kill filecheck --- salt/strelka/filestream/config.sls | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/salt/strelka/filestream/config.sls b/salt/strelka/filestream/config.sls index 9c0ef1357..193241f32 100644 --- a/salt/strelka/filestream/config.sls +++ b/salt/strelka/filestream/config.sls @@ -78,15 +78,6 @@ filecheck_script: - group: 939 - mode: 755 -filecheck_restart: - cmd.run: - - name: pkill -f "python3 /opt/so/conf/strelka/filecheck" - - hide_output: True - - success_retcodes: [0,1] - - onchanges: - - file: filecheck_script - - file: filecheck_conf - {% if GLOBALS.md_engine == 'ZEEK' %} filecheck_run_socore: @@ -115,6 +106,15 @@ remove_filecheck_run_socore: {% endif %} +filecheck_restart: + cmd.run: + - name: pkill -f "python3 /opt/so/conf/strelka/filecheck" + - hide_output: True + - success_retcodes: [0,1] + - onchanges: + - file: filecheck_script + - file: filecheck_conf + filcheck_history_clean: cron.present: - name: '/usr/bin/find /nsm/strelka/history/ -type f -mtime +2 -exec rm {} + > /dev/null 2>&1' From 58fe25623b6ece773278c293727428059e8944fb Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 17:48:34 -0400 Subject: [PATCH 11/12] ensure ownership of /opt/so/log/strelka/filecheck_stdout.log --- salt/strelka/filestream/config.sls | 7 +++++++ salt/strelka/map.jinja | 2 ++ 2 files changed, 9 insertions(+) diff --git a/salt/strelka/filestream/config.sls b/salt/strelka/filestream/config.sls index 193241f32..c827ff5fb 100644 --- a/salt/strelka/filestream/config.sls +++ b/salt/strelka/filestream/config.sls @@ -7,6 +7,7 @@ {% if sls.split('.')[0] in allowed_states %} {% from 'strelka/map.jinja' import STRELKAMERGED %} {% from 'vars/globals.map.jinja' import GLOBALS %} +{% from 'strelka/map.jinja' import filecheck_runas %} include: - strelka.config @@ -78,6 +79,12 @@ filecheck_script: - group: 939 - mode: 755 +filecheck_stdout.log: + file.managed: + - name: /opt/so/log/strelka/filecheck_stdout.log + - user: {{ filecheck_runas }} + - group: {{ filecheck_runas }} + {% if GLOBALS.md_engine == 'ZEEK' %} filecheck_run_socore: diff --git a/salt/strelka/map.jinja b/salt/strelka/map.jinja index 387036248..646f7a746 100644 --- a/salt/strelka/map.jinja +++ b/salt/strelka/map.jinja @@ -24,8 +24,10 @@ {% if GLOBALS.md_engine == "SURICATA" %} {% set extract_path = '/nsm/suricata/extracted' %} +{% set filecheck_runas = 'suricata' %} {% else %} {% set extract_path = '/nsm/zeek/extracted/complete' %} +{% set filecheck_runas = 'socore' %} {% endif %} {% do STRELKADEFAULTS.strelka.filecheck.update({'extract_path': extract_path}) %} From 789fff561efdbccd10497b7cf66e496cfeb543f8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 17:55:30 -0400 Subject: [PATCH 12/12] ensure ownership of /opt/so/log/strelka/filecheck.log --- salt/strelka/filestream/config.sls | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/strelka/filestream/config.sls b/salt/strelka/filestream/config.sls index c827ff5fb..993a59650 100644 --- a/salt/strelka/filestream/config.sls +++ b/salt/strelka/filestream/config.sls @@ -79,6 +79,12 @@ filecheck_script: - group: 939 - mode: 755 +filecheck.log: + file.managed: + - name: /opt/so/log/strelka/filecheck.log + - user: {{ filecheck_runas }} + - group: {{ filecheck_runas }} + filecheck_stdout.log: file.managed: - name: /opt/so/log/strelka/filecheck_stdout.log