From 592bbf42173fbe73046def9656ebb4637e42aacd Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 5 Dec 2022 09:21:08 -0500 Subject: [PATCH 1/9] fix filecheck for suricata deployments --- salt/strelka/filecheck/filecheck | 43 +++++++++++++++++++++----------- salt/strelka/init.sls | 20 +++++++++++---- 2 files changed, 44 insertions(+), 19 deletions(-) diff --git a/salt/strelka/filecheck/filecheck b/salt/strelka/filecheck/filecheck index d91c75900..c6148a953 100644 --- a/salt/strelka/filecheck/filecheck +++ b/salt/strelka/filecheck/filecheck @@ -31,14 +31,21 @@ extract_path = cfg["filecheck"]["extract_path"] historypath = cfg["filecheck"]["historypath"] strelkapath = cfg["filecheck"]["strelkapath"] logfile = cfg["filecheck"]["logfile"] +recycle_secs = cfg["filecheck"].get("recycle_secs", 300) logging.basicConfig(filename=logfile, filemode='w', format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S', level=logging.INFO) def checkexisting(): - for file in os.listdir(extract_path): - filename = os.path.join(extract_path, file) - logging.info("Processing existing file " + filename) - checksum(filename) + for root, dirs, files in os.walk(extract_path): + for file in files: + try: + path = os.path.join(root, file) + filename = os.path.join(extract_path, path) + if os.path.isfile(filename): + logging.info("Processing existing file: " + filename) + checksum(filename) + except Exception as err: + logging.error("Failed to process file: " + file) def checksum(filename): with open(filename, 'rb') as afile: @@ -67,24 +74,32 @@ def process(filename, hizash): class CreatedEventHandler(FileSystemEventHandler): def on_created(self, event): filename = event.src_path - logging.info("Found new file") + logging.info("Found new file: " + filename) checksum(filename) if __name__ == "__main__": + logging.info("Starting filecheck") checkexisting() - event_handler =CreatedEventHandler() + event_handler =CreatedEventHandler() observer = Observer() - logging.info("Starting filecheck") - observer.schedule(event_handler, extract_path, recursive=True) - observer.start() - try: - while True: - time.sleep(1) - except KeyboardInterrupt: + shutdown = False + while not shutdown: + logging.info("Scheduling observer") + observer.schedule(event_handler, extract_path, recursive=True) + observer.start() + try: + time.sleep(recycle_secs) + except KeyboardInterrupt: + logging.warn("User requested shutdown") + shutdown = True + observer.stop() - observer.join() + observer.join() + + if not shutdown: + logging.info("Recycling observer to pick up new subdirectories") logging.info("Exiting filecheck") diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls index 24be1852a..9e7adb57e 100644 --- a/salt/strelka/init.sls +++ b/salt/strelka/init.sls @@ -24,6 +24,15 @@ {% import_yaml 'strelka/defaults.yaml' as strelka_config with context %} {% set IGNORELIST = salt['pillar.get']('strelka:ignore', strelka_config.strelka.ignore, merge=True, merge_nested_lists=True) %} +{% if ENGINE == "SURICATA" %} + {% set filecheck_runas = 'suricata' %} + {% set filecheck_groupid = 940 %} +{% else %} + {% set filecheck_runas = 'socore' %} + {% set filecheck_groupid = 939 %} +{% endif %} + + {% if grains['os'] != 'CentOS' %} strelkapkgs: pkg.installed: @@ -124,7 +133,8 @@ strelkaunprocessed: file.directory: - name: /nsm/strelka/unprocessed - user: 939 - - group: 939 + - group: {{ filecheck_groupid }} + - mode 775 - makedirs: True # Check to see if Strelka frontend port is available @@ -137,14 +147,14 @@ filecheck_logdir: file.directory: - name: /opt/so/log/strelka - user: 939 - - group: 939 + - group: {{ filecheck_groupid }} - makedirs: True filecheck_history: file.directory: - name: /nsm/strelka/history - user: 939 - - group: 939 + - group: {{ filecheck_groupid }} - makedirs: True filecheck_conf: @@ -158,13 +168,13 @@ filecheck_script: - name: /opt/so/conf/strelka/filecheck - source: salt://strelka/filecheck/filecheck - user: 939 - - group: 939 + - group: {{ filecheck_groupid }} - mode: 755 filecheck_run: cron.present: - name: 'ps -ef | grep filecheck | grep -v grep || python3 /opt/so/conf/strelka/filecheck >> /opt/so/log/strelka/filecheck_stdout.log 2>&1 &' - - user: socore + - user: {{ filecheck_runas }} filcheck_history_clean: cron.present: From 992ced685f65c861b9352e1af5149d0c989b03b0 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 5 Dec 2022 09:27:31 -0500 Subject: [PATCH 2/9] fix filecheck for suricata deployments --- salt/strelka/filecheck/filecheck | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/strelka/filecheck/filecheck b/salt/strelka/filecheck/filecheck index c6148a953..314f73a70 100644 --- a/salt/strelka/filecheck/filecheck +++ b/salt/strelka/filecheck/filecheck @@ -83,11 +83,11 @@ if __name__ == "__main__": checkexisting() event_handler =CreatedEventHandler() - observer = Observer() shutdown = False while not shutdown: logging.info("Scheduling observer") + observer = Observer() observer.schedule(event_handler, extract_path, recursive=True) observer.start() try: From e9bb60dedb828b7560ba587d01934ec153d257b5 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 5 Dec 2022 09:28:25 -0500 Subject: [PATCH 3/9] fix filecheck for suricata deployments --- salt/strelka/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls index 9e7adb57e..61926265f 100644 --- a/salt/strelka/init.sls +++ b/salt/strelka/init.sls @@ -134,7 +134,7 @@ strelkaunprocessed: - name: /nsm/strelka/unprocessed - user: 939 - group: {{ filecheck_groupid }} - - mode 775 + - mode: 775 - makedirs: True # Check to see if Strelka frontend port is available From fe798138e3546132ac3d348782f8fc3cc3066a5e Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 5 Dec 2022 09:50:35 -0500 Subject: [PATCH 4/9] add suricata to socore group --- salt/strelka/init.sls | 15 +++++++-------- salt/suricata/init.sls | 6 ++++++ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls index 61926265f..ffb6b7380 100644 --- a/salt/strelka/init.sls +++ b/salt/strelka/init.sls @@ -26,13 +26,10 @@ {% if ENGINE == "SURICATA" %} {% set filecheck_runas = 'suricata' %} - {% set filecheck_groupid = 940 %} {% else %} {% set filecheck_runas = 'socore' %} - {% set filecheck_groupid = 939 %} {% endif %} - {% if grains['os'] != 'CentOS' %} strelkapkgs: pkg.installed: @@ -133,7 +130,7 @@ strelkaunprocessed: file.directory: - name: /nsm/strelka/unprocessed - user: 939 - - group: {{ filecheck_groupid }} + - group: 939 - mode: 775 - makedirs: True @@ -147,14 +144,16 @@ filecheck_logdir: file.directory: - name: /opt/so/log/strelka - user: 939 - - group: {{ filecheck_groupid }} + - group: 939 + - mode: 775 - makedirs: True filecheck_history: file.directory: - name: /nsm/strelka/history - user: 939 - - group: {{ filecheck_groupid }} + - group: 939 + - mode: 775 - makedirs: True filecheck_conf: @@ -165,10 +164,10 @@ filecheck_conf: filecheck_script: file.managed: - - name: /opt/so/conf/strelka/filecheck + - name: /usr/sbin/filecheck - source: salt://strelka/filecheck/filecheck - user: 939 - - group: {{ filecheck_groupid }} + - group: 939 - mode: 755 filecheck_run: diff --git a/salt/suricata/init.sls b/salt/suricata/init.sls index d6333c65a..4bdee1550 100644 --- a/salt/suricata/init.sls +++ b/salt/suricata/init.sls @@ -44,6 +44,12 @@ suricata: - home: /nsm/suricata - createhome: False +suricatasocoregroup: + group.present: + - gid: 939 + - addusers: + - suricata + suridir: file.directory: - name: /opt/so/conf/suricata From f06443f3dd066c8727bfe2ee0c12e6a3f71d27b3 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 5 Dec 2022 09:57:24 -0500 Subject: [PATCH 5/9] add suricata to socore group --- salt/suricata/init.sls | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/suricata/init.sls b/salt/suricata/init.sls index 4bdee1550..cef56bdb1 100644 --- a/salt/suricata/init.sls +++ b/salt/suricata/init.sls @@ -44,8 +44,9 @@ suricata: - home: /nsm/suricata - createhome: False -suricatasocoregroup: +socoregroupwithsuricata: group.present: + - name: socore - gid: 939 - addusers: - suricata From d7f60a0e5801e73beedcd005cc86561c8d7984bb Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 5 Dec 2022 10:01:40 -0500 Subject: [PATCH 6/9] only check files on inotify --- salt/strelka/filecheck/filecheck | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/strelka/filecheck/filecheck b/salt/strelka/filecheck/filecheck index 314f73a70..bfea89007 100644 --- a/salt/strelka/filecheck/filecheck +++ b/salt/strelka/filecheck/filecheck @@ -74,8 +74,9 @@ def process(filename, hizash): class CreatedEventHandler(FileSystemEventHandler): def on_created(self, event): filename = event.src_path - logging.info("Found new file: " + filename) - checksum(filename) + if os.path.isfile(filename): + logging.info("Found new file: " + filename) + checksum(filename) if __name__ == "__main__": logging.info("Starting filecheck") From 483a9d477ffa98e5b1ea33858625517cca7adb98 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 5 Dec 2022 10:15:15 -0500 Subject: [PATCH 7/9] undo filecheck location move --- salt/strelka/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls index ffb6b7380..b372e6f6c 100644 --- a/salt/strelka/init.sls +++ b/salt/strelka/init.sls @@ -164,7 +164,7 @@ filecheck_conf: filecheck_script: file.managed: - - name: /usr/sbin/filecheck + - name: /opt/so/conf/strelka/filecheck - source: salt://strelka/filecheck/filecheck - user: 939 - group: 939 From 86c31c129af10eb3f3d6545026b96b05f99a7bad Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 5 Dec 2022 10:27:42 -0500 Subject: [PATCH 8/9] add suricata to socore group --- salt/strelka/filecheck/filecheck | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/salt/strelka/filecheck/filecheck b/salt/strelka/filecheck/filecheck index bfea89007..73f24c7aa 100644 --- a/salt/strelka/filecheck/filecheck +++ b/salt/strelka/filecheck/filecheck @@ -36,26 +36,27 @@ recycle_secs = cfg["filecheck"].get("recycle_secs", 300) logging.basicConfig(filename=logfile, filemode='w', format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S', level=logging.INFO) def checkexisting(): + logging.info("Checking for existing files"); for root, dirs, files in os.walk(extract_path): for file in files: try: path = os.path.join(root, file) filename = os.path.join(extract_path, path) - if os.path.isfile(filename): - logging.info("Processing existing file: " + filename) - checksum(filename) + checksum(filename) except Exception as err: logging.error("Failed to process file: " + file) def checksum(filename): - with open(filename, 'rb') as afile: - shawnuff = hashlib.sha1() - buf = afile.read(8192) - while len(buf) > 0: - shawnuff.update(buf) + if os.path.isfile(filename) && "/tmp/" not in filename: + with open(filename, 'rb') as afile: + logging.info("Processing file: " + filename) + shawnuff = hashlib.sha1() buf = afile.read(8192) - hizash=shawnuff.hexdigest() - process(filename, hizash) + while len(buf) > 0: + shawnuff.update(buf) + buf = afile.read(8192) + hizash=shawnuff.hexdigest() + process(filename, hizash) def process(filename, hizash): if os.path.exists(historypath + hizash): @@ -73,10 +74,7 @@ def process(filename, hizash): class CreatedEventHandler(FileSystemEventHandler): def on_created(self, event): - filename = event.src_path - if os.path.isfile(filename): - logging.info("Found new file: " + filename) - checksum(filename) + checksum(event.src_path) if __name__ == "__main__": logging.info("Starting filecheck") From 69c5a9dd90e7ce9eb20d707a3e991c23488fcb26 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 5 Dec 2022 10:31:09 -0500 Subject: [PATCH 9/9] ensure tmp files are not processed --- salt/strelka/filecheck/filecheck | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/strelka/filecheck/filecheck b/salt/strelka/filecheck/filecheck index 73f24c7aa..146625552 100644 --- a/salt/strelka/filecheck/filecheck +++ b/salt/strelka/filecheck/filecheck @@ -47,7 +47,7 @@ def checkexisting(): logging.error("Failed to process file: " + file) def checksum(filename): - if os.path.isfile(filename) && "/tmp/" not in filename: + if os.path.isfile(filename) and "/tmp/" not in filename: with open(filename, 'rb') as afile: logging.info("Processing file: " + filename) shawnuff = hashlib.sha1()