From e9e7362005d5c77b07701826e21798ca142a96fa Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 31 Oct 2022 12:57:08 -0400 Subject: [PATCH] Add Filechecks --- salt/common/init.sls | 2 + salt/strelka/filecheck/filecheck | 68 +++++++++++++++++++++++++++ salt/strelka/filecheck/filecheck.yaml | 11 +++++ salt/strelka/init.sls | 44 +++++++++++++---- 4 files changed, 117 insertions(+), 8 deletions(-) create mode 100644 salt/strelka/filecheck/filecheck create mode 100644 salt/strelka/filecheck/filecheck.yaml diff --git a/salt/common/init.sls b/salt/common/init.sls index 0eaf5e77e..d6c8e0103 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -113,6 +113,7 @@ commonpkgs: - python3-mysqldb - python3-packaging - python3-lxml + - python3-watchdog - git - vim @@ -156,6 +157,7 @@ commonpkgs: - python36-mysql - python36-packaging - python36-lxml + - python36-watchdog - yum-utils - device-mapper-persistent-data - lvm2 diff --git a/salt/strelka/filecheck/filecheck b/salt/strelka/filecheck/filecheck new file mode 100644 index 000000000..5f08f88b3 --- /dev/null +++ b/salt/strelka/filecheck/filecheck @@ -0,0 +1,68 @@ +import os +import time +import hashlib +import logging +import yaml +from watchdog.observers import Observer +from watchdog.events import FileSystemEventHandler + +with open("/opt/so/conf/filecheck.yaml", "r") as ymlfile: + cfg = yaml.load(ymlfile) + +extract_path = cfg["filecheck.extract_path"] +historypath = cfg["filecheck.historypath"] +strelkapath = cfg["filecheck.strelkapath"] +logfile = cfg["filecheck.logfile"] + +logging.basicConfig(filename=logfile, filemode='w', format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S', level=logging.INFO) + +def checkexisting(): + for file in os.listdir(extract_path): + filename = os.path.join(extract_path, file) + logging.info("Processing existing file " + filename) + checksum(filename) + +def checksum(filename): + with open(filename, 'rb') as afile: + shawnuff = hashlib.sha1() + buf = afile.read(8192) + while len(buf) > 0: + shawnuff.update(buf) + buf = afile.read(8192) + hizash=shawnuff.hexdigest() + process(filename, hizash) + +def process(filename, hizash): + if os.path.exists(historypath + hizash): + logging.info(filename + " Already exists.. removing") + os.remove(filename) + else: + # Write the file + logging.info(filename + " is new. Creating a record and sending to Strelka") + with open(os.path.join(historypath + hizash), 'w') as fp: + pass + head, tail = os.path.split(filename) + + # Move the file + os.rename(filename, strelkapath + tail) + +class CreatedEventHandler(FileSystemEventHandler): + def on_created(self, event): + filename = event.src_path + logging.info("Found new file") + checksum(filename) + +if __name__ == "__main__": + + checkexisting() + event_handler =CreatedEventHandler() + + observer = Observer() + observer.schedule(event_handler, extract_path, recursive=True) + observer.start() + try: + while True: + time.sleep(1) + except KeyboardInterrupt: + observer.stop() + observer.join() diff --git a/salt/strelka/filecheck/filecheck.yaml b/salt/strelka/filecheck/filecheck.yaml new file mode 100644 index 000000000..b6f16b3ea --- /dev/null +++ b/salt/strelka/filecheck/filecheck.yaml @@ -0,0 +1,11 @@ +{%- set ENGINE = salt['pillar.get']('global:mdengine', '') %} +filecheck: + {% if ENGINE == "SURICATA" %} + extract_path = '/nsm/suricata/extracted' + {% else %} + extract_path = '/nsm/zeek/extracted/complete' + {% endif %} + historypath = '/nsm/strelka/history/' + strelkapath = '/nsm/strelka/unprocessed/' + logfile = '/opt/so/log/strelka/filecheck.log' + diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls index 56a5b9dcc..9e192609a 100644 --- a/salt/strelka/init.sls +++ b/salt/strelka/init.sls @@ -79,7 +79,7 @@ strelkarepos: {% endif %} strelkadatadir: - file.directory: + file.directory: - name: /nsm/strelka - user: 939 - group: 939 @@ -93,21 +93,21 @@ strelkalogdir: - makedirs: True strelkaprocessed: - file.directory: + file.directory: - name: /nsm/strelka/processed - user: 939 - group: 939 - makedirs: True strelkastaging: - file.directory: + file.directory: - name: /nsm/strelka/staging - user: 939 - group: 939 - makedirs: True strelkaunprocessed: - file.directory: + file.directory: - name: /nsm/strelka/unprocessed - user: 939 - group: 939 @@ -115,8 +115,36 @@ strelkaunprocessed: # Check to see if Strelka frontend port is available strelkaportavailable: - cmd.run: - - name: netstat -utanp | grep ":57314" | grep -qvE 'docker|TIME_WAIT' && PROCESS=$(netstat -utanp | grep ":57314" | uniq) && echo "Another process ($PROCESS) appears to be using port 57314. Please terminate this process, or reboot to ensure a clean state so that Strelka can start properly." && exit 1 || exit 0 + cmd.run: + - name: netstat -utanp | grep ":57314" | grep -qvE 'docker|TIME_WAIT' && PROCESS=$(netstat -utanp | grep ":57314" | uniq) && echo "Another process ($PROCESS) appears to be using port 57314. Please terminate this process, or reboot to ensure a clean state so that Strelka can start properly." && exit 1 || exit 0 + +# Filecheck Section +filecheck_history: + file.directory: + - name: /nsm/strelka/history + - user: 939 + - group: 939 + +filecheck_conf: + file.managed: + - name: /opt/so/conf/strelka/filecheck.yaml + - source: salt://strelka/filecheck/filecheck.yaml + - template: jinja + +filecheck_script: + file.managed: + - name: /opt/so/conf/strelka/filecheck + - source: salt://strelka/filecheck/filecheck + - user: 939 + - group: 939 + - mode: 755 + +filecheck_run: + cmd.run: + - name: 'python3 /opt/so/conf/strelka/filecheck &' + - unless: ps -ef | grep filecheck | grep -v grep + +# End Filecheck Section strelka_coordinator: docker_container.running: @@ -212,7 +240,7 @@ strelka_zeek_extracted_sync_old: {% if ENGINE == "SURICATA" %} strelka_suricata_extracted_sync: - cron.present: + cron.absent: - user: root - identifier: zeek-extracted-strelka-sync - name: '[ -d /nsm/suricata/extracted/ ] && find /nsm/suricata/extracted/* -not \( -path /nsm/suricata/extracted/tmp -prune \) -type f -print0 | xargs -0 -I {} mv {} /nsm/strelka/unprocessed/ > /dev/null 2>&1' @@ -220,7 +248,7 @@ strelka_suricata_extracted_sync: {% else %} strelka_zeek_extracted_sync: - cron.present: + cron.absent: - user: root - identifier: zeek-extracted-strelka-sync - name: '[ -d /nsm/zeek/extracted/complete/ ] && mv /nsm/zeek/extracted/complete/* /nsm/strelka/unprocessed/ > /dev/null 2>&1'