Add Filechecks

salt/common/init.sls

@@ -113,6 +113,7 @@ commonpkgs:
       - python3-mysqldb
       - python3-packaging
       - python3-lxml
+      - python3-watchdog
       - git
       - vim
 
@@ -156,6 +157,7 @@ commonpkgs:
       - python36-mysql
       - python36-packaging
       - python36-lxml
+      - python36-watchdog
       - yum-utils
       - device-mapper-persistent-data
       - lvm2

salt/strelka/filecheck/filecheck

@@ -0,0 +1,68 @@
+import os
+import time
+import hashlib
+import logging
+import yaml
+from watchdog.observers import Observer
+from watchdog.events import FileSystemEventHandler
+
+with open("/opt/so/conf/filecheck.yaml", "r") as ymlfile:
+    cfg = yaml.load(ymlfile)
+
+extract_path = cfg["filecheck.extract_path"]
+historypath = cfg["filecheck.historypath"]
+strelkapath = cfg["filecheck.strelkapath"]
+logfile = cfg["filecheck.logfile"]
+
+logging.basicConfig(filename=logfile, filemode='w', format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S', level=logging.INFO)
+
+def checkexisting():
+    for file in os.listdir(extract_path):
+        filename = os.path.join(extract_path, file)
+        logging.info("Processing existing file " + filename)
+        checksum(filename) 
+
+def checksum(filename):
+    with open(filename, 'rb') as afile:
+        shawnuff = hashlib.sha1()
+        buf = afile.read(8192)
+        while len(buf) > 0:
+            shawnuff.update(buf)
+            buf = afile.read(8192)
+        hizash=shawnuff.hexdigest()
+        process(filename, hizash)
+
+def process(filename, hizash):
+    if os.path.exists(historypath + hizash):
+        logging.info(filename + " Already exists.. removing")
+        os.remove(filename)
+    else:
+        # Write the file
+        logging.info(filename + " is new. Creating a record and sending to Strelka")
+        with open(os.path.join(historypath + hizash), 'w') as fp:
+            pass
+        head, tail = os.path.split(filename)
+
+        # Move the file
+        os.rename(filename, strelkapath + tail)
+
+class CreatedEventHandler(FileSystemEventHandler):
+    def on_created(self, event):
+        filename = event.src_path
+        logging.info("Found new file")
+        checksum(filename)
+
+if __name__ == "__main__":
+
+    checkexisting()
+    event_handler =CreatedEventHandler()
+
+    observer = Observer()
+    observer.schedule(event_handler, extract_path, recursive=True)
+    observer.start()
+    try:
+        while True:
+            time.sleep(1)
+    except KeyboardInterrupt:
+        observer.stop()
+    observer.join()

salt/strelka/filecheck/filecheck.yaml

@@ -0,0 +1,11 @@
+{%- set ENGINE = salt['pillar.get']('global:mdengine', '') %}
+filecheck:
+    {% if ENGINE == "SURICATA" %}
+    extract_path = '/nsm/suricata/extracted'
+    {% else %}
+    extract_path = '/nsm/zeek/extracted/complete'
+    {% endif %}
+    historypath = '/nsm/strelka/history/'
+    strelkapath = '/nsm/strelka/unprocessed/'
+    logfile = '/opt/so/log/strelka/filecheck.log'
+

salt/strelka/init.sls

@@ -79,7 +79,7 @@ strelkarepos:
 {% endif %}
 
 strelkadatadir:
-   file.directory:
+  file.directory:
     - name: /nsm/strelka
     - user: 939
     - group: 939
@@ -93,21 +93,21 @@ strelkalogdir:
     - makedirs: True
 
 strelkaprocessed:
-   file.directory:
+  file.directory:
     - name: /nsm/strelka/processed
     - user: 939
     - group: 939
     - makedirs: True
 
 strelkastaging:
-   file.directory:
+  file.directory:
     - name: /nsm/strelka/staging
     - user: 939
     - group: 939
     - makedirs: True
 
 strelkaunprocessed:
-   file.directory:
+  file.directory:
     - name: /nsm/strelka/unprocessed
     - user: 939
     - group: 939
@@ -115,8 +115,36 @@ strelkaunprocessed:
 
 # Check to see if Strelka frontend port is available
 strelkaportavailable:
-    cmd.run:
+  cmd.run:
-      - name: netstat -utanp | grep ":57314" | grep -qvE 'docker|TIME_WAIT' && PROCESS=$(netstat -utanp | grep ":57314" | uniq) && echo "Another process ($PROCESS) appears to be using port 57314.  Please terminate this process, or reboot to ensure a clean state so that Strelka can start properly." && exit 1 || exit 0
+    - name: netstat -utanp | grep ":57314" | grep -qvE 'docker|TIME_WAIT' && PROCESS=$(netstat -utanp | grep ":57314" | uniq) && echo "Another process ($PROCESS) appears to be using port 57314.  Please terminate this process, or reboot to ensure a clean state so that Strelka can start properly." && exit 1 || exit 0
+
+# Filecheck Section
+filecheck_history:
+  file.directory:
+    - name: /nsm/strelka/history
+    - user: 939
+    - group: 939
+
+filecheck_conf:
+  file.managed:
+    - name: /opt/so/conf/strelka/filecheck.yaml
+    - source: salt://strelka/filecheck/filecheck.yaml
+    - template: jinja
+
+filecheck_script:
+  file.managed:
+    - name: /opt/so/conf/strelka/filecheck
+    - source: salt://strelka/filecheck/filecheck
+    - user: 939
+    - group: 939
+    - mode: 755
+
+filecheck_run:
+  cmd.run:
+    - name: 'python3 /opt/so/conf/strelka/filecheck &'
+    - unless: ps -ef | grep filecheck | grep -v grep
+
+# End Filecheck Section
 
 strelka_coordinator:
   docker_container.running:
@@ -212,7 +240,7 @@ strelka_zeek_extracted_sync_old:
 {% if ENGINE == "SURICATA" %}
 
 strelka_suricata_extracted_sync:
-  cron.present:
+  cron.absent:
     - user: root
     - identifier: zeek-extracted-strelka-sync
     - name: '[ -d /nsm/suricata/extracted/ ] && find /nsm/suricata/extracted/* -not \( -path /nsm/suricata/extracted/tmp -prune \) -type f -print0 | xargs -0 -I {} mv {} /nsm/strelka/unprocessed/ > /dev/null 2>&1'
@@ -220,7 +248,7 @@ strelka_suricata_extracted_sync:
 
 {% else %}
 strelka_zeek_extracted_sync:
-  cron.present:
+  cron.absent:
     - user: root
     - identifier: zeek-extracted-strelka-sync
     - name: '[ -d /nsm/zeek/extracted/complete/ ] && mv /nsm/zeek/extracted/complete/* /nsm/strelka/unprocessed/ > /dev/null 2>&1'