Merge pull request #9033 from Security-Onion-Solutions/strelkafix

Add Filechecks

salt/common/files/sensor-rotate.conf

@@ -19,4 +19,17 @@
     extension .log
     dateext
     dateyesterday
-}
+}
+
+/opt/so/log/strelka/filecheck.log
+{
+    daily
+    rotate 14
+    missingok
+    copytruncate
+    compress
+    create
+    extension .log
+    dateext
+    dateyesterday
+}

salt/common/init.sls

@@ -38,15 +38,15 @@ socore:
 soconfperms:
   file.directory:
     - name: /opt/so/conf
-    - uid: 939
-    - gid: 939
+    - user: 939
+    - group: 939
     - dir_mode: 770
 
 sostatusconf:
   file.directory:
     - name: /opt/so/conf/so-status
-    - uid: 939
-    - gid: 939
+    - user: 939
+    - group: 939
     - dir_mode: 770
 
 so-status.conf:
@@ -57,8 +57,8 @@ so-status.conf:
 sosaltstackperms:
   file.directory:
     - name: /opt/so/saltstack
-    - uid: 939
-    - gid: 939
+    - user: 939
+    - group: 939
     - dir_mode: 770
 
 so_log_perms:
@@ -113,6 +113,7 @@ commonpkgs:
       - python3-mysqldb
       - python3-packaging
       - python3-lxml
+      - python3-watchdog
       - git
       - vim
 
@@ -156,6 +157,7 @@ commonpkgs:
       - python36-mysql
       - python36-packaging
       - python36-lxml
+      - python36-watchdog
       - yum-utils
       - device-mapper-persistent-data
       - lvm2

salt/strelka/filecheck/filecheck

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+
+# Copyright 2014-2022 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import time
+import hashlib
+import logging
+import yaml
+from watchdog.observers import Observer
+from watchdog.events import FileSystemEventHandler
+
+with open("/opt/so/conf/strelka/filecheck.yaml", "r") as ymlfile:
+    cfg = yaml.load(ymlfile)
+
+extract_path = cfg["filecheck"]["extract_path"]
+historypath = cfg["filecheck"]["historypath"]
+strelkapath = cfg["filecheck"]["strelkapath"]
+logfile = cfg["filecheck"]["logfile"]
+
+logging.basicConfig(filename=logfile, filemode='w', format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S', level=logging.INFO)
+
+def checkexisting():
+    for file in os.listdir(extract_path):
+        filename = os.path.join(extract_path, file)
+        logging.info("Processing existing file " + filename)
+        checksum(filename) 
+
+def checksum(filename):
+    with open(filename, 'rb') as afile:
+        shawnuff = hashlib.sha1()
+        buf = afile.read(8192)
+        while len(buf) > 0:
+            shawnuff.update(buf)
+            buf = afile.read(8192)
+        hizash=shawnuff.hexdigest()
+        process(filename, hizash)
+
+def process(filename, hizash):
+    if os.path.exists(historypath + hizash):
+        logging.info(filename + " Already exists.. removing")
+        os.remove(filename)
+    else:
+        # Write the file
+        logging.info(filename + " is new. Creating a record and sending to Strelka")
+        with open(os.path.join(historypath + hizash), 'w') as fp:
+            pass
+        head, tail = os.path.split(filename)
+
+        # Move the file
+        os.rename(filename, strelkapath + tail)
+
+class CreatedEventHandler(FileSystemEventHandler):
+    def on_created(self, event):
+        filename = event.src_path
+        logging.info("Found new file")
+        checksum(filename)
+
+if __name__ == "__main__":
+
+    checkexisting()
+    event_handler =CreatedEventHandler()
+
+    observer = Observer()
+    observer.schedule(event_handler, extract_path, recursive=True)
+    observer.start()
+    try:
+        while True:
+            time.sleep(1)
+    except KeyboardInterrupt:
+        observer.stop()
+    observer.join()

salt/strelka/filecheck/filecheck.yaml

@@ -0,0 +1,11 @@
+{%- set ENGINE = salt['pillar.get']('global:mdengine', '') %}
+filecheck:
+    {%- if ENGINE == "SURICATA" %}
+    extract_path: '/nsm/suricata/extracted'
+    {%- else %}
+    extract_path: '/nsm/zeek/extracted/complete'
+    {%- endif %}
+    historypath: '/nsm/strelka/history/'
+    strelkapath: '/nsm/strelka/unprocessed/'
+    logfile: '/opt/so/log/strelka/filecheck.log'
+

salt/strelka/init.sls

@@ -79,7 +79,7 @@ strelkarepos:
 {% endif %}
 
 strelkadatadir:
-   file.directory:
+  file.directory:
     - name: /nsm/strelka
     - user: 939
     - group: 939
@@ -93,21 +93,21 @@ strelkalogdir:
     - makedirs: True
 
 strelkaprocessed:
-   file.directory:
+  file.directory:
     - name: /nsm/strelka/processed
     - user: 939
     - group: 939
     - makedirs: True
 
 strelkastaging:
-   file.directory:
+  file.directory:
     - name: /nsm/strelka/staging
     - user: 939
     - group: 939
     - makedirs: True
 
 strelkaunprocessed:
-   file.directory:
+  file.directory:
     - name: /nsm/strelka/unprocessed
     - user: 939
     - group: 939
@@ -115,8 +115,50 @@ strelkaunprocessed:
 
 # Check to see if Strelka frontend port is available
 strelkaportavailable:
-    cmd.run:
-      - name: netstat -utanp | grep ":57314" | grep -qvE 'docker|TIME_WAIT' && PROCESS=$(netstat -utanp | grep ":57314" | uniq) && echo "Another process ($PROCESS) appears to be using port 57314.  Please terminate this process, or reboot to ensure a clean state so that Strelka can start properly." && exit 1 || exit 0
+  cmd.run:
+    - name: netstat -utanp | grep ":57314" | grep -qvE 'docker|TIME_WAIT' && PROCESS=$(netstat -utanp | grep ":57314" | uniq) && echo "Another process ($PROCESS) appears to be using port 57314.  Please terminate this process, or reboot to ensure a clean state so that Strelka can start properly." && exit 1 || exit 0
+
+# Filecheck Section
+filecheck_logdir:
+  file.directory:
+    - name: /opt/so/log/strelka
+    - user: 939
+    - group: 939
+    - makedirs: True
+    
+filecheck_history:
+  file.directory:
+    - name: /nsm/strelka/history
+    - user: 939
+    - group: 939
+    - makedirs: True
+
+filecheck_conf:
+  file.managed:
+    - name: /opt/so/conf/strelka/filecheck.yaml
+    - source: salt://strelka/filecheck/filecheck.yaml
+    - template: jinja
+
+filecheck_script:
+  file.managed:
+    - name: /opt/so/conf/strelka/filecheck
+    - source: salt://strelka/filecheck/filecheck
+    - user: 939
+    - group: 939
+    - mode: 755
+
+filecheck_run:
+  cmd.run:
+    - name: 'python3 /opt/so/conf/strelka/filecheck'
+    - bg: True
+    - runas: socore
+    - unless: ps -ef | grep filecheck | grep -v grep
+
+filcheck_history_clean:
+  cron.present:
+    - name: '/usr/bin/find /nsm/strelka/history/ -type f -mtime +2 -exec rm {} + > /dev/null 2>&1>'
+    - minute: '33'
+# End Filecheck Section
 
 strelka_coordinator:
   docker_container.running:
@@ -212,7 +254,7 @@ strelka_zeek_extracted_sync_old:
 {% if ENGINE == "SURICATA" %}
 
 strelka_suricata_extracted_sync:
-  cron.present:
+  cron.absent:
     - user: root
     - identifier: zeek-extracted-strelka-sync
     - name: '[ -d /nsm/suricata/extracted/ ] && find /nsm/suricata/extracted/* -not \( -path /nsm/suricata/extracted/tmp -prune \) -type f -print0 | xargs -0 -I {} mv {} /nsm/strelka/unprocessed/ > /dev/null 2>&1'
@@ -220,7 +262,7 @@ strelka_suricata_extracted_sync:
 
 {% else %}
 strelka_zeek_extracted_sync:
-  cron.present:
+  cron.absent:
     - user: root
     - identifier: zeek-extracted-strelka-sync
     - name: '[ -d /nsm/zeek/extracted/complete/ ] && mv /nsm/zeek/extracted/complete/* /nsm/strelka/unprocessed/ > /dev/null 2>&1'

salt/zeek/init.sls

@@ -76,6 +76,7 @@ zeekextractcompletedir:
   file.directory:
     - name: /nsm/zeek/extracted/complete
     - user: 937
+    - group: 939
     - makedirs: True
 
 # Sync the policies