diff --git a/salt/common/files/sensor-rotate.conf b/salt/common/files/sensor-rotate.conf index cefd3944e..797eb8a45 100644 --- a/salt/common/files/sensor-rotate.conf +++ b/salt/common/files/sensor-rotate.conf @@ -19,4 +19,17 @@ extension .log dateext dateyesterday -} \ No newline at end of file +} + +/opt/so/log/strelka/filecheck.log +{ + daily + rotate 14 + missingok + copytruncate + compress + create + extension .log + dateext + dateyesterday +} diff --git a/salt/common/init.sls b/salt/common/init.sls index 0eaf5e77e..2040edc97 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -38,15 +38,15 @@ socore: soconfperms: file.directory: - name: /opt/so/conf - - uid: 939 - - gid: 939 + - user: 939 + - group: 939 - dir_mode: 770 sostatusconf: file.directory: - name: /opt/so/conf/so-status - - uid: 939 - - gid: 939 + - user: 939 + - group: 939 - dir_mode: 770 so-status.conf: @@ -57,8 +57,8 @@ so-status.conf: sosaltstackperms: file.directory: - name: /opt/so/saltstack - - uid: 939 - - gid: 939 + - user: 939 + - group: 939 - dir_mode: 770 so_log_perms: @@ -113,6 +113,7 @@ commonpkgs: - python3-mysqldb - python3-packaging - python3-lxml + - python3-watchdog - git - vim @@ -156,6 +157,7 @@ commonpkgs: - python36-mysql - python36-packaging - python36-lxml + - python36-watchdog - yum-utils - device-mapper-persistent-data - lvm2 diff --git a/salt/strelka/filecheck/filecheck b/salt/strelka/filecheck/filecheck new file mode 100644 index 000000000..816125fcb --- /dev/null +++ b/salt/strelka/filecheck/filecheck @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 + +# Copyright 2014-2022 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import time +import hashlib +import logging +import yaml +from watchdog.observers import Observer +from watchdog.events import FileSystemEventHandler + +with open("/opt/so/conf/strelka/filecheck.yaml", "r") as ymlfile: + cfg = yaml.load(ymlfile) + +extract_path = cfg["filecheck"]["extract_path"] +historypath = cfg["filecheck"]["historypath"] +strelkapath = cfg["filecheck"]["strelkapath"] +logfile = cfg["filecheck"]["logfile"] + +logging.basicConfig(filename=logfile, filemode='w', format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S', level=logging.INFO) + +def checkexisting(): + for file in os.listdir(extract_path): + filename = os.path.join(extract_path, file) + logging.info("Processing existing file " + filename) + checksum(filename) + +def checksum(filename): + with open(filename, 'rb') as afile: + shawnuff = hashlib.sha1() + buf = afile.read(8192) + while len(buf) > 0: + shawnuff.update(buf) + buf = afile.read(8192) + hizash=shawnuff.hexdigest() + process(filename, hizash) + +def process(filename, hizash): + if os.path.exists(historypath + hizash): + logging.info(filename + " Already exists.. removing") + os.remove(filename) + else: + # Write the file + logging.info(filename + " is new. Creating a record and sending to Strelka") + with open(os.path.join(historypath + hizash), 'w') as fp: + pass + head, tail = os.path.split(filename) + + # Move the file + os.rename(filename, strelkapath + tail) + +class CreatedEventHandler(FileSystemEventHandler): + def on_created(self, event): + filename = event.src_path + logging.info("Found new file") + checksum(filename) + +if __name__ == "__main__": + + checkexisting() + event_handler =CreatedEventHandler() + + observer = Observer() + observer.schedule(event_handler, extract_path, recursive=True) + observer.start() + try: + while True: + time.sleep(1) + except KeyboardInterrupt: + observer.stop() + observer.join() diff --git a/salt/strelka/filecheck/filecheck.yaml b/salt/strelka/filecheck/filecheck.yaml new file mode 100644 index 000000000..2b46afdf5 --- /dev/null +++ b/salt/strelka/filecheck/filecheck.yaml @@ -0,0 +1,11 @@ +{%- set ENGINE = salt['pillar.get']('global:mdengine', '') %} +filecheck: + {%- if ENGINE == "SURICATA" %} + extract_path: '/nsm/suricata/extracted' + {%- else %} + extract_path: '/nsm/zeek/extracted/complete' + {%- endif %} + historypath: '/nsm/strelka/history/' + strelkapath: '/nsm/strelka/unprocessed/' + logfile: '/opt/so/log/strelka/filecheck.log' + diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls index 56a5b9dcc..ff4727126 100644 --- a/salt/strelka/init.sls +++ b/salt/strelka/init.sls @@ -79,7 +79,7 @@ strelkarepos: {% endif %} strelkadatadir: - file.directory: + file.directory: - name: /nsm/strelka - user: 939 - group: 939 @@ -93,21 +93,21 @@ strelkalogdir: - makedirs: True strelkaprocessed: - file.directory: + file.directory: - name: /nsm/strelka/processed - user: 939 - group: 939 - makedirs: True strelkastaging: - file.directory: + file.directory: - name: /nsm/strelka/staging - user: 939 - group: 939 - makedirs: True strelkaunprocessed: - file.directory: + file.directory: - name: /nsm/strelka/unprocessed - user: 939 - group: 939 @@ -115,8 +115,50 @@ strelkaunprocessed: # Check to see if Strelka frontend port is available strelkaportavailable: - cmd.run: - - name: netstat -utanp | grep ":57314" | grep -qvE 'docker|TIME_WAIT' && PROCESS=$(netstat -utanp | grep ":57314" | uniq) && echo "Another process ($PROCESS) appears to be using port 57314. Please terminate this process, or reboot to ensure a clean state so that Strelka can start properly." && exit 1 || exit 0 + cmd.run: + - name: netstat -utanp | grep ":57314" | grep -qvE 'docker|TIME_WAIT' && PROCESS=$(netstat -utanp | grep ":57314" | uniq) && echo "Another process ($PROCESS) appears to be using port 57314. Please terminate this process, or reboot to ensure a clean state so that Strelka can start properly." && exit 1 || exit 0 + +# Filecheck Section +filecheck_logdir: + file.directory: + - name: /opt/so/log/strelka + - user: 939 + - group: 939 + - makedirs: True + +filecheck_history: + file.directory: + - name: /nsm/strelka/history + - user: 939 + - group: 939 + - makedirs: True + +filecheck_conf: + file.managed: + - name: /opt/so/conf/strelka/filecheck.yaml + - source: salt://strelka/filecheck/filecheck.yaml + - template: jinja + +filecheck_script: + file.managed: + - name: /opt/so/conf/strelka/filecheck + - source: salt://strelka/filecheck/filecheck + - user: 939 + - group: 939 + - mode: 755 + +filecheck_run: + cmd.run: + - name: 'python3 /opt/so/conf/strelka/filecheck' + - bg: True + - runas: socore + - unless: ps -ef | grep filecheck | grep -v grep + +filcheck_history_clean: + cron.present: + - name: '/usr/bin/find /nsm/strelka/history/ -type f -mtime +2 -exec rm {} + > /dev/null 2>&1>' + - minute: '33' +# End Filecheck Section strelka_coordinator: docker_container.running: @@ -212,7 +254,7 @@ strelka_zeek_extracted_sync_old: {% if ENGINE == "SURICATA" %} strelka_suricata_extracted_sync: - cron.present: + cron.absent: - user: root - identifier: zeek-extracted-strelka-sync - name: '[ -d /nsm/suricata/extracted/ ] && find /nsm/suricata/extracted/* -not \( -path /nsm/suricata/extracted/tmp -prune \) -type f -print0 | xargs -0 -I {} mv {} /nsm/strelka/unprocessed/ > /dev/null 2>&1' @@ -220,7 +262,7 @@ strelka_suricata_extracted_sync: {% else %} strelka_zeek_extracted_sync: - cron.present: + cron.absent: - user: root - identifier: zeek-extracted-strelka-sync - name: '[ -d /nsm/zeek/extracted/complete/ ] && mv /nsm/zeek/extracted/complete/* /nsm/strelka/unprocessed/ > /dev/null 2>&1' diff --git a/salt/zeek/init.sls b/salt/zeek/init.sls index ff91762f5..4640f0fea 100644 --- a/salt/zeek/init.sls +++ b/salt/zeek/init.sls @@ -76,6 +76,7 @@ zeekextractcompletedir: file.directory: - name: /nsm/zeek/extracted/complete - user: 937 + - group: 939 - makedirs: True # Sync the policies