From e9e7362005d5c77b07701826e21798ca142a96fa Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 31 Oct 2022 12:57:08 -0400 Subject: [PATCH 01/15] Add Filechecks --- salt/common/init.sls | 2 + salt/strelka/filecheck/filecheck | 68 +++++++++++++++++++++++++++ salt/strelka/filecheck/filecheck.yaml | 11 +++++ salt/strelka/init.sls | 44 +++++++++++++---- 4 files changed, 117 insertions(+), 8 deletions(-) create mode 100644 salt/strelka/filecheck/filecheck create mode 100644 salt/strelka/filecheck/filecheck.yaml diff --git a/salt/common/init.sls b/salt/common/init.sls index 0eaf5e77e..d6c8e0103 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -113,6 +113,7 @@ commonpkgs: - python3-mysqldb - python3-packaging - python3-lxml + - python3-watchdog - git - vim @@ -156,6 +157,7 @@ commonpkgs: - python36-mysql - python36-packaging - python36-lxml + - python36-watchdog - yum-utils - device-mapper-persistent-data - lvm2 diff --git a/salt/strelka/filecheck/filecheck b/salt/strelka/filecheck/filecheck new file mode 100644 index 000000000..5f08f88b3 --- /dev/null +++ b/salt/strelka/filecheck/filecheck @@ -0,0 +1,68 @@ +import os +import time +import hashlib +import logging +import yaml +from watchdog.observers import Observer +from watchdog.events import FileSystemEventHandler + +with open("/opt/so/conf/filecheck.yaml", "r") as ymlfile: + cfg = yaml.load(ymlfile) + +extract_path = cfg["filecheck.extract_path"] +historypath = cfg["filecheck.historypath"] +strelkapath = cfg["filecheck.strelkapath"] +logfile = cfg["filecheck.logfile"] + +logging.basicConfig(filename=logfile, filemode='w', format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S', level=logging.INFO) + +def checkexisting(): + for file in os.listdir(extract_path): + filename = os.path.join(extract_path, file) + logging.info("Processing existing file " + filename) + checksum(filename) + +def checksum(filename): + with open(filename, 'rb') as afile: + shawnuff = hashlib.sha1() + buf = afile.read(8192) + while len(buf) > 0: + shawnuff.update(buf) + buf = afile.read(8192) + hizash=shawnuff.hexdigest() + process(filename, hizash) + +def process(filename, hizash): + if os.path.exists(historypath + hizash): + logging.info(filename + " Already exists.. removing") + os.remove(filename) + else: + # Write the file + logging.info(filename + " is new. Creating a record and sending to Strelka") + with open(os.path.join(historypath + hizash), 'w') as fp: + pass + head, tail = os.path.split(filename) + + # Move the file + os.rename(filename, strelkapath + tail) + +class CreatedEventHandler(FileSystemEventHandler): + def on_created(self, event): + filename = event.src_path + logging.info("Found new file") + checksum(filename) + +if __name__ == "__main__": + + checkexisting() + event_handler =CreatedEventHandler() + + observer = Observer() + observer.schedule(event_handler, extract_path, recursive=True) + observer.start() + try: + while True: + time.sleep(1) + except KeyboardInterrupt: + observer.stop() + observer.join() diff --git a/salt/strelka/filecheck/filecheck.yaml b/salt/strelka/filecheck/filecheck.yaml new file mode 100644 index 000000000..b6f16b3ea --- /dev/null +++ b/salt/strelka/filecheck/filecheck.yaml @@ -0,0 +1,11 @@ +{%- set ENGINE = salt['pillar.get']('global:mdengine', '') %} +filecheck: + {% if ENGINE == "SURICATA" %} + extract_path = '/nsm/suricata/extracted' + {% else %} + extract_path = '/nsm/zeek/extracted/complete' + {% endif %} + historypath = '/nsm/strelka/history/' + strelkapath = '/nsm/strelka/unprocessed/' + logfile = '/opt/so/log/strelka/filecheck.log' + diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls index 56a5b9dcc..9e192609a 100644 --- a/salt/strelka/init.sls +++ b/salt/strelka/init.sls @@ -79,7 +79,7 @@ strelkarepos: {% endif %} strelkadatadir: - file.directory: + file.directory: - name: /nsm/strelka - user: 939 - group: 939 @@ -93,21 +93,21 @@ strelkalogdir: - makedirs: True strelkaprocessed: - file.directory: + file.directory: - name: /nsm/strelka/processed - user: 939 - group: 939 - makedirs: True strelkastaging: - file.directory: + file.directory: - name: /nsm/strelka/staging - user: 939 - group: 939 - makedirs: True strelkaunprocessed: - file.directory: + file.directory: - name: /nsm/strelka/unprocessed - user: 939 - group: 939 @@ -115,8 +115,36 @@ strelkaunprocessed: # Check to see if Strelka frontend port is available strelkaportavailable: - cmd.run: - - name: netstat -utanp | grep ":57314" | grep -qvE 'docker|TIME_WAIT' && PROCESS=$(netstat -utanp | grep ":57314" | uniq) && echo "Another process ($PROCESS) appears to be using port 57314. Please terminate this process, or reboot to ensure a clean state so that Strelka can start properly." && exit 1 || exit 0 + cmd.run: + - name: netstat -utanp | grep ":57314" | grep -qvE 'docker|TIME_WAIT' && PROCESS=$(netstat -utanp | grep ":57314" | uniq) && echo "Another process ($PROCESS) appears to be using port 57314. Please terminate this process, or reboot to ensure a clean state so that Strelka can start properly." && exit 1 || exit 0 + +# Filecheck Section +filecheck_history: + file.directory: + - name: /nsm/strelka/history + - user: 939 + - group: 939 + +filecheck_conf: + file.managed: + - name: /opt/so/conf/strelka/filecheck.yaml + - source: salt://strelka/filecheck/filecheck.yaml + - template: jinja + +filecheck_script: + file.managed: + - name: /opt/so/conf/strelka/filecheck + - source: salt://strelka/filecheck/filecheck + - user: 939 + - group: 939 + - mode: 755 + +filecheck_run: + cmd.run: + - name: 'python3 /opt/so/conf/strelka/filecheck &' + - unless: ps -ef | grep filecheck | grep -v grep + +# End Filecheck Section strelka_coordinator: docker_container.running: @@ -212,7 +240,7 @@ strelka_zeek_extracted_sync_old: {% if ENGINE == "SURICATA" %} strelka_suricata_extracted_sync: - cron.present: + cron.absent: - user: root - identifier: zeek-extracted-strelka-sync - name: '[ -d /nsm/suricata/extracted/ ] && find /nsm/suricata/extracted/* -not \( -path /nsm/suricata/extracted/tmp -prune \) -type f -print0 | xargs -0 -I {} mv {} /nsm/strelka/unprocessed/ > /dev/null 2>&1' @@ -220,7 +248,7 @@ strelka_suricata_extracted_sync: {% else %} strelka_zeek_extracted_sync: - cron.present: + cron.absent: - user: root - identifier: zeek-extracted-strelka-sync - name: '[ -d /nsm/zeek/extracted/complete/ ] && mv /nsm/zeek/extracted/complete/* /nsm/strelka/unprocessed/ > /dev/null 2>&1' From 4c5a2c0610057ea0bbe42dab3d6e06f67b9b87dd Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 31 Oct 2022 13:36:42 -0400 Subject: [PATCH 02/15] Update filecheck --- salt/strelka/filecheck/filecheck | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/strelka/filecheck/filecheck b/salt/strelka/filecheck/filecheck index 5f08f88b3..2957c2e8b 100644 --- a/salt/strelka/filecheck/filecheck +++ b/salt/strelka/filecheck/filecheck @@ -6,7 +6,7 @@ import yaml from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler -with open("/opt/so/conf/filecheck.yaml", "r") as ymlfile: +with open("/opt/so/conf/strelka/filecheck.yaml", "r") as ymlfile: cfg = yaml.load(ymlfile) extract_path = cfg["filecheck.extract_path"] From d2eb61a830af2c396171d62ccabd3fcac3c726dd Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 31 Oct 2022 13:41:45 -0400 Subject: [PATCH 03/15] Update filecheck.yaml --- salt/strelka/filecheck/filecheck.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/strelka/filecheck/filecheck.yaml b/salt/strelka/filecheck/filecheck.yaml index b6f16b3ea..cf7dcc199 100644 --- a/salt/strelka/filecheck/filecheck.yaml +++ b/salt/strelka/filecheck/filecheck.yaml @@ -1,10 +1,10 @@ -{%- set ENGINE = salt['pillar.get']('global:mdengine', '') %} +{%- set ENGINE = salt['pillar.get']('global:mdengine', '') -%} filecheck: - {% if ENGINE == "SURICATA" %} + {%- if ENGINE == "SURICATA" -%} extract_path = '/nsm/suricata/extracted' - {% else %} + {%- else -%} extract_path = '/nsm/zeek/extracted/complete' - {% endif %} + {%- endif -%} historypath = '/nsm/strelka/history/' strelkapath = '/nsm/strelka/unprocessed/' logfile = '/opt/so/log/strelka/filecheck.log' From e93e2995b7ea1b474c8c6dd8dcd2a4348b3d9685 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 31 Oct 2022 13:42:18 -0400 Subject: [PATCH 04/15] Update filecheck --- salt/strelka/filecheck/filecheck | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/strelka/filecheck/filecheck b/salt/strelka/filecheck/filecheck index 2957c2e8b..b74cefaed 100644 --- a/salt/strelka/filecheck/filecheck +++ b/salt/strelka/filecheck/filecheck @@ -9,7 +9,7 @@ from watchdog.events import FileSystemEventHandler with open("/opt/so/conf/strelka/filecheck.yaml", "r") as ymlfile: cfg = yaml.load(ymlfile) -extract_path = cfg["filecheck.extract_path"] +extract_path = cfg["filecheck"]["extract_path"] historypath = cfg["filecheck.historypath"] strelkapath = cfg["filecheck.strelkapath"] logfile = cfg["filecheck.logfile"] From 518d2aaa9cc7d3140ebfcc536ab4511de40ec8e6 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 31 Oct 2022 13:45:00 -0400 Subject: [PATCH 05/15] Update filecheck.yaml --- salt/strelka/filecheck/filecheck.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/salt/strelka/filecheck/filecheck.yaml b/salt/strelka/filecheck/filecheck.yaml index cf7dcc199..2b46afdf5 100644 --- a/salt/strelka/filecheck/filecheck.yaml +++ b/salt/strelka/filecheck/filecheck.yaml @@ -1,11 +1,11 @@ -{%- set ENGINE = salt['pillar.get']('global:mdengine', '') -%} +{%- set ENGINE = salt['pillar.get']('global:mdengine', '') %} filecheck: - {%- if ENGINE == "SURICATA" -%} - extract_path = '/nsm/suricata/extracted' - {%- else -%} - extract_path = '/nsm/zeek/extracted/complete' - {%- endif -%} - historypath = '/nsm/strelka/history/' - strelkapath = '/nsm/strelka/unprocessed/' - logfile = '/opt/so/log/strelka/filecheck.log' + {%- if ENGINE == "SURICATA" %} + extract_path: '/nsm/suricata/extracted' + {%- else %} + extract_path: '/nsm/zeek/extracted/complete' + {%- endif %} + historypath: '/nsm/strelka/history/' + strelkapath: '/nsm/strelka/unprocessed/' + logfile: '/opt/so/log/strelka/filecheck.log' From 07e72e40132a09d109af5d5f618b3ae861bf97b3 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 31 Oct 2022 13:47:49 -0400 Subject: [PATCH 06/15] Update filecheck --- salt/strelka/filecheck/filecheck | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/strelka/filecheck/filecheck b/salt/strelka/filecheck/filecheck index b74cefaed..8b1ddd446 100644 --- a/salt/strelka/filecheck/filecheck +++ b/salt/strelka/filecheck/filecheck @@ -10,9 +10,9 @@ with open("/opt/so/conf/strelka/filecheck.yaml", "r") as ymlfile: cfg = yaml.load(ymlfile) extract_path = cfg["filecheck"]["extract_path"] -historypath = cfg["filecheck.historypath"] -strelkapath = cfg["filecheck.strelkapath"] -logfile = cfg["filecheck.logfile"] +historypath = cfg["filecheck"]["historypath"] +strelkapath = cfg["filecheck"]["strelkapath"] +logfile = cfg["filecheck"]["logfile"] logging.basicConfig(filename=logfile, filemode='w', format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S', level=logging.INFO) From 5635375d8dadb249c16ba35623a257a9c1a0c820 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 31 Oct 2022 14:30:11 -0400 Subject: [PATCH 07/15] Update init.sls --- salt/strelka/init.sls | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls index 9e192609a..d93bb1540 100644 --- a/salt/strelka/init.sls +++ b/salt/strelka/init.sls @@ -141,7 +141,8 @@ filecheck_script: filecheck_run: cmd.run: - - name: 'python3 /opt/so/conf/strelka/filecheck &' + - name: 'python3 /opt/so/conf/strelka/filecheck' + - bg: True - unless: ps -ef | grep filecheck | grep -v grep # End Filecheck Section From db9b93a96c05afd51cbdc28cdfd76bbfd9a016b5 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 31 Oct 2022 14:35:02 -0400 Subject: [PATCH 08/15] Update init.sls --- salt/strelka/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls index d93bb1540..3bc7ddf0e 100644 --- a/salt/strelka/init.sls +++ b/salt/strelka/init.sls @@ -143,6 +143,7 @@ filecheck_run: cmd.run: - name: 'python3 /opt/so/conf/strelka/filecheck' - bg: True + - runas: 939 - unless: ps -ef | grep filecheck | grep -v grep # End Filecheck Section From 297373877abb3f7c5db64a61182bf148c89a9aca Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 31 Oct 2022 14:36:40 -0400 Subject: [PATCH 09/15] Update init.sls --- salt/zeek/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/zeek/init.sls b/salt/zeek/init.sls index ff91762f5..4640f0fea 100644 --- a/salt/zeek/init.sls +++ b/salt/zeek/init.sls @@ -76,6 +76,7 @@ zeekextractcompletedir: file.directory: - name: /nsm/zeek/extracted/complete - user: 937 + - group: 939 - makedirs: True # Sync the policies From 416c28fded8c7c482594dfa93f321990ee20e20f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 31 Oct 2022 14:42:23 -0400 Subject: [PATCH 10/15] Update init.sls --- salt/strelka/init.sls | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls index 3bc7ddf0e..dac07f717 100644 --- a/salt/strelka/init.sls +++ b/salt/strelka/init.sls @@ -119,11 +119,19 @@ strelkaportavailable: - name: netstat -utanp | grep ":57314" | grep -qvE 'docker|TIME_WAIT' && PROCESS=$(netstat -utanp | grep ":57314" | uniq) && echo "Another process ($PROCESS) appears to be using port 57314. Please terminate this process, or reboot to ensure a clean state so that Strelka can start properly." && exit 1 || exit 0 # Filecheck Section +filecheck_logdir: + file.directory: + - name: /opt/so/log/strelka + - user: 939 + - group: 939 + - makedirs: True + filecheck_history: file.directory: - name: /nsm/strelka/history - user: 939 - group: 939 + - makedirs: True filecheck_conf: file.managed: From 86ca3602f3e5b07ef5895b2c56f947ffe29a9253 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 31 Oct 2022 14:44:01 -0400 Subject: [PATCH 11/15] Update init.sls --- salt/strelka/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls index dac07f717..deb1c5096 100644 --- a/salt/strelka/init.sls +++ b/salt/strelka/init.sls @@ -151,7 +151,7 @@ filecheck_run: cmd.run: - name: 'python3 /opt/so/conf/strelka/filecheck' - bg: True - - runas: 939 + - runas: socore - unless: ps -ef | grep filecheck | grep -v grep # End Filecheck Section From bf41f2984ac00662240acd534e9aae99b8c24b7e Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 31 Oct 2022 14:58:55 -0400 Subject: [PATCH 12/15] Update init.sls --- salt/common/init.sls | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index d6c8e0103..2040edc97 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -38,15 +38,15 @@ socore: soconfperms: file.directory: - name: /opt/so/conf - - uid: 939 - - gid: 939 + - user: 939 + - group: 939 - dir_mode: 770 sostatusconf: file.directory: - name: /opt/so/conf/so-status - - uid: 939 - - gid: 939 + - user: 939 + - group: 939 - dir_mode: 770 so-status.conf: @@ -57,8 +57,8 @@ so-status.conf: sosaltstackperms: file.directory: - name: /opt/so/saltstack - - uid: 939 - - gid: 939 + - user: 939 + - group: 939 - dir_mode: 770 so_log_perms: From f7043f3f62fb11fe21baf369bdb27ba2533ac156 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 31 Oct 2022 15:25:38 -0400 Subject: [PATCH 13/15] Update init.sls --- salt/strelka/init.sls | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls index deb1c5096..ff4727126 100644 --- a/salt/strelka/init.sls +++ b/salt/strelka/init.sls @@ -154,6 +154,10 @@ filecheck_run: - runas: socore - unless: ps -ef | grep filecheck | grep -v grep +filcheck_history_clean: + cron.present: + - name: '/usr/bin/find /nsm/strelka/history/ -type f -mtime +2 -exec rm {} + > /dev/null 2>&1>' + - minute: '33' # End Filecheck Section strelka_coordinator: From 16d3dead04f782d1e7d9efd9e77134aaee637443 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 31 Oct 2022 15:33:10 -0400 Subject: [PATCH 14/15] Update sensor-rotate.conf --- salt/common/files/sensor-rotate.conf | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/salt/common/files/sensor-rotate.conf b/salt/common/files/sensor-rotate.conf index cefd3944e..797eb8a45 100644 --- a/salt/common/files/sensor-rotate.conf +++ b/salt/common/files/sensor-rotate.conf @@ -19,4 +19,17 @@ extension .log dateext dateyesterday -} \ No newline at end of file +} + +/opt/so/log/strelka/filecheck.log +{ + daily + rotate 14 + missingok + copytruncate + compress + create + extension .log + dateext + dateyesterday +} From 06ddae13b51b26b38441a62846d94e0467d2689e Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 31 Oct 2022 15:41:57 -0400 Subject: [PATCH 15/15] Update filecheck --- salt/strelka/filecheck/filecheck | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/salt/strelka/filecheck/filecheck b/salt/strelka/filecheck/filecheck index 8b1ddd446..816125fcb 100644 --- a/salt/strelka/filecheck/filecheck +++ b/salt/strelka/filecheck/filecheck @@ -1,3 +1,20 @@ +#!/usr/bin/env python3 + +# Copyright 2014-2022 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + import os import time import hashlib