From 4187363451cbd7be32dc6a2c6d49972e7721f876 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 2 Nov 2022 09:44:08 -0400 Subject: [PATCH 01/13] Add Strelka Filecheck --- salt/strelka/filecheck/filecheck | 75 +++++++++++++++++++++++++++ salt/strelka/filecheck/filecheck.yaml | 10 ++++ 2 files changed, 85 insertions(+) create mode 100644 salt/strelka/filecheck/filecheck create mode 100644 salt/strelka/filecheck/filecheck.yaml diff --git a/salt/strelka/filecheck/filecheck b/salt/strelka/filecheck/filecheck new file mode 100644 index 000000000..3d498ce62 --- /dev/null +++ b/salt/strelka/filecheck/filecheck @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +import os +import time +import hashlib +import logging +import yaml +from watchdog.observers import Observer +from watchdog.events import FileSystemEventHandler + +with open("/opt/so/conf/strelka/filecheck.yaml", "r") as ymlfile: + cfg = yaml.load(ymlfile) + +extract_path = cfg["filecheck"]["extract_path"] +historypath = cfg["filecheck"]["historypath"] +strelkapath = cfg["filecheck"]["strelkapath"] +logfile = cfg["filecheck"]["logfile"] + +logging.basicConfig(filename=logfile, filemode='w', format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S', level=logging.INFO) + +def checkexisting(): + for file in os.listdir(extract_path): + filename = os.path.join(extract_path, file) + logging.info("Processing existing file " + filename) + checksum(filename) + +def checksum(filename): + with open(filename, 'rb') as afile: + shawnuff = hashlib.sha1() + buf = afile.read(8192) + while len(buf) > 0: + shawnuff.update(buf) + buf = afile.read(8192) + hizash=shawnuff.hexdigest() + process(filename, hizash) + +def process(filename, hizash): + if os.path.exists(historypath + hizash): + logging.info(filename + " Already exists.. removing") + os.remove(filename) + else: + # Write the file + logging.info(filename + " is new. Creating a record and sending to Strelka") + with open(os.path.join(historypath + hizash), 'w') as fp: + pass + head, tail = os.path.split(filename) + + # Move the file + os.rename(filename, strelkapath + tail) + +class CreatedEventHandler(FileSystemEventHandler): + def on_created(self, event): + filename = event.src_path + logging.info("Found new file") + checksum(filename) + +if __name__ == "__main__": + + checkexisting() + event_handler =CreatedEventHandler() + + observer = Observer() + observer.schedule(event_handler, extract_path, recursive=True) + observer.start() + try: + while True: + time.sleep(1) + except KeyboardInterrupt: + observer.stop() + observer.join() \ No newline at end of file diff --git a/salt/strelka/filecheck/filecheck.yaml b/salt/strelka/filecheck/filecheck.yaml new file mode 100644 index 000000000..1c156fc3d --- /dev/null +++ b/salt/strelka/filecheck/filecheck.yaml @@ -0,0 +1,10 @@ +{%- set ENGINE = salt['pillar.get']('global:mdengine', '') %} +filecheck: + {%- if ENGINE == "SURICATA" %} + extract_path: '/nsm/suricata/extracted' + {%- else %} + extract_path: '/nsm/zeek/extracted/complete' + {%- endif %} + historypath: '/nsm/strelka/history/' + strelkapath: '/nsm/strelka/unprocessed/' + logfile: '/opt/so/log/strelka/filecheck.log' From 225c33e5c93c41c7fd456c093ec4bf663fe0a4b2 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 2 Nov 2022 09:46:23 -0400 Subject: [PATCH 02/13] Add Strelka Filecheck --- salt/strelka/init.sls | 67 +++++++++++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 24 deletions(-) diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls index 8fae4be57..e3477dd9e 100644 --- a/salt/strelka/init.sls +++ b/salt/strelka/init.sls @@ -105,6 +105,49 @@ strelkaportavailable: cmd.run: - name: netstat -utanp | grep ":57314" | grep -qvE 'docker|TIME_WAIT' && PROCESS=$(netstat -utanp | grep ":57314" | uniq) && echo "Another process ($PROCESS) appears to be using port 57314. Please terminate this process, or reboot to ensure a clean state so that Strelka can start properly." && exit 1 || exit 0 +# Filecheck Section +filecheck_logdir: + file.directory: + - name: /opt/so/log/strelka + - user: 939 + - group: 939 + - makedirs: True + +filecheck_history: + file.directory: + - name: /nsm/strelka/history + - user: 939 + - group: 939 + - makedirs: True + +filecheck_conf: + file.managed: + - name: /opt/so/conf/strelka/filecheck.yaml + - source: salt://strelka/filecheck/filecheck.yaml + - template: jinja + +filecheck_script: + file.managed: + - name: /opt/so/conf/strelka/filecheck + - source: salt://strelka/filecheck/filecheck + - user: 939 + - group: 939 + - mode: 755 + +filecheck_run: + cmd.run: + - name: 'python3 /opt/so/conf/strelka/filecheck' + - bg: True + - runas: socore + - unless: ps -ef | grep filecheck | grep -v grep + +filcheck_history_clean: + cron.present: + - name: '/usr/bin/find /nsm/strelka/history/ -type f -mtime +2 -exec rm {} + > /dev/null 2>&1>' + - minute: '33' +# End Filecheck Section + + strelka_coordinator: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-redis:{{ GLOBALS.so_version }} @@ -190,30 +233,6 @@ append_so-strelka-filestream_so-status.conf: - name: /opt/so/conf/so-status/so-status.conf - text: so-strelka-filestream -strelka_zeek_extracted_sync_old: - cron.absent: - - user: root - - name: '[ -d /nsm/zeek/extracted/complete/ ] && mv /nsm/zeek/extracted/complete/* /nsm/strelka/ > /dev/null 2>&1' - - minute: '*' - -{% if GLOBALS.md_engine == "SURICATA" %} - -strelka_suricata_extracted_sync: - cron.present: - - user: root - - identifier: zeek-extracted-strelka-sync - - name: '[ -d /nsm/suricata/extracted/ ] && find /nsm/suricata/extracted/* -not \( -path /nsm/suricata/extracted/tmp -prune \) -type f -print0 | xargs -0 -I {} mv {} /nsm/strelka/unprocessed/ > /dev/null 2>&1' - - minute: '*' - -{% else %} -strelka_zeek_extracted_sync: - cron.present: - - user: root - - identifier: zeek-extracted-strelka-sync - - name: '[ -d /nsm/zeek/extracted/complete/ ] && mv /nsm/zeek/extracted/complete/* /nsm/strelka/unprocessed/ > /dev/null 2>&1' - - minute: '*' - -{% endif %} {% else %} {{sls}}_state_not_allowed: From bf5df1ac51a5bcd19e41a7f1ad5fde87b82275c5 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 2 Nov 2022 09:57:07 -0400 Subject: [PATCH 03/13] Add Strelka Filecheck --- salt/common/init.sls | 12 ++++++------ salt/zeek/init.sls | 3 +++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 5bb1051b6..a03eaacde 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -33,15 +33,15 @@ socore: soconfperms: file.directory: - name: /opt/so/conf - - uid: 939 - - gid: 939 + - user: 939 + - group: 939 - dir_mode: 770 sostatusconf: file.directory: - name: /opt/so/conf/so-status - - uid: 939 - - gid: 939 + - user: 939 + - group: 939 - dir_mode: 770 so-status.conf: @@ -52,8 +52,8 @@ so-status.conf: sosaltstackperms: file.directory: - name: /opt/so/saltstack - - uid: 939 - - gid: 939 + - user: 939 + - group: 939 - dir_mode: 770 so_log_perms: diff --git a/salt/zeek/init.sls b/salt/zeek/init.sls index b1130ee43..5cc6310b9 100644 --- a/salt/zeek/init.sls +++ b/salt/zeek/init.sls @@ -56,12 +56,15 @@ zeekextractdir: - name: /nsm/zeek/extracted - user: 937 - group: 939 + - mode: 770 - makedirs: True zeekextractcompletedir: file.directory: - name: /nsm/zeek/extracted/complete - user: 937 + - group: 939 + - mode: 770 - makedirs: True # Sync the policies From d97de9fd0d3c953eea71812caea7ff09f3776dac Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 2 Nov 2022 10:02:21 -0400 Subject: [PATCH 04/13] Add Strelka Filecheck --- salt/common/packages.sls | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/common/packages.sls b/salt/common/packages.sls index d3e158cc2..e52e8174f 100644 --- a/salt/common/packages.sls +++ b/salt/common/packages.sls @@ -18,6 +18,7 @@ commonpkgs: - createrepo - python3-lxml - python3-packaging + - python3-watchdog - yum-utils - device-mapper-persistent-data - lvm2 @@ -50,6 +51,7 @@ commonpkgs: - python36-mysql - python36-packaging - python36-lxml + - securityonion-python36-watchdog - yum-utils - device-mapper-persistent-data - lvm2 From de19a4dc536086b51d43a65dfb603613bff35193 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 2 Nov 2022 10:04:33 -0400 Subject: [PATCH 05/13] Add Strelka Filecheck --- salt/suricata/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/suricata/init.sls b/salt/suricata/init.sls index 5af9f591d..e4992890b 100644 --- a/salt/suricata/init.sls +++ b/salt/suricata/init.sls @@ -57,6 +57,7 @@ suridatadir: - name: /nsm/suricata/extracted - user: 940 - group: 939 + - mode: 770 - makedirs: True surirulesync: From 1a678064dc3c805862321311b1494d0314eecb8f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Nov 2022 13:42:24 -0500 Subject: [PATCH 06/13] upgrade to salt 3005.1 --- salt/salt/master.defaults.yaml | 2 +- salt/salt/minion.defaults.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/salt/master.defaults.yaml b/salt/salt/master.defaults.yaml index 3e3510c8c..aff6ad54d 100644 --- a/salt/salt/master.defaults.yaml +++ b/salt/salt/master.defaults.yaml @@ -2,4 +2,4 @@ # When updating the salt version, also update the version in securityonion-builds/images/iso-task/Dockerfile and saltify function in so-functions salt: master: - version: 3004.2 + version: 3005.1 diff --git a/salt/salt/minion.defaults.yaml b/salt/salt/minion.defaults.yaml index e4ffe5fcb..4e4e2c319 100644 --- a/salt/salt/minion.defaults.yaml +++ b/salt/salt/minion.defaults.yaml @@ -2,6 +2,6 @@ # When updating the salt version, also update the version in securityonion-builds/images/iso-task/Dockerfile and saltify function in so-functions salt: minion: - version: 3004.2 + version: 3005.1 check_threshold: 3600 # in seconds, threshold used for so-salt-minion-check. any value less than 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default service_start_delay: 30 # in seconds. From 00cb0f5abb7a7075da3b41a9ed5695186b9571ce Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Nov 2022 15:45:18 -0500 Subject: [PATCH 07/13] roll back to salt 3004.2 --- salt/salt/master.defaults.yaml | 2 +- salt/salt/minion.defaults.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/salt/master.defaults.yaml b/salt/salt/master.defaults.yaml index aff6ad54d..3e3510c8c 100644 --- a/salt/salt/master.defaults.yaml +++ b/salt/salt/master.defaults.yaml @@ -2,4 +2,4 @@ # When updating the salt version, also update the version in securityonion-builds/images/iso-task/Dockerfile and saltify function in so-functions salt: master: - version: 3005.1 + version: 3004.2 diff --git a/salt/salt/minion.defaults.yaml b/salt/salt/minion.defaults.yaml index 4e4e2c319..e4ffe5fcb 100644 --- a/salt/salt/minion.defaults.yaml +++ b/salt/salt/minion.defaults.yaml @@ -2,6 +2,6 @@ # When updating the salt version, also update the version in securityonion-builds/images/iso-task/Dockerfile and saltify function in so-functions salt: minion: - version: 3005.1 + version: 3004.2 check_threshold: 3600 # in seconds, threshold used for so-salt-minion-check. any value less than 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default service_start_delay: 30 # in seconds. From 40f5bb25eff83eee05a8638a392d53900d8c8d9e Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 11 Nov 2022 16:28:23 -0500 Subject: [PATCH 08/13] FIX: Avoid deprecation warning in Zeek file extraction script #9123 --- salt/zeek/policy/securityonion/file-extraction/extract.zeek | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/zeek/policy/securityonion/file-extraction/extract.zeek b/salt/zeek/policy/securityonion/file-extraction/extract.zeek index d4ba0551e..36c211938 100644 --- a/salt/zeek/policy/securityonion/file-extraction/extract.zeek +++ b/salt/zeek/policy/securityonion/file-extraction/extract.zeek @@ -45,7 +45,7 @@ event file_state_remove(f: fa_file) # Delete the file if it didn't pass our requirements check. local nuke = fmt("rm %s/%s", FileExtract::prefix, f$info$extracted); - when ( local nukeit = Exec::run([$cmd=nuke]) ) + when [nuke] ( local nukeit = Exec::run([$cmd=nuke]) ) { } return; @@ -56,7 +56,7 @@ event file_state_remove(f: fa_file) local dest = fmt("%scomplete/%s-%s-%s.%s", FileExtract::prefix, f$source, f$id, f$info$md5, extension); # Copy it to the $prefix/complete folder then delete it. I got some weird results with moving when it came to watchdog in python. local cmd = fmt("cp %s/%s %s && rm %s/%s", FileExtract::prefix, orig, dest, FileExtract::prefix, orig); - when ( local result = Exec::run([$cmd=cmd]) ) + when [cmd] ( local result = Exec::run([$cmd=cmd]) ) { } f$info$extracted = dest; From a15ca3cc491f9e60f01678c9857b1960a44d15d2 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Sat, 12 Nov 2022 13:11:38 -0500 Subject: [PATCH 09/13] fix descriptions in files related to analyzers --- salt/sensoroni/files/analyzers/emailrep/emailrep.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/sensoroni/files/analyzers/emailrep/emailrep.py b/salt/sensoroni/files/analyzers/emailrep/emailrep.py index d48977a07..0897c541a 100755 --- a/salt/sensoroni/files/analyzers/emailrep/emailrep.py +++ b/salt/sensoroni/files/analyzers/emailrep/emailrep.py @@ -53,7 +53,7 @@ def analyze(conf, input): def main(): dir = os.path.dirname(os.path.realpath(__file__)) - parser = argparse.ArgumentParser(description='Search Greynoise for a given artifact') + parser = argparse.ArgumentParser(description='Search EmailRep for a given artifact') parser.add_argument('artifact', help='the artifact represented in JSON format') parser.add_argument('-c', '--config', metavar="CONFIG_FILE", default=dir + "/emailrep.yaml", help='optional config file to use instead of the default config file') From 154dff98dea59d062bab2d4463aa1020b67a50ea Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Sat, 12 Nov 2022 13:12:23 -0500 Subject: [PATCH 10/13] fix descriptions in files related to analyzers --- salt/sensoroni/files/analyzers/pulsedive/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/sensoroni/files/analyzers/pulsedive/README.md b/salt/sensoroni/files/analyzers/pulsedive/README.md index d3879fb8d..7550457a8 100644 --- a/salt/sensoroni/files/analyzers/pulsedive/README.md +++ b/salt/sensoroni/files/analyzers/pulsedive/README.md @@ -5,7 +5,7 @@ Search Pulsedive for a domain, hash, IP, URI, URL, or User Agent. ## Configuration Requirements -``api_key`` - API key used for communication with the Virustotal API +``api_key`` - API key used for communication with the Pulsedive API This value should be set in the ``sensoroni`` pillar, like so: From 2f4ce916780e5fb9a256e114cadc4f0dbac310f8 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Sat, 12 Nov 2022 13:12:58 -0500 Subject: [PATCH 11/13] fix descriptions in files related to analyzers --- salt/sensoroni/files/analyzers/pulsedive/pulsedive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/sensoroni/files/analyzers/pulsedive/pulsedive.py b/salt/sensoroni/files/analyzers/pulsedive/pulsedive.py index fd9e0072f..68e08bfa2 100644 --- a/salt/sensoroni/files/analyzers/pulsedive/pulsedive.py +++ b/salt/sensoroni/files/analyzers/pulsedive/pulsedive.py @@ -91,7 +91,7 @@ def analyze(conf, input): def main(): dir = os.path.dirname(os.path.realpath(__file__)) - parser = argparse.ArgumentParser(description='Search VirusTotal for a given artifact') + parser = argparse.ArgumentParser(description='Search Pulsedive for a given artifact') parser.add_argument('artifact', help='the artifact represented in JSON format') parser.add_argument('-c', '--config', metavar="CONFIG_FILE", default=dir + "/pulsedive.yaml", help='optional config file to use instead of the default config file') From f77db78219eca2f88d6256ab25b74631a01fd275 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Sat, 12 Nov 2022 13:13:30 -0500 Subject: [PATCH 12/13] fix descriptions in files related to analyzers --- salt/sensoroni/files/analyzers/urlscan/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/sensoroni/files/analyzers/urlscan/README.md b/salt/sensoroni/files/analyzers/urlscan/README.md index 9f33c3106..cab1e7aa6 100644 --- a/salt/sensoroni/files/analyzers/urlscan/README.md +++ b/salt/sensoroni/files/analyzers/urlscan/README.md @@ -5,7 +5,7 @@ Submit a URL to Urlscan for analysis. ## Configuration Requirements -``api_key`` - API key used for communication with the Virustotal API +``api_key`` - API key used for communication with the urlscan API ``enabled`` - Determines whether or not the analyzer is enabled. Defaults to ``False`` ``visibility`` - Determines whether or not scan results are visibile publicly. Defaults to ``public`` ``timeout`` - Time to wait for scan results. Defaults to ``180``s From 632464335f6e160161d76878389cf5ce6d6910bf Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Sat, 12 Nov 2022 13:14:02 -0500 Subject: [PATCH 13/13] fix descriptions in files related to analyzers --- salt/sensoroni/files/analyzers/urlscan/urlscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/sensoroni/files/analyzers/urlscan/urlscan.py b/salt/sensoroni/files/analyzers/urlscan/urlscan.py index a07e61c89..1f226da53 100755 --- a/salt/sensoroni/files/analyzers/urlscan/urlscan.py +++ b/salt/sensoroni/files/analyzers/urlscan/urlscan.py @@ -77,7 +77,7 @@ def analyze(conf, input): def main(): dir = os.path.dirname(os.path.realpath(__file__)) - parser = argparse.ArgumentParser(description='Search Alienvault OTX for a given artifact') + parser = argparse.ArgumentParser(description='Search urlscan for a given artifact') parser.add_argument('artifact', help='the artifact represented in JSON format') parser.add_argument('-c', '--config', metavar="CONFIG_FILE", default=dir + "/urlscan.yaml", help='optional config file to use instead of the default config file')