Merge pull request #9071 from Security-Onion-Solutions/dev

Merge Dev into Foxtrot

salt/common/files/sensor-rotate.conf

@@ -19,4 +19,17 @@
     extension .log
     dateext
     dateyesterday
-}
+}
+
+/opt/so/log/strelka/filecheck.log
+{
+    daily
+    rotate 14
+    missingok
+    copytruncate
+    compress
+    create
+    extension .log
+    dateext
+    dateyesterday
+}

salt/common/init.sls

@@ -38,15 +38,15 @@ socore:
 soconfperms:
   file.directory:
     - name: /opt/so/conf
-    - uid: 939
-    - gid: 939
+    - user: 939
+    - group: 939
     - dir_mode: 770
 
 sostatusconf:
   file.directory:
     - name: /opt/so/conf/so-status
-    - uid: 939
-    - gid: 939
+    - user: 939
+    - group: 939
     - dir_mode: 770
 
 so-status.conf:
@@ -57,8 +57,8 @@ so-status.conf:
 sosaltstackperms:
   file.directory:
     - name: /opt/so/saltstack
-    - uid: 939
-    - gid: 939
+    - user: 939
+    - group: 939
     - dir_mode: 770
 
 so_log_perms:

salt/common/tools/sbin/so-pcap-export

@@ -20,7 +20,7 @@ if [ $# -lt 2 ]; then
   exit 1
 fi
 
-docker exec -it so-sensoroni scripts/stenoquery.sh "$1" -w /nsm/pcapout/$2.pcap
+docker exec -t so-sensoroni scripts/stenoquery.sh "$1" -w /nsm/pcapout/$2.pcap
 
 echo ""
 echo "If successful, the output was written to: /nsm/pcapout/$2.pcap"

salt/common/tools/sbin/soup

@@ -549,6 +549,7 @@ preupgrade_changes() {
     [[ "$INSTALLEDVERSION" == 2.3.160 ]] && up_to_2.3.170
     [[ "$INSTALLEDVERSION" == 2.3.170 ]] && up_to_2.3.180
     [[ "$INSTALLEDVERSION" == 2.3.180 ]] && up_to_2.3.181
+    [[ "$INSTALLEDVERSION" == 2.3.181 ]] && up_to_2.3.190
     true
 }
 
@@ -570,6 +571,7 @@ postupgrade_changes() {
     [[ "$POSTVERSION" == 2.3.160 ]] && post_to_2.3.170
     [[ "$POSTVERSION" == 2.3.170 ]] && post_to_2.3.180
     [[ "$POSTVERSION" == 2.3.180 ]] && post_to_2.3.181
+    [[ "$POSTVERSION" == 2.3.181 ]] && post_to_2.3.190
 
     true
 }
@@ -655,22 +657,32 @@ post_to_2.3.140() {
 
 post_to_2.3.150() {
   echo "Nothing to do for .150"
+  POSTVERSION=2.3.150
 }
 
 post_to_2.3.160() {
   echo "Nothing to do for .160"
+  POSTVERSION=2.3.160
 }
 
 post_to_2.3.170() {
   echo "Nothing to do for .170"
+  POSTVERSION=2.3.170
 }
 
 post_to_2.3.180() {
   echo "Nothing to do for .180"
+  POSTVERSION=2.3.180
 }
 
 post_to_2.3.181() {
   echo "Nothing to do for .181"
+  POSTVERSION=2.3.181
+}
+
+post_to_2.3.190() {
+  echo "Nothing to do for .190"
+  POSTVERSION=2.3.190
 }
 
 stop_salt_master() {
@@ -972,6 +984,13 @@ up_to_2.3.181() {
   INSTALLEDVERSION=2.3.181
 }
 
+up_to_2.3.190() {
+  echo "Upgrading to 2.3.190"
+  chown -R zeek:socore /nsm/zeek/extracted/complete
+  chmod 770 /nsm/zeek/extracted/complete 
+  INSTALLEDVERSION=2.3.190
+}
+
 verify_upgradespace() {
   CURRENTSPACE=$(df -BG / | grep -v Avail | awk '{print $4}' | sed 's/.$//')
   if [ "$CURRENTSPACE" -lt "10" ]; then

salt/sensoroni/files/analyzers/README.md

@@ -5,20 +5,19 @@ Security Onion provides a means for performing data analysis on varying inputs.
 ## Supported Observable Types
 The built-in analyzers support the following observable types:
 
-| Name                    | Domain | Hash  | IP    | JA3   | Mail  | Other | URI   |  URL  | User Agent |
-| ------------------------|--------|-------|-------|-------|-------|-------|-------|-------|------------
-| Alienvault OTX          |✓ |✓|✓|✗|✗|✗|✗|✓|✗|
-| EmailRep                |✗ |✗|✗|✗|✓|✗|✗|✗|✗|
-| Greynoise               |✗ |✗|✓|✗|✗|✗|✗|✗|✗|
-| JA3er                   |✗ |✗|✗|✓|✗|✗|✗|✗|✗|
-| LocalFile               |✓ |✓|✓|✓|✗|✓|✗|✓|✗|
-| Malware Hash Registry   |✗ |✓|✗|✗|✗|✗|✗|✓|✗|
-| Pulsedive               |✓ |✓|✓|✗|✗|✗|✓|✓|✓|
-| Spamhaus                |✗ |✗|✓|✗|✗|✗|✗|✗|✗|
-| Urlhaus                 |✗ |✗|✗|✗|✗|✗|✗|✓|✗|
-| Urlscan                 |✗ |✗|✗|✗|✗|✗|✗|✓|✗|
-| Virustotal              |✓ |✓|✓|✗|✗|✗|✗|✓|✗|
-| WhoisLookup             |✓ |✗|✗|✗|✗|✗|✓|✗|✗|
+| Name                    | Domain | Hash  | IP    | Mail  | Other | URI   |  URL  | User Agent |
+| ------------------------|--------|-------|-------|-------|-------|-------|-------|-------|
+| Alienvault OTX          |✓ |✓|✓|✗|✗|✗|✓|✗|
+| EmailRep                |✗ |✗|✗|✓|✗|✗|✗|✗|
+| Greynoise               |✗ |✗|✓|✗|✗|✗|✗|✗|
+| LocalFile               |✓ |✓|✓|✗|✓|✗|✓|✗|
+| Malware Hash Registry   |✗ |✓|✗|✗|✗|✗|✓|✗|
+| Pulsedive               |✓ |✓|✓|✗|✗|✓|✓|✓|
+| Spamhaus                |✗ |✗|✓|✗|✗|✗|✗|✗|
+| Urlhaus                 |✗ |✗|✗|✗|✗|✗|✓|✗|
+| Urlscan                 |✗ |✗|✗|✗|✗|✗|✓|✗|
+| Virustotal              |✓ |✓|✓|✗|✗|✗|✓|✗|
+| WhoisLookup             |✓ |✗|✗|✗|✗|✓|✗|✗|
 
 ## Authentication
 Many analyzers require authentication, via an API key or similar. The table below illustrates which analyzers require authentication.
@@ -28,7 +27,6 @@ Many analyzers require authentication, via an API key or similar. The table belo
 [AlienVault OTX](https://otx.alienvault.com/api)            |✓|
 [EmailRep](https://emailrep.io/key)                  |✓|
 [GreyNoise](https://www.greynoise.io/plans/community)                 |✓|
-[JA3er](https://ja3er.com/)                     |✗|
 LocalFile                 |✗|
 [Malware Hash Registry](https://hash.cymru.com/docs_whois)    |✗|
 [Pulsedive](https://pulsedive.com/api/)                 |✓|

salt/sensoroni/files/analyzers/ja3er/ja3er.json

@@ -1,7 +0,0 @@
-{
-  "name": "JA3er Hash Search",
-  "version": "0.1",
-  "author": "Security Onion Solutions",
-  "description": "This analyzer queries JA3er user agents and sightings",
-  "supportedTypes" :  ["ja3"]
-}

salt/sensoroni/files/analyzers/ja3er/ja3er.py

@@ -1,53 +0,0 @@
-import json
-import os
-import requests
-import helpers
-import argparse
-
-
-def sendReq(conf, meta, hash):
-    url = conf['base_url'] + hash
-    response = requests.request('GET', url)
-    return response.json()
-
-
-def prepareResults(raw):
-    if "error" in raw:
-        if "Sorry" in raw["error"]:
-            status = "ok"
-            summary = "no_results"
-        elif "Invalid hash" in raw["error"]:
-            status = "caution"
-            summary = "invalid_input"
-        else:
-            status = "caution"
-            summary = "internal_failure"
-    else:
-        status = "info"
-        summary = "suspicious"
-    results = {'response': raw, 'summary': summary, 'status': status}
-    return results
-
-
-def analyze(conf, input):
-    meta = helpers.loadMetadata(__file__)
-    data = helpers.parseArtifact(input)
-    helpers.checkSupportedType(meta, data["artifactType"])
-    response = sendReq(conf, meta, data["value"])
-    return prepareResults(response)
-
-
-def main():
-    dir = os.path.dirname(os.path.realpath(__file__))
-    parser = argparse.ArgumentParser(description='Search JA3er for a given artifact')
-    parser.add_argument('artifact', help='the artifact represented in JSON format')
-    parser.add_argument('-c', '--config', metavar="CONFIG_FILE", default=dir + "/ja3er.yaml", help='optional config file to use instead of the default config file')
-
-    args = parser.parse_args()
-    if args.artifact:
-        results = analyze(helpers.loadConfig(args.config), args.artifact)
-        print(json.dumps(results))
-
-
-if __name__ == "__main__":
-    main()

salt/sensoroni/files/analyzers/ja3er/ja3er.yaml

@@ -1 +0,0 @@
-base_url: https://ja3er.com/search/

salt/sensoroni/files/analyzers/ja3er/ja3er_test.py

@@ -1,72 +0,0 @@
-from io import StringIO
-import sys
-from unittest.mock import patch, MagicMock
-from ja3er import ja3er
-import unittest
-
-
-class TestJa3erMethods(unittest.TestCase):
-
-    def test_main_missing_input(self):
-        with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stderr', new=StringIO()) as mock_stderr:
-                sys.argv = ["cmd"]
-                ja3er.main()
-                self.assertEqual(mock_stderr.getvalue(), "usage: cmd [-h] [-c CONFIG_FILE] artifact\ncmd: error: the following arguments are required: artifact\n")
-                sysmock.assert_called_once_with(2)
-
-    def test_main_success(self):
-        output = {"foo": "bar"}
-        with patch('sys.stdout', new=StringIO()) as mock_stdout:
-            with patch('ja3er.ja3er.analyze', new=MagicMock(return_value=output)) as mock:
-                sys.argv = ["cmd", "input"]
-                ja3er.main()
-                expected = '{"foo": "bar"}\n'
-                self.assertEqual(mock_stdout.getvalue(), expected)
-                mock.assert_called_once()
-
-    def test_sendReq(self):
-        with patch('requests.request', new=MagicMock(return_value=MagicMock())) as mock:
-            meta = {}
-            conf = {"base_url": "myurl/"}
-            hash = "abcd1234"
-            response = ja3er.sendReq(conf=conf, meta=meta, hash=hash)
-            mock.assert_called_once_with("GET", "myurl/abcd1234")
-            self.assertIsNotNone(response)
-
-    def test_prepareResults_none(self):
-        raw = {"error": "Sorry no values found"}
-        results = ja3er.prepareResults(raw)
-        self.assertEqual(results["response"], raw)
-        self.assertEqual(results["summary"], "no_results")
-        self.assertEqual(results["status"], "ok")
-
-    def test_prepareResults_invalidHash(self):
-        raw = {"error": "Invalid hash"}
-        results = ja3er.prepareResults(raw)
-        self.assertEqual(results["response"], raw)
-        self.assertEqual(results["summary"], "invalid_input")
-        self.assertEqual(results["status"], "caution")
-
-    def test_prepareResults_internal_failure(self):
-        raw = {"error": "unknown"}
-        results = ja3er.prepareResults(raw)
-        self.assertEqual(results["response"], raw)
-        self.assertEqual(results["summary"], "internal_failure")
-        self.assertEqual(results["status"], "caution")
-
-    def test_prepareResults_info(self):
-        raw = [{"User-Agent": "Blah/5.0", "Count": 24874, "Last_seen": "2022-04-08 16:18:38"}, {"Comment": "Brave browser v1.36.122\n\n", "Reported": "2022-03-28 20:26:42"}]
-        results = ja3er.prepareResults(raw)
-        self.assertEqual(results["response"], raw)
-        self.assertEqual(results["summary"], "suspicious")
-        self.assertEqual(results["status"], "info")
-
-    def test_analyze(self):
-        output = {"info": "Results found."}
-        artifactInput = '{"value":"abcd1234","artifactType":"ja3"}'
-        conf = {"base_url": "myurl/"}
-        with patch('ja3er.ja3er.sendReq', new=MagicMock(return_value=output)) as mock:
-            results = ja3er.analyze(conf, artifactInput)
-            self.assertEqual(results["summary"], "suspicious")
-            mock.assert_called_once()

salt/sensoroni/files/analyzers/ja3er/requirements.txt

@@ -1,2 +0,0 @@
-requests>=2.27.1
-pyyaml>=6.0

salt/strelka/filecheck/filecheck

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+
+# Copyright 2014-2022 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import time
+import hashlib
+import logging
+import yaml
+from watchdog.observers import Observer
+from watchdog.events import FileSystemEventHandler
+
+with open("/opt/so/conf/strelka/filecheck.yaml", "r") as ymlfile:
+    cfg = yaml.load(ymlfile)
+
+extract_path = cfg["filecheck"]["extract_path"]
+historypath = cfg["filecheck"]["historypath"]
+strelkapath = cfg["filecheck"]["strelkapath"]
+logfile = cfg["filecheck"]["logfile"]
+
+logging.basicConfig(filename=logfile, filemode='w', format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S', level=logging.INFO)
+
+def checkexisting():
+    for file in os.listdir(extract_path):
+        filename = os.path.join(extract_path, file)
+        logging.info("Processing existing file " + filename)
+        checksum(filename) 
+
+def checksum(filename):
+    with open(filename, 'rb') as afile:
+        shawnuff = hashlib.sha1()
+        buf = afile.read(8192)
+        while len(buf) > 0:
+            shawnuff.update(buf)
+            buf = afile.read(8192)
+        hizash=shawnuff.hexdigest()
+        process(filename, hizash)
+
+def process(filename, hizash):
+    if os.path.exists(historypath + hizash):
+        logging.info(filename + " Already exists.. removing")
+        os.remove(filename)
+    else:
+        # Write the file
+        logging.info(filename + " is new. Creating a record and sending to Strelka")
+        with open(os.path.join(historypath + hizash), 'w') as fp:
+            pass
+        head, tail = os.path.split(filename)
+
+        # Move the file
+        os.rename(filename, strelkapath + tail)
+
+class CreatedEventHandler(FileSystemEventHandler):
+    def on_created(self, event):
+        filename = event.src_path
+        logging.info("Found new file")
+        checksum(filename)
+
+if __name__ == "__main__":
+
+    checkexisting()
+    event_handler =CreatedEventHandler()
+
+    observer = Observer()
+    observer.schedule(event_handler, extract_path, recursive=True)
+    observer.start()
+    try:
+        while True:
+            time.sleep(1)
+    except KeyboardInterrupt:
+        observer.stop()
+    observer.join()

salt/strelka/filecheck/filecheck.yaml

@@ -0,0 +1,11 @@
+{%- set ENGINE = salt['pillar.get']('global:mdengine', '') %}
+filecheck:
+    {%- if ENGINE == "SURICATA" %}
+    extract_path: '/nsm/suricata/extracted'
+    {%- else %}
+    extract_path: '/nsm/zeek/extracted/complete'
+    {%- endif %}
+    historypath: '/nsm/strelka/history/'
+    strelkapath: '/nsm/strelka/unprocessed/'
+    logfile: '/opt/so/log/strelka/filecheck.log'
+

salt/strelka/init.sls

@@ -24,6 +24,20 @@
 {% import_yaml 'strelka/defaults.yaml' as strelka_config with context %}
 {% set IGNORELIST = salt['pillar.get']('strelka:ignore', strelka_config.strelka.ignore, merge=True, merge_nested_lists=True) %}
 
+{% if grains['os'] != 'CentOS' %}     
+strelkapkgs:
+  pkg.installed:
+    - skip_suggestions: True
+    - pkgs:
+      - python3-watchdog
+{% else %}
+strelkapkgs:
+  pkg.installed:
+    - skip_suggestions: True
+    - pkgs:
+      - securityonion-python36-watchdog
+{% endif %}
+    
 # Strelka config
 strelkaconfdir:
   file.directory:
@@ -79,7 +93,7 @@ strelkarepos:
 {% endif %}
 
 strelkadatadir:
-   file.directory:
+  file.directory:
     - name: /nsm/strelka
     - user: 939
     - group: 939
@@ -93,21 +107,21 @@ strelkalogdir:
     - makedirs: True
 
 strelkaprocessed:
-   file.directory:
+  file.directory:
     - name: /nsm/strelka/processed
     - user: 939
     - group: 939
     - makedirs: True
 
 strelkastaging:
-   file.directory:
+  file.directory:
     - name: /nsm/strelka/staging
     - user: 939
     - group: 939
     - makedirs: True
 
 strelkaunprocessed:
-   file.directory:
+  file.directory:
     - name: /nsm/strelka/unprocessed
     - user: 939
     - group: 939
@@ -115,8 +129,50 @@ strelkaunprocessed:
 
 # Check to see if Strelka frontend port is available
 strelkaportavailable:
-    cmd.run:
-      - name: netstat -utanp | grep ":57314" | grep -qvE 'docker|TIME_WAIT' && PROCESS=$(netstat -utanp | grep ":57314" | uniq) && echo "Another process ($PROCESS) appears to be using port 57314.  Please terminate this process, or reboot to ensure a clean state so that Strelka can start properly." && exit 1 || exit 0
+  cmd.run:
+    - name: netstat -utanp | grep ":57314" | grep -qvE 'docker|TIME_WAIT' && PROCESS=$(netstat -utanp | grep ":57314" | uniq) && echo "Another process ($PROCESS) appears to be using port 57314.  Please terminate this process, or reboot to ensure a clean state so that Strelka can start properly." && exit 1 || exit 0
+
+# Filecheck Section
+filecheck_logdir:
+  file.directory:
+    - name: /opt/so/log/strelka
+    - user: 939
+    - group: 939
+    - makedirs: True
+    
+filecheck_history:
+  file.directory:
+    - name: /nsm/strelka/history
+    - user: 939
+    - group: 939
+    - makedirs: True
+
+filecheck_conf:
+  file.managed:
+    - name: /opt/so/conf/strelka/filecheck.yaml
+    - source: salt://strelka/filecheck/filecheck.yaml
+    - template: jinja
+
+filecheck_script:
+  file.managed:
+    - name: /opt/so/conf/strelka/filecheck
+    - source: salt://strelka/filecheck/filecheck
+    - user: 939
+    - group: 939
+    - mode: 755
+
+filecheck_run:
+  cmd.run:
+    - name: 'python3 /opt/so/conf/strelka/filecheck'
+    - bg: True
+    - runas: socore
+    - unless: ps -ef | grep filecheck | grep -v grep
+
+filcheck_history_clean:
+  cron.present:
+    - name: '/usr/bin/find /nsm/strelka/history/ -type f -mtime +2 -exec rm {} + > /dev/null 2>&1>'
+    - minute: '33'
+# End Filecheck Section
 
 strelka_coordinator:
   docker_container.running:
@@ -212,7 +268,7 @@ strelka_zeek_extracted_sync_old:
 {% if ENGINE == "SURICATA" %}
 
 strelka_suricata_extracted_sync:
-  cron.present:
+  cron.absent:
     - user: root
     - identifier: zeek-extracted-strelka-sync
     - name: '[ -d /nsm/suricata/extracted/ ] && find /nsm/suricata/extracted/* -not \( -path /nsm/suricata/extracted/tmp -prune \) -type f -print0 | xargs -0 -I {} mv {} /nsm/strelka/unprocessed/ > /dev/null 2>&1'
@@ -220,7 +276,7 @@ strelka_suricata_extracted_sync:
 
 {% else %}
 strelka_zeek_extracted_sync:
-  cron.present:
+  cron.absent:
     - user: root
     - identifier: zeek-extracted-strelka-sync
     - name: '[ -d /nsm/zeek/extracted/complete/ ] && mv /nsm/zeek/extracted/complete/* /nsm/strelka/unprocessed/ > /dev/null 2>&1'

salt/suricata/init.sls

@@ -68,6 +68,7 @@ suridatadir:
     - name: /nsm/suricata/extracted
     - user: 940
     - group: 939
+    - mode: 770
     - makedirs: True
 
 surirulesync:

salt/zeek/init.sls

@@ -70,12 +70,15 @@ zeekextractdir:
     - name: /nsm/zeek/extracted
     - user: 937
     - group: 939
+    - mode: 770
     - makedirs: True
 
 zeekextractcompletedir:
   file.directory:
     - name: /nsm/zeek/extracted/complete
     - user: 937
+    - group: 939
+    - mode: 770
     - makedirs: True
 
 # Sync the policies