Merge pull request #9071 from Security-Onion-Solutions/dev

Merge Dev into Foxtrot

VERSION

@@ -1 +1 @@
-2.3.181
+2.3.190

salt/common/files/sensor-rotate.conf

@@ -19,4 +19,17 @@
     extension .log
     dateext
     dateyesterday
-}
+}
+
+/opt/so/log/strelka/filecheck.log
+{
+    daily
+    rotate 14
+    missingok
+    copytruncate
+    compress
+    create
+    extension .log
+    dateext
+    dateyesterday
+}

salt/common/init.sls

@@ -38,15 +38,15 @@ socore:
 soconfperms:
   file.directory:
     - name: /opt/so/conf
-    - uid: 939
-    - gid: 939
+    - user: 939
+    - group: 939
     - dir_mode: 770
 
 sostatusconf:
   file.directory:
     - name: /opt/so/conf/so-status
-    - uid: 939
-    - gid: 939
+    - user: 939
+    - group: 939
     - dir_mode: 770
 
 so-status.conf:
@@ -57,8 +57,8 @@ so-status.conf:
 sosaltstackperms:
   file.directory:
     - name: /opt/so/saltstack
-    - uid: 939
-    - gid: 939
+    - user: 939
+    - group: 939
     - dir_mode: 770
 
 so_log_perms:

salt/common/tools/sbin/so-pcap-export

@@ -20,7 +20,7 @@ if [ $# -lt 2 ]; then
   exit 1
 fi
 
-docker exec -it so-sensoroni scripts/stenoquery.sh "$1" -w /nsm/pcapout/$2.pcap
+docker exec -t so-sensoroni scripts/stenoquery.sh "$1" -w /nsm/pcapout/$2.pcap
 
 echo ""
 echo "If successful, the output was written to: /nsm/pcapout/$2.pcap"

salt/common/tools/sbin/soup

@@ -549,6 +549,7 @@ preupgrade_changes() {
     [[ "$INSTALLEDVERSION" == 2.3.160 ]] && up_to_2.3.170
     [[ "$INSTALLEDVERSION" == 2.3.170 ]] && up_to_2.3.180
     [[ "$INSTALLEDVERSION" == 2.3.180 ]] && up_to_2.3.181
+    [[ "$INSTALLEDVERSION" == 2.3.181 ]] && up_to_2.3.190
     true
 }
 
@@ -570,6 +571,7 @@ postupgrade_changes() {
     [[ "$POSTVERSION" == 2.3.160 ]] && post_to_2.3.170
     [[ "$POSTVERSION" == 2.3.170 ]] && post_to_2.3.180
     [[ "$POSTVERSION" == 2.3.180 ]] && post_to_2.3.181
+    [[ "$POSTVERSION" == 2.3.181 ]] && post_to_2.3.190
 
     true
 }
@@ -655,22 +657,32 @@ post_to_2.3.140() {
 
 post_to_2.3.150() {
   echo "Nothing to do for .150"
+  POSTVERSION=2.3.150
 }
 
 post_to_2.3.160() {
   echo "Nothing to do for .160"
+  POSTVERSION=2.3.160
 }
 
 post_to_2.3.170() {
   echo "Nothing to do for .170"
+  POSTVERSION=2.3.170
 }
 
 post_to_2.3.180() {
   echo "Nothing to do for .180"
+  POSTVERSION=2.3.180
 }
 
 post_to_2.3.181() {
   echo "Nothing to do for .181"
+  POSTVERSION=2.3.181
+}
+
+post_to_2.3.190() {
+  echo "Nothing to do for .190"
+  POSTVERSION=2.3.190
 }
 
 stop_salt_master() {
@@ -972,6 +984,13 @@ up_to_2.3.181() {
   INSTALLEDVERSION=2.3.181
 }
 
+up_to_2.3.190() {
+  echo "Upgrading to 2.3.190"
+  chown -R zeek:socore /nsm/zeek/extracted/complete
+  chmod 770 /nsm/zeek/extracted/complete 
+  INSTALLEDVERSION=2.3.190
+}
+
 verify_upgradespace() {
   CURRENTSPACE=$(df -BG / | grep -v Avail | awk '{print $4}' | sed 's/.$//')
   if [ "$CURRENTSPACE" -lt "10" ]; then

salt/sensoroni/files/analyzers/README.md

@@ -5,20 +5,19 @@ Security Onion provides a means for performing data analysis on varying inputs.
 ## Supported Observable Types
 The built-in analyzers support the following observable types:
 
-| Name                    | Domain | Hash  | IP    | JA3   | Mail  | Other | URI   |  URL  | User Agent |
-| ------------------------|--------|-------|-------|-------|-------|-------|-------|-------|------------
-| Alienvault OTX          |✓ |✓|✓|✗|✗|✗|✗|✓|✗|
-| EmailRep                |✗ |✗|✗|✗|✓|✗|✗|✗|✗|
-| Greynoise               |✗ |✗|✓|✗|✗|✗|✗|✗|✗|
-| JA3er                   |✗ |✗|✗|✓|✗|✗|✗|✗|✗|
-| LocalFile               |✓ |✓|✓|✓|✗|✓|✗|✓|✗|
-| Malware Hash Registry   |✗ |✓|✗|✗|✗|✗|✗|✓|✗|
-| Pulsedive               |✓ |✓|✓|✗|✗|✗|✓|✓|✓|
-| Spamhaus                |✗ |✗|✓|✗|✗|✗|✗|✗|✗|
-| Urlhaus                 |✗ |✗|✗|✗|✗|✗|✗|✓|✗|
-| Urlscan                 |✗ |✗|✗|✗|✗|✗|✗|✓|✗|
-| Virustotal              |✓ |✓|✓|✗|✗|✗|✗|✓|✗|
-| WhoisLookup             |✓ |✗|✗|✗|✗|✗|✓|✗|✗|
+| Name                    | Domain | Hash  | IP    | Mail  | Other | URI   |  URL  | User Agent |
+| ------------------------|--------|-------|-------|-------|-------|-------|-------|-------|
+| Alienvault OTX          |✓ |✓|✓|✗|✗|✗|✓|✗|
+| EmailRep                |✗ |✗|✗|✓|✗|✗|✗|✗|
+| Greynoise               |✗ |✗|✓|✗|✗|✗|✗|✗|
+| LocalFile               |✓ |✓|✓|✗|✓|✗|✓|✗|
+| Malware Hash Registry   |✗ |✓|✗|✗|✗|✗|✓|✗|
+| Pulsedive               |✓ |✓|✓|✗|✗|✓|✓|✓|
+| Spamhaus                |✗ |✗|✓|✗|✗|✗|✗|✗|
+| Urlhaus                 |✗ |✗|✗|✗|✗|✗|✓|✗|
+| Urlscan                 |✗ |✗|✗|✗|✗|✗|✓|✗|
+| Virustotal              |✓ |✓|✓|✗|✗|✗|✓|✗|
+| WhoisLookup             |✓ |✗|✗|✗|✗|✓|✗|✗|
 
 ## Authentication
 Many analyzers require authentication, via an API key or similar. The table below illustrates which analyzers require authentication.
@@ -28,7 +27,6 @@ Many analyzers require authentication, via an API key or similar. The table belo
 [AlienVault OTX](https://otx.alienvault.com/api)            |✓|
 [EmailRep](https://emailrep.io/key)                  |✓|
 [GreyNoise](https://www.greynoise.io/plans/community)                 |✓|
-[JA3er](https://ja3er.com/)                     |✗|
 LocalFile                 |✗|
 [Malware Hash Registry](https://hash.cymru.com/docs_whois)    |✗|
 [Pulsedive](https://pulsedive.com/api/)                 |✓|

salt/sensoroni/files/analyzers/ja3er/ja3er.json

@@ -1,7 +0,0 @@
-{
-  "name": "JA3er Hash Search",
-  "version": "0.1",
-  "author": "Security Onion Solutions",
-  "description": "This analyzer queries JA3er user agents and sightings",
-  "supportedTypes" :  ["ja3"]
-}

salt/sensoroni/files/analyzers/ja3er/ja3er.py

@@ -1,53 +0,0 @@
-import json
-import os
-import requests
-import helpers
-import argparse
-
-
-def sendReq(conf, meta, hash):
-    url = conf['base_url'] + hash
-    response = requests.request('GET', url)
-    return response.json()
-
-
-def prepareResults(raw):
-    if "error" in raw:
-        if "Sorry" in raw["error"]:
-            status = "ok"
-            summary = "no_results"
-        elif "Invalid hash" in raw["error"]:
-            status = "caution"
-            summary = "invalid_input"
-        else:
-            status = "caution"
-            summary = "internal_failure"
-    else:
-        status = "info"
-        summary = "suspicious"
-    results = {'response': raw, 'summary': summary, 'status': status}
-    return results
-
-
-def analyze(conf, input):
-    meta = helpers.loadMetadata(__file__)
-    data = helpers.parseArtifact(input)
-    helpers.checkSupportedType(meta, data["artifactType"])
-    response = sendReq(conf, meta, data["value"])
-    return prepareResults(response)
-
-
-def main():
-    dir = os.path.dirname(os.path.realpath(__file__))
-    parser = argparse.ArgumentParser(description='Search JA3er for a given artifact')
-    parser.add_argument('artifact', help='the artifact represented in JSON format')
-    parser.add_argument('-c', '--config', metavar="CONFIG_FILE", default=dir + "/ja3er.yaml", help='optional config file to use instead of the default config file')
-
-    args = parser.parse_args()
-    if args.artifact:
-        results = analyze(helpers.loadConfig(args.config), args.artifact)
-        print(json.dumps(results))
-
-
-if __name__ == "__main__":
-    main()

salt/sensoroni/files/analyzers/ja3er/ja3er.yaml

@@ -1 +0,0 @@
-base_url: https://ja3er.com/search/

salt/sensoroni/files/analyzers/ja3er/ja3er_test.py

@@ -1,72 +0,0 @@
-from io import StringIO
-import sys
-from unittest.mock import patch, MagicMock
-from ja3er import ja3er
-import unittest
-
-
-class TestJa3erMethods(unittest.TestCase):
-
-    def test_main_missing_input(self):
-        with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stderr', new=StringIO()) as mock_stderr:
-                sys.argv = ["cmd"]
-                ja3er.main()
-                self.assertEqual(mock_stderr.getvalue(), "usage: cmd [-h] [-c CONFIG_FILE] artifact\ncmd: error: the following arguments are required: artifact\n")
-                sysmock.assert_called_once_with(2)
-
-    def test_main_success(self):
-        output = {"foo": "bar"}
-        with patch('sys.stdout', new=StringIO()) as mock_stdout:
-            with patch('ja3er.ja3er.analyze', new=MagicMock(return_value=output)) as mock:
-                sys.argv = ["cmd", "input"]
-                ja3er.main()
-                expected = '{"foo": "bar"}\n'
-                self.assertEqual(mock_stdout.getvalue(), expected)
-                mock.assert_called_once()
-
-    def test_sendReq(self):
-        with patch('requests.request', new=MagicMock(return_value=MagicMock())) as mock:
-            meta = {}
-            conf = {"base_url": "myurl/"}
-            hash = "abcd1234"
-            response = ja3er.sendReq(conf=conf, meta=meta, hash=hash)
-            mock.assert_called_once_with("GET", "myurl/abcd1234")
-            self.assertIsNotNone(response)
-
-    def test_prepareResults_none(self):
-        raw = {"error": "Sorry no values found"}
-        results = ja3er.prepareResults(raw)
-        self.assertEqual(results["response"], raw)
-        self.assertEqual(results["summary"], "no_results")
-        self.assertEqual(results["status"], "ok")
-
-    def test_prepareResults_invalidHash(self):
-        raw = {"error": "Invalid hash"}
-        results = ja3er.prepareResults(raw)
-        self.assertEqual(results["response"], raw)
-        self.assertEqual(results["summary"], "invalid_input")
-        self.assertEqual(results["status"], "caution")
-
-    def test_prepareResults_internal_failure(self):
-        raw = {"error": "unknown"}
-        results = ja3er.prepareResults(raw)
-        self.assertEqual(results["response"], raw)
-        self.assertEqual(results["summary"], "internal_failure")
-        self.assertEqual(results["status"], "caution")
-
-    def test_prepareResults_info(self):
-        raw = [{"User-Agent": "Blah/5.0", "Count": 24874, "Last_seen": "2022-04-08 16:18:38"}, {"Comment": "Brave browser v1.36.122\n\n", "Reported": "2022-03-28 20:26:42"}]
-        results = ja3er.prepareResults(raw)
-        self.assertEqual(results["response"], raw)
-        self.assertEqual(results["summary"], "suspicious")
-        self.assertEqual(results["status"], "info")
-
-    def test_analyze(self):
-        output = {"info": "Results found."}
-        artifactInput = '{"value":"abcd1234","artifactType":"ja3"}'
-        conf = {"base_url": "myurl/"}
-        with patch('ja3er.ja3er.sendReq', new=MagicMock(return_value=output)) as mock:
-            results = ja3er.analyze(conf, artifactInput)
-            self.assertEqual(results["summary"], "suspicious")
-            mock.assert_called_once()

salt/sensoroni/files/analyzers/ja3er/requirements.txt

@@ -1,2 +0,0 @@
-requests>=2.27.1
-pyyaml>=6.0

salt/strelka/filecheck/filecheck

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+
+# Copyright 2014-2022 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import time
+import hashlib
+import logging
+import yaml
+from watchdog.observers import Observer
+from watchdog.events import FileSystemEventHandler
+
+with open("/opt/so/conf/strelka/filecheck.yaml", "r") as ymlfile:
+    cfg = yaml.load(ymlfile)
+
+extract_path = cfg["filecheck"]["extract_path"]
+historypath = cfg["filecheck"]["historypath"]
+strelkapath = cfg["filecheck"]["strelkapath"]
+logfile = cfg["filecheck"]["logfile"]
+
+logging.basicConfig(filename=logfile, filemode='w', format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S', level=logging.INFO)
+
+def checkexisting():
+    for file in os.listdir(extract_path):
+        filename = os.path.join(extract_path, file)
+        logging.info("Processing existing file " + filename)
+        checksum(filename) 
+
+def checksum(filename):
+    with open(filename, 'rb') as afile:
+        shawnuff = hashlib.sha1()
+        buf = afile.read(8192)
+        while len(buf) > 0:
+            shawnuff.update(buf)
+            buf = afile.read(8192)
+        hizash=shawnuff.hexdigest()
+        process(filename, hizash)
+
+def process(filename, hizash):
+    if os.path.exists(historypath + hizash):
+        logging.info(filename + " Already exists.. removing")
+        os.remove(filename)
+    else:
+        # Write the file
+        logging.info(filename + " is new. Creating a record and sending to Strelka")
+        with open(os.path.join(historypath + hizash), 'w') as fp:
+            pass
+        head, tail = os.path.split(filename)
+
+        # Move the file
+        os.rename(filename, strelkapath + tail)
+
+class CreatedEventHandler(FileSystemEventHandler):
+    def on_created(self, event):
+        filename = event.src_path
+        logging.info("Found new file")
+        checksum(filename)
+
+if __name__ == "__main__":
+
+    checkexisting()
+    event_handler =CreatedEventHandler()
+
+    observer = Observer()
+    observer.schedule(event_handler, extract_path, recursive=True)
+    observer.start()
+    try:
+        while True:
+            time.sleep(1)
+    except KeyboardInterrupt:
+        observer.stop()
+    observer.join()

salt/strelka/filecheck/filecheck.yaml

@@ -0,0 +1,11 @@
+{%- set ENGINE = salt['pillar.get']('global:mdengine', '') %}
+filecheck:
+    {%- if ENGINE == "SURICATA" %}
+    extract_path: '/nsm/suricata/extracted'
+    {%- else %}
+    extract_path: '/nsm/zeek/extracted/complete'
+    {%- endif %}
+    historypath: '/nsm/strelka/history/'
+    strelkapath: '/nsm/strelka/unprocessed/'
+    logfile: '/opt/so/log/strelka/filecheck.log'
+

salt/strelka/init.sls

@@ -24,6 +24,20 @@
 {% import_yaml 'strelka/defaults.yaml' as strelka_config with context %}
 {% set IGNORELIST = salt['pillar.get']('strelka:ignore', strelka_config.strelka.ignore, merge=True, merge_nested_lists=True) %}
 
+{% if grains['os'] != 'CentOS' %}     
+strelkapkgs:
+  pkg.installed:
+    - skip_suggestions: True
+    - pkgs:
+      - python3-watchdog
+{% else %}
+strelkapkgs:
+  pkg.installed:
+    - skip_suggestions: True
+    - pkgs:
+      - securityonion-python36-watchdog
+{% endif %}
+    
 # Strelka config
 strelkaconfdir:
   file.directory:
@@ -79,7 +93,7 @@ strelkarepos:
 {% endif %}
 
 strelkadatadir:
-   file.directory:
+  file.directory:
     - name: /nsm/strelka
     - user: 939
     - group: 939
@@ -93,21 +107,21 @@ strelkalogdir:
     - makedirs: True
 
 strelkaprocessed:
-   file.directory:
+  file.directory:
     - name: /nsm/strelka/processed
     - user: 939
     - group: 939
     - makedirs: True
 
 strelkastaging:
-   file.directory:
+  file.directory:
     - name: /nsm/strelka/staging
     - user: 939
     - group: 939
     - makedirs: True
 
 strelkaunprocessed:
-   file.directory:
+  file.directory:
     - name: /nsm/strelka/unprocessed
     - user: 939
     - group: 939
@@ -115,8 +129,50 @@ strelkaunprocessed:
 
 # Check to see if Strelka frontend port is available
 strelkaportavailable:
-    cmd.run:
-      - name: netstat -utanp | grep ":57314" | grep -qvE 'docker|TIME_WAIT' && PROCESS=$(netstat -utanp | grep ":57314" | uniq) && echo "Another process ($PROCESS) appears to be using port 57314.  Please terminate this process, or reboot to ensure a clean state so that Strelka can start properly." && exit 1 || exit 0
+  cmd.run:
+    - name: netstat -utanp | grep ":57314" | grep -qvE 'docker|TIME_WAIT' && PROCESS=$(netstat -utanp | grep ":57314" | uniq) && echo "Another process ($PROCESS) appears to be using port 57314.  Please terminate this process, or reboot to ensure a clean state so that Strelka can start properly." && exit 1 || exit 0
+
+# Filecheck Section
+filecheck_logdir:
+  file.directory:
+    - name: /opt/so/log/strelka
+    - user: 939
+    - group: 939
+    - makedirs: True
+    
+filecheck_history:
+  file.directory:
+    - name: /nsm/strelka/history
+    - user: 939
+    - group: 939
+    - makedirs: True
+
+filecheck_conf:
+  file.managed:
+    - name: /opt/so/conf/strelka/filecheck.yaml
+    - source: salt://strelka/filecheck/filecheck.yaml
+    - template: jinja
+
+filecheck_script:
+  file.managed:
+    - name: /opt/so/conf/strelka/filecheck
+    - source: salt://strelka/filecheck/filecheck
+    - user: 939
+    - group: 939
+    - mode: 755
+
+filecheck_run:
+  cmd.run:
+    - name: 'python3 /opt/so/conf/strelka/filecheck'
+    - bg: True
+    - runas: socore
+    - unless: ps -ef | grep filecheck | grep -v grep
+
+filcheck_history_clean:
+  cron.present:
+    - name: '/usr/bin/find /nsm/strelka/history/ -type f -mtime +2 -exec rm {} + > /dev/null 2>&1>'
+    - minute: '33'
+# End Filecheck Section
 
 strelka_coordinator:
   docker_container.running:
@@ -212,7 +268,7 @@ strelka_zeek_extracted_sync_old:
 {% if ENGINE == "SURICATA" %}
 
 strelka_suricata_extracted_sync:
-  cron.present:
+  cron.absent:
     - user: root
     - identifier: zeek-extracted-strelka-sync
     - name: '[ -d /nsm/suricata/extracted/ ] && find /nsm/suricata/extracted/* -not \( -path /nsm/suricata/extracted/tmp -prune \) -type f -print0 | xargs -0 -I {} mv {} /nsm/strelka/unprocessed/ > /dev/null 2>&1'
@@ -220,7 +276,7 @@ strelka_suricata_extracted_sync:
 
 {% else %}
 strelka_zeek_extracted_sync:
-  cron.present:
+  cron.absent:
     - user: root
     - identifier: zeek-extracted-strelka-sync
     - name: '[ -d /nsm/zeek/extracted/complete/ ] && mv /nsm/zeek/extracted/complete/* /nsm/strelka/unprocessed/ > /dev/null 2>&1'

salt/suricata/init.sls

@@ -68,6 +68,7 @@ suridatadir:
     - name: /nsm/suricata/extracted
     - user: 940
     - group: 939
+    - mode: 770
     - makedirs: True
 
 surirulesync:

salt/zeek/init.sls

@@ -70,12 +70,15 @@ zeekextractdir:
     - name: /nsm/zeek/extracted
     - user: 937
     - group: 939
+    - mode: 770
     - makedirs: True
 
 zeekextractcompletedir:
   file.directory:
     - name: /nsm/zeek/extracted/complete
     - user: 937
+    - group: 939
+    - mode: 770
     - makedirs: True
 
 # Sync the policies

setup/automation/README.txt

@@ -0,0 +1,3 @@
+These automation files were designed for internal Security Onion testing. 
+
+While you may be able to make them work for your use case, we do not provide free support for them, and support for paying customers is limited to best effort.