diff --git a/VERSION b/VERSION index 1380e3d15..eba1e6a4f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3.181 +2.3.190 diff --git a/salt/common/files/sensor-rotate.conf b/salt/common/files/sensor-rotate.conf index cefd3944e..797eb8a45 100644 --- a/salt/common/files/sensor-rotate.conf +++ b/salt/common/files/sensor-rotate.conf @@ -19,4 +19,17 @@ extension .log dateext dateyesterday -} \ No newline at end of file +} + +/opt/so/log/strelka/filecheck.log +{ + daily + rotate 14 + missingok + copytruncate + compress + create + extension .log + dateext + dateyesterday +} diff --git a/salt/common/init.sls b/salt/common/init.sls index 0eaf5e77e..94a1b4869 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -38,15 +38,15 @@ socore: soconfperms: file.directory: - name: /opt/so/conf - - uid: 939 - - gid: 939 + - user: 939 + - group: 939 - dir_mode: 770 sostatusconf: file.directory: - name: /opt/so/conf/so-status - - uid: 939 - - gid: 939 + - user: 939 + - group: 939 - dir_mode: 770 so-status.conf: @@ -57,8 +57,8 @@ so-status.conf: sosaltstackperms: file.directory: - name: /opt/so/saltstack - - uid: 939 - - gid: 939 + - user: 939 + - group: 939 - dir_mode: 770 so_log_perms: diff --git a/salt/common/tools/sbin/so-pcap-export b/salt/common/tools/sbin/so-pcap-export index 25b89d4b7..bdeb76917 100755 --- a/salt/common/tools/sbin/so-pcap-export +++ b/salt/common/tools/sbin/so-pcap-export @@ -20,7 +20,7 @@ if [ $# -lt 2 ]; then exit 1 fi -docker exec -it so-sensoroni scripts/stenoquery.sh "$1" -w /nsm/pcapout/$2.pcap +docker exec -t so-sensoroni scripts/stenoquery.sh "$1" -w /nsm/pcapout/$2.pcap echo "" echo "If successful, the output was written to: /nsm/pcapout/$2.pcap" diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index c96d4e904..2edbcd35c 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -549,6 +549,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" == 2.3.160 ]] && up_to_2.3.170 [[ "$INSTALLEDVERSION" == 2.3.170 ]] && up_to_2.3.180 [[ "$INSTALLEDVERSION" == 2.3.180 ]] && up_to_2.3.181 + [[ "$INSTALLEDVERSION" == 2.3.181 ]] && up_to_2.3.190 true } @@ -570,6 +571,7 @@ postupgrade_changes() { [[ "$POSTVERSION" == 2.3.160 ]] && post_to_2.3.170 [[ "$POSTVERSION" == 2.3.170 ]] && post_to_2.3.180 [[ "$POSTVERSION" == 2.3.180 ]] && post_to_2.3.181 + [[ "$POSTVERSION" == 2.3.181 ]] && post_to_2.3.190 true } @@ -655,22 +657,32 @@ post_to_2.3.140() { post_to_2.3.150() { echo "Nothing to do for .150" + POSTVERSION=2.3.150 } post_to_2.3.160() { echo "Nothing to do for .160" + POSTVERSION=2.3.160 } post_to_2.3.170() { echo "Nothing to do for .170" + POSTVERSION=2.3.170 } post_to_2.3.180() { echo "Nothing to do for .180" + POSTVERSION=2.3.180 } post_to_2.3.181() { echo "Nothing to do for .181" + POSTVERSION=2.3.181 +} + +post_to_2.3.190() { + echo "Nothing to do for .190" + POSTVERSION=2.3.190 } stop_salt_master() { @@ -972,6 +984,13 @@ up_to_2.3.181() { INSTALLEDVERSION=2.3.181 } +up_to_2.3.190() { + echo "Upgrading to 2.3.190" + chown -R zeek:socore /nsm/zeek/extracted/complete + chmod 770 /nsm/zeek/extracted/complete + INSTALLEDVERSION=2.3.190 +} + verify_upgradespace() { CURRENTSPACE=$(df -BG / | grep -v Avail | awk '{print $4}' | sed 's/.$//') if [ "$CURRENTSPACE" -lt "10" ]; then diff --git a/salt/sensoroni/files/analyzers/README.md b/salt/sensoroni/files/analyzers/README.md index a86730734..2025b85cb 100644 --- a/salt/sensoroni/files/analyzers/README.md +++ b/salt/sensoroni/files/analyzers/README.md @@ -5,20 +5,19 @@ Security Onion provides a means for performing data analysis on varying inputs. ## Supported Observable Types The built-in analyzers support the following observable types: -| Name | Domain | Hash | IP | JA3 | Mail | Other | URI | URL | User Agent | -| ------------------------|--------|-------|-------|-------|-------|-------|-------|-------|------------ -| Alienvault OTX |✓ |✓|✓|✗|✗|✗|✗|✓|✗| -| EmailRep |✗ |✗|✗|✗|✓|✗|✗|✗|✗| -| Greynoise |✗ |✗|✓|✗|✗|✗|✗|✗|✗| -| JA3er |✗ |✗|✗|✓|✗|✗|✗|✗|✗| -| LocalFile |✓ |✓|✓|✓|✗|✓|✗|✓|✗| -| Malware Hash Registry |✗ |✓|✗|✗|✗|✗|✗|✓|✗| -| Pulsedive |✓ |✓|✓|✗|✗|✗|✓|✓|✓| -| Spamhaus |✗ |✗|✓|✗|✗|✗|✗|✗|✗| -| Urlhaus |✗ |✗|✗|✗|✗|✗|✗|✓|✗| -| Urlscan |✗ |✗|✗|✗|✗|✗|✗|✓|✗| -| Virustotal |✓ |✓|✓|✗|✗|✗|✗|✓|✗| -| WhoisLookup |✓ |✗|✗|✗|✗|✗|✓|✗|✗| +| Name | Domain | Hash | IP | Mail | Other | URI | URL | User Agent | +| ------------------------|--------|-------|-------|-------|-------|-------|-------|-------| +| Alienvault OTX |✓ |✓|✓|✗|✗|✗|✓|✗| +| EmailRep |✗ |✗|✗|✓|✗|✗|✗|✗| +| Greynoise |✗ |✗|✓|✗|✗|✗|✗|✗| +| LocalFile |✓ |✓|✓|✗|✓|✗|✓|✗| +| Malware Hash Registry |✗ |✓|✗|✗|✗|✗|✓|✗| +| Pulsedive |✓ |✓|✓|✗|✗|✓|✓|✓| +| Spamhaus |✗ |✗|✓|✗|✗|✗|✗|✗| +| Urlhaus |✗ |✗|✗|✗|✗|✗|✓|✗| +| Urlscan |✗ |✗|✗|✗|✗|✗|✓|✗| +| Virustotal |✓ |✓|✓|✗|✗|✗|✓|✗| +| WhoisLookup |✓ |✗|✗|✗|✗|✓|✗|✗| ## Authentication Many analyzers require authentication, via an API key or similar. The table below illustrates which analyzers require authentication. @@ -28,7 +27,6 @@ Many analyzers require authentication, via an API key or similar. The table belo [AlienVault OTX](https://otx.alienvault.com/api) |✓| [EmailRep](https://emailrep.io/key) |✓| [GreyNoise](https://www.greynoise.io/plans/community) |✓| -[JA3er](https://ja3er.com/) |✗| LocalFile |✗| [Malware Hash Registry](https://hash.cymru.com/docs_whois) |✗| [Pulsedive](https://pulsedive.com/api/) |✓| diff --git a/salt/sensoroni/files/analyzers/emailrep/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl b/salt/sensoroni/files/analyzers/emailrep/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl similarity index 100% rename from salt/sensoroni/files/analyzers/emailrep/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl rename to salt/sensoroni/files/analyzers/emailrep/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl diff --git a/salt/sensoroni/files/analyzers/greynoise/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl b/salt/sensoroni/files/analyzers/greynoise/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl similarity index 100% rename from salt/sensoroni/files/analyzers/greynoise/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl rename to salt/sensoroni/files/analyzers/greynoise/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl diff --git a/salt/sensoroni/files/analyzers/ja3er/__init__.py b/salt/sensoroni/files/analyzers/ja3er/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/salt/sensoroni/files/analyzers/ja3er/ja3er.json b/salt/sensoroni/files/analyzers/ja3er/ja3er.json deleted file mode 100644 index de072d0b7..000000000 --- a/salt/sensoroni/files/analyzers/ja3er/ja3er.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "name": "JA3er Hash Search", - "version": "0.1", - "author": "Security Onion Solutions", - "description": "This analyzer queries JA3er user agents and sightings", - "supportedTypes" : ["ja3"] -} diff --git a/salt/sensoroni/files/analyzers/ja3er/ja3er.py b/salt/sensoroni/files/analyzers/ja3er/ja3er.py deleted file mode 100755 index c1018a880..000000000 --- a/salt/sensoroni/files/analyzers/ja3er/ja3er.py +++ /dev/null @@ -1,53 +0,0 @@ -import json -import os -import requests -import helpers -import argparse - - -def sendReq(conf, meta, hash): - url = conf['base_url'] + hash - response = requests.request('GET', url) - return response.json() - - -def prepareResults(raw): - if "error" in raw: - if "Sorry" in raw["error"]: - status = "ok" - summary = "no_results" - elif "Invalid hash" in raw["error"]: - status = "caution" - summary = "invalid_input" - else: - status = "caution" - summary = "internal_failure" - else: - status = "info" - summary = "suspicious" - results = {'response': raw, 'summary': summary, 'status': status} - return results - - -def analyze(conf, input): - meta = helpers.loadMetadata(__file__) - data = helpers.parseArtifact(input) - helpers.checkSupportedType(meta, data["artifactType"]) - response = sendReq(conf, meta, data["value"]) - return prepareResults(response) - - -def main(): - dir = os.path.dirname(os.path.realpath(__file__)) - parser = argparse.ArgumentParser(description='Search JA3er for a given artifact') - parser.add_argument('artifact', help='the artifact represented in JSON format') - parser.add_argument('-c', '--config', metavar="CONFIG_FILE", default=dir + "/ja3er.yaml", help='optional config file to use instead of the default config file') - - args = parser.parse_args() - if args.artifact: - results = analyze(helpers.loadConfig(args.config), args.artifact) - print(json.dumps(results)) - - -if __name__ == "__main__": - main() diff --git a/salt/sensoroni/files/analyzers/ja3er/ja3er.yaml b/salt/sensoroni/files/analyzers/ja3er/ja3er.yaml deleted file mode 100644 index 40d6f64dd..000000000 --- a/salt/sensoroni/files/analyzers/ja3er/ja3er.yaml +++ /dev/null @@ -1 +0,0 @@ -base_url: https://ja3er.com/search/ diff --git a/salt/sensoroni/files/analyzers/ja3er/ja3er_test.py b/salt/sensoroni/files/analyzers/ja3er/ja3er_test.py deleted file mode 100644 index 41de4e9c7..000000000 --- a/salt/sensoroni/files/analyzers/ja3er/ja3er_test.py +++ /dev/null @@ -1,72 +0,0 @@ -from io import StringIO -import sys -from unittest.mock import patch, MagicMock -from ja3er import ja3er -import unittest - - -class TestJa3erMethods(unittest.TestCase): - - def test_main_missing_input(self): - with patch('sys.exit', new=MagicMock()) as sysmock: - with patch('sys.stderr', new=StringIO()) as mock_stderr: - sys.argv = ["cmd"] - ja3er.main() - self.assertEqual(mock_stderr.getvalue(), "usage: cmd [-h] [-c CONFIG_FILE] artifact\ncmd: error: the following arguments are required: artifact\n") - sysmock.assert_called_once_with(2) - - def test_main_success(self): - output = {"foo": "bar"} - with patch('sys.stdout', new=StringIO()) as mock_stdout: - with patch('ja3er.ja3er.analyze', new=MagicMock(return_value=output)) as mock: - sys.argv = ["cmd", "input"] - ja3er.main() - expected = '{"foo": "bar"}\n' - self.assertEqual(mock_stdout.getvalue(), expected) - mock.assert_called_once() - - def test_sendReq(self): - with patch('requests.request', new=MagicMock(return_value=MagicMock())) as mock: - meta = {} - conf = {"base_url": "myurl/"} - hash = "abcd1234" - response = ja3er.sendReq(conf=conf, meta=meta, hash=hash) - mock.assert_called_once_with("GET", "myurl/abcd1234") - self.assertIsNotNone(response) - - def test_prepareResults_none(self): - raw = {"error": "Sorry no values found"} - results = ja3er.prepareResults(raw) - self.assertEqual(results["response"], raw) - self.assertEqual(results["summary"], "no_results") - self.assertEqual(results["status"], "ok") - - def test_prepareResults_invalidHash(self): - raw = {"error": "Invalid hash"} - results = ja3er.prepareResults(raw) - self.assertEqual(results["response"], raw) - self.assertEqual(results["summary"], "invalid_input") - self.assertEqual(results["status"], "caution") - - def test_prepareResults_internal_failure(self): - raw = {"error": "unknown"} - results = ja3er.prepareResults(raw) - self.assertEqual(results["response"], raw) - self.assertEqual(results["summary"], "internal_failure") - self.assertEqual(results["status"], "caution") - - def test_prepareResults_info(self): - raw = [{"User-Agent": "Blah/5.0", "Count": 24874, "Last_seen": "2022-04-08 16:18:38"}, {"Comment": "Brave browser v1.36.122\n\n", "Reported": "2022-03-28 20:26:42"}] - results = ja3er.prepareResults(raw) - self.assertEqual(results["response"], raw) - self.assertEqual(results["summary"], "suspicious") - self.assertEqual(results["status"], "info") - - def test_analyze(self): - output = {"info": "Results found."} - artifactInput = '{"value":"abcd1234","artifactType":"ja3"}' - conf = {"base_url": "myurl/"} - with patch('ja3er.ja3er.sendReq', new=MagicMock(return_value=output)) as mock: - results = ja3er.analyze(conf, artifactInput) - self.assertEqual(results["summary"], "suspicious") - mock.assert_called_once() diff --git a/salt/sensoroni/files/analyzers/ja3er/requirements.txt b/salt/sensoroni/files/analyzers/ja3er/requirements.txt deleted file mode 100644 index a8980057f..000000000 --- a/salt/sensoroni/files/analyzers/ja3er/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -requests>=2.27.1 -pyyaml>=6.0 diff --git a/salt/sensoroni/files/analyzers/ja3er/source-packages/certifi-2021.10.8-py2.py3-none-any.whl b/salt/sensoroni/files/analyzers/ja3er/source-packages/certifi-2021.10.8-py2.py3-none-any.whl deleted file mode 100644 index fbcb86b5f..000000000 Binary files a/salt/sensoroni/files/analyzers/ja3er/source-packages/certifi-2021.10.8-py2.py3-none-any.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/ja3er/source-packages/charset_normalizer-2.0.12-py3-none-any.whl b/salt/sensoroni/files/analyzers/ja3er/source-packages/charset_normalizer-2.0.12-py3-none-any.whl deleted file mode 100644 index 17a2dfbeb..000000000 Binary files a/salt/sensoroni/files/analyzers/ja3er/source-packages/charset_normalizer-2.0.12-py3-none-any.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/ja3er/source-packages/idna-3.3-py3-none-any.whl b/salt/sensoroni/files/analyzers/ja3er/source-packages/idna-3.3-py3-none-any.whl deleted file mode 100644 index 060541bc9..000000000 Binary files a/salt/sensoroni/files/analyzers/ja3er/source-packages/idna-3.3-py3-none-any.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/ja3er/source-packages/requests-2.27.1-py2.py3-none-any.whl b/salt/sensoroni/files/analyzers/ja3er/source-packages/requests-2.27.1-py2.py3-none-any.whl deleted file mode 100644 index 807fc6110..000000000 Binary files a/salt/sensoroni/files/analyzers/ja3er/source-packages/requests-2.27.1-py2.py3-none-any.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/ja3er/source-packages/urllib3-1.26.9-py2.py3-none-any.whl b/salt/sensoroni/files/analyzers/ja3er/source-packages/urllib3-1.26.9-py2.py3-none-any.whl deleted file mode 100644 index 5019453dd..000000000 Binary files a/salt/sensoroni/files/analyzers/ja3er/source-packages/urllib3-1.26.9-py2.py3-none-any.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/ja3er/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl b/salt/sensoroni/files/analyzers/localfile/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl similarity index 100% rename from salt/sensoroni/files/analyzers/ja3er/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl rename to salt/sensoroni/files/analyzers/localfile/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl diff --git a/salt/sensoroni/files/analyzers/localfile/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl b/salt/sensoroni/files/analyzers/otx/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl similarity index 100% rename from salt/sensoroni/files/analyzers/localfile/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl rename to salt/sensoroni/files/analyzers/otx/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl diff --git a/salt/sensoroni/files/analyzers/otx/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl b/salt/sensoroni/files/analyzers/pulsedive/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl similarity index 100% rename from salt/sensoroni/files/analyzers/otx/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl rename to salt/sensoroni/files/analyzers/pulsedive/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl diff --git a/salt/sensoroni/files/analyzers/pulsedive/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl b/salt/sensoroni/files/analyzers/spamhaus/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl similarity index 100% rename from salt/sensoroni/files/analyzers/pulsedive/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl rename to salt/sensoroni/files/analyzers/spamhaus/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl diff --git a/salt/sensoroni/files/analyzers/spamhaus/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl b/salt/sensoroni/files/analyzers/urlhaus/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl similarity index 100% rename from salt/sensoroni/files/analyzers/spamhaus/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl rename to salt/sensoroni/files/analyzers/urlhaus/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl diff --git a/salt/sensoroni/files/analyzers/urlhaus/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl b/salt/sensoroni/files/analyzers/urlscan/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl similarity index 100% rename from salt/sensoroni/files/analyzers/urlhaus/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl rename to salt/sensoroni/files/analyzers/urlscan/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl diff --git a/salt/sensoroni/files/analyzers/virustotal/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl b/salt/sensoroni/files/analyzers/virustotal/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl deleted file mode 100644 index d2b6c37f9..000000000 Binary files a/salt/sensoroni/files/analyzers/virustotal/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/urlscan/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl b/salt/sensoroni/files/analyzers/virustotal/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl similarity index 100% rename from salt/sensoroni/files/analyzers/urlscan/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl rename to salt/sensoroni/files/analyzers/virustotal/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl diff --git a/salt/strelka/filecheck/filecheck b/salt/strelka/filecheck/filecheck new file mode 100644 index 000000000..816125fcb --- /dev/null +++ b/salt/strelka/filecheck/filecheck @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 + +# Copyright 2014-2022 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import time +import hashlib +import logging +import yaml +from watchdog.observers import Observer +from watchdog.events import FileSystemEventHandler + +with open("/opt/so/conf/strelka/filecheck.yaml", "r") as ymlfile: + cfg = yaml.load(ymlfile) + +extract_path = cfg["filecheck"]["extract_path"] +historypath = cfg["filecheck"]["historypath"] +strelkapath = cfg["filecheck"]["strelkapath"] +logfile = cfg["filecheck"]["logfile"] + +logging.basicConfig(filename=logfile, filemode='w', format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S', level=logging.INFO) + +def checkexisting(): + for file in os.listdir(extract_path): + filename = os.path.join(extract_path, file) + logging.info("Processing existing file " + filename) + checksum(filename) + +def checksum(filename): + with open(filename, 'rb') as afile: + shawnuff = hashlib.sha1() + buf = afile.read(8192) + while len(buf) > 0: + shawnuff.update(buf) + buf = afile.read(8192) + hizash=shawnuff.hexdigest() + process(filename, hizash) + +def process(filename, hizash): + if os.path.exists(historypath + hizash): + logging.info(filename + " Already exists.. removing") + os.remove(filename) + else: + # Write the file + logging.info(filename + " is new. Creating a record and sending to Strelka") + with open(os.path.join(historypath + hizash), 'w') as fp: + pass + head, tail = os.path.split(filename) + + # Move the file + os.rename(filename, strelkapath + tail) + +class CreatedEventHandler(FileSystemEventHandler): + def on_created(self, event): + filename = event.src_path + logging.info("Found new file") + checksum(filename) + +if __name__ == "__main__": + + checkexisting() + event_handler =CreatedEventHandler() + + observer = Observer() + observer.schedule(event_handler, extract_path, recursive=True) + observer.start() + try: + while True: + time.sleep(1) + except KeyboardInterrupt: + observer.stop() + observer.join() diff --git a/salt/strelka/filecheck/filecheck.yaml b/salt/strelka/filecheck/filecheck.yaml new file mode 100644 index 000000000..2b46afdf5 --- /dev/null +++ b/salt/strelka/filecheck/filecheck.yaml @@ -0,0 +1,11 @@ +{%- set ENGINE = salt['pillar.get']('global:mdengine', '') %} +filecheck: + {%- if ENGINE == "SURICATA" %} + extract_path: '/nsm/suricata/extracted' + {%- else %} + extract_path: '/nsm/zeek/extracted/complete' + {%- endif %} + historypath: '/nsm/strelka/history/' + strelkapath: '/nsm/strelka/unprocessed/' + logfile: '/opt/so/log/strelka/filecheck.log' + diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls index 56a5b9dcc..ec49787f7 100644 --- a/salt/strelka/init.sls +++ b/salt/strelka/init.sls @@ -24,6 +24,20 @@ {% import_yaml 'strelka/defaults.yaml' as strelka_config with context %} {% set IGNORELIST = salt['pillar.get']('strelka:ignore', strelka_config.strelka.ignore, merge=True, merge_nested_lists=True) %} +{% if grains['os'] != 'CentOS' %} +strelkapkgs: + pkg.installed: + - skip_suggestions: True + - pkgs: + - python3-watchdog +{% else %} +strelkapkgs: + pkg.installed: + - skip_suggestions: True + - pkgs: + - securityonion-python36-watchdog +{% endif %} + # Strelka config strelkaconfdir: file.directory: @@ -79,7 +93,7 @@ strelkarepos: {% endif %} strelkadatadir: - file.directory: + file.directory: - name: /nsm/strelka - user: 939 - group: 939 @@ -93,21 +107,21 @@ strelkalogdir: - makedirs: True strelkaprocessed: - file.directory: + file.directory: - name: /nsm/strelka/processed - user: 939 - group: 939 - makedirs: True strelkastaging: - file.directory: + file.directory: - name: /nsm/strelka/staging - user: 939 - group: 939 - makedirs: True strelkaunprocessed: - file.directory: + file.directory: - name: /nsm/strelka/unprocessed - user: 939 - group: 939 @@ -115,8 +129,50 @@ strelkaunprocessed: # Check to see if Strelka frontend port is available strelkaportavailable: - cmd.run: - - name: netstat -utanp | grep ":57314" | grep -qvE 'docker|TIME_WAIT' && PROCESS=$(netstat -utanp | grep ":57314" | uniq) && echo "Another process ($PROCESS) appears to be using port 57314. Please terminate this process, or reboot to ensure a clean state so that Strelka can start properly." && exit 1 || exit 0 + cmd.run: + - name: netstat -utanp | grep ":57314" | grep -qvE 'docker|TIME_WAIT' && PROCESS=$(netstat -utanp | grep ":57314" | uniq) && echo "Another process ($PROCESS) appears to be using port 57314. Please terminate this process, or reboot to ensure a clean state so that Strelka can start properly." && exit 1 || exit 0 + +# Filecheck Section +filecheck_logdir: + file.directory: + - name: /opt/so/log/strelka + - user: 939 + - group: 939 + - makedirs: True + +filecheck_history: + file.directory: + - name: /nsm/strelka/history + - user: 939 + - group: 939 + - makedirs: True + +filecheck_conf: + file.managed: + - name: /opt/so/conf/strelka/filecheck.yaml + - source: salt://strelka/filecheck/filecheck.yaml + - template: jinja + +filecheck_script: + file.managed: + - name: /opt/so/conf/strelka/filecheck + - source: salt://strelka/filecheck/filecheck + - user: 939 + - group: 939 + - mode: 755 + +filecheck_run: + cmd.run: + - name: 'python3 /opt/so/conf/strelka/filecheck' + - bg: True + - runas: socore + - unless: ps -ef | grep filecheck | grep -v grep + +filcheck_history_clean: + cron.present: + - name: '/usr/bin/find /nsm/strelka/history/ -type f -mtime +2 -exec rm {} + > /dev/null 2>&1>' + - minute: '33' +# End Filecheck Section strelka_coordinator: docker_container.running: @@ -212,7 +268,7 @@ strelka_zeek_extracted_sync_old: {% if ENGINE == "SURICATA" %} strelka_suricata_extracted_sync: - cron.present: + cron.absent: - user: root - identifier: zeek-extracted-strelka-sync - name: '[ -d /nsm/suricata/extracted/ ] && find /nsm/suricata/extracted/* -not \( -path /nsm/suricata/extracted/tmp -prune \) -type f -print0 | xargs -0 -I {} mv {} /nsm/strelka/unprocessed/ > /dev/null 2>&1' @@ -220,7 +276,7 @@ strelka_suricata_extracted_sync: {% else %} strelka_zeek_extracted_sync: - cron.present: + cron.absent: - user: root - identifier: zeek-extracted-strelka-sync - name: '[ -d /nsm/zeek/extracted/complete/ ] && mv /nsm/zeek/extracted/complete/* /nsm/strelka/unprocessed/ > /dev/null 2>&1' diff --git a/salt/suricata/init.sls b/salt/suricata/init.sls index db09e310b..d6333c65a 100644 --- a/salt/suricata/init.sls +++ b/salt/suricata/init.sls @@ -68,6 +68,7 @@ suridatadir: - name: /nsm/suricata/extracted - user: 940 - group: 939 + - mode: 770 - makedirs: True surirulesync: diff --git a/salt/zeek/init.sls b/salt/zeek/init.sls index ff91762f5..dda1bed52 100644 --- a/salt/zeek/init.sls +++ b/salt/zeek/init.sls @@ -70,12 +70,15 @@ zeekextractdir: - name: /nsm/zeek/extracted - user: 937 - group: 939 + - mode: 770 - makedirs: True zeekextractcompletedir: file.directory: - name: /nsm/zeek/extracted/complete - user: 937 + - group: 939 + - mode: 770 - makedirs: True # Sync the policies diff --git a/setup/automation/README.txt b/setup/automation/README.txt new file mode 100644 index 000000000..8fa208885 --- /dev/null +++ b/setup/automation/README.txt @@ -0,0 +1,3 @@ +These automation files were designed for internal Security Onion testing. + +While you may be able to make them work for your use case, we do not provide free support for them, and support for paying customers is limited to best effort.