diff --git a/salt/sensoroni/files/analyzers/README.md b/salt/sensoroni/files/analyzers/README.md index a86730734..e87a95638 100644 --- a/salt/sensoroni/files/analyzers/README.md +++ b/salt/sensoroni/files/analyzers/README.md @@ -5,20 +5,19 @@ Security Onion provides a means for performing data analysis on varying inputs. ## Supported Observable Types The built-in analyzers support the following observable types: -| Name | Domain | Hash | IP | JA3 | Mail | Other | URI | URL | User Agent | -| ------------------------|--------|-------|-------|-------|-------|-------|-------|-------|------------ -| Alienvault OTX |✓ |✓|✓|✗|✗|✗|✗|✓|✗| -| EmailRep |✗ |✗|✗|✗|✓|✗|✗|✗|✗| -| Greynoise |✗ |✗|✓|✗|✗|✗|✗|✗|✗| -| JA3er |✗ |✗|✗|✓|✗|✗|✗|✗|✗| -| LocalFile |✓ |✓|✓|✓|✗|✓|✗|✓|✗| -| Malware Hash Registry |✗ |✓|✗|✗|✗|✗|✗|✓|✗| -| Pulsedive |✓ |✓|✓|✗|✗|✗|✓|✓|✓| -| Spamhaus |✗ |✗|✓|✗|✗|✗|✗|✗|✗| -| Urlhaus |✗ |✗|✗|✗|✗|✗|✗|✓|✗| -| Urlscan |✗ |✗|✗|✗|✗|✗|✗|✓|✗| -| Virustotal |✓ |✓|✓|✗|✗|✗|✗|✓|✗| -| WhoisLookup |✓ |✗|✗|✗|✗|✗|✓|✗|✗| +| Name | Domain | Hash | IP | Mail | Other | URI | URL | User Agent | +| ------------------------|--------|-------|-------|-------|-------|-------|-------|-------| +| Alienvault OTX |✓ |✓|✓|✗|✗|✗|✓|✗| +| EmailRep |✗ |✗|✗|✓|✗|✗|✗|✗| +| Greynoise |✗ |✗|✓|✗|✗|✗|✗|✗| +| LocalFile |✓ |✓|✓|✗|✓|✗|✓|✗| +| Malware Hash Registry |✗ |✓|✗|✗|✗|✗|✓|✗| +| Pulsedive |✓ |✓|✓|✗|✗|✓|✓|✓| +| Spamhaus |✗ |✗|✓|✗|✗|✗|✗|✗| +| Urlhaus |✗ |✗|✗|✗|✗|✗|✓|✗| +| Urlscan |✗ |✗|✗|✗|✗|✗|✓|✗| +| Virustotal |✓ |✓|✓|✗|✗|✗|✓|✗| +| WhoisLookup |✓ |✗|✗|✗|✗|✓|✗|✗| ## Authentication Many analyzers require authentication, via an API key or similar. The table below illustrates which analyzers require authentication. @@ -28,7 +27,6 @@ Many analyzers require authentication, via an API key or similar. The table belo [AlienVault OTX](https://otx.alienvault.com/api) |✓| [EmailRep](https://emailrep.io/key) |✓| [GreyNoise](https://www.greynoise.io/plans/community) |✓| -[JA3er](https://ja3er.com/) |✗| LocalFile |✗| [Malware Hash Registry](https://hash.cymru.com/docs_whois) |✗| [Pulsedive](https://pulsedive.com/api/) |✓| @@ -265,5 +263,3 @@ The following requirements must be satisfied in order for analyzer pull requests - All source code must include accompanying unit test coverage. The Security Onion project will automatically run the unit tests after each push to a `securityonion` repository fork, and again when submitting a pull request. Failed unit tests, or insufficient unit test coverage, will result in the submitter being sent an automated email message. - Documentation of the analyzer, its input requirements, conditions for operation, and other relevant information must be clearly written in an accompanying analyzer metadata file. This file is described in more detail earlier in this document. - Source code must be well-written and be free of security defects that can put users or their data at unnecessary risk. - - diff --git a/salt/sensoroni/files/analyzers/emailrep/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl b/salt/sensoroni/files/analyzers/emailrep/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl similarity index 100% rename from salt/sensoroni/files/analyzers/emailrep/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl rename to salt/sensoroni/files/analyzers/emailrep/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl diff --git a/salt/sensoroni/files/analyzers/greynoise/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl b/salt/sensoroni/files/analyzers/greynoise/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl similarity index 100% rename from salt/sensoroni/files/analyzers/greynoise/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl rename to salt/sensoroni/files/analyzers/greynoise/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl diff --git a/salt/sensoroni/files/analyzers/ja3er/__init__.py b/salt/sensoroni/files/analyzers/ja3er/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/salt/sensoroni/files/analyzers/ja3er/ja3er.json b/salt/sensoroni/files/analyzers/ja3er/ja3er.json deleted file mode 100644 index de072d0b7..000000000 --- a/salt/sensoroni/files/analyzers/ja3er/ja3er.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "name": "JA3er Hash Search", - "version": "0.1", - "author": "Security Onion Solutions", - "description": "This analyzer queries JA3er user agents and sightings", - "supportedTypes" : ["ja3"] -} diff --git a/salt/sensoroni/files/analyzers/ja3er/ja3er.py b/salt/sensoroni/files/analyzers/ja3er/ja3er.py deleted file mode 100755 index c1018a880..000000000 --- a/salt/sensoroni/files/analyzers/ja3er/ja3er.py +++ /dev/null @@ -1,53 +0,0 @@ -import json -import os -import requests -import helpers -import argparse - - -def sendReq(conf, meta, hash): - url = conf['base_url'] + hash - response = requests.request('GET', url) - return response.json() - - -def prepareResults(raw): - if "error" in raw: - if "Sorry" in raw["error"]: - status = "ok" - summary = "no_results" - elif "Invalid hash" in raw["error"]: - status = "caution" - summary = "invalid_input" - else: - status = "caution" - summary = "internal_failure" - else: - status = "info" - summary = "suspicious" - results = {'response': raw, 'summary': summary, 'status': status} - return results - - -def analyze(conf, input): - meta = helpers.loadMetadata(__file__) - data = helpers.parseArtifact(input) - helpers.checkSupportedType(meta, data["artifactType"]) - response = sendReq(conf, meta, data["value"]) - return prepareResults(response) - - -def main(): - dir = os.path.dirname(os.path.realpath(__file__)) - parser = argparse.ArgumentParser(description='Search JA3er for a given artifact') - parser.add_argument('artifact', help='the artifact represented in JSON format') - parser.add_argument('-c', '--config', metavar="CONFIG_FILE", default=dir + "/ja3er.yaml", help='optional config file to use instead of the default config file') - - args = parser.parse_args() - if args.artifact: - results = analyze(helpers.loadConfig(args.config), args.artifact) - print(json.dumps(results)) - - -if __name__ == "__main__": - main() diff --git a/salt/sensoroni/files/analyzers/ja3er/ja3er.yaml b/salt/sensoroni/files/analyzers/ja3er/ja3er.yaml deleted file mode 100644 index 40d6f64dd..000000000 --- a/salt/sensoroni/files/analyzers/ja3er/ja3er.yaml +++ /dev/null @@ -1 +0,0 @@ -base_url: https://ja3er.com/search/ diff --git a/salt/sensoroni/files/analyzers/ja3er/ja3er_test.py b/salt/sensoroni/files/analyzers/ja3er/ja3er_test.py deleted file mode 100644 index 41de4e9c7..000000000 --- a/salt/sensoroni/files/analyzers/ja3er/ja3er_test.py +++ /dev/null @@ -1,72 +0,0 @@ -from io import StringIO -import sys -from unittest.mock import patch, MagicMock -from ja3er import ja3er -import unittest - - -class TestJa3erMethods(unittest.TestCase): - - def test_main_missing_input(self): - with patch('sys.exit', new=MagicMock()) as sysmock: - with patch('sys.stderr', new=StringIO()) as mock_stderr: - sys.argv = ["cmd"] - ja3er.main() - self.assertEqual(mock_stderr.getvalue(), "usage: cmd [-h] [-c CONFIG_FILE] artifact\ncmd: error: the following arguments are required: artifact\n") - sysmock.assert_called_once_with(2) - - def test_main_success(self): - output = {"foo": "bar"} - with patch('sys.stdout', new=StringIO()) as mock_stdout: - with patch('ja3er.ja3er.analyze', new=MagicMock(return_value=output)) as mock: - sys.argv = ["cmd", "input"] - ja3er.main() - expected = '{"foo": "bar"}\n' - self.assertEqual(mock_stdout.getvalue(), expected) - mock.assert_called_once() - - def test_sendReq(self): - with patch('requests.request', new=MagicMock(return_value=MagicMock())) as mock: - meta = {} - conf = {"base_url": "myurl/"} - hash = "abcd1234" - response = ja3er.sendReq(conf=conf, meta=meta, hash=hash) - mock.assert_called_once_with("GET", "myurl/abcd1234") - self.assertIsNotNone(response) - - def test_prepareResults_none(self): - raw = {"error": "Sorry no values found"} - results = ja3er.prepareResults(raw) - self.assertEqual(results["response"], raw) - self.assertEqual(results["summary"], "no_results") - self.assertEqual(results["status"], "ok") - - def test_prepareResults_invalidHash(self): - raw = {"error": "Invalid hash"} - results = ja3er.prepareResults(raw) - self.assertEqual(results["response"], raw) - self.assertEqual(results["summary"], "invalid_input") - self.assertEqual(results["status"], "caution") - - def test_prepareResults_internal_failure(self): - raw = {"error": "unknown"} - results = ja3er.prepareResults(raw) - self.assertEqual(results["response"], raw) - self.assertEqual(results["summary"], "internal_failure") - self.assertEqual(results["status"], "caution") - - def test_prepareResults_info(self): - raw = [{"User-Agent": "Blah/5.0", "Count": 24874, "Last_seen": "2022-04-08 16:18:38"}, {"Comment": "Brave browser v1.36.122\n\n", "Reported": "2022-03-28 20:26:42"}] - results = ja3er.prepareResults(raw) - self.assertEqual(results["response"], raw) - self.assertEqual(results["summary"], "suspicious") - self.assertEqual(results["status"], "info") - - def test_analyze(self): - output = {"info": "Results found."} - artifactInput = '{"value":"abcd1234","artifactType":"ja3"}' - conf = {"base_url": "myurl/"} - with patch('ja3er.ja3er.sendReq', new=MagicMock(return_value=output)) as mock: - results = ja3er.analyze(conf, artifactInput) - self.assertEqual(results["summary"], "suspicious") - mock.assert_called_once() diff --git a/salt/sensoroni/files/analyzers/ja3er/requirements.txt b/salt/sensoroni/files/analyzers/ja3er/requirements.txt deleted file mode 100644 index a8980057f..000000000 --- a/salt/sensoroni/files/analyzers/ja3er/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -requests>=2.27.1 -pyyaml>=6.0 diff --git a/salt/sensoroni/files/analyzers/ja3er/source-packages/certifi-2021.10.8-py2.py3-none-any.whl b/salt/sensoroni/files/analyzers/ja3er/source-packages/certifi-2021.10.8-py2.py3-none-any.whl deleted file mode 100644 index fbcb86b5f..000000000 Binary files a/salt/sensoroni/files/analyzers/ja3er/source-packages/certifi-2021.10.8-py2.py3-none-any.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/ja3er/source-packages/charset_normalizer-2.0.12-py3-none-any.whl b/salt/sensoroni/files/analyzers/ja3er/source-packages/charset_normalizer-2.0.12-py3-none-any.whl deleted file mode 100644 index 17a2dfbeb..000000000 Binary files a/salt/sensoroni/files/analyzers/ja3er/source-packages/charset_normalizer-2.0.12-py3-none-any.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/ja3er/source-packages/idna-3.3-py3-none-any.whl b/salt/sensoroni/files/analyzers/ja3er/source-packages/idna-3.3-py3-none-any.whl deleted file mode 100644 index 060541bc9..000000000 Binary files a/salt/sensoroni/files/analyzers/ja3er/source-packages/idna-3.3-py3-none-any.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/ja3er/source-packages/requests-2.27.1-py2.py3-none-any.whl b/salt/sensoroni/files/analyzers/ja3er/source-packages/requests-2.27.1-py2.py3-none-any.whl deleted file mode 100644 index 807fc6110..000000000 Binary files a/salt/sensoroni/files/analyzers/ja3er/source-packages/requests-2.27.1-py2.py3-none-any.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/ja3er/source-packages/urllib3-1.26.9-py2.py3-none-any.whl b/salt/sensoroni/files/analyzers/ja3er/source-packages/urllib3-1.26.9-py2.py3-none-any.whl deleted file mode 100644 index 5019453dd..000000000 Binary files a/salt/sensoroni/files/analyzers/ja3er/source-packages/urllib3-1.26.9-py2.py3-none-any.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/ja3er/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl b/salt/sensoroni/files/analyzers/localfile/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl similarity index 100% rename from salt/sensoroni/files/analyzers/ja3er/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl rename to salt/sensoroni/files/analyzers/localfile/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl diff --git a/salt/sensoroni/files/analyzers/localfile/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl b/salt/sensoroni/files/analyzers/otx/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl similarity index 100% rename from salt/sensoroni/files/analyzers/localfile/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl rename to salt/sensoroni/files/analyzers/otx/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl diff --git a/salt/sensoroni/files/analyzers/otx/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl b/salt/sensoroni/files/analyzers/pulsedive/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl similarity index 100% rename from salt/sensoroni/files/analyzers/otx/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl rename to salt/sensoroni/files/analyzers/pulsedive/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl diff --git a/salt/sensoroni/files/analyzers/pulsedive/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl b/salt/sensoroni/files/analyzers/spamhaus/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl similarity index 100% rename from salt/sensoroni/files/analyzers/pulsedive/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl rename to salt/sensoroni/files/analyzers/spamhaus/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl diff --git a/salt/sensoroni/files/analyzers/spamhaus/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl b/salt/sensoroni/files/analyzers/urlhaus/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl similarity index 100% rename from salt/sensoroni/files/analyzers/spamhaus/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl rename to salt/sensoroni/files/analyzers/urlhaus/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl diff --git a/salt/sensoroni/files/analyzers/urlhaus/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl b/salt/sensoroni/files/analyzers/urlscan/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl similarity index 100% rename from salt/sensoroni/files/analyzers/urlhaus/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl rename to salt/sensoroni/files/analyzers/urlscan/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl diff --git a/salt/sensoroni/files/analyzers/virustotal/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl b/salt/sensoroni/files/analyzers/virustotal/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl deleted file mode 100644 index d2b6c37f9..000000000 Binary files a/salt/sensoroni/files/analyzers/virustotal/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl and /dev/null differ diff --git a/salt/sensoroni/files/analyzers/urlscan/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl b/salt/sensoroni/files/analyzers/virustotal/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl similarity index 100% rename from salt/sensoroni/files/analyzers/urlscan/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl rename to salt/sensoroni/files/analyzers/virustotal/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl