Merge pull request #15024 from Security-Onion-Solutions/reyesj2/pypy

fix analyzers and upgrade deps

salt/sensoroni/defaults.yaml

@@ -34,6 +34,8 @@ sensoroni:
         api_version: community
       localfile:
         file_path: []
+      malwarebazaar:
+        api_key:
       otx:
         base_url: https://otx.alienvault.com/api/v1/
         api_key:
@@ -49,12 +51,16 @@ sensoroni:
         live_flow: False
         mailbox_email_address:
         message_source_id:
+      threatfox:
+        api_key:
       urlscan:
         base_url: https://urlscan.io/api/v1/
         api_key:
         enabled: False
         visibility: public
         timeout: 180
+      urlhaus:
+        api_key:
       virustotal:
         base_url: https://www.virustotal.com/api/v3/search?query=
         api_key:

salt/sensoroni/files/analyzers/README.md

@@ -35,15 +35,15 @@ Many analyzers require authentication, via an API key or similar. The table belo
 [EchoTrail](https://www.echotrail.io/docs/quickstart)            |✓|
 [EmailRep](https://emailrep.io/key)                  |✓|
 [Elasticsearch](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/setting-up-authentication.html)            |✓|
-[GreyNoise](https://www.greynoise.io/plans/community)                 |✓|
+[GreyNoise (community)](https://www.greynoise.io/plans/community)                 |✗|
 [LocalFile](https://github.com/Security-Onion-Solutions/securityonion/tree/fix/sublime_analyzer_documentation/salt/sensoroni/files/analyzers/localfile)                 |✗|
 [Malware Hash Registry](https://hash.cymru.com/docs_whois)    |✗|
-[MalwareBazaar](https://bazaar.abuse.ch/)            |✗|
+[MalwareBazaar](https://bazaar.abuse.ch/)            |✓|
 [Pulsedive](https://pulsedive.com/api/)                 |✓|
 [Spamhaus](https://www.spamhaus.org/dbl/)                  |✗|
 [Sublime Platform](https://sublime.security)              |✓| 
-[ThreatFox](https://threatfox.abuse.ch/)            |✗|
-[Urlhaus](https://urlhaus.abuse.ch/)                   |✗|
+[ThreatFox](https://threatfox.abuse.ch/)            |✓|
+[Urlhaus](https://urlhaus.abuse.ch/)                   |✓|
 [Urlscan](https://urlscan.io/docs/api/)                   |✓|
 [VirusTotal](https://developers.virustotal.com/reference/overview)                |✓|
 [WhoisLookup](https://github.com/meeb/whoisit)           |✗|

salt/sensoroni/files/analyzers/echotrail/README.md

@@ -1,24 +0,0 @@
-# EchoTrail
-
-
-## Description
-Submit a filename, hash, commandline to EchoTrail for analysis
-
-## Configuration Requirements
-
-In SOC, navigate to `Administration`, toggle `Show all configurable settings, including advanced settings.`, and navigate to `sensoroni` -> `analyzers` -> `echotrail`.
-![echotrail](https://github.com/Security-Onion-Solutions/securityonion/blob/2.4/dev/assets/images/screenshots/analyzers/echotrail.png?raw=true)
-
-
-The following configuration options are available for:
-
-``api_key`` - API key used for communication with the Echotrail API (Required)
-
-This value should be set in the ``sensoroni`` pillar, like so:
-
-```
-sensoroni:
-  analyzers:
-    echotrail:
-      api_key: $yourapikey
-```

salt/sensoroni/files/analyzers/echotrail/echotrail.json

@@ -1,10 +0,0 @@
-{
-    "name": "Echotrail",
-    "version": "0.1",
-    "author": "Security Onion Solutions",
-    "description": "This analyzer queries Echotrail to see if a related filename, hash, or commandline is considered malicious.",
-    "supportedTypes" :  ["filename","hash","commandline"],
-    "baseUrl": "https://api.echotrail.io/insights/"
-  }
-  
-    

salt/sensoroni/files/analyzers/echotrail/echotrail.py

@@ -1,67 +0,0 @@
-import json
-import os
-import sys
-import requests
-import helpers
-import argparse
-
-
-# for test usage:
-# python3 echotrail.py '{"artifactType":"hash", "value":"438b6ccd84f4dd32d9684ed7d58fd7d1e5a75fe3f3d12ab6c788e6bb0ffad5e7"}'
-# You will need to provide an API key in the .yaml file.
-def checkConfigRequirements(conf):
-    if not conf['api_key']:
-        sys.exit(126)
-    else:
-        return True
-
-
-def sendReq(conf, observ_value):
-    # send a get requests using a user-provided API key and the API url
-    url = conf['base_url'] + observ_value
-    headers = {'x-api-key': conf['api_key']}
-    response = requests.request('GET', url=url, headers=headers)
-    return response.json()
-
-
-def prepareResults(raw):
-    # checking for the 'filenames' key alone does
-    # not work when querying by filename.
-    # So, we can account for a hash query, a filename query,
-    # and anything else with these if statements.
-    if 'filenames' in raw.keys():
-        summary = raw['filenames'][0][0]
-    elif 'tags' in raw.keys():
-        summary = raw['tags'][0][0]
-    else:
-        summary = 'inconclusive'
-    status = 'info'
-    return {'response': raw, 'summary': summary, 'status': status}
-
-
-def analyze(conf, input):
-    # put all of our methods together and return a properly formatted output.
-    checkConfigRequirements(conf)
-    meta = helpers.loadMetadata(__file__)
-    data = helpers.parseArtifact(input)
-    helpers.checkSupportedType(meta, data['artifactType'])
-    response = sendReq(conf, data['value'])
-    return prepareResults(response)
-
-
-def main():
-    dir = os.path.dirname(os.path.realpath(__file__))
-    parser = argparse.ArgumentParser(
-        description='Search Echotrail for a given artifact')
-    parser.add_argument(
-        'artifact', help='the artifact represented in JSON format')
-    parser.add_argument('-c', '--config', metavar='CONFIG_FILE', default=dir + '/echotrail.yaml',
-                        help='optional config file to use instead of the default config file')
-    args = parser.parse_args()
-    if args.artifact:
-        results = analyze(helpers.loadConfig(args.config), args.artifact)
-        print(json.dumps(results))
-
-
-if __name__ == '__main__':
-    main()

salt/sensoroni/files/analyzers/echotrail/echotrail.yaml

@@ -1,3 +0,0 @@
-base_url: "{{ salt['pillar.get']('sensoroni:analyzers:echotrail:base_url', 'https://api.echotrail.io/insights/') }}"
-api_key: "{{ salt['pillar.get']('sensoroni:analyzers:echotrail:api_key', '') }}"
-

salt/sensoroni/files/analyzers/echotrail/echotrail_test.py

@@ -1,78 +0,0 @@
-from io import StringIO
-import sys
-from unittest.mock import patch, MagicMock
-import unittest
-import echotrail
-
-
-class TestEchoTrailMethods(unittest.TestCase):
-    def test_main_success(self):
-        with patch('sys.stdout', new=StringIO()) as mock_cmd:
-            with patch('echotrail.analyze', new=MagicMock(return_value={'test': 'val'})) as mock:
-                sys.argv = ["test", "test"]
-                echotrail.main()
-                expected = '{"test": "val"}\n'
-                self.assertEqual(mock_cmd.getvalue(), expected)
-                mock.assert_called_once()
-
-    def test_main_missing_input(self):
-        with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stderr', new=StringIO()) as mock_stderr:
-                sys.argv = ["cmd"]
-                echotrail.main()
-                self.assertEqual(mock_stderr.getvalue(), "usage: cmd [-h] [-c CONFIG_FILE] artifact\ncmd: error: the following arguments are required: artifact\n")
-                sysmock.assert_called_once()
-
-    def test_checkConfigRequirements(self):
-        conf = {'base_url': 'https://www.randurl.xyz/', 'api_key': ''}
-        with self.assertRaises(SystemExit) as cm:
-            echotrail.checkConfigRequirements(conf)
-        self.assertEqual(cm.exception.code, 126)
-
-    def test_sendReq(self):
-        with patch('requests.request', new=MagicMock(return_value=MagicMock())) as mock:
-            response = echotrail.sendReq(conf={'base_url': 'https://www.randurl.xyz/', 'api_key': 'randkey'}, observ_value='example_data')
-            self.assertIsNotNone(response)
-            mock.assert_called_once()
-
-    def test_prepareResults_noinput(self):
-        raw = {}
-        sim_results = {'response': raw,
-                       'status': 'info', 'summary': 'inconclusive'}
-        results = echotrail.prepareResults(raw)
-        self.assertEqual(results, sim_results)
-
-    def test_prepareResults_none(self):
-        raw = {'query_status': 'no_result'}
-        sim_results = {'response': raw,
-                       'status': 'info', 'summary': 'inconclusive'}
-        results = echotrail.prepareResults(raw)
-        self.assertEqual(results, sim_results)
-
-    def test_prepareResults_filenames(self):
-        raw = {'filenames': [["abc.exe", "def.exe"], ["abc.exe", "def.exe"]]}
-        sim_results = {'response': raw,
-                       'status': 'info', 'summary': 'abc.exe'}
-        results = echotrail.prepareResults(raw)
-        self.assertEqual(results, sim_results)
-
-    def test_prepareResults_tags(self):
-        raw = {'tags': [["tag1", "tag2"], ["tag1", "tag2"]]}
-        sim_results = {'response': raw,
-                       'status': 'info', 'summary': 'tag1'}
-        results = echotrail.prepareResults(raw)
-        self.assertEqual(results, sim_results)
-
-    def test_analyze(self):
-        sendReqOutput = {'threat': 'no_result'}
-        input = '{"artifactType":"hash", "value":"1234"}'
-        prepareResultOutput = {'response': '',
-                               'summary': 'inconclusive', 'status': 'info'}
-        conf = {"api_key": "xyz"}
-
-        with patch('echotrail.sendReq', new=MagicMock(return_value=sendReqOutput)) as mock:
-            with patch('echotrail.prepareResults', new=MagicMock(return_value=prepareResultOutput)) as mock2:
-                results = echotrail.analyze(conf, input)
-                self.assertEqual(results["summary"], "inconclusive")
-                mock2.assert_called_once()
-                mock.assert_called_once()

salt/sensoroni/files/analyzers/echotrail/requirements.txt

@@ -1,2 +0,0 @@
-requests>=2.31.0
-pyyaml>=6.0

salt/sensoroni/files/analyzers/greynoise/greynoise.json

@@ -1,6 +1,6 @@
 {
   "name": "Greynoise IP Analyzer",
-  "version": "0.1",
+  "version": "0.2",
   "author": "Security Onion Solutions",
   "description": "This analyzer queries Greynoise for context around an IP address",
   "supportedTypes" :  ["ip"]

salt/sensoroni/files/analyzers/greynoise/greynoise.py

@@ -7,6 +7,10 @@ import argparse
 
 
 def checkConfigRequirements(conf):
+    # Community API doesn't require API key
+    if conf.get('api_version') == 'community':
+        return True
+    # Other API versions require API key
     if "api_key" not in conf or len(conf['api_key']) == 0:
         sys.exit(126)
     else:
@@ -17,10 +21,12 @@ def sendReq(conf, meta, ip):
     url = conf['base_url']
     if conf['api_version'] == 'community':
         url = url + 'v3/community/' + ip
-    elif conf['api_version'] == 'investigate' or 'automate':
+        # Community API doesn't use API key
+        response = requests.request('GET', url=url)
+    elif conf['api_version'] in ['investigate', 'automate']:
         url = url + 'v2/noise/context/' + ip
-    headers = {"key": conf['api_key']}
-    response = requests.request('GET', url=url, headers=headers)
+        headers = {"key": conf['api_key']}
+        response = requests.request('GET', url=url, headers=headers)
     return response.json()
 
 

salt/sensoroni/files/analyzers/greynoise/greynoise_test.py

@@ -31,13 +31,31 @@ class TestGreynoiseMethods(unittest.TestCase):
             greynoise.checkConfigRequirements(conf)
         self.assertEqual(cm.exception.code, 126)
 
+    def test_checkConfigRequirements_community_no_key(self):
+        conf = {"api_version": "community"}
+        # Should not raise exception for community version
+        result = greynoise.checkConfigRequirements(conf)
+        self.assertTrue(result)
+
+    def test_checkConfigRequirements_investigate_no_key(self):
+        conf = {"api_version": "investigate"}
+        with self.assertRaises(SystemExit) as cm:
+            greynoise.checkConfigRequirements(conf)
+        self.assertEqual(cm.exception.code, 126)
+
+    def test_checkConfigRequirements_investigate_with_key(self):
+        conf = {"api_version": "investigate", "api_key": "test_key"}
+        result = greynoise.checkConfigRequirements(conf)
+        self.assertTrue(result)
+
     def test_sendReq_community(self):
         with patch('requests.request', new=MagicMock(return_value=MagicMock())) as mock:
             meta = {}
-            conf = {"base_url": "https://myurl/", "api_key": "abcd1234", "api_version": "community"}
+            conf = {"base_url": "https://myurl/", "api_version": "community"}
             ip = "192.168.1.1"
             response = greynoise.sendReq(conf=conf, meta=meta, ip=ip)
-            mock.assert_called_once_with("GET", headers={'key': 'abcd1234'}, url="https://myurl/v3/community/192.168.1.1")
+            # Community API should not include headers
+            mock.assert_called_once_with("GET", url="https://myurl/v3/community/192.168.1.1")
             self.assertIsNotNone(response)
 
     def test_sendReq_investigate(self):
@@ -115,3 +133,16 @@ class TestGreynoiseMethods(unittest.TestCase):
             results = greynoise.analyze(conf, artifactInput)
             self.assertEqual(results["summary"], "suspicious")
             mock.assert_called_once()
+
+    def test_analyze_community_no_key(self):
+        output = {"ip": "8.8.8.8", "noise": "false", "riot": "true",
+                  "classification": "benign", "name": "Google Public DNS",
+                  "link": "https://viz.gn.io", "last_seen": "2022-04-26",
+                  "message": "Success"}
+        artifactInput = '{"value":"8.8.8.8","artifactType":"ip"}'
+        conf = {"base_url": "myurl/", "api_version": "community"}
+        with patch('greynoise.greynoise.sendReq', new=MagicMock(return_value=output)) as mock:
+            results = greynoise.analyze(conf, artifactInput)
+            self.assertEqual(results["summary"], "harmless")
+            self.assertEqual(results["status"], "ok")
+            mock.assert_called_once()

salt/sensoroni/files/analyzers/malwarebazaar/malwarebazaar.json

@@ -1,6 +1,6 @@
 {
     "name": "Malwarebazaar",
-    "version": "0.1",
+    "version": "0.2",
     "author": "Security Onion Solutions",
     "description": "This analyzer queries Malwarebazaar to see if a hash, gimphash, tlsh, or telfhash is considered malicious.",
     "supportedTypes" :  ["gimphash","hash","tlsh", "telfhash"],

salt/sensoroni/files/analyzers/malwarebazaar/malwarebazaar.py

@@ -2,12 +2,21 @@ import requests
 import helpers
 import json
 import sys
+import os
+import argparse
 
 # supports querying for hash, gimphash, tlsh, and telfhash
 # usage is as follows:
 # python3 malwarebazaar.py '{"artifactType":"x", "value":"y"}'
 
 
+def checkConfigRequirements(conf):
+    if not conf.get('api_key'):
+        sys.exit(126)
+    else:
+        return True
+
+
 def buildReq(observ_type, observ_value):
     # determine correct query type to send based off of observable type
     unique_types = {'gimphash': 1, 'telfhash': 1, 'tlsh': 1}
@@ -18,10 +27,13 @@ def buildReq(observ_type, observ_value):
     return {'query': qtype, observ_type: observ_value}
 
 
-def sendReq(meta, query):
+def sendReq(conf, meta, query):
     # send a post request with our compiled query to the API
     url = meta['baseUrl']
-    response = requests.post(url, query)
+    headers = {}
+    if conf.get('api_key'):
+        headers['Auth-Key'] = conf['api_key']
+    response = requests.post(url, query, headers=headers)
     return response.json()
 
 
@@ -113,10 +125,11 @@ def prepareResults(raw):
     return {'response': raw, 'summary': summary, 'status': status}
 
 
-def analyze(input):
+def analyze(conf, input):
     # put all of our methods together, pass them input, and return
     # properly formatted json/python dict output
-    data = json.loads(input)
+    checkConfigRequirements(conf)
+    data = helpers.parseArtifact(input)
     meta = helpers.loadMetadata(__file__)
     helpers.checkSupportedType(meta, data["artifactType"])
 
@@ -127,7 +140,7 @@ def analyze(input):
         # twice for the sake of retrieving more specific data.
 
         initialQuery = buildReq(data['artifactType'], data['value'])
-        initialRaw = sendReq(meta, initialQuery)
+        initialRaw = sendReq(conf, meta, initialQuery)
 
         # To prevent double-querying when a tlsh/gimphash is invalid,
         # this if statement is necessary.
@@ -140,16 +153,22 @@ def analyze(input):
             return prepareResults(initialRaw)
 
     query = buildReq(data['artifactType'], data['value'])
-    response = sendReq(meta, query)
+    response = sendReq(conf, meta, query)
     return prepareResults(response)
 
 
 def main():
-    if len(sys.argv) == 2:
-        results = analyze(sys.argv[1])
+    dir = os.path.dirname(os.path.realpath(__file__))
+    parser = argparse.ArgumentParser(
+        description='Search MalwareBazaar for a given artifact')
+    parser.add_argument(
+        'artifact', help='the artifact represented in JSON format')
+    parser.add_argument('-c', '--config', metavar='CONFIG_FILE', default=dir + '/malwarebazaar.yaml',
+                        help='optional config file to use instead of the default config file')
+    args = parser.parse_args()
+    if args.artifact:
+        results = analyze(helpers.loadConfig(args.config), args.artifact)
         print(json.dumps(results))
-    else:
-        print("ERROR: Input is not in proper JSON format")
 
 
 if __name__ == '__main__':

salt/sensoroni/files/analyzers/malwarebazaar/malwarebazaar.yaml

@@ -0,0 +1 @@
+api_key: "{{ salt['pillar.get']('sensoroni:analyzers:malwarebazaar:api_key', '') }}"

salt/sensoroni/files/analyzers/malwarebazaar/malwarebazaar_test.py

@@ -6,22 +6,18 @@ import unittest
 
 
 class TestMalwarebazaarMethods(unittest.TestCase):
-    def test_main_missing_input(self):
-        with patch('sys.stdout', new=StringIO()) as mock_cmd:
-            sys.argv = ["cmd"]
-            malwarebazaar.main()
-            self.assertEqual(mock_cmd.getvalue(),
-                             'ERROR: Input is not in proper JSON format\n')
-
     def test_main_success(self):
-        with patch('sys.stdout', new=StringIO()) as mock_cmd:
-            with patch('malwarebazaar.malwarebazaar.analyze',
-                       new=MagicMock(return_value={'test': 'val'})) as mock:
-                sys.argv = ["cmd", "input"]
-                malwarebazaar.main()
-                expected = '{"test": "val"}\n'
-                self.assertEqual(mock_cmd.getvalue(), expected)
-                mock.assert_called_once()
+        output = {"test": "val"}
+        config = {"api_key": "test_key"}
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            with patch('malwarebazaar.malwarebazaar.analyze', new=MagicMock(return_value=output)) as mock_analyze:
+                with patch('helpers.loadConfig', new=MagicMock(return_value=config)) as mock_config:
+                    sys.argv = ["cmd", "input"]
+                    malwarebazaar.main()
+                    expected = '{"test": "val"}\n'
+                    self.assertEqual(mock_stdout.getvalue(), expected)
+                    mock_analyze.assert_called_once()
+                    mock_config.assert_called_once()
 
     def test_isInJson_tail_greater_than_max_depth(self):
         max_depth = 1000
@@ -84,6 +80,7 @@ class TestMalwarebazaarMethods(unittest.TestCase):
             and then we compared results['summary'] with 'no result' """
         sendReqOutput = {'threat': 'no_result', "query_status": "ok",
                          'data': [{'sha256_hash': 'notavalidhash'}]}
+        config = {"api_key": "test_key"}
         input = '{"artifactType": "hash", "value": "1234"}'
         input2 = '{"artifactType": "tlsh", "value": "1234"}'
         input3 = '{"artifactType": "gimphash", "value": "1234"}'
@@ -94,9 +91,9 @@ class TestMalwarebazaarMethods(unittest.TestCase):
                    new=MagicMock(return_value=sendReqOutput)) as mock:
             with patch('malwarebazaar.malwarebazaar.prepareResults',
                        new=MagicMock(return_value=prep_res_sim)) as mock2:
-                results = malwarebazaar.analyze(input)
-                results2 = malwarebazaar.analyze(input2)
-                results3 = malwarebazaar.analyze(input3)
+                results = malwarebazaar.analyze(config, input)
+                results2 = malwarebazaar.analyze(config, input2)
+                results3 = malwarebazaar.analyze(config, input3)
                 self.assertEqual(results["summary"], prep_res_sim['summary'])
                 self.assertEqual(results2["summary"], prep_res_sim['summary'])
                 self.assertEqual(results3["summary"], prep_res_sim['summary'])
@@ -113,6 +110,7 @@ class TestMalwarebazaarMethods(unittest.TestCase):
             and then we compared results['summary'] with 'no result' """
         sendReqOutput = {'threat': 'threat', "query_status": "notok", 'data': [
             {'sha256_hash': 'validhash'}]}
+        config = {"api_key": "test_key"}
         input = '{"artifactType": "hash", "value": "1234"}'
         input2 = '{"artifactType": "tlsh", "value": "1234"}'
         input3 = '{"artifactType": "gimphash", "value": "1234"}'
@@ -123,9 +121,9 @@ class TestMalwarebazaarMethods(unittest.TestCase):
                    new=MagicMock(return_value=sendReqOutput)) as mock:
             with patch('malwarebazaar.malwarebazaar.prepareResults',
                        new=MagicMock(return_value=prep_res_sim)) as mock2:
-                results = malwarebazaar.analyze(input)
-                results2 = malwarebazaar.analyze(input2)
-                results3 = malwarebazaar.analyze(input3)
+                results = malwarebazaar.analyze(config, input)
+                results2 = malwarebazaar.analyze(config, input2)
+                results3 = malwarebazaar.analyze(config, input3)
                 self.assertEqual(results["summary"], prep_res_sim['summary'])
                 self.assertEqual(results2["summary"], prep_res_sim['summary'])
                 self.assertEqual(results3["summary"], prep_res_sim['summary'])
@@ -239,7 +237,18 @@ class TestMalwarebazaarMethods(unittest.TestCase):
     def test_sendReq(self):
         with patch('requests.post',
                    new=MagicMock(return_value=MagicMock())) as mock:
+            conf = {"api_key": "test_key"}
             response = malwarebazaar.sendReq(
-                {'baseUrl': 'https://www.randurl.xyz'}, 'example_data')
+                conf, {'baseUrl': 'https://www.randurl.xyz'}, 'example_data')
             self.assertIsNotNone(response)
             mock.assert_called_once()
+
+    def test_checkConfigRequirements_valid(self):
+        config = {"api_key": "test_key"}
+        self.assertTrue(malwarebazaar.checkConfigRequirements(config))
+
+    def test_checkConfigRequirements_missing_key(self):
+        config = {}
+        with self.assertRaises(SystemExit) as cm:
+            malwarebazaar.checkConfigRequirements(config)
+        self.assertEqual(cm.exception.code, 126)