urlhaus dep upgrades + update to use authenticated abusech api

salt/sensoroni/defaults.yaml

@@ -55,6 +55,8 @@ sensoroni:
         enabled: False
         visibility: public
         timeout: 180
+      urlhaus:
+        api_key:
       virustotal:
         base_url: https://www.virustotal.com/api/v3/search?query=
         api_key:

salt/sensoroni/files/analyzers/README.md

@@ -43,7 +43,7 @@ Many analyzers require authentication, via an API key or similar. The table belo
 [Spamhaus](https://www.spamhaus.org/dbl/)                  |✗|
 [Sublime Platform](https://sublime.security)              |✓| 
 [ThreatFox](https://threatfox.abuse.ch/)            |✗|
-[Urlhaus](https://urlhaus.abuse.ch/)                   |✗|
+[Urlhaus](https://urlhaus.abuse.ch/)                   |✓|
 [Urlscan](https://urlscan.io/docs/api/)                   |✓|
 [VirusTotal](https://developers.virustotal.com/reference/overview)                |✓|
 [WhoisLookup](https://github.com/meeb/whoisit)           |✗|

salt/sensoroni/files/analyzers/urlhaus/urlhaus.json

@@ -1,6 +1,6 @@
 {
   "name": "Urlhaus",
-  "version": "0.1",
+  "version": "0.2",
   "author": "Security Onion Solutions",
   "description": "This analyzer queries URLHaus to see if a URL is considered malicious.",
   "supportedTypes" :  ["url"],

salt/sensoroni/files/analyzers/urlhaus/urlhaus.py

@@ -1,16 +1,28 @@
 import json
+import os
 import requests
 import sys
 import helpers
+import argparse
+
+
+def checkConfigRequirements(conf):
+    if not conf.get('api_key'):
+        sys.exit(126)
+    else:
+        return True
 
 
 def buildReq(artifact_value):
     return {"url": artifact_value}
 
 
-def sendReq(meta, payload):
+def sendReq(conf, meta, payload):
     url = meta['baseUrl']
-    response = requests.request('POST', url, data=payload)
+    headers = {}
+    if conf.get('api_key'):
+        headers['Auth-Key'] = conf['api_key']
+    response = requests.request('POST', url, data=payload, headers=headers)
     return response.json()
 
 
@@ -31,21 +43,28 @@ def prepareResults(raw):
     return results
 
 
-def analyze(input):
+def analyze(conf, input):
+    checkConfigRequirements(conf)
     meta = helpers.loadMetadata(__file__)
     data = helpers.parseArtifact(input)
     helpers.checkSupportedType(meta, data["artifactType"])
     payload = buildReq(data["value"])
-    response = sendReq(meta, payload)
+    response = sendReq(conf, meta, payload)
     return prepareResults(response)
 
 
 def main():
-    if len(sys.argv) == 2:
-        results = analyze(sys.argv[1])
+    dir = os.path.dirname(os.path.realpath(__file__))
+    parser = argparse.ArgumentParser(
+        description='Search URLhaus for a given artifact')
+    parser.add_argument(
+        'artifact', help='the artifact represented in JSON format')
+    parser.add_argument('-c', '--config', metavar='CONFIG_FILE', default=dir + '/urlhaus.yaml',
+                        help='optional config file to use instead of the default config file')
+    args = parser.parse_args()
+    if args.artifact:
+        results = analyze(helpers.loadConfig(args.config), args.artifact)
         print(json.dumps(results))
-    else:
-        print("ERROR: Missing input JSON")
 
 
 if __name__ == "__main__":

salt/sensoroni/files/analyzers/urlhaus/urlhaus.yaml

@@ -0,0 +1 @@
+api_key: "{{ salt['pillar.get']('sensoroni:analyzers:urlhaus:api_key', '') }}"

salt/sensoroni/files/analyzers/urlhaus/urlhaus_test.py

@@ -1,27 +1,24 @@
 from io import StringIO
 import sys
 from unittest.mock import patch, MagicMock
-from urlhaus import urlhaus
 import unittest
+from urlhaus import urlhaus
 
 
 class TestUrlhausMethods(unittest.TestCase):
 
-    def test_main_missing_input(self):
-        with patch('sys.stdout', new=StringIO()) as mock_stdout:
-            sys.argv = ["cmd"]
-            urlhaus.main()
-            self.assertEqual(mock_stdout.getvalue(), "ERROR: Missing input JSON\n")
-
     def test_main_success(self):
         output = {"foo": "bar"}
+        config = {"api_key": "test_key"}
         with patch('sys.stdout', new=StringIO()) as mock_stdout:
-            with patch('urlhaus.urlhaus.analyze', new=MagicMock(return_value=output)) as mock:
-                sys.argv = ["cmd", "input"]
-                urlhaus.main()
-                expected = '{"foo": "bar"}\n'
-                self.assertEqual(mock_stdout.getvalue(), expected)
-                mock.assert_called_once()
+            with patch('urlhaus.urlhaus.analyze', new=MagicMock(return_value=output)) as mock_analyze:
+                with patch('helpers.loadConfig', new=MagicMock(return_value=config)) as mock_config:
+                    sys.argv = ["cmd", "input"]
+                    urlhaus.main()
+                    expected = '{"foo": "bar"}\n'
+                    self.assertEqual(mock_stdout.getvalue(), expected)
+                    mock_analyze.assert_called_once()
+                    mock_config.assert_called_once()
 
     def test_buildReq(self):
         result = urlhaus.buildReq("test")
@@ -29,9 +26,10 @@ class TestUrlhausMethods(unittest.TestCase):
 
     def test_sendReq(self):
         with patch('requests.request', new=MagicMock(return_value=MagicMock())) as mock:
+            conf = {"api_key": "test_key"}
             meta = {"baseUrl": "myurl"}
-            response = urlhaus.sendReq(meta, "mypayload")
-            mock.assert_called_once_with("POST", "myurl", data="mypayload")
+            response = urlhaus.sendReq(conf, meta, "mypayload")
+            mock.assert_called_once_with("POST", "myurl", data="mypayload", headers={"Auth-Key": "test_key"})
             self.assertIsNotNone(response)
 
     def test_prepareResults_none(self):
@@ -65,8 +63,19 @@ class TestUrlhausMethods(unittest.TestCase):
 
     def test_analyze(self):
         output = {"threat": "malware_download"}
+        config = {"api_key": "test_key"}
         artifactInput = '{"value":"foo","artifactType":"url"}'
         with patch('urlhaus.urlhaus.sendReq', new=MagicMock(return_value=output)) as mock:
-            results = urlhaus.analyze(artifactInput)
+            results = urlhaus.analyze(config, artifactInput)
             self.assertEqual(results["summary"], "malware_download")
             mock.assert_called_once()
+
+    def test_checkConfigRequirements_valid(self):
+        config = {"api_key": "test_key"}
+        self.assertTrue(urlhaus.checkConfigRequirements(config))
+
+    def test_checkConfigRequirements_missing_key(self):
+        config = {}
+        with self.assertRaises(SystemExit) as cm:
+            urlhaus.checkConfigRequirements(config)
+        self.assertEqual(cm.exception.code, 126)

salt/sensoroni/soc_sensoroni.yaml

@@ -291,6 +291,14 @@ sensoroni:
         sensitive: False
         advanced: True
         forcedType: string
+    urlhaus:
+      api_key:
+        description: API key for the urlhaus analyzer.
+        helpLink: sensoroni.html
+        global: False
+        sensitive: True
+        advanced: False
+        forcedType: string
     virustotal:
       api_key:
         description: API key for the VirusTotal analyzer.