diff --git a/salt/sensoroni/files/analyzers/urlscan/__init__.py b/salt/sensoroni/files/analyzers/urlscan/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/salt/sensoroni/files/analyzers/urlscan/requirements.txt b/salt/sensoroni/files/analyzers/urlscan/requirements.txt new file mode 100644 index 000000000..a8980057f --- /dev/null +++ b/salt/sensoroni/files/analyzers/urlscan/requirements.txt @@ -0,0 +1,2 @@ +requests>=2.27.1 +pyyaml>=6.0 diff --git a/salt/sensoroni/files/analyzers/urlscan/urlscan.json b/salt/sensoroni/files/analyzers/urlscan/urlscan.json new file mode 100644 index 000000000..75e85bdbf --- /dev/null +++ b/salt/sensoroni/files/analyzers/urlscan/urlscan.json @@ -0,0 +1,7 @@ +{ + "name": "Urlscan", + "version": "0.1", + "author": "Security Onion Solutions", + "description": "This analyzer submits a URL to Urlscan for context around an observable.", + "supportedTypes" : ["url"] +} diff --git a/salt/sensoroni/files/analyzers/urlscan/urlscan.py b/salt/sensoroni/files/analyzers/urlscan/urlscan.py new file mode 100755 index 000000000..94c3ec8db --- /dev/null +++ b/salt/sensoroni/files/analyzers/urlscan/urlscan.py @@ -0,0 +1,91 @@ +import json +import requests +import helpers +import sys +import os +import argparse +import time + + +def checkConfigRequirements(conf): + if "enabled" in conf: + if "api_key" not in conf or len(conf['api_key']) == 0: + sys.exit(126) + else: + return True + else: + sys.exit(126) + + +def buildReq(conf, artifact_type, artifact_value): + headers = {"API-Key": conf["api_key"]} + url = conf['base_url'] + 'scan/' + visibility = conf['visibility'] + data = {"url": artifact_value, "visibility": visibility} + return url, headers, data + + +def getReport(conf, report_url): + report = requests.request('GET', report_url) + timeout = conf.get('timeout', 300) + counter = 0 + while report.status_code == 404: + time.sleep(2) + counter += 2 + if counter >= timeout: + break + report = requests.request('GET', report_url) + return report + + +def sendReq(url, headers, data): + submission = requests.request('POST', url=url, headers=headers, data=data).json() + report_url = submission['api'] + return report_url + + +def prepareResults(raw): + if raw and "verdicts" in raw: + if raw["verdicts"]["overall"]["malicious"] is True: + status = "threat" + summary = "malicious" + elif raw["verdicts"]["overall"]["score"] > 0: + status = "caution" + summary = "suspicious" + else: + status = "info" + summary = "Scan complete." + else: + status = "caution" + summary = "internal_failure" + + results = {'response': raw, 'status': status, 'summary': summary} + return results + + +def analyze(conf, input): + checkConfigRequirements(conf) + meta = helpers.loadMetadata(__file__) + data = helpers.parseArtifact(input) + helpers.checkSupportedType(meta, data["artifactType"]) + request = buildReq(conf, data["artifactType"], data["value"]) + report_url = sendReq(request[0], request[1], request[2]) + time.sleep(10) + report = getReport(conf, report_url) + return prepareResults(report.json()) + + +def main(): + dir = os.path.dirname(os.path.realpath(__file__)) + parser = argparse.ArgumentParser(description='Search Alienvault OTX for a given artifact') + parser.add_argument('artifact', help='the artifact represented in JSON format') + parser.add_argument('-c', '--config', metavar="CONFIG_FILE", default=dir + "/urlscan.yaml", help='optional config file to use instead of the default config file') + + args = parser.parse_args() + if args.artifact: + results = analyze(helpers.loadConfig(args.config), args.artifact) + print(json.dumps(results)) + + +if __name__ == "__main__": + main() diff --git a/salt/sensoroni/files/analyzers/urlscan/urlscan.yaml b/salt/sensoroni/files/analyzers/urlscan/urlscan.yaml new file mode 100644 index 000000000..986a61359 --- /dev/null +++ b/salt/sensoroni/files/analyzers/urlscan/urlscan.yaml @@ -0,0 +1,5 @@ +base_url: https://urlscan.io/api/v1/ +api_key: "{{ salt['pillar.get']('sensoroni:analyzers:urlscan:api_key', '') }}" +enabled: "{{ salt['pillar.get']('sensoroni:analyzers:urlscan:enabled', 'False') }}" +visibility: "{{ salt['pillar.get']('sensoroni:analyzers:urlscan:visibility', 'public') }}" +timeout: "{{ salt['pillar.get']('sensoroni:analyzers:urlscan:visibility', '180') }}" diff --git a/salt/sensoroni/files/analyzers/urlscan/urlscan_test.py b/salt/sensoroni/files/analyzers/urlscan/urlscan_test.py new file mode 100644 index 000000000..487e6dbe3 --- /dev/null +++ b/salt/sensoroni/files/analyzers/urlscan/urlscan_test.py @@ -0,0 +1,121 @@ +from io import StringIO +import sys +from unittest.mock import patch, MagicMock, PropertyMock, call +from urlscan import urlscan +import unittest + + +class TestUrlScanMethods(unittest.TestCase): + + def test_main_missing_input(self): + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stderr', new=StringIO()) as mock_stderr: + sys.argv = ["cmd"] + urlscan.main() + self.assertEqual(mock_stderr.getvalue(), "usage: cmd [-h] [-c CONFIG_FILE] artifact\ncmd: error: the following arguments are required: artifact\n") + sysmock.assert_called_once_with(2) + + def test_main_success(self): + output = {"foo": "bar"} + with patch('sys.stdout', new=StringIO()) as mock_stdout: + with patch('urlscan.urlscan.analyze', new=MagicMock(return_value=output)) as mock: + sys.argv = ["cmd", "input"] + urlscan.main() + expected = '{"foo": "bar"}\n' + self.assertEqual(mock_stdout.getvalue(), expected) + mock.assert_called_once() + + def test_checkConfigRequirements_notEnabled(self): + conf = {"not_a_key": "abcd12345"} + with self.assertRaises(SystemExit) as cm: + urlscan.checkConfigRequirements(conf) + self.assertEqual(cm.exception.code, 126) + + def test_checkConfigRequirements_noApikey(self): + conf = {"enabled": True, "not_a_key": "abcd12345"} + with self.assertRaises(SystemExit) as cm: + urlscan.checkConfigRequirements(conf) + self.assertEqual(cm.exception.code, 126) + + def test_checkConfigRequirements_Exist(self): + conf = {"enabled": True, "api_key": "abcd12345"} + config_exists = urlscan.checkConfigRequirements(conf) + self.assertTrue(config_exists) + + def test_buildReq(self): + conf = {'base_url': 'https://myurl/api/v1/', 'api_key': 'abcd12345', 'visibility': 'public'} + artifact_type = "url" + artifact_value = "https://abc.com" + result = urlscan.buildReq(conf, artifact_type, artifact_value) + self.assertEqual("https://myurl/api/v1/scan/", result[0]) + self.assertEqual({'API-Key': 'abcd12345'}, result[1]) + + def test_sendReq(self): + with patch('requests.request', new=MagicMock(return_value=MagicMock())) as mock: + headers = {"API-Key": "abcd1234"} + data = {"url": "https://urlscan.io", "visibility": "public"} + response = urlscan.sendReq("https://myurl", headers=headers, data=data) + mock.assert_called_once_with("POST", url="https://myurl", headers={"API-Key": "abcd1234"}, data={"url": "https://urlscan.io", "visibility": "public"}) + self.assertIsNotNone(response) + + def test_getReport_noRetry(self): + output_report = MagicMock() + type(output_report).status_code = PropertyMock(return_value=404) + output_report_body = {"requests": "body"} + output_report.json.return_value = output_report_body + with patch('requests.request', new=MagicMock(return_value=output_report)) as mock: + result = urlscan.getReport({'timeout': 0}, "https://abc.com/report") + self.assertEqual(404, result.status_code) + mock.assert_called_once() + + def test_getReport_withRetry(self): + output_report = MagicMock() + type(output_report).status_code = PropertyMock(return_value=404) + output_report_body = {"requests": "body"} + output_report.json.return_value = output_report_body + with patch('requests.request', new=MagicMock(return_value=output_report)) as mock: + result = urlscan.getReport({'timeout': 3}, "https://abc.com/report") + self.assertEqual(404, result.status_code) + mock.assert_has_calls([call('GET', 'https://abc.com/report'), call('GET', 'https://abc.com/report')]) + + def test_prepareResults_sus(self): + raw = {"requests": [{"request": {"requestId": "1"}}], "verdicts": {"overall": {"score": 50, "malicious": False, "hasVerdicts": False}}} + results = urlscan.prepareResults(raw) + self.assertEqual(results["response"], raw) + self.assertEqual(results["summary"], "suspicious") + self.assertEqual(results["status"], "caution") + + def test_prepareResults_mal(self): + raw = {"requests": [{"request": {"requestId": "2"}}], "verdicts": {"overall": {"score": 100, "malicious": True, "hasVerdicts": False}}} + results = urlscan.prepareResults(raw) + self.assertEqual(results["response"], raw) + self.assertEqual(results["summary"], "malicious") + self.assertEqual(results["status"], "threat") + + def test_prepareResults_info(self): + raw = {"requests": [{"request": {"requestId": "3"}}], "verdicts": {"overall": {"score": 0, "malicious": False, "hasVerdicts": False}}} + results = urlscan.prepareResults(raw) + self.assertEqual(results["response"], raw) + self.assertEqual(results["summary"], "Scan complete.") + self.assertEqual(results["status"], "info") + + def test_prepareResults_error(self): + raw = {} + results = urlscan.prepareResults(raw) + self.assertEqual(results["response"], raw) + self.assertEqual(results["summary"], "internal_failure") + self.assertEqual(results["status"], "caution") + + def test_analyze(self): + output_req = "https://myurl/report" + output_report = MagicMock() + output_report_body = {"requests": [{"request": {"requestId": "3"}}], "verdicts": {"overall": {"score": 0, "malicious": False, "hasVerdicts": False}}} + output_report.json.return_value = output_report_body + artifactInput = '{"value":"https://abc.com","artifactType":"url"}' + conf = {'enabled': True, 'base_url': 'https://myurl/api/v1/', 'api_key': 'abcd12345', 'visibility': 'public'} + with patch('urlscan.urlscan.sendReq', new=MagicMock(return_value=output_req)) as mock_req: + with patch('urlscan.urlscan.getReport', new=MagicMock(return_value=output_report)) as mock_report: + results = urlscan.analyze(conf, artifactInput) + self.assertEqual(results["summary"], "Scan complete.") + mock_req.assert_called_once() + mock_report.assert_called_once()