Merge remote-tracking branch 'remotes/origin/2.4/dev' into patch2.4

salt/sensoroni/files/analyzers/README.md

@@ -5,20 +5,19 @@ Security Onion provides a means for performing data analysis on varying inputs.
 ## Supported Observable Types
 The built-in analyzers support the following observable types:
 
-| Name                    | Domain | Hash  | IP    | JA3   | Mail  | Other | URI   |  URL  | User Agent |
-| ------------------------|--------|-------|-------|-------|-------|-------|-------|-------|------------
-| Alienvault OTX          |✓ |✓|✓|✗|✗|✗|✗|✓|✗|
-| EmailRep                |✗ |✗|✗|✗|✓|✗|✗|✗|✗|
-| Greynoise               |✗ |✗|✓|✗|✗|✗|✗|✗|✗|
-| JA3er                   |✗ |✗|✗|✓|✗|✗|✗|✗|✗|
-| LocalFile               |✓ |✓|✓|✓|✗|✓|✗|✓|✗|
-| Malware Hash Registry   |✗ |✓|✗|✗|✗|✗|✗|✓|✗|
-| Pulsedive               |✓ |✓|✓|✗|✗|✗|✓|✓|✓|
-| Spamhaus                |✗ |✗|✓|✗|✗|✗|✗|✗|✗|
-| Urlhaus                 |✗ |✗|✗|✗|✗|✗|✗|✓|✗|
-| Urlscan                 |✗ |✗|✗|✗|✗|✗|✗|✓|✗|
-| Virustotal              |✓ |✓|✓|✗|✗|✗|✗|✓|✗|
-| WhoisLookup             |✓ |✗|✗|✗|✗|✗|✓|✗|✗|
+| Name                    | Domain | Hash  | IP    | Mail  | Other | URI   |  URL  | User Agent |
+| ------------------------|--------|-------|-------|-------|-------|-------|-------|-------|
+| Alienvault OTX          |✓ |✓|✓|✗|✗|✗|✓|✗|
+| EmailRep                |✗ |✗|✗|✓|✗|✗|✗|✗|
+| Greynoise               |✗ |✗|✓|✗|✗|✗|✗|✗|
+| LocalFile               |✓ |✓|✓|✗|✓|✗|✓|✗|
+| Malware Hash Registry   |✗ |✓|✗|✗|✗|✗|✓|✗|
+| Pulsedive               |✓ |✓|✓|✗|✗|✓|✓|✓|
+| Spamhaus                |✗ |✗|✓|✗|✗|✗|✗|✗|
+| Urlhaus                 |✗ |✗|✗|✗|✗|✗|✓|✗|
+| Urlscan                 |✗ |✗|✗|✗|✗|✗|✓|✗|
+| Virustotal              |✓ |✓|✓|✗|✗|✗|✓|✗|
+| WhoisLookup             |✓ |✗|✗|✗|✗|✓|✗|✗|
 
 ## Authentication
 Many analyzers require authentication, via an API key or similar. The table below illustrates which analyzers require authentication.
@@ -28,7 +27,6 @@ Many analyzers require authentication, via an API key or similar. The table belo
 [AlienVault OTX](https://otx.alienvault.com/api)            |✓|
 [EmailRep](https://emailrep.io/key)                  |✓|
 [GreyNoise](https://www.greynoise.io/plans/community)                 |✓|
-[JA3er](https://ja3er.com/)                     |✗|
 LocalFile                 |✗|
 [Malware Hash Registry](https://hash.cymru.com/docs_whois)    |✗|
 [Pulsedive](https://pulsedive.com/api/)                 |✓|
@@ -265,5 +263,3 @@ The following requirements must be satisfied in order for analyzer pull requests
 - All source code must include accompanying unit test coverage. The Security Onion project will automatically run the unit tests after each push to a `securityonion` repository fork, and again when submitting a pull request. Failed unit tests, or insufficient unit test coverage, will result in the submitter being sent an automated email message.
 - Documentation of the analyzer, its input requirements, conditions for operation, and other relevant information must be clearly written in an accompanying analyzer metadata file. This file is described in more detail earlier in this document.
 - Source code must be well-written and be free of security defects that can put users or their data at unnecessary risk.
-
-

salt/sensoroni/files/analyzers/ja3er/ja3er.json

@@ -1,7 +0,0 @@
-{
-  "name": "JA3er Hash Search",
-  "version": "0.1",
-  "author": "Security Onion Solutions",
-  "description": "This analyzer queries JA3er user agents and sightings",
-  "supportedTypes" :  ["ja3"]
-}

salt/sensoroni/files/analyzers/ja3er/ja3er.py

@@ -1,53 +0,0 @@
-import json
-import os
-import requests
-import helpers
-import argparse
-
-
-def sendReq(conf, meta, hash):
-    url = conf['base_url'] + hash
-    response = requests.request('GET', url)
-    return response.json()
-
-
-def prepareResults(raw):
-    if "error" in raw:
-        if "Sorry" in raw["error"]:
-            status = "ok"
-            summary = "no_results"
-        elif "Invalid hash" in raw["error"]:
-            status = "caution"
-            summary = "invalid_input"
-        else:
-            status = "caution"
-            summary = "internal_failure"
-    else:
-        status = "info"
-        summary = "suspicious"
-    results = {'response': raw, 'summary': summary, 'status': status}
-    return results
-
-
-def analyze(conf, input):
-    meta = helpers.loadMetadata(__file__)
-    data = helpers.parseArtifact(input)
-    helpers.checkSupportedType(meta, data["artifactType"])
-    response = sendReq(conf, meta, data["value"])
-    return prepareResults(response)
-
-
-def main():
-    dir = os.path.dirname(os.path.realpath(__file__))
-    parser = argparse.ArgumentParser(description='Search JA3er for a given artifact')
-    parser.add_argument('artifact', help='the artifact represented in JSON format')
-    parser.add_argument('-c', '--config', metavar="CONFIG_FILE", default=dir + "/ja3er.yaml", help='optional config file to use instead of the default config file')
-
-    args = parser.parse_args()
-    if args.artifact:
-        results = analyze(helpers.loadConfig(args.config), args.artifact)
-        print(json.dumps(results))
-
-
-if __name__ == "__main__":
-    main()

salt/sensoroni/files/analyzers/ja3er/ja3er.yaml

@@ -1 +0,0 @@
-base_url: https://ja3er.com/search/

salt/sensoroni/files/analyzers/ja3er/ja3er_test.py

@@ -1,72 +0,0 @@
-from io import StringIO
-import sys
-from unittest.mock import patch, MagicMock
-from ja3er import ja3er
-import unittest
-
-
-class TestJa3erMethods(unittest.TestCase):
-
-    def test_main_missing_input(self):
-        with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stderr', new=StringIO()) as mock_stderr:
-                sys.argv = ["cmd"]
-                ja3er.main()
-                self.assertEqual(mock_stderr.getvalue(), "usage: cmd [-h] [-c CONFIG_FILE] artifact\ncmd: error: the following arguments are required: artifact\n")
-                sysmock.assert_called_once_with(2)
-
-    def test_main_success(self):
-        output = {"foo": "bar"}
-        with patch('sys.stdout', new=StringIO()) as mock_stdout:
-            with patch('ja3er.ja3er.analyze', new=MagicMock(return_value=output)) as mock:
-                sys.argv = ["cmd", "input"]
-                ja3er.main()
-                expected = '{"foo": "bar"}\n'
-                self.assertEqual(mock_stdout.getvalue(), expected)
-                mock.assert_called_once()
-
-    def test_sendReq(self):
-        with patch('requests.request', new=MagicMock(return_value=MagicMock())) as mock:
-            meta = {}
-            conf = {"base_url": "myurl/"}
-            hash = "abcd1234"
-            response = ja3er.sendReq(conf=conf, meta=meta, hash=hash)
-            mock.assert_called_once_with("GET", "myurl/abcd1234")
-            self.assertIsNotNone(response)
-
-    def test_prepareResults_none(self):
-        raw = {"error": "Sorry no values found"}
-        results = ja3er.prepareResults(raw)
-        self.assertEqual(results["response"], raw)
-        self.assertEqual(results["summary"], "no_results")
-        self.assertEqual(results["status"], "ok")
-
-    def test_prepareResults_invalidHash(self):
-        raw = {"error": "Invalid hash"}
-        results = ja3er.prepareResults(raw)
-        self.assertEqual(results["response"], raw)
-        self.assertEqual(results["summary"], "invalid_input")
-        self.assertEqual(results["status"], "caution")
-
-    def test_prepareResults_internal_failure(self):
-        raw = {"error": "unknown"}
-        results = ja3er.prepareResults(raw)
-        self.assertEqual(results["response"], raw)
-        self.assertEqual(results["summary"], "internal_failure")
-        self.assertEqual(results["status"], "caution")
-
-    def test_prepareResults_info(self):
-        raw = [{"User-Agent": "Blah/5.0", "Count": 24874, "Last_seen": "2022-04-08 16:18:38"}, {"Comment": "Brave browser v1.36.122\n\n", "Reported": "2022-03-28 20:26:42"}]
-        results = ja3er.prepareResults(raw)
-        self.assertEqual(results["response"], raw)
-        self.assertEqual(results["summary"], "suspicious")
-        self.assertEqual(results["status"], "info")
-
-    def test_analyze(self):
-        output = {"info": "Results found."}
-        artifactInput = '{"value":"abcd1234","artifactType":"ja3"}'
-        conf = {"base_url": "myurl/"}
-        with patch('ja3er.ja3er.sendReq', new=MagicMock(return_value=output)) as mock:
-            results = ja3er.analyze(conf, artifactInput)
-            self.assertEqual(results["summary"], "suspicious")
-            mock.assert_called_once()

salt/sensoroni/files/analyzers/ja3er/requirements.txt

@@ -1,2 +0,0 @@
-requests>=2.27.1
-pyyaml>=6.0