diff --git a/HOTFIX b/HOTFIX index 39e4800ae..e69de29bb 100644 --- a/HOTFIX +++ b/HOTFIX @@ -1 +0,0 @@ -20231204 diff --git a/VERSION b/VERSION index 8ea99f559..29630cd6d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.30 +2.4.40 diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index dc84ba5bd..d2582ff94 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -109,6 +109,7 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|timeout exceeded" # server not yet ready (telegraf waiting on elasticsearch) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|influxsize kbytes" # server not yet ready (telegraf waiting on influx) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|expected field at" # server not yet ready (telegraf waiting on health data) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|connection timed out" # server not yet ready (telegraf plugin unable to connect) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|cached the public key" # server not yet ready (salt minion waiting on key acceptance) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|no ingest nodes" # server not yet ready (logstash waiting on elastic) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to poll" # server not yet ready (sensoroni waiting on soc) diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 004dea94d..7d3883895 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -10,6 +10,7 @@ elasticfleet: logging: zeek: excluded: + - analyzer - broker - capture_loss - cluster diff --git a/salt/elasticsearch/files/ingest/common.nids b/salt/elasticsearch/files/ingest/common.nids index 53a3f7b79..84d170c93 100644 --- a/salt/elasticsearch/files/ingest/common.nids +++ b/salt/elasticsearch/files/ingest/common.nids @@ -1,17 +1,17 @@ { "description" : "common.nids", "processors" : [ - { "convert": { "if": "ctx.rule.uuid != null", "field": "rule.uuid", "type": "integer" } }, - { "set": { "if": "ctx.rule?.uuid < 1000000", "field": "rule.reference", "value": "https://www.snort.org/search?query={{rule.gid}}-{{rule.uuid}}" } }, - { "set": { "if": "ctx.rule?.uuid > 1999999", "field": "rule.reference", "value": "https://doc.emergingthreats.net/{{rule.uuid}}" } }, - { "convert": { "if": "ctx.rule.uuid != null", "field": "rule.uuid", "type": "string" } }, - { "dissect": { "if": "ctx.rule.name != null", "field": "rule.name", "pattern" : "%{rule_type} %{rest_of_rulename} ", "ignore_failure": true } }, - { "set": { "if": "ctx.rule_type == 'GPL'", "field": "rule.ruleset", "value": "Snort GPL" } }, - { "set": { "if": "ctx.rule_type == 'ET'", "field": "rule.ruleset", "value": "Emerging Threats" } }, - { "set": { "if": "ctx.rule.severity == 3", "field": "event.severity", "value": 1, "override": true } }, - { "set": { "if": "ctx.rule.severity == 2", "field": "event.severity", "value": 2, "override": true } }, - { "set": { "if": "ctx.rule.severity == 1", "field": "event.severity", "value": 3, "override": true } }, - { "remove": { "field": ["rule_type", "rest_of_rulename", "host"], "ignore_failure": true } }, - { "pipeline": { "name": "common" } } + { "convert": { "if": "ctx.rule.uuid != null", "field": "rule.uuid", "type": "integer" } }, + { "set": { "if": "ctx.rule?.uuid < 1000000", "field": "rule.reference", "value": "https://www.snort.org/rule_docs/{{rule.gid}}-{{rule.uuid}}" } }, + { "set": { "if": "ctx.rule?.uuid > 1999999", "field": "rule.reference", "value": "https://community.emergingthreats.net" } }, + { "convert": { "if": "ctx.rule.uuid != null", "field": "rule.uuid", "type": "string" } }, + { "dissect": { "if": "ctx.rule.name != null", "field": "rule.name", "pattern" : "%{rule_type} %{rest_of_rulename} ", "ignore_failure": true } }, + { "set": { "if": "ctx.rule_type == 'GPL'", "field": "rule.ruleset", "value": "Snort GPL" } }, + { "set": { "if": "ctx.rule_type == 'ET'", "field": "rule.ruleset", "value": "Emerging Threats" } }, + { "set": { "if": "ctx.rule.severity == 3", "field": "event.severity", "value": 1, "override": true } }, + { "set": { "if": "ctx.rule.severity == 2", "field": "event.severity", "value": 2, "override": true } }, + { "set": { "if": "ctx.rule.severity == 1", "field": "event.severity", "value": 3, "override": true } }, + { "remove": { "field": ["rule_type", "rest_of_rulename", "host"], "ignore_failure": true } }, + { "pipeline": { "name": "common" } } ] } diff --git a/salt/elasticsearch/files/ingest/zeek.ssl b/salt/elasticsearch/files/ingest/zeek.ssl index 8ae7c8728..87174d3d2 100644 --- a/salt/elasticsearch/files/ingest/zeek.ssl +++ b/salt/elasticsearch/files/ingest/zeek.ssl @@ -1,26 +1,40 @@ { "description" : "zeek.ssl", "processors" : [ - { "set": { "field": "event.dataset", "value": "ssl" } }, - { "remove": { "field": ["host"], "ignore_failure": true } }, - { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, - { "rename": { "field": "message2.version", "target_field": "ssl.version", "ignore_missing": true } }, - { "rename": { "field": "message2.cipher", "target_field": "ssl.cipher", "ignore_missing": true } }, - { "rename": { "field": "message2.curve", "target_field": "ssl.curve", "ignore_missing": true } }, - { "rename": { "field": "message2.server_name", "target_field": "ssl.server_name", "ignore_missing": true } }, - { "rename": { "field": "message2.resumed", "target_field": "ssl.resumed", "ignore_missing": true } }, - { "rename": { "field": "message2.last_alert", "target_field": "ssl.last_alert", "ignore_missing": true } }, - { "rename": { "field": "message2.next_protocol", "target_field": "ssl.next_protocol", "ignore_missing": true } }, - { "rename": { "field": "message2.established", "target_field": "ssl.established", "ignore_missing": true } }, - { "rename": { "field": "message2.cert_chain_fuids", "target_field": "ssl.certificate.chain_fuids", "ignore_missing": true } }, - { "rename": { "field": "message2.client_cert_chain_fuids", "target_field": "ssl.client.certificate.chain_fuids", "ignore_missing": true } }, - { "rename": { "field": "message2.subject", "target_field": "ssl.certificate.subject", "ignore_missing": true } }, - { "rename": { "field": "message2.issuer", "target_field": "ssl.certificate.issuer", "ignore_missing": true } }, - { "rename": { "field": "message2.client_subject", "target_field": "ssl.client.subject", "ignore_missing": true } }, - { "rename": { "field": "message2.client_issuer", "target_field": "ssl.client.issuer", "ignore_missing": true } }, - { "rename": { "field": "message2.validation_status","target_field": "ssl.validation_status", "ignore_missing": true } }, - { "rename": { "field": "message2.ja3", "target_field": "hash.ja3", "ignore_missing": true } }, - { "rename": { "field": "message2.ja3s", "target_field": "hash.ja3s", "ignore_missing": true } }, + { "set": { "field": "event.dataset", "value": "ssl" } }, + { "remove": { "field": ["host"], "ignore_failure": true } }, + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.version", "target_field": "ssl.version", "ignore_missing": true } }, + { "rename": { "field": "message2.cipher", "target_field": "ssl.cipher", "ignore_missing": true } }, + { "rename": { "field": "message2.curve", "target_field": "ssl.curve", "ignore_missing": true } }, + { "rename": { "field": "message2.server_name", "target_field": "ssl.server_name", "ignore_missing": true } }, + { "rename": { "field": "message2.resumed", "target_field": "ssl.resumed", "ignore_missing": true } }, + { "rename": { "field": "message2.last_alert", "target_field": "ssl.last_alert", "ignore_missing": true } }, + { "rename": { "field": "message2.next_protocol", "target_field": "ssl.next_protocol", "ignore_missing": true } }, + { "rename": { "field": "message2.established", "target_field": "ssl.established", "ignore_missing": true } }, + { "rename": { "if": "ctx.message2?.cert_chain_fps != null", "field": "message2.cert_chain_fps", "target_field": "tls.server.hash.sha256", "ignore_missing": true } }, + { "rename": { "field": "message2?.cert_chain_fuids", "target_field": "ssl.certificate.chain_fuids", "ignore_missing": true } }, + { "rename": { "if": "ctx.message2?.client_cert_chain_fps != null", "field": "message2.client_cert_chain_fps", "target_field": "tls.client.hash.sha256", "ignore_failure": true, "ignore_missing": true } }, + { "rename": { "field": "message2.client_cert_chain_fuids", "target_field": "ssl.client.certificate.chain_fuids", "ignore_missing": true } }, + { "rename": { "field": "message2.subject", "target_field": "ssl.certificate.subject", "ignore_missing": true } }, + { "rename": { "field": "message2.issuer", "target_field": "ssl.certificate.issuer", "ignore_missing": true } }, + { "rename": { "field": "message2.client_subject", "target_field": "ssl.client.subject", "ignore_missing": true } }, + { "rename": { "field": "message2.client_issuer", "target_field": "ssl.client.issuer", "ignore_missing": true } }, + { "rename": { "field": "message2.validation_status","target_field": "ssl.validation_status", "ignore_missing": true } }, + { "rename": { "field": "message2.ja3", "target_field": "hash.ja3", "ignore_missing": true } }, + { "rename": { "field": "message2.ja3s", "target_field": "hash.ja3s", "ignore_missing": true } }, + { "foreach": + { + "if": "ctx?.tls?.client?.hash?.sha256 !=null", + "field": "tls.client.hash.sha256", + "processor": { + "append": { + "field": "hash.sha256", + "value": "{{_ingest._value}}" + } + } + } + }, { "pipeline": { "name": "zeek.common_ssl" } } ] } diff --git a/salt/elasticsearch/files/ingest/zeek.x509 b/salt/elasticsearch/files/ingest/zeek.x509 index 640ea81e3..64d06131a 100644 --- a/salt/elasticsearch/files/ingest/zeek.x509 +++ b/salt/elasticsearch/files/ingest/zeek.x509 @@ -3,44 +3,45 @@ "processors" : [ { "set": { "field": "event.dataset", "value": "x509" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, - { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, - { "rename": { "field": "message2.id", "target_field": "log.id.fuid", "ignore_missing": true } }, - { "dot_expander": { "field": "certificate.version", "path": "message2", "ignore_failure": true } }, - { "rename": { "field": "message2.certificate.version", "target_field": "x509.certificate.version", "ignore_missing": true } }, - { "dot_expander": { "field": "certificate.serial", "path": "message2", "ignore_failure": true } }, - { "rename": { "field": "message2.certificate.serial", "target_field": "x509.certificate.serial", "ignore_missing": true } }, - { "dot_expander": { "field": "certificate.subject", "path": "message2", "ignore_failure": true } }, - { "rename": { "field": "message2.certificate.subject", "target_field": "x509.certificate.subject", "ignore_missing": true } }, - { "dot_expander": { "field": "certificate.issuer", "path": "message2", "ignore_failure": true } }, - { "rename": { "field": "message2.certificate.issuer", "target_field": "x509.certificate.issuer", "ignore_missing": true } }, - { "dot_expander": { "field": "certificate.not_valid_before", "path": "message2", "ignore_failure": true } }, - { "rename": { "field": "message2.certificate.not_valid_before", "target_field": "x509.certificate.not_valid_before", "ignore_missing": true } }, - { "dot_expander": { "field": "certificate.not_valid_after", "path": "message2", "ignore_failure": true } }, - { "rename": { "field": "message2.certificate.not_valid_after", "target_field": "x509.certificate.not_valid_after", "ignore_missing": true } }, - { "dot_expander": { "field": "certificate.key_alg", "path": "message2", "ignore_failure": true } }, - { "rename": { "field": "message2.certificate.key_alg", "target_field": "x509.certificate.key.algorithm", "ignore_missing": true } }, - { "dot_expander": { "field": "certificate.sig_alg", "path": "message2", "ignore_failure": true } }, - { "rename": { "field": "message2.certificate.sig_alg", "target_field": "x509.certificate.signing_algorithm", "ignore_missing": true } }, - { "dot_expander": { "field": "certificate.key_type", "path": "message2", "ignore_failure": true } }, - { "rename": { "field": "message2.certificate.key_type", "target_field": "x509.certificate.key.type", "ignore_missing": true } }, - { "dot_expander": { "field": "certificate.key_length", "path": "message2", "ignore_failure": true } }, - { "rename": { "field": "message2.certificate.key_length", "target_field": "x509.certificate.key.length", "ignore_missing": true } }, - { "dot_expander": { "field": "certificate.exponent", "path": "message2", "ignore_failure": true } }, - { "rename": { "field": "message2.certificate.exponent", "target_field": "x509.certificate.exponent", "ignore_missing": true } }, - { "dot_expander": { "field": "certificate.curve", "path": "message2", "ignore_failure": true } }, - { "rename": { "field": "message2.certificate.curve", "target_field": "x509.certificate.curve", "ignore_missing": true } }, - { "dot_expander": { "field": "san.dns", "path": "message2", "ignore_failure": true } }, - { "rename": { "field": "message2.san.dns", "target_field": "x509.san_dns", "ignore_missing": true } }, - { "dot_expander": { "field": "san.uri", "path": "message2", "ignore_failure": true } }, - { "rename": { "field": "message2.san.uri", "target_field": "x509.san_uri", "ignore_missing": true } }, - { "dot_expander": { "field": "san.email", "path": "message2", "ignore_failure": true } }, - { "rename": { "field": "message2.san.email", "target_field": "x509.san_email", "ignore_missing": true } }, - { "dot_expander": { "field": "san.ip", "path": "message2", "ignore_failure": true } }, - { "rename": { "field": "message2.san.ip", "target_field": "x509.san_ip", "ignore_missing": true } }, - { "dot_expander": { "field": "basic_constraints.ca", "path": "message2", "ignore_failure": true } }, - { "rename": { "field": "message2.basic_constraints.ca", "target_field": "x509.basic_constraints.ca", "ignore_missing": true } }, - { "dot_expander": { "field": "basic_constraints.path_length", "path": "message2", "ignore_failure": true } }, - { "rename": { "field": "message2.basic_constraints.path_length", "target_field": "x509.basic_constraints.path_length", "ignore_missing": true } }, - { "pipeline": { "name": "zeek.common_ssl" } } + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id", "target_field": "log.id.fuid", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.version", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.version", "target_field": "x509.certificate.version", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.serial", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.serial", "target_field": "x509.certificate.serial", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.subject", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.subject", "target_field": "x509.certificate.subject", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.issuer", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.issuer", "target_field": "x509.certificate.issuer", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.not_valid_before", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.not_valid_before", "target_field": "x509.certificate.not_valid_before", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.not_valid_after", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.not_valid_after", "target_field": "x509.certificate.not_valid_after", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.key_alg", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.key_alg", "target_field": "x509.certificate.key.algorithm", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.sig_alg", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.sig_alg", "target_field": "x509.certificate.signing_algorithm", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.key_type", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.key_type", "target_field": "x509.certificate.key.type", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.key_length", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.key_length", "target_field": "x509.certificate.key.length", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.exponent", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.exponent", "target_field": "x509.certificate.exponent", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.curve", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.curve", "target_field": "x509.certificate.curve", "ignore_missing": true } }, + { "dot_expander": { "field": "san.dns", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.san.dns", "target_field": "x509.san_dns", "ignore_missing": true } }, + { "dot_expander": { "field": "san.uri", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.san.uri", "target_field": "x509.san_uri", "ignore_missing": true } }, + { "dot_expander": { "field": "san.email", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.san.email", "target_field": "x509.san_email", "ignore_missing": true } }, + { "dot_expander": { "field": "san.ip", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.san.ip", "target_field": "x509.san_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "basic_constraints.ca", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.basic_constraints.ca", "target_field": "x509.basic_constraints.ca", "ignore_missing": true } }, + { "dot_expander": { "field": "basic_constraints.path_length", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.basic_constraints.path_length", "target_field": "x509.basic_constraints.path_length", "ignore_missing": true } }, + { "rename": { "field": "message2.fingerprint", "target_field": "hash.sha256", "ignore_missing": true } }, + { "pipeline": { "name": "zeek.common_ssl" } } ] } diff --git a/salt/manager/tools/sbin/so-yaml.py b/salt/manager/tools/sbin/so-yaml.py index 68f9b43fe..874fc9e0f 100755 --- a/salt/manager/tools/sbin/so-yaml.py +++ b/salt/manager/tools/sbin/so-yaml.py @@ -16,12 +16,12 @@ lockFile = "/tmp/so-yaml.lock" def showUsage(args): print('Usage: {} [ARGS...]'.format(sys.argv[0])) print(' General commands:') - print(' remove - Removes a yaml top-level key, if it exists. Requires KEY arg.') + print(' remove - Removes a yaml key, if it exists. Requires KEY arg.') print(' help - Prints this usage information.') print('') print(' Where:') print(' YAML_FILE - Path to the file that will be modified. Ex: /opt/so/conf/service/conf.yaml') - print(' KEY - Top level key only, does not support dot-notations for nested keys at this time. Ex: level1') + print(' KEY - YAML key, does not support \' or " characters at this time. Ex: level1.level2') sys.exit(1) @@ -36,6 +36,14 @@ def writeYaml(filename, content): return yaml.dump(content, file) +def removeKey(content, key): + pieces = key.split(".", 1) + if len(pieces) > 1: + removeKey(content[pieces[0]], pieces[1]) + else: + content.pop(key, None) + + def remove(args): if len(args) != 2: print('Missing filename or key arg', file=sys.stderr) @@ -43,11 +51,12 @@ def remove(args): return filename = args[0] + key = args[1] + content = loadYaml(filename) - - content.pop(args[1], None) - + removeKey(content, key) writeYaml(filename, content) + return 0 diff --git a/salt/manager/tools/sbin/so-yaml_test.py b/salt/manager/tools/sbin/so-yaml_test.py index 0f2ac7d81..7d0ed1a8e 100644 --- a/salt/manager/tools/sbin/so-yaml_test.py +++ b/salt/manager/tools/sbin/so-yaml_test.py @@ -57,6 +57,36 @@ class TestRemove(unittest.TestCase): expected = "key2: false\n" self.assertEqual(actual, expected) + def test_remove_nested(self): + filename = "/tmp/so-yaml_test-remove.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: abc }, key2: false}") + file.close() + + soyaml.remove([filename, "key1.child2"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n child1: 123\nkey2: false\n" + self.assertEqual(actual, expected) + + def test_remove_nested_deep(self): + filename = "/tmp/so-yaml_test-remove.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: ab } }, key2: false}") + file.close() + + soyaml.remove([filename, "key1.child2.deep1"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n child1: 123\n child2:\n deep2: ab\nkey2: false\n" + self.assertEqual(actual, expected) + def test_remove_missing_args(self): with patch('sys.exit', new=MagicMock()) as sysmock: with patch('sys.stderr', new=StringIO()) as mock_stdout: diff --git a/salt/sensoroni/defaults.yaml b/salt/sensoroni/defaults.yaml index f53646ac2..1a5c7beef 100644 --- a/salt/sensoroni/defaults.yaml +++ b/salt/sensoroni/defaults.yaml @@ -22,11 +22,17 @@ sensoroni: base_url: https://otx.alienvault.com/api/v1/ api_key: pulsedive: - base_url: https://pulsedive.com/api/ + base_url: https://pulsedive.com/api/ api_key: spamhaus: lookup_host: zen.spamhaus.org nameservers: [] + sublime_platform: + base_url: https://api.platform.sublimesecurity.com + api_key: + live_flow: False + mailbox_email_address: + message_source_id: urlscan: base_url: https://urlscan.io/api/v1/ api_key: diff --git a/salt/sensoroni/files/analyzers/README.md b/salt/sensoroni/files/analyzers/README.md index 19335a545..a968fdc57 100644 --- a/salt/sensoroni/files/analyzers/README.md +++ b/salt/sensoroni/files/analyzers/README.md @@ -6,19 +6,20 @@ Security Onion provides a means for performing data analysis on varying inputs. The built-in analyzers support the following observable types: -| Name | Domain | Hash | IP | Mail | Other | URI | URL | User Agent | -| ------------------------|--------|-------|-------|-------|-------|-------|-------|-------| -| Alienvault OTX |✓ |✓|✓|✗|✗|✗|✓|✗| -| EmailRep |✗ |✗|✗|✓|✗|✗|✗|✗| -| Greynoise |✗ |✗|✓|✗|✗|✗|✗|✗| -| LocalFile |✓ |✓|✓|✗|✓|✗|✓|✗| -| Malware Hash Registry |✗ |✓|✗|✗|✗|✗|✓|✗| -| Pulsedive |✓ |✓|✓|✗|✗|✓|✓|✓| -| Spamhaus |✗ |✗|✓|✗|✗|✗|✗|✗| -| Urlhaus |✗ |✗|✗|✗|✗|✗|✓|✗| -| Urlscan |✗ |✗|✗|✗|✗|✗|✓|✗| -| Virustotal |✓ |✓|✓|✗|✗|✗|✓|✗| -| WhoisLookup |✓ |✗|✗|✗|✗|✓|✗|✗| +| Name | Domain | EML | Hash | IP | Mail | Other | URI | URL | User Agent | +| ------------------------|--------|-------|-------|-------|-------|-------|-------|-------|-------| +| Alienvault OTX |✓ |✗|✓|✓|✗|✗|✗|✓|✗| +| EmailRep |✗ |✗|✗|✗|✓|✗|✗|✗|✗| +| Greynoise |✗ |✗|✗|✓|✗|✗|✗|✗|✗| +| LocalFile |✓ |✗|✓|✓|✗|✓|✗|✓|✗| +| Malware Hash Registry |✗ |✗|✓|✗|✗|✗|✗|✓|✗| +| Pulsedive |✓ |✗|✓|✓|✗|✗|✓|✓|✓| +| Spamhaus |✗ |✗|✗|✓|✗|✗|✗|✗|✗| +| Sublime Platform |✗ |✓|✗|✗|✗|✗|✗|✗|✗| +| Urlhaus |✗ |✗|✗|✗|✗|✗|✗|✓|✗| +| Urlscan |✗ |✗|✗|✗|✗|✗|✗|✓|✗| +| Virustotal |✓ |✗|✓|✓|✗|✗|✗|✓|✗| +| WhoisLookup |✓ |✗|✗|✗|✗|✗|✓|✗|✗| ## Authentication @@ -29,10 +30,11 @@ Many analyzers require authentication, via an API key or similar. The table belo [AlienVault OTX](https://otx.alienvault.com/api) |✓| [EmailRep](https://emailrep.io/key) |✓| [GreyNoise](https://www.greynoise.io/plans/community) |✓| -LocalFile |✗| +[LocalFile](https://github.com/Security-Onion-Solutions/securityonion/tree/fix/sublime_analyzer_documentation/salt/sensoroni/files/analyzers/localfile) |✗| [Malware Hash Registry](https://hash.cymru.com/docs_whois) |✗| [Pulsedive](https://pulsedive.com/api/) |✓| [Spamhaus](https://www.spamhaus.org/dbl/) |✗| +[Sublime Platform](https://sublime.security) |✓| [Urlhaus](https://urlhaus.abuse.ch/) |✗| [Urlscan](https://urlscan.io/docs/api/) |✓| [VirusTotal](https://developers.virustotal.com/reference/overview) |✓| diff --git a/salt/sensoroni/files/analyzers/sublime/README.md b/salt/sensoroni/files/analyzers/sublime/README.md new file mode 100644 index 000000000..77894a2b1 --- /dev/null +++ b/salt/sensoroni/files/analyzers/sublime/README.md @@ -0,0 +1,24 @@ +# Sublime + +## Description +Submit a base64-encoded EML file to Sublime Platform for analysis. + +## Configuration Requirements +In SOC, navigate to `Administration`, toggle `Show all configurable settings, including advanced settings.`, and navigate to `sensoroni` -> `analyzers` -> `sublime_platform`. + +![image](https://github.com/Security-Onion-Solutions/securityonion/assets/16829864/a914f59d-c09f-40b6-ae8b-d644df236b81) + + +The following configuration options are available for: + +``api_key`` - API key used for communication with the Sublime Platform API (Required) + +``base_url`` - URL used for communication with Sublime Platform. If no value is supplied, the default of `https://api.platform.sublimesecurity.com` will be used. + +The following options relate to [Live Flow](https://docs.sublimesecurity.com/reference/analyzerawmessageliveflow-1) analysis only: + +``live_flow`` - Determines if live flow analysis should be used. Defaults to `False`. + +``mailbox_email_address`` - The mailbox address to use for during live flow analysis. (Required for live flow analysis) + +``message_source_id`` - The ID of the message source to use during live flow analysis. (Required for live flow analysis) diff --git a/salt/sensoroni/files/analyzers/sublime/__init__.py b/salt/sensoroni/files/analyzers/sublime/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/salt/sensoroni/files/analyzers/sublime/requirements.txt b/salt/sensoroni/files/analyzers/sublime/requirements.txt new file mode 100644 index 000000000..a8980057f --- /dev/null +++ b/salt/sensoroni/files/analyzers/sublime/requirements.txt @@ -0,0 +1,2 @@ +requests>=2.27.1 +pyyaml>=6.0 diff --git a/salt/sensoroni/files/analyzers/sublime/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl b/salt/sensoroni/files/analyzers/sublime/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl new file mode 100644 index 000000000..d2b6c37f9 Binary files /dev/null and b/salt/sensoroni/files/analyzers/sublime/source-packages/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.whl differ diff --git a/salt/sensoroni/files/analyzers/sublime/source-packages/certifi-2021.10.8-py2.py3-none-any.whl b/salt/sensoroni/files/analyzers/sublime/source-packages/certifi-2021.10.8-py2.py3-none-any.whl new file mode 100644 index 000000000..fbcb86b5f Binary files /dev/null and b/salt/sensoroni/files/analyzers/sublime/source-packages/certifi-2021.10.8-py2.py3-none-any.whl differ diff --git a/salt/sensoroni/files/analyzers/sublime/source-packages/charset_normalizer-2.0.12-py3-none-any.whl b/salt/sensoroni/files/analyzers/sublime/source-packages/charset_normalizer-2.0.12-py3-none-any.whl new file mode 100644 index 000000000..17a2dfbeb Binary files /dev/null and b/salt/sensoroni/files/analyzers/sublime/source-packages/charset_normalizer-2.0.12-py3-none-any.whl differ diff --git a/salt/sensoroni/files/analyzers/sublime/source-packages/idna-3.3-py3-none-any.whl b/salt/sensoroni/files/analyzers/sublime/source-packages/idna-3.3-py3-none-any.whl new file mode 100644 index 000000000..060541bc9 Binary files /dev/null and b/salt/sensoroni/files/analyzers/sublime/source-packages/idna-3.3-py3-none-any.whl differ diff --git a/salt/sensoroni/files/analyzers/sublime/source-packages/requests-2.27.1-py2.py3-none-any.whl b/salt/sensoroni/files/analyzers/sublime/source-packages/requests-2.27.1-py2.py3-none-any.whl new file mode 100644 index 000000000..807fc6110 Binary files /dev/null and b/salt/sensoroni/files/analyzers/sublime/source-packages/requests-2.27.1-py2.py3-none-any.whl differ diff --git a/salt/sensoroni/files/analyzers/sublime/source-packages/urllib3-1.26.9-py2.py3-none-any.whl b/salt/sensoroni/files/analyzers/sublime/source-packages/urllib3-1.26.9-py2.py3-none-any.whl new file mode 100644 index 000000000..5019453dd Binary files /dev/null and b/salt/sensoroni/files/analyzers/sublime/source-packages/urllib3-1.26.9-py2.py3-none-any.whl differ diff --git a/salt/sensoroni/files/analyzers/sublime/sublime.json b/salt/sensoroni/files/analyzers/sublime/sublime.json new file mode 100644 index 000000000..4d1770872 --- /dev/null +++ b/salt/sensoroni/files/analyzers/sublime/sublime.json @@ -0,0 +1,7 @@ +{ + "name": "Sublime", + "version": "0.1", + "author": "Security Onion Solutions", + "description": "This analyzer analyzes an email with Sublime Security to determine if it is considered malicious.", + "supportedTypes" : ["eml"] +} diff --git a/salt/sensoroni/files/analyzers/sublime/sublime.py b/salt/sensoroni/files/analyzers/sublime/sublime.py new file mode 100644 index 000000000..ad1774745 --- /dev/null +++ b/salt/sensoroni/files/analyzers/sublime/sublime.py @@ -0,0 +1,83 @@ +import json +import requests +import sys +import os +import helpers +import argparse + + +def checkConfigRequirements(conf): + if "api_key" not in conf or len(conf['api_key']) == 0: + sys.exit(126) + + +def buildReq(conf, artifact_value): + headers = {"Authorization": "Bearer " + conf['api_key']} + base_url = conf['base_url'] + if str(conf['live_flow']).lower() == "true": + uri = "/v1/live-flow/raw-messages/analyze" + data = {"create_mailbox": True, "mailbox_email_address": str(conf['mailbox_email_address']), "message_source_id": str(conf['message_source_id']), "raw_message": artifact_value} + else: + uri = "/v0/messages/analyze" + data = {"raw_message": artifact_value, + "run_active_detection_rules": True} + url = base_url + uri + return url, headers, data + + +def sendReq(url, headers, data): + response = requests.request('POST', + url=url, + headers=headers, + data=json.dumps(data)).json() + return response + + +def prepareResults(raw): + matched = [] + if "rule_results" in raw: + for r in raw["rule_results"]: + if r["matched"] is True: + matched.append(r) + if len(matched) > 0: + raw = matched + status = "threat" + summary = "malicious" + else: + raw = "No rules matched." + status = "ok" + summary = "harmless" + elif "flagged_rules" in raw: + if raw["flagged_rules"] is not None: + status = "threat" + summary = "malicious" + else: + status = "ok" + summary = "harmless" + results = {'response': raw, 'status': status, 'summary': summary} + return results + + +def analyze(conf, input): + checkConfigRequirements(conf) + meta = helpers.loadMetadata(__file__) + data = helpers.parseArtifact(input) + helpers.checkSupportedType(meta, data["artifactType"]) + request = buildReq(conf, data["value"]) + response = sendReq(request[0], request[1], request[2]) + return prepareResults(response) + + +def main(): + dir = os.path.dirname(os.path.realpath(__file__)) + parser = argparse.ArgumentParser(description="Submit an email to Sublime Platform's EML Analyzer") + parser.add_argument('artifact', help='the artifact represented in JSON format') + parser.add_argument('-c', '--config', metavar="CONFIG_FILE", default=dir + "/sublime.yaml", help='optional config file to use instead of the default config file') + args = parser.parse_args() + if args.artifact: + results = analyze(helpers.loadConfig(args.config), args.artifact) + print(json.dumps(results)) + + +if __name__ == "__main__": + main() diff --git a/salt/sensoroni/files/analyzers/sublime/sublime.yaml b/salt/sensoroni/files/analyzers/sublime/sublime.yaml new file mode 100644 index 000000000..0776050a8 --- /dev/null +++ b/salt/sensoroni/files/analyzers/sublime/sublime.yaml @@ -0,0 +1,5 @@ +base_url: "{{ salt['pillar.get']('sensoroni:analyzers:sublime_platform:base_url', 'https://api.platform.sublimesecurity.com') }}" +api_key: "{{ salt['pillar.get']('sensoroni:analyzers:sublime_platform:api_key', '') }}" +live_flow: "{{ salt['pillar.get']('sensoroni:analyzers:sublime_platform:live_flow', 'False') }}" +mailbox_email_address: "{{ salt['pillar.get']('sensoroni:analyzers:sublime_platform:mailbox_email_address', '') }}" +message_source_id: "{{ salt['pillar.get']('sensoroni:analyzers:sublime_platform:message_source_id', '') }}" diff --git a/salt/sensoroni/files/analyzers/sublime/sublime_test.py b/salt/sensoroni/files/analyzers/sublime/sublime_test.py new file mode 100755 index 000000000..27c9465d7 --- /dev/null +++ b/salt/sensoroni/files/analyzers/sublime/sublime_test.py @@ -0,0 +1,188 @@ +from io import StringIO +import sys +from unittest.mock import patch, MagicMock +from sublime import sublime +import json +import unittest + + +class TestSublimePlatformMethods(unittest.TestCase): + + def test_main_missing_input(self): + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stderr', new=StringIO()) as mock_stderr: + sys.argv = ["cmd"] + sublime.main() + self.assertEqual(mock_stderr.getvalue(), '''usage: cmd [-h] [-c CONFIG_FILE] artifact\ncmd: error: the following arguments are required: artifact\n''') + sysmock.assert_called_once_with(2) + + def test_main_success(self): + output = {"foo": "bar"} + with patch('sys.stdout', new=StringIO()) as mock_stdout: + with patch('sublime.sublime.analyze', new=MagicMock(return_value=output)) as mock: + sys.argv = ["cmd", "input"] + sublime.main() + expected = '{"foo": "bar"}\n' + self.assertEqual(mock_stdout.getvalue(), expected) + mock.assert_called_once() + + def test_checkKeyNonexistent(self): + conf = {"not_a_key": "abcd12345"} + with self.assertRaises(SystemExit) as cm: + sublime.checkConfigRequirements(conf) + self.assertEqual(cm.exception.code, 126) + + def test_buildReqLiveFlow(self): + conf = {'base_url': 'https://api.platform.sublimesecurity.com', 'api_key': 'abcd12345', 'live_flow': True, 'mailbox_email_address': 'user@test.local', 'message_source_id': 'abcd1234'} + artifact_value = "abcd1234" + result = sublime.buildReq(conf, artifact_value) + self.assertEqual("https://api.platform.sublimesecurity.com/v1/live-flow/raw-messages/analyze", result[0]) + self.assertEqual({'Authorization': 'Bearer abcd12345'}, result[1]) + + def test_buildReqNotLiveFlow(self): + conf = {'base_url': 'https://api.platform.sublimesecurity.com', 'api_key': 'abcd12345', 'live_flow': False, 'mailbox_email_address': 'user@test.local'} + artifact_value = "abcd1234" + result = sublime.buildReq(conf, artifact_value) + self.assertEqual("https://api.platform.sublimesecurity.com/v0/messages/analyze", result[0]) + self.assertEqual({'Authorization': 'Bearer abcd12345'}, result[1]) + + def test_prepareResultsRuleResultsMatched(self): + raw = ''' { + "rule_results": [{ + "rule": { + "id": "9147f589-39d5-4dd0-a0ee-00433a6e2632", + "name": "AnonymousFox Indicators", + "source": "type.inbound\\nand regex.icontains(sender.email.email, \\"(anonymous|smtp)fox-\\"))\\n", + "severity": "medium" + }, + "matched": true, + "success": true, + "error": null, + "external_errors": null, + "execution_time": 0.000071679 + }]}''' + results = sublime.prepareResults(json.loads(raw)) + print(results) + self.assertEqual(results["response"], json.loads(raw)["rule_results"]) + self.assertEqual(results["summary"], "malicious") + self.assertEqual(results["status"], "threat") + + def test_prepareResultsRuleResultsNotMatched(self): + raw = ''' { + "rule_results": [{ + "rule": { + "id": "9147f589-39d5-4dd0-a0ee-00433a6e2632", + "name": "AnonymousFox Indicators", + "source": "type.inbound and regex.icontains(.value, \\"(anonymous|smtp)fox-\\"))\\n", + "severity": "medium" + }, + "matched": false, + "success": true, + "error": null, + "external_errors": null, + "execution_time": 0.000071679 + }]}''' + results = sublime.prepareResults(json.loads(raw)) + print(results) + self.assertEqual(results["response"], "No rules matched.") + self.assertEqual(results["summary"], "harmless") + self.assertEqual(results["status"], "ok") + + def test_prepareResultsLiveFlowMatched(self): + raw = '''{ + "canonical_id": "fb8b46e3317ac7d5036c6b21517d363634293c6d4f6bf1b1e67548c80948a1c6", + "flagged_rules": [ + { + "actions": null, + "active": true, + "active_updated_at": "2023-08-09T14:58:25.669495Z", + "attack_types": [ + "Credential Phishing", + "Malware/Ransomware" + ], + "authors": null, + "created_at": "2023-08-09 01:00:25.642489+00", + "created_by_api_request_id": null, + "created_by_org_id": null, + "created_by_org_name": null, + "created_by_user_id": null, + "created_by_user_name": null, + "description": "Recursively scans files and archives to detect HTML smuggling techniques.\\n", + "detection_methods": [ + "Archive analysis", + "File analysis", + "HTML analysis", + "Javascript analysis" + ], + "exclusion_mql": null, + "false_positives": null, + "feed_external_rule_id": "0b0fed36-735a-50f1-bf10-6673237a4623", + "feed_id": "4e5d7da3-d566-4910-a613-f00709702240", + "full_type": "detection_rule", + "id": "537bf73d-a4f0-4389-b2a1-272192efa0d5", + "immutable": true, + "internal_type": null, + "label": null, + "maturity": null, + "name": "Attachment: HTML smuggling with unescape", + "org_id": "dac92af8-2bd6-4861-9ee1-a04e713e3ae2", + "references": [ + "https://www.microsoft.com/security/blog/2021/11/11/html-smuggling-surges-highly-evasive-loader" + ], + "severity": "high", + "source_md5": "b68388617d78ccc20075ca8fffc7e3f8", + "tactics_and_techniques": [ + "Evasion", + "HTML smuggling", + "Scripting" + ], + "tags": null, + "type": "detection", + "updated_at": "2023-11-01 15:25:47.212056+00", + "user_provided_tags": [ + + ] + } + ], + "message_id": "0071b1ac-d7ca-4e37-91c5-068a96b9dda8", + "raw_message_id": "1dc90473-b028-4754-942c-476cfb1ca2ff" + }''' + + results = sublime.prepareResults(json.loads(raw)) + print(results) + self.assertEqual(results["response"], json.loads(raw)) + self.assertEqual(results["summary"], "malicious") + self.assertEqual(results["status"], "threat") + + def test_prepareResultsLiveFlowNotMatched(self): + raw = '''{ + "canonical_id": "092459fa0d9edd5d8e2d0ccf3af50120c63ec58717a8cfdeb15854706940346f", + "flagged_rules": null, + "message_id": "1e8693b4-bf44-4cb9-ac9a-85fc2a99eeb8", + "raw_message_id": "5d2f03c2-86e1-47d8-81ae-620ecb5c6553" + }''' + + results = sublime.prepareResults(json.loads(raw)) + self.assertEqual(results["response"], json.loads(raw)) + self.assertEqual(results["summary"], "harmless") + self.assertEqual(results["status"], "ok") + + def test_sendReq(self): + with patch('requests.request', new=MagicMock(return_value=MagicMock())) as mock: + url = "https://api.platform.sublimesecurity.com/v1/live-flow/raw-messages/analyze" + headers = {'Authorization': 'Bearer abcd12345'} + data = {"create_mailbox": True, "mailbox_email_address": "user@test.local", "message_source_id": "abcd1234", "raw_message": "abcd1234"} + url = "https://api.platform.sublimesecurity.com/v1/live-flow/raw-messages/analyze" + response = sublime.sendReq(url=url, headers=headers, data=data) + mock.assert_called_once_with('POST', url=url, headers=headers, data=json.dumps(data)) + self.assertIsNotNone(response) + + def test_analyze(self): + output = '{"message_id":"abcd1234","raw_message_id":"abcd1234","canonical_id":"abcd1234","flagged_rules":null}' + artifactInput = '{"value":"RnJvbTogQWxpY2UgPGFsaWNlQGV4YW1wbGUuY29tPgpUbzogQm9iIDxib2JA","artifactType":"eml"}' + conf = {'base_url': 'https://api.platform.sublimesecurity.com', 'api_key': 'abcd12345', 'live_flow': False, 'mailbox_email_address': 'user@test.local'} + with patch('sublime.sublime.sendReq', new=MagicMock(return_value=json.loads(output))) as mock: + results = sublime.analyze(conf, artifactInput) + print(results) + self.assertEqual(results["summary"], "harmless") + mock.assert_called_once() diff --git a/salt/sensoroni/soc_sensoroni.yaml b/salt/sensoroni/soc_sensoroni.yaml index db51da358..9c2304d6c 100644 --- a/salt/sensoroni/soc_sensoroni.yaml +++ b/salt/sensoroni/soc_sensoroni.yaml @@ -128,6 +128,42 @@ sensoroni: sensitive: False advanced: True forcedTypes: "[]string" + sublime_platform: + api_key: + description: API key for the Sublime Platform analyzer. + helpLink: cases.html + global: False + sensitive: True + advanced: True + forcedType: string + base_url: + description: Base URL for the Sublime Platform analyzer. + helpLink: cases.html + global: False + sensitive: False + advanced: True + forcedType: string + live_flow: + description: Determines if live flow analysis is used. + helpLink: cases.html + global: False + sensitive: False + advanced: True + forcedType: bool + mailbox_email_address: + description: Source mailbox address used for live flow analysis. + helpLink: cases.html + global: False + sensitive: False + advanced: True + forcedType: string + message_source_id: + description: ID of the message source used for live flow analysis. + helpLink: cases.html + global: False + sensitive: False + advanced: True + forcedType: string urlscan: api_key: description: API key for the Urlscan analyzer. diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index ceca9ef31..3655019e4 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1240,7 +1240,7 @@ soc: showSubtitle: true - name: HTTP description: HTTP with exe downloads - query: 'tags:http AND (file.resp_mime_types:dosexec OR file.resp_mime_types:executable) | groupby http.virtual_host' + query: 'tags:http AND file.resp_mime_types:*exec* | groupby http.virtual_host' showSubtitle: true - name: Intel description: Intel framework hits grouped by indicator @@ -1675,6 +1675,7 @@ soc: labels: - autonomous-system - domain + - eml - file - filename - fqdn diff --git a/salt/soc/files/bin/salt-relay.sh b/salt/soc/files/bin/salt-relay.sh index fea81728d..4b183b20a 100755 --- a/salt/soc/files/bin/salt-relay.sh +++ b/salt/soc/files/bin/salt-relay.sh @@ -37,12 +37,14 @@ function poll() { function respond() { file="$QUEUE_DIR/$1.response" + tmpfile="${file}.tmp" response=$2 - touch "$file" - chmod 660 "$file" - chown "$QUEUE_OWNER:$QUEUE_GROUP" "$file" - echo "$response" > "$file" + touch "$tmpfile" + chmod 660 "$tmpfile" + chown "$QUEUE_OWNER:$QUEUE_GROUP" "$tmpfile" + echo "$response" > "$tmpfile" + mv $tmpfile $file } function list_minions() {