Merge branch '2.4/dev' of github.com:Security-Onion-Solutions/securityonion into 2.4/dev

This commit is contained in:
Mike Reeves
2024-12-18 10:17:04 -05:00
16 changed files with 235 additions and 15 deletions

View File

@@ -150,6 +150,7 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|app_layer.error" # false positive (suricata 7) in stats.log e.g. app_layer.error.imap.parser | Total | 0
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|is not an ip string literal" # false positive (Open Canary logging out blank IP addresses)
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|syncing rule" # false positive (rule sync log line includes rule name which can contain 'error')
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|request_unauthorized" # false positive (login failures to Hydra result in an 'error' log)
fi
if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
@@ -210,6 +211,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|integrity check failed" # Detections: Exclude false positive due to automated testing
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|syncErrors" # Detections: Not an actual error
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Initialized license manager" # SOC log: before fields.status was changed to fields.licenseStatus
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|from NIC checksum offloading" # zeek reporter.log
fi
RESULT=0
@@ -248,6 +250,9 @@ exclude_log "agentstatus.log" # ignore this log since it tracks agents in error
exclude_log "detections_runtime-status_yara.log" # temporarily ignore this log until Detections is more stable
exclude_log "/nsm/kafka/data/" # ignore Kafka data directory from log check.
# Include Zeek reporter.log to detect errors after running known good pcap(s) through sensor
echo "/nsm/zeek/spool/logger/reporter.log" >> /tmp/log_check_files
for log_file in $(cat /tmp/log_check_files); do
status "Checking log file $log_file"
tail -n $RECENT_LOG_LINES $log_file > /tmp/log_check

View File

@@ -82,6 +82,7 @@ docker:
- 443:443
- 8443:8443
- 7788:7788
- 7789:7789
custom_bind_mounts: []
extra_hosts: []
extra_env: []

View File

@@ -76,5 +76,11 @@ do
printf "\n### $GOOS/$GOARCH Installer Generated...\n"
done
printf "\n\n### Generating MSI...\n"
docker run \
--mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ \
{{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o /output/so-elastic-agent_windows_amd64_msi --arch x64 /workspace/so-elastic-agent.wxs
printf "\n### MSI Generated...\n"
printf "\n### Cleaning up temp files in /nsm/elastic-agent-workspace\n"
rm -rf /nsm/elastic-agent-workspace

View File

@@ -3230,6 +3230,8 @@ elasticsearch:
template:
settings:
index:
lifecycle:
name: so-logs-cisco_secure_email_gateway.log-logs
number_of_replicas: 0
policy:
phases:
@@ -10462,6 +10464,8 @@ elasticsearch:
template:
settings:
index:
lifecycle:
name: so-logs-ti_rapid7_threat_command.alert-logs
number_of_replicas: 0
policy:
phases:
@@ -10506,6 +10510,8 @@ elasticsearch:
template:
settings:
index:
lifecycle:
name: so-logs-ti_rapid7_threat_command.ioc-logs
number_of_replicas: 0
policy:
phases:
@@ -10550,6 +10556,8 @@ elasticsearch:
template:
settings:
index:
lifecycle:
name: so-logs-ti_rapid7_threat_command.vulnerability-logs
number_of_replicas: 0
policy:
phases:

View File

@@ -10,7 +10,7 @@
{ "split": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "module_temp" } },
{ "set": { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}" } },
{ "gsub": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "pattern": "^[^.]*.", "replacement": "", "target_field": "dataset_tag_temp" } },
{ "append": { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp}}" } },
{ "append": { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp}}", "allow_duplicates": false } },
{ "set": { "if": "ctx.network?.direction == 'egress'", "override": true, "field": "network.initiated", "value": "true" } },
{ "set": { "if": "ctx.network?.direction == 'ingress'", "override": true, "field": "network.initiated", "value": "false" } },
{ "set": { "if": "ctx.network?.type == 'ipv4'", "override": true, "field": "destination.ipv6", "value": "false" } },

View File

@@ -18,6 +18,7 @@
{ "set": { "if": "ctx.destination?.ip != null", "field": "server.ip", "value": "{{destination.ip}}" } },
{ "set": { "if": "ctx.destination?.port != null", "field": "server.port", "value": "{{destination.port}}" } },
{ "set": { "field": "observer.name", "value": "{{agent.name}}" } },
{ "append": { "if": "ctx.network?.protocol != null && ctx.network?.protocol.contains(\"openvpn\")","field": "tags","value": ["{{network.protocol}}"],"allow_duplicates": false,"ignore_failure": true}},
{ "date": { "field": "message2.ts", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "ignore_failure": true } },
{ "remove": { "field": ["agent"], "ignore_failure": true } },
{ "pipeline": { "name": "common" } }

View File

@@ -38,6 +38,8 @@
{ "set": { "if": "ctx.connection?.state == 'SH'", "field": "connection.state_description", "value": "Originator sent a SYN followed by a FIN, we never saw a SYN ACK from the responder (hence the connection was 'half' open)" } },
{ "set": { "if": "ctx.connection?.state == 'SHR'", "field": "connection.state_description", "value": "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator" } },
{ "set": { "if": "ctx.connection?.state == 'OTH'", "field": "connection.state_description", "value": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed)" } },
{ "set": { "if": "ctx.network?.protocol != null && ctx.network?.protocol.contains(\"ipsec\")", "field": "network.protocol", "value": "ipsec"}},
{ "set": { "if": "ctx.network?.protocol != null && ctx.network?.protocol.contains(\"openvpn\")", "field": "network.protocol", "value": "openvpn"}},
{ "pipeline": { "name": "zeek.common" } }
]
}

View File

@@ -0,0 +1,38 @@
{
"description": "zeek.ipsec",
"processors": [
{"set": { "field": "event.dataset","value": "ipsec"}},
{"json": { "field": "message","target_field": "message2","ignore_failure": true}},
{"rename": {"field": "message2.initiator_spi","target_field": "ipsec.initiator_spi","ignore_missing": true}},
{"rename": {"field": "message2.responder_spi","target_field": "ipsec.responder_spi","ignore_missing": true}},
{"rename": {"field": "message2.maj_ver","target_field": "ipsec.maj_version","ignore_missing": true}},
{"rename": {"field": "message2.min_ver","target_field": "ipsec.min_version","ignore_missing": true}},
{"set": {"ignore_failure": true,"field": "ipsec.version","value": "{{ipsec.maj_version}}.{{ipsec.min_version}}"}},
{"rename": {"field": "message2.exchange_type","target_field": "ipsec.exchange_type","ignore_missing": true}},
{"rename": {"field": "message2.flag_e","target_field": "ipsec.flag_e","ignore_missing": true}},
{"rename": {"field": "message2.flag_c","target_field": "ipsec.flag_c","ignore_missing": true}},
{"rename": {"field": "message2.flag_a","target_field": "ipsec.flag_a","ignore_missing": true}},
{"rename": {"field": "message2.flag_i","target_field": "ipsec.flag_i","ignore_missing": true}},
{"rename": {"field": "message2.flag_v","target_field": "ipsec.flag_v","ignore_missing": true}},
{"rename": {"field": "message2.flag_r","target_field": "ipsec.flag_r","ignore_missing": true}},
{"rename": {"field": "message2.message_id","target_field": "ipsec.message_id","ignore_missing": true}},
{"rename": {"field": "message2.vendor_ids","target_field": "ipsec.vendor_ids","ignore_missing": true}},
{"rename": {"field": "message2.notify_messages","target_field": "ipsec.notify_messages","ignore_missing": true}},
{"rename": {"field": "message2.transforms","target_field": "ipsec.transforms","ignore_missing": true}},
{"rename": {"field": "message2.ke_dh_groups","target_field": "ipsec.ke_dh_groups","ignore_missing": true}},
{"rename": {"field": "message2.proposals","target_field": "ipsec.proposals","ignore_missing": true}},
{"rename": {"field": "message2.certificates","target_field": "ipsec.certificates","ignore_missing": true}},
{"rename": {"field": "message2.transform_attributes","target_field": "ipsec.transform_attributes","ignore_missing": true}},
{"rename": {"field": "message2.length","target_field": "ipsec.length","ignore_missing": true}},
{"rename": {"field": "message2.hash","target_field": "ipsec.hash","ignore_missing": true}},
{"rename": {"field": "message2.doi","target_field": "ipsec.doi","ignore_missing": true}},
{"rename": {"field": "message2.situation","target_field": "ipsec.situation","ignore_missing": true}},
{"script": {
"lang": "painless",
"description": "Remove ipsec fields with empty arrays",
"source": "if (ctx.containsKey('ipsec') && ctx.ipsec instanceof Map) {\n for (String field : ['certificates', 'ke_dh_groups', 'notify_messages', 'proposals', 'transforms', 'transform_attributes', 'vendor_ids']) {\n if (ctx.ipsec[field] instanceof List && ctx.ipsec[field].isEmpty()) {\n ctx.ipsec.remove(field);\n }\n }\n }",
"ignore_failure": true
}},
{"pipeline": {"name": "zeek.common"}}
]
}

View File

@@ -603,6 +603,89 @@
}
}
},
"ipsec": {
"properties": {
"certificates": {
"ignore_above": 1024,
"type": "keyword"
},
"exchange_type": {
"type": "short"
},
"flag_a": {
"type": "boolean"
},
"flag_c": {
"type": "boolean"
},
"flag_e": {
"type": "boolean"
},
"flag_i": {
"type": "boolean"
},
"flag_r": {
"type": "boolean"
},
"flag_v": {
"type": "boolean"
},
"hash": {
"ignore_above": 1024,
"type": "keyword"
},
"initiator_spi": {
"ignore_above": 1024,
"type": "keyword"
},
"ke_dh_groups": {
"type": "short"
},
"length": {
"type": "long"
},
"maj_version": {
"type": "short"
},
"message_id": {
"type": "long"
},
"min_version": {
"type": "short"
},
"notify_messages": {
"ignore_above": 1024,
"type": "keyword"
},
"proposals": {
"type": "long"
},
"responder_spi": {
"ignore_above": 1024,
"type": "keyword"
},
"situation": {
"ignore_above": 1024,
"type": "keyword"
},
"transform_attributes": {
"ignore_above": 1024,
"type": "keyword"
},
"transforms": {
"ignore_above": 1024,
"type": "keyword"
},
"vendor_ids": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"irc": {
"properties": {
"addl": {

View File

@@ -126,7 +126,6 @@ kratos:
issuer:
description: The name to show in the MFA authenticator app. Useful for differentiating between installations that share the same user email address.
global: True
advanced: True
helpLink: kratos.html
webauthn:
enabled:

View File

@@ -1380,6 +1380,10 @@ main() {
echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes."
(wait_for_salt_minion "$MINIONID" "5" '/dev/stdout' || fail "Salt minion was not running or ready.") 2>&1 | tee -a "$SOUP_LOG"
# Stop long-running scripts to allow potentially updated scripts to load on the next execution.
killall salt-relay.sh
highstate
postupgrade_changes
[[ $is_airgap -eq 0 ]] && unmount_update

View File

@@ -1327,16 +1327,49 @@ soc:
showAiSummaries: true
autoUpdateEnabled: true
autoEnabledSigmaRules:
default:
- core+critical
- securityonion-resources+critical
- securityonion-resources+high
so-eval:
- securityonion-resources+critical
- securityonion-resources+high
so-import:
- securityonion-resources+critical
- securityonion-resources+high
default: []
so-eval: []
so-import: []
enabledSigmaRules:
default: |-
# SOS - resources ruleset
- ruleset: ["securityonion-resources"]
level: ["critical", "high"]
product: ["*"]
category: ["*"]
service: ["*"]
# SigmaHQ - Core ruleset - Logsource: System events supported by Elastic Agent
- ruleset: ["core"]
level: ["critical"]
product: ["*"]
category: ["process_creation", "file_event", "registry_event", "network_connection", "dns_query"]
service: ["*"]
# SigmaHQ - Core ruleset - Logsource: Windows eventlogs
- ruleset: ["core"]
level: ["critical"]
product: ["windows"]
category: ["*"]
service: ["security", "system", "dns-client", "application"]
# SigmaHQ - Core ruleset - Logsource: misc
- ruleset: ["core"]
level: ["critical"]
product: ["*"]
category: ["antivirus"]
service: ["*"]
so-eval: |-
# SOS - resources ruleset
- ruleset: ["securityonion-resources"]
level: ["critical", "high"]
product: ["*"]
category: ["*"]
service: ["*"]
so-import: |-
# SOS - resources ruleset
- ruleset: ["securityonion-resources"]
level: ["critical", "high"]
product: ["*"]
category: ["*"]
service: ["*"]
communityRulesImportFrequencySeconds: 86400
communityRulesImportErrorSeconds: 300
failAfterConsecutiveErrorCount: 10
@@ -1896,6 +1929,9 @@ soc:
- name: Intel
description: Zeek Intel framework hits
query: 'tags:intel | groupby intel.indicator | groupby -sankey intel.indicator source.ip | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby intel.indicator_type | groupby intel.seen_where'
- name: IPSec
description: IPSec VPN connection metadata
query: 'tags:ipsec | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby destination.geo.country_name | groupby ipsec.version'
- name: IRC
description: IRC (Internet Relay Chat) network metadata
query: 'tags:irc | groupby irc.command.type | groupby -sankey irc.command.type irc.username | groupby irc.username | groupby irc.nickname | groupby irc.command.value | groupby irc.command.info | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name'
@@ -1908,6 +1944,9 @@ soc:
- name: NTLM
description: NTLM (New Technology LAN Manager) network metadata
query: 'tags:ntlm | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby ntlm.server.dns.name | groupby ntlm.server.nb.name | groupby ntlm.server.tree.name | groupby ntlm.success | groupby source.ip | groupby destination.ip'
- name: OpenVPN
description: OpenVPN connection metadata
query: 'tags:openvpn | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby destination.geo.country_name'
- name: PE
description: PE (Portable Executable) files transferred via network traffic
query: 'tags:pe | groupby file.machine | groupby -sankey file.machine file.os | groupby file.os | groupby -sankey file.os file.subsystem | groupby file.subsystem | groupby file.section_names | groupby file.is_exe | groupby file.is_64bit'

View File

@@ -35,13 +35,21 @@
{# since cases is not a valid soc config item and only used for the map files, remove it from being placed in the config #}
{% do SOCMERGED.config.server.modules.pop('cases') %}
{# set Sigma rules based on role if defined and default if not #}
{# set enabled Sigma rules based on role if defined and default if not #}
{# this particular config is deprecated as of 2.4.120 - use enabledSigmaRules instead #}
{% if GLOBALS.role in SOCMERGED.config.server.modules.elastalertengine.autoEnabledSigmaRules %}
{% do SOCMERGED.config.server.modules.elastalertengine.update({'autoEnabledSigmaRules': SOCMERGED.config.server.modules.elastalertengine.autoEnabledSigmaRules[GLOBALS.role]}) %}
{% else %}
{% do SOCMERGED.config.server.modules.elastalertengine.update({'autoEnabledSigmaRules': SOCMERGED.config.server.modules.elastalertengine.autoEnabledSigmaRules.default}) %}
{% endif %}
{# set enabled Sigma rules based on role if defined and default if not #}
{% if GLOBALS.role in SOCMERGED.config.server.modules.elastalertengine.enabledSigmaRules %}
{% do SOCMERGED.config.server.modules.elastalertengine.update({'enabledSigmaRules': SOCMERGED.config.server.modules.elastalertengine.enabledSigmaRules[GLOBALS.role]}) %}
{% else %}
{% do SOCMERGED.config.server.modules.elastalertengine.update({'enabledSigmaRules': SOCMERGED.config.server.modules.elastalertengine.enabledSigmaRules.default}) %}
{% endif %}
{# set elastalertengine.rulesRepos and strelkaengine.rulesRepos based on airgap or not #}
{% if GLOBALS.airgap %}
{% do SOCMERGED.config.server.modules.elastalertengine.update({'rulesRepos': SOCMERGED.config.server.modules.elastalertengine.rulesRepos.airgap}) %}

View File

@@ -215,14 +215,29 @@ soc:
duplicates: True
forcedType: string
jinjaEscaped: True
enabledSigmaRules:
default: &enabledSigmaRules
description: 'Sigma rules to automatically enable on initial import. The format is a YAML list, with the ability to filter for ruleset, level, product, category and service. Refer to the documentation for further details. These will be applied based on role if defined and default if not.'
global: True
helpLink: sigma.html
multiline: True
syntax: yaml
forcedType: string
jinjaEscaped: True
so-eval: *enabledSigmaRules
so-import: *enabledSigmaRules
autoEnabledSigmaRules:
default: &autoEnabledSigmaRules
description: 'Sigma rules to automatically enable on initial import. Format is $Ruleset+$Level - for example, for the core community ruleset and critical level rules: core+critical. These will be applied based on role if defined and default if not.'
description: 'DEPRECATED: Will be removed in a future release - use enabledSigmaRules instead.'
global: True
advanced: True
helpLink: sigma.html
so-eval: *autoEnabledSigmaRules
so-import: *autoEnabledSigmaRules
autoUpdateEnabled:
description: 'Automatically update Sigma rules on a regular basis. This will update the rules based on the configured frequency.'
global: True
advanced: True
communityRulesImportFrequencySeconds:
description: 'How often to check for new Sigma rules (in seconds). This applies to both Community Rule Packages and any configured Git repos.'
global: True
@@ -329,6 +344,10 @@ soc:
showAiSummaries:
description: Show AI summaries for Strelka rules.
global: True
autoUpdateEnabled:
description: 'Automatically update YARA rules on a regular basis. This will update the rules based on the configured frequency.'
global: True
advanced: True
autoEnabledYaraRules:
description: 'YARA rules to automatically enable on initial import. Format is $Ruleset - for example, for the default shipped ruleset: securityonion-yara'
global: True
@@ -367,6 +386,10 @@ soc:
showAiSummaries:
description: Show AI summaries for Suricata rules.
global: True
autoUpdateEnabled:
description: 'Automatically update Suricata rules on a regular basis. This will update the rules based on the configured frequency.'
global: True
advanced: True
communityRulesImportFrequencySeconds:
description: 'How often to check for new Suricata rules (in seconds).'
global: True

View File

@@ -23,6 +23,7 @@ zeek:
CfgDir: /opt/zeek/etc
CompressLogs: 1
ZeekPort: 27760
FileExtractDir: ""
local:
load:
- misc/loaded-scripts
@@ -70,6 +71,8 @@ zeek:
- zeek-spicy-wireguard
- zeek-spicy-stun
- http2
- zeek-spicy-ipsec
- zeek-spicy-openvpn
load-sigs:
- frameworks/signatures/detect-windows-shells
redef:

View File

@@ -1,6 +1,6 @@
## Global ZeekControl configuration file.
{%- set ALLOWEDOPTIONS = ['commtimeout','commandtimeout','compresscmd','compressextension','compresslogs','compresslogsinflight','controltopic','crashexpireinterval','croncmd','debug','env_vars','havenfs','keeplogs','logdir','logexpireinterval','logrotationinterval','mailalarmsinterval','mailalarmsto','mailarchivelogfail','mailconnectionsummary','mailfrom','mailhostupdown','mailreceivingpackets','mailreplyto','mailsubjectprefix','mailto','makearchivename','memlimit','mindiskspace','pfringclusterid','pfringclustertype','pfringfirstappinstance','prefixes','savetraces','sendmail','sitepluginpath','sitepolicypath','sitepolicyscripts','statslogenable','statslogexpireinterval','statuscmdshowall','stoptimeout','stopwait','timefmt','timemachinehost','timemachineport','zeekargs','zeekport','bindir','capstatspath','cfgdir','debuglog','defaultstoredir','helperdir','libdir','libdir64','libdirinternal','localnetscfg','lockfile','logexpireminutes','nodecfg','os','pcapbufsize','pcapsnaplen','plugindir','pluginzeekdir','policydir','policydirsiteinstall','policydirsiteinstallauto','postprocdir','scriptsdir','spooldir','standalone','statefile','staticdir','statsdir','statslog','time','tmpdir','tmpexecdir','tracesummary','version','zeek','zeekbase'] %}
{%- set ALLOWEDOPTIONS = ['commtimeout','commandtimeout','compresscmd','compressextension','compresslogs','compresslogsinflight','controltopic','crashexpireinterval','croncmd','debug','env_vars','fileextractdir','havenfs','keeplogs','logdir','logexpireinterval','logrotationinterval','mailalarmsinterval','mailalarmsto','mailarchivelogfail','mailconnectionsummary','mailfrom','mailhostupdown','mailreceivingpackets','mailreplyto','mailsubjectprefix','mailto','makearchivename','memlimit','mindiskspace','pfringclusterid','pfringclustertype','pfringfirstappinstance','prefixes','savetraces','sendmail','sitepluginpath','sitepolicypath','sitepolicyscripts','statslogenable','statslogexpireinterval','statuscmdshowall','stoptimeout','stopwait','timefmt','timemachinehost','timemachineport','zeekargs','zeekport','bindir','capstatspath','cfgdir','debuglog','defaultstoredir','helperdir','libdir','libdir64','libdirinternal','localnetscfg','lockfile','logexpireminutes','nodecfg','os','pcapbufsize','pcapsnaplen','plugindir','pluginzeekdir','policydir','policydirsiteinstall','policydirsiteinstallauto','postprocdir','scriptsdir','spooldir','standalone','statefile','staticdir','statsdir','statslog','time','tmpdir','tmpexecdir','tracesummary','version','zeek','zeekbase'] %}
{%- for option in ZEEKCTL|sort %}
{%- if option|lower in ALLOWEDOPTIONS %}