From 5b9f6b2d52fcbd2c37493d50531b50d0cf10c1f1 Mon Sep 17 00:00:00 2001 From: defensivedepth Date: Mon, 2 Dec 2024 14:42:56 -0500 Subject: [PATCH 1/5] fix path --- .../tools/sbin_jinja/so-elastic-agent-gen-installers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers index 7647086ad..850669ba0 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers @@ -79,7 +79,7 @@ done printf "\n\n### Generating MSI...\n" docker run \ --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ \ -{{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o /output/so-elastic-agent_msi --arch x64 /workspace/so-elastic-agent.wxs +{{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o /output/so-elastic-agent_windows_amd64_msi --arch x64 /workspace/so-elastic-agent.wxs printf "\n### MSI Generated...\n" printf "\n### Cleaning up temp files in /nsm/elastic-agent-workspace\n" From 754d28e95db9ab58241e82704b12a6e5e8322480 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 5 Dec 2024 09:52:55 -0600 Subject: [PATCH 2/5] add openvpn & ipsec support to Zeek --- salt/elasticsearch/files/ingest/global@custom | 2 +- salt/elasticsearch/files/ingest/zeek.common | 1 + salt/elasticsearch/files/ingest/zeek.conn | 2 + salt/elasticsearch/files/ingest/zeek.ipsec | 38 +++++++++ .../templates/component/ecs/zeek.json | 83 +++++++++++++++++++ salt/soc/defaults.yaml | 6 ++ salt/zeek/defaults.yaml | 2 + 7 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 salt/elasticsearch/files/ingest/zeek.ipsec diff --git a/salt/elasticsearch/files/ingest/global@custom b/salt/elasticsearch/files/ingest/global@custom index dbf215fb1..27acce74c 100644 --- a/salt/elasticsearch/files/ingest/global@custom +++ b/salt/elasticsearch/files/ingest/global@custom @@ -10,7 +10,7 @@ { "split": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "module_temp" } }, { "set": { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}" } }, { "gsub": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "pattern": "^[^.]*.", "replacement": "", "target_field": "dataset_tag_temp" } }, - { "append": { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp}}" } }, + { "append": { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp}}", "allow_duplicates": false } }, { "set": { "if": "ctx.network?.direction == 'egress'", "override": true, "field": "network.initiated", "value": "true" } }, { "set": { "if": "ctx.network?.direction == 'ingress'", "override": true, "field": "network.initiated", "value": "false" } }, { "set": { "if": "ctx.network?.type == 'ipv4'", "override": true, "field": "destination.ipv6", "value": "false" } }, diff --git a/salt/elasticsearch/files/ingest/zeek.common b/salt/elasticsearch/files/ingest/zeek.common index 3265e20be..8cf4df9be 100644 --- a/salt/elasticsearch/files/ingest/zeek.common +++ b/salt/elasticsearch/files/ingest/zeek.common @@ -18,6 +18,7 @@ { "set": { "if": "ctx.destination?.ip != null", "field": "server.ip", "value": "{{destination.ip}}" } }, { "set": { "if": "ctx.destination?.port != null", "field": "server.port", "value": "{{destination.port}}" } }, { "set": { "field": "observer.name", "value": "{{agent.name}}" } }, + { "append": { "if": "ctx.network?.protocol != null && ctx.network?.protocol.contains(\"openvpn\")","field": "tags","value": ["{{network.protocol}}"],"allow_duplicates": false,"ignore_failure": true}}, { "date": { "field": "message2.ts", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "ignore_failure": true } }, { "remove": { "field": ["agent"], "ignore_failure": true } }, { "pipeline": { "name": "common" } } diff --git a/salt/elasticsearch/files/ingest/zeek.conn b/salt/elasticsearch/files/ingest/zeek.conn index 8b458e032..bd98192d8 100644 --- a/salt/elasticsearch/files/ingest/zeek.conn +++ b/salt/elasticsearch/files/ingest/zeek.conn @@ -38,6 +38,8 @@ { "set": { "if": "ctx.connection?.state == 'SH'", "field": "connection.state_description", "value": "Originator sent a SYN followed by a FIN, we never saw a SYN ACK from the responder (hence the connection was 'half' open)" } }, { "set": { "if": "ctx.connection?.state == 'SHR'", "field": "connection.state_description", "value": "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator" } }, { "set": { "if": "ctx.connection?.state == 'OTH'", "field": "connection.state_description", "value": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed)" } }, + { "set": { "if": "ctx.network?.protocol.contains(\"ipsec\")", "field": "network.protocol", "value": "ipsec"}}, + { "set": { "if": "ctx.network?.protocol.contains(\"openvpn\")", "field": "network.protocol", "value": "openvpn"}}, { "pipeline": { "name": "zeek.common" } } ] } diff --git a/salt/elasticsearch/files/ingest/zeek.ipsec b/salt/elasticsearch/files/ingest/zeek.ipsec new file mode 100644 index 000000000..bf2872ebc --- /dev/null +++ b/salt/elasticsearch/files/ingest/zeek.ipsec @@ -0,0 +1,38 @@ +{ + "description": "zeek.ipsec", + "processors": [ + {"set": { "field": "event.dataset","value": "ipsec"}}, + {"json": { "field": "message","target_field": "message2","ignore_failure": true}}, + {"rename": {"field": "message2.initiator_spi","target_field": "ipsec.initiator_spi","ignore_missing": true}}, + {"rename": {"field": "message2.responder_spi","target_field": "ipsec.responder_spi","ignore_missing": true}}, + {"rename": {"field": "message2.maj_ver","target_field": "ipsec.maj_version","ignore_missing": true}}, + {"rename": {"field": "message2.min_ver","target_field": "ipsec.min_version","ignore_missing": true}}, + {"set": {"ignore_failure": true,"field": "ipsec.version","value": "{{ipsec.maj_version}}.{{ipsec.min_version}}"}}, + {"rename": {"field": "message2.exchange_type","target_field": "ipsec.exchange_type","ignore_missing": true}}, + {"rename": {"field": "message2.flag_e","target_field": "ipsec.flag_e","ignore_missing": true}}, + {"rename": {"field": "message2.flag_c","target_field": "ipsec.flag_c","ignore_missing": true}}, + {"rename": {"field": "message2.flag_a","target_field": "ipsec.flag_a","ignore_missing": true}}, + {"rename": {"field": "message2.flag_i","target_field": "ipsec.flag_i","ignore_missing": true}}, + {"rename": {"field": "message2.flag_v","target_field": "ipsec.flag_v","ignore_missing": true}}, + {"rename": {"field": "message2.flag_r","target_field": "ipsec.flag_r","ignore_missing": true}}, + {"rename": {"field": "message2.message_id","target_field": "ipsec.message_id","ignore_missing": true}}, + {"rename": {"field": "message2.vendor_ids","target_field": "ipsec.vendor_ids","ignore_missing": true}}, + {"rename": {"field": "message2.notify_messages","target_field": "ipsec.notify_messages","ignore_missing": true}}, + {"rename": {"field": "message2.transforms","target_field": "ipsec.transforms","ignore_missing": true}}, + {"rename": {"field": "message2.ke_dh_groups","target_field": "ipsec.ke_dh_groups","ignore_missing": true}}, + {"rename": {"field": "message2.proposals","target_field": "ipsec.proposals","ignore_missing": true}}, + {"rename": {"field": "message2.certificates","target_field": "ipsec.certificates","ignore_missing": true}}, + {"rename": {"field": "message2.transform_attributes","target_field": "ipsec.transform_attributes","ignore_missing": true}}, + {"rename": {"field": "message2.length","target_field": "ipsec.length","ignore_missing": true}}, + {"rename": {"field": "message2.hash","target_field": "ipsec.hash","ignore_missing": true}}, + {"rename": {"field": "message2.doi","target_field": "ipsec.doi","ignore_missing": true}}, + {"rename": {"field": "message2.situation","target_field": "ipsec.situation","ignore_missing": true}}, + {"script": { + "lang": "painless", + "description": "Remove ipsec fields with empty arrays", + "source": "if (ctx.containsKey('ipsec') && ctx.ipsec instanceof Map) {\n for (String field : ['certificates', 'ke_dh_groups', 'notify_messages', 'proposals', 'transforms', 'transform_attributes', 'vendor_ids']) {\n if (ctx.ipsec[field] instanceof List && ctx.ipsec[field].isEmpty()) {\n ctx.ipsec.remove(field);\n }\n }\n }", + "ignore_failure": true + }}, + {"pipeline": {"name": "zeek.common"}} + ] +} \ No newline at end of file diff --git a/salt/elasticsearch/templates/component/ecs/zeek.json b/salt/elasticsearch/templates/component/ecs/zeek.json index f14349263..0f8fb2cfa 100644 --- a/salt/elasticsearch/templates/component/ecs/zeek.json +++ b/salt/elasticsearch/templates/component/ecs/zeek.json @@ -603,6 +603,89 @@ } } }, + "ipsec": { + "properties": { + "certificates": { + "ignore_above": 1024, + "type": "keyword" + }, + "exchange_type": { + "type": "short" + }, + "flag_a": { + "type": "boolean" + }, + "flag_c": { + "type": "boolean" + }, + "flag_e": { + "type": "boolean" + }, + "flag_i": { + "type": "boolean" + }, + "flag_r": { + "type": "boolean" + }, + "flag_v": { + "type": "boolean" + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "initiator_spi": { + "ignore_above": 1024, + "type": "keyword" + }, + "ke_dh_groups": { + "type": "short" + }, + "length": { + "type": "long" + }, + "maj_version": { + "type": "short" + }, + "message_id": { + "type": "long" + }, + "min_version": { + "type": "short" + }, + "notify_messages": { + "ignore_above": 1024, + "type": "keyword" + }, + "proposals": { + "type": "long" + }, + "responder_spi": { + "ignore_above": 1024, + "type": "keyword" + }, + "situation": { + "ignore_above": 1024, + "type": "keyword" + }, + "transform_attributes": { + "ignore_above": 1024, + "type": "keyword" + }, + "transforms": { + "ignore_above": 1024, + "type": "keyword" + }, + "vendor_ids": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "irc": { "properties": { "addl": { diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 7521c9582..580b6993f 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1929,6 +1929,9 @@ soc: - name: Intel description: Zeek Intel framework hits query: 'tags:intel | groupby intel.indicator | groupby -sankey intel.indicator source.ip | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby intel.indicator_type | groupby intel.seen_where' + - name: IPSec + description: IPSec VPN connection metadata + query: 'tags:ipsec | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby destination.geo.country_name | groupby ipsec.version' - name: IRC description: IRC (Internet Relay Chat) network metadata query: 'tags:irc | groupby irc.command.type | groupby -sankey irc.command.type irc.username | groupby irc.username | groupby irc.nickname | groupby irc.command.value | groupby irc.command.info | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' @@ -1941,6 +1944,9 @@ soc: - name: NTLM description: NTLM (New Technology LAN Manager) network metadata query: 'tags:ntlm | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby ntlm.server.dns.name | groupby ntlm.server.nb.name | groupby ntlm.server.tree.name | groupby ntlm.success | groupby source.ip | groupby destination.ip' + - name: OpenVPN + description: OpenVPN connection metadata + query: 'tags:openvpn | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby destination.geo.country_name' - name: PE description: PE (Portable Executable) files transferred via network traffic query: 'tags:pe | groupby file.machine | groupby -sankey file.machine file.os | groupby file.os | groupby -sankey file.os file.subsystem | groupby file.subsystem | groupby file.section_names | groupby file.is_exe | groupby file.is_64bit' diff --git a/salt/zeek/defaults.yaml b/salt/zeek/defaults.yaml index b4291640a..83d8baff0 100644 --- a/salt/zeek/defaults.yaml +++ b/salt/zeek/defaults.yaml @@ -70,6 +70,8 @@ zeek: - zeek-spicy-wireguard - zeek-spicy-stun - http2 + - zeek-spicy-ipsec + - zeek-spicy-openvpn load-sigs: - frameworks/signatures/detect-windows-shells redef: From 9532f21c7b037522bf1b1c6f51fe58a00c1d5a39 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 5 Dec 2024 13:49:44 -0600 Subject: [PATCH 3/5] check zeek reporter.log --- salt/common/tools/sbin/so-log-check | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index fc855e9f7..9b54cf586 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -211,6 +211,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|integrity check failed" # Detections: Exclude false positive due to automated testing EXCLUDED_ERRORS="$EXCLUDED_ERRORS|syncErrors" # Detections: Not an actual error EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Initialized license manager" # SOC log: before fields.status was changed to fields.licenseStatus + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|from NIC checksum offloading" # zeek reporter.log fi RESULT=0 @@ -249,6 +250,9 @@ exclude_log "agentstatus.log" # ignore this log since it tracks agents in error exclude_log "detections_runtime-status_yara.log" # temporarily ignore this log until Detections is more stable exclude_log "/nsm/kafka/data/" # ignore Kafka data directory from log check. +# Include Zeek reporter.log to detect errors after running known good pcap(s) through sensor +echo "/nsm/zeek/spool/logger/reporter.log" >> /tmp/log_check_files + for log_file in $(cat /tmp/log_check_files); do status "Checking log file $log_file" tail -n $RECENT_LOG_LINES $log_file > /tmp/log_check From ad8b339a3b658947e0658db44d877eaac5f970c1 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 6 Dec 2024 09:07:16 -0600 Subject: [PATCH 4/5] fix error due to null reference Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/elasticsearch/files/ingest/zeek.conn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/files/ingest/zeek.conn b/salt/elasticsearch/files/ingest/zeek.conn index bd98192d8..6051d93a1 100644 --- a/salt/elasticsearch/files/ingest/zeek.conn +++ b/salt/elasticsearch/files/ingest/zeek.conn @@ -38,8 +38,8 @@ { "set": { "if": "ctx.connection?.state == 'SH'", "field": "connection.state_description", "value": "Originator sent a SYN followed by a FIN, we never saw a SYN ACK from the responder (hence the connection was 'half' open)" } }, { "set": { "if": "ctx.connection?.state == 'SHR'", "field": "connection.state_description", "value": "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator" } }, { "set": { "if": "ctx.connection?.state == 'OTH'", "field": "connection.state_description", "value": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed)" } }, - { "set": { "if": "ctx.network?.protocol.contains(\"ipsec\")", "field": "network.protocol", "value": "ipsec"}}, - { "set": { "if": "ctx.network?.protocol.contains(\"openvpn\")", "field": "network.protocol", "value": "openvpn"}}, + { "set": { "if": "ctx.network?.protocol != null && ctx.network?.protocol.contains(\"ipsec\")", "field": "network.protocol", "value": "ipsec"}}, + { "set": { "if": "ctx.network?.protocol != null && ctx.network?.protocol.contains(\"openvpn\")", "field": "network.protocol", "value": "openvpn"}}, { "pipeline": { "name": "zeek.common" } } ] } From 1de20e9d43b1a0d2e94b628d5b1d5d443a0b4665 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 6 Dec 2024 09:55:56 -0600 Subject: [PATCH 5/5] fix zeek file extract Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/zeek/defaults.yaml | 1 + salt/zeek/files/zeekctl.cfg.jinja | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/zeek/defaults.yaml b/salt/zeek/defaults.yaml index b4291640a..c66124a86 100644 --- a/salt/zeek/defaults.yaml +++ b/salt/zeek/defaults.yaml @@ -23,6 +23,7 @@ zeek: CfgDir: /opt/zeek/etc CompressLogs: 1 ZeekPort: 27760 + FileExtractDir: "" local: load: - misc/loaded-scripts diff --git a/salt/zeek/files/zeekctl.cfg.jinja b/salt/zeek/files/zeekctl.cfg.jinja index 0a6be371c..018ac738c 100644 --- a/salt/zeek/files/zeekctl.cfg.jinja +++ b/salt/zeek/files/zeekctl.cfg.jinja @@ -1,6 +1,6 @@ ## Global ZeekControl configuration file. -{%- set ALLOWEDOPTIONS = ['commtimeout','commandtimeout','compresscmd','compressextension','compresslogs','compresslogsinflight','controltopic','crashexpireinterval','croncmd','debug','env_vars','havenfs','keeplogs','logdir','logexpireinterval','logrotationinterval','mailalarmsinterval','mailalarmsto','mailarchivelogfail','mailconnectionsummary','mailfrom','mailhostupdown','mailreceivingpackets','mailreplyto','mailsubjectprefix','mailto','makearchivename','memlimit','mindiskspace','pfringclusterid','pfringclustertype','pfringfirstappinstance','prefixes','savetraces','sendmail','sitepluginpath','sitepolicypath','sitepolicyscripts','statslogenable','statslogexpireinterval','statuscmdshowall','stoptimeout','stopwait','timefmt','timemachinehost','timemachineport','zeekargs','zeekport','bindir','capstatspath','cfgdir','debuglog','defaultstoredir','helperdir','libdir','libdir64','libdirinternal','localnetscfg','lockfile','logexpireminutes','nodecfg','os','pcapbufsize','pcapsnaplen','plugindir','pluginzeekdir','policydir','policydirsiteinstall','policydirsiteinstallauto','postprocdir','scriptsdir','spooldir','standalone','statefile','staticdir','statsdir','statslog','time','tmpdir','tmpexecdir','tracesummary','version','zeek','zeekbase'] %} +{%- set ALLOWEDOPTIONS = ['commtimeout','commandtimeout','compresscmd','compressextension','compresslogs','compresslogsinflight','controltopic','crashexpireinterval','croncmd','debug','env_vars','fileextractdir','havenfs','keeplogs','logdir','logexpireinterval','logrotationinterval','mailalarmsinterval','mailalarmsto','mailarchivelogfail','mailconnectionsummary','mailfrom','mailhostupdown','mailreceivingpackets','mailreplyto','mailsubjectprefix','mailto','makearchivename','memlimit','mindiskspace','pfringclusterid','pfringclustertype','pfringfirstappinstance','prefixes','savetraces','sendmail','sitepluginpath','sitepolicypath','sitepolicyscripts','statslogenable','statslogexpireinterval','statuscmdshowall','stoptimeout','stopwait','timefmt','timemachinehost','timemachineport','zeekargs','zeekport','bindir','capstatspath','cfgdir','debuglog','defaultstoredir','helperdir','libdir','libdir64','libdirinternal','localnetscfg','lockfile','logexpireminutes','nodecfg','os','pcapbufsize','pcapsnaplen','plugindir','pluginzeekdir','policydir','policydirsiteinstall','policydirsiteinstallauto','postprocdir','scriptsdir','spooldir','standalone','statefile','staticdir','statsdir','statslog','time','tmpdir','tmpexecdir','tracesummary','version','zeek','zeekbase'] %} {%- for option in ZEEKCTL|sort %} {%- if option|lower in ALLOWEDOPTIONS %}