From 3d1ce4ef10f5124e356ac51cff1bf0a153e91da1 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Wed, 21 Dec 2022 13:26:04 -0500 Subject: [PATCH 01/39] Improve SOC dashboards --- salt/soc/defaults.yaml | 64 +++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index ec78bff8b..90c4c2422 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1374,19 +1374,19 @@ soc: queries: - name: Overview description: Overview of all events - query: '* | groupby -sankey event.dataset event.category* | groupby -pie event.category | groupby -bar event.module | groupby event.dataset | groupby event.module | groupby event.category | groupby observer.name | groupby source.ip | groupby destination.ip | groupby destination.port' + query: '* | groupby -sankey event.dataset event.category* | groupby -pie event.category | groupby -bar event.module | groupby event.dataset | groupby event.module | groupby event.category | groupby observer.name | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - name: SOC Auth description: SOC (Security Onion Console) authentication logs - query: 'event.module:kratos AND event.dataset:audit AND msg:authenticated | groupby http_request.headers.x-real-ip | groupby identity_id | groupby http_request.headers.user-agent' + query: 'event.module:kratos AND event.dataset:audit AND msg:authenticated | groupby -sankey http_request.headers.x-real-ip identity_id | groupby http_request.headers.x-real-ip | groupby identity_id | groupby http_request.headers.user-agent' - name: Elastalerts description: Elastalert logs query: '_index: "*:elastalert*" | groupby rule_name | groupby alert_info.type' - name: Alerts description: Overview of all alerts - query: 'event.dataset: alert | groupby event.module | groupby rule.name | groupby event.severity | groupby source.ip | groupby destination.ip | groupby destination.port' + query: 'event.dataset:alert | groupby event.module | groupby rule.name | groupby event.severity | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - name: NIDS Alerts description: NIDS (Network Intrusion Detection System) alerts - query: 'event.category: network AND event.dataset: alert | groupby rule.category | groupby rule.gid | groupby rule.uuid | groupby rule.name | groupby source.ip | groupby destination.ip | groupby destination.port' + query: 'event.category:network AND event.dataset:alert | groupby rule.category | groupby -sankey source.ip destination.ip | groupby rule.gid | groupby rule.uuid | groupby rule.name | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - name: Sysmon Overview description: Overview of all Sysmon data types query: 'event.module:sysmon | groupby -sankey event.dataset winlog.computer_name | groupby -sankey winlog.computer_name user.name | groupby winlog.computer_name | groupby event.dataset | groupby user.name | groupby dns.query.name | groupby process.executable | groupby winlog.event_data.TargetObject | groupby file.target | groupby source.ip | groupby destination.ip | groupby destination.port' @@ -1410,106 +1410,106 @@ soc: query: 'event.module:strelka | groupby file.mime_type | groupby file.name | groupby file.source' - name: Zeek Notice description: Zeek notice logs - query: 'event.dataset:notice | groupby notice.note | groupby notice.message | groupby notice.sub_message | groupby source.ip | groupby destination.ip | groupby destination.port' + query: 'event.dataset:notice | groupby -sankey notice.note destination.ip | groupby notice.note | groupby notice.message | groupby notice.sub_message | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - name: Connections description: Network connection metadata - query: 'event.dataset:conn | groupby source.ip | groupby destination.ip | groupby destination.port | groupby network.protocol | groupby network.transport | groupby connection.history | groupby connection.state | groupby connection.state_description | groupby source.geo.country_name | groupby destination.geo.country_name | groupby client.ip_bytes | groupby server.ip_bytes' + query: 'event.dataset:conn | groupby source.ip | groupby destination.ip | groupby destination.port | groupby -sankey destination.port network.protocol | groupby network.protocol | groupby network.transport | groupby connection.history | groupby connection.state | groupby connection.state_description | groupby source.geo.country_name | groupby destination.geo.country_name | groupby client.ip_bytes | groupby server.ip_bytes | groupby client.oui' - name: DCE_RPC description: DCE_RPC (Distributed Computing Environment / Remote Procedure Calls) network metadata - query: 'event.dataset:dce_rpc | groupby dce_rpc.operation | groupby dce_rpc.endpoint | groupby dce_rpc.named_pipe | groupby source.ip | groupby destination.ip | groupby destination.port' + query: 'event.dataset:dce_rpc | groupby -sankey dce_rpc.endpoint dce_rpc.operation | groupby dce_rpc.endpoint | groupby dce_rpc.operation | groupby dce_rpc.named_pipe | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - name: DHCP description: DHCP (Dynamic Host Configuration Protocol) leases - query: 'event.dataset:dhcp | groupby host.hostname | groupby host.domain | groupby dhcp.message_types | groupby client.address | groupby server.address' + query: 'event.dataset:dhcp | groupby host.hostname | groupby host.domain | groupby dhcp.message_types | groupby -sankey client.address server.address | groupby client.address | groupby server.address' - name: DNS description: DNS (Domain Name System) queries - query: 'event.dataset:dns | groupby dns.query.name | groupby dns.highest_registered_domain | groupby dns.parent_domain | groupby dns.answers.name | groupby dns.query.type_name | groupby dns.response.code_name | groupby source.ip | groupby destination.ip | groupby destination.port' + query: 'event.dataset:dns | groupby dns.query.name | groupby dns.highest_registered_domain | groupby dns.parent_domain | groupby -sankey source.ip destination.ip | groupby dns.answers.name | groupby dns.query.type_name | groupby dns.response.code_name | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - name: DPD description: DPD (Dynamic Protocol Detection) errors - query: 'event.dataset:dpd | groupby error.reason | groupby source.ip | groupby destination.ip | groupby destination.port | groupby network.protocol' + query: 'event.dataset:dpd | groupby error.reason | groupby network.protocol | groupby -sankey source.ip destination.ip | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - name: Files description: Files seen in network traffic - query: 'event.dataset:file | groupby file.mime_type | groupby file.source | groupby file.bytes.total | groupby source.ip | groupby destination.ip' + query: 'event.dataset:file | groupby file.mime_type | groupby -sankey file.mime_type file.source | groupby file.source | groupby file.bytes.total | groupby source.ip | groupby destination.ip | groupby destination_geo.organization_name' - name: FTP description: FTP (File Transfer Protocol) network metadata - query: 'event.dataset:ftp | groupby ftp.command | groupby ftp.argument | groupby ftp.user | groupby source.ip | groupby destination.ip | groupby destination.port' + query: 'event.dataset:ftp | groupby ftp.command | groupby ftp.argument | groupby ftp.user | groupby -sankey source.ip destination.ip | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - name: HTTP description: HTTP (Hyper Text Transport Protocol) network metadata - query: 'event.dataset:http | groupby http.method | groupby http.virtual_host | groupby http.uri | groupby http.useragent | groupby http.status_code | groupby http.status_message | groupby file.resp_mime_types | groupby source.ip | groupby destination.ip | groupby destination.port' + query: 'event.dataset:http | groupby http.method | groupby -sankey http.method http.virtual_host | groupby http.virtual_host | groupby http.uri | groupby http.useragent | groupby http.status_code | groupby http.status_message | groupby file.resp_mime_types | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - name: Intel description: Zeek Intel framework hits query: 'event.dataset:intel | groupby intel.indicator | groupby intel.indicator_type | groupby intel.seen_where | groupby source.ip | groupby destination.ip | groupby destination.port' - name: IRC description: IRC (Internet Relay Chat) network metadata - query: 'event.dataset:irc | groupby irc.command.type | groupby irc.username | groupby irc.nickname | groupby irc.command.value | groupby irc.command.info | groupby source.ip | groupby destination.ip | groupby destination.port' + query: 'event.dataset:irc | groupby irc.command.type | groupby -sankey irc.command.type irc.username | groupby irc.username | groupby irc.nickname | groupby irc.command.value | groupby irc.command.info | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - name: Kerberos description: Kerberos network metadata - query: 'event.dataset:kerberos | groupby kerberos.service | groupby kerberos.client | groupby kerberos.request_type | groupby source.ip | groupby destination.ip | groupby destination.port' + query: 'event.dataset:kerberos | groupby kerberos.service | groupby -sankey kerberos.service destination.ip | groupby kerberos.client | groupby kerberos.request_type | groupby source.ip | groupby destination.ip | groupby destination.port' - name: MySQL description: MySQL network metadata - query: 'event.dataset:mysql | groupby mysql.command | groupby mysql.argument | groupby mysql.success | groupby mysql.response | groupby mysql.rows | groupby source.ip | groupby destination.ip | groupby destination.port' + query: 'event.dataset:mysql | groupby mysql.command | groupby -sankey mysql.command destination.ip | groupby mysql.argument | groupby mysql.success | groupby mysql.response | groupby mysql.rows | groupby source.ip | groupby destination.ip | groupby destination.port' - name: NTLM description: NTLM (New Technology LAN Manager) network metadata - query: 'event.dataset:ntlm | groupby ntlm.server.dns.name | groupby ntlm.server.nb.name | groupby ntlm.server.tree.name | groupby ntlm.success | groupby source.ip | groupby destination.ip | groupby destination.port' + query: 'event.dataset:ntlm | groupby ntlm.server.dns.name | groupby ntlm.server.nb.name | groupby -sankey source.ip destination.ip | groupby ntlm.server.tree.name | groupby ntlm.success | groupby source.ip | groupby destination.ip | groupby destination.port' - name: PE description: PE (Portable Executable) files transferred via network traffic - query: 'event.dataset:pe | groupby file.machine | groupby file.os | groupby file.subsystem | groupby file.section_names | groupby file.is_exe | groupby file.is_64bit' + query: 'event.dataset:pe | groupby file.machine | groupby -sankey file.machine file.os | groupby file.os | groupby file.subsystem | groupby file.section_names | groupby file.is_exe | groupby file.is_64bit' - name: RADIUS description: RADIUS (Remote Authentication Dial-In User Service) network metadata - query: 'event.dataset:radius | groupby user.name.keyword | groupby source.ip | groupby destination.ip | groupby destination.port' + query: 'event.dataset:radius | groupby -sankey user.name.keyword destination.ip | groupby user.name.keyword | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - name: RDP description: RDP (Remote Desktop Protocol) network metadata - query: 'event.dataset:rdp | groupby client.name | groupby source.ip | groupby destination.ip | groupby destination.port' + query: 'event.dataset:rdp | groupby client.name | groupby -sankey source.ip destination.ip | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - name: RFB description: RFB (Remote Frame Buffer) network metadata - query: 'event.dataset:rfb | groupby rfb.desktop.name.keyword | groupby source.ip | groupby destination.ip | groupby destination.port' + query: 'event.dataset:rfb | groupby rfb.desktop.name.keyword | groupby -sankey source.ip destination.ip | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - name: Signatures description: Zeek signatures query: 'event.dataset:signatures | groupby signature_id' - name: SIP description: SIP (Session Initiation Protocol) network metadata - query: 'event.dataset:sip | groupby client.user_agent | groupby sip.method | groupby sip.uri | groupby source.ip | groupby destination.ip | groupby destination.port' + query: 'event.dataset:sip | groupby client.user_agent | groupby sip.method | groupby sip.uri | groupby -sankey source.ip destination.ip | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - name: SMB_Files description: Files transferred via SMB (Server Message Block) - query: 'event.dataset:smb_files | groupby file.action | groupby file.path | groupby file.name | groupby source.ip | groupby destination.ip | groupby destination.port' + query: 'event.dataset:smb_files | groupby file.action | groupby file.path | groupby file.name | groupby -sankey source.ip destination.ip | groupby source.ip | groupby destination.ip | groupby destination.port' - name: SMB_Mapping description: SMB (Server Message Block) mapping network metadata - query: 'event.dataset:smb_mapping | groupby smb.share_type | groupby smb.path | groupby smb.service | groupby source.ip | groupby destination.ip | groupby destination.port' + query: 'event.dataset:smb_mapping | groupby smb.share_type | groupby smb.path | groupby smb.service | groupby -sankey source.ip destination.ip | groupby source.ip | groupby destination.ip | groupby destination.port' - name: SMTP description: SMTP (Simple Mail Transfer Protocol) network metadata - query: 'event.dataset:smtp | groupby smtp.from | groupby smtp.recipient_to | groupby smtp.subject | groupby source.ip | groupby destination.ip | groupby destination.port' + query: 'event.dataset:smtp | groupby smtp.from | groupby smtp.recipient_to | groupby -sankey source.ip destination.ip | groupby smtp.subject | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - name: SNMP description: SNMP (Simple Network Management Protocol) network metadat - query: 'event.dataset:snmp | groupby snmp.community | groupby snmp.version | groupby source.ip | groupby destination.ip | groupby destination.port' + query: 'event.dataset:snmp | groupby snmp.community | groupby snmp.version | groupby -sankey source.ip destination.ip | groupby source.ip | groupby destination.ip | groupby destination.port' - name: Software description: Software seen by Zeek via network traffic query: 'event.dataset:software | groupby software.type | groupby software.name | groupby source.ip' - name: SSH description: SSH (Secure Shell) connections seen by Zeek - query: 'event.dataset:ssh | groupby ssh.client | groupby ssh.server | groupby ssh.direction | groupby ssh.version | groupby ssh.hassh_version | groupby source.ip | groupby destination.ip | groupby destination.port' + query: 'event.dataset:ssh | groupby ssh.client | groupby ssh.server | groupby -sankey source.ip destination.ip | groupby ssh.direction | groupby ssh.version | groupby ssh.hassh_version | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - name: SSL description: SSL/TLS network metadata - query: 'event.dataset:ssl | groupby ssl.version | groupby ssl.validation_status | groupby ssl.server_name | groupby ssl.certificate.issuer | groupby ssl.certificate.subject | groupby source.ip | groupby destination.ip | groupby destination.port' + query: 'event.dataset:ssl | groupby ssl.version | groupby ssl.validation_status | groupby -sankey source.ip ssl.server_name | groupby ssl.server_name | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name | groupby ssl.certificate.issuer | groupby ssl.certificate.subject' - name: STUN description: STUN (Session Traversal Utilities for NAT) network metadata query: 'event.dataset:stun* | groupby -sankey source.ip destination.ip | groupby destination.geo.country_name | groupby source.ip | groupby destination.ip | groupby destination.port | groupby event.dataset' - name: Syslog description: Syslog logs - query: 'event.dataset:syslog | groupby syslog.severity_label | groupby syslog.facility_label | groupby network.protocol | groupby source.ip | groupby destination.ip | groupby destination.port' + query: 'event.dataset:syslog | groupby syslog.severity_label | groupby syslog.facility_label | groupby -sankey source.ip destination.ip | groupby source.ip | groupby destination.ip | groupby destination.port | groupby network.protocol' - name: TDS description: TDS (Tabular Data Stream) network metadata query: 'event.dataset:tds* | groupby -sankey event.dataset source.ip destination.ip | groupby event.dataset | groupby tds.command | groupby tds.header_type | groupby tds.procedure_name | groupby source.ip | groupby destination.ip | groupby destination.port | groupby tds.query' - name: Tunnel description: Tunnels seen by Zeek - query: 'event.dataset:tunnel | groupby tunnel.type | groupby event.action | groupby source.ip | groupby destination.ip | groupby destination.port' + query: 'event.dataset:tunnel | groupby -sankey source.ip destination.ip | groupby tunnel.type | groupby event.action | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination.geo.country_name' - name: Weird description: Weird network traffic seen by Zeek - query: 'event.dataset:weird | groupby weird.name | groupby weird.additional_info | groupby source.ip | groupby destination.ip | groupby destination.port ' + query: 'event.dataset:weird | groupby -sankey weird.name destination.ip | groupby weird.name | groupby weird.additional_info | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - name: WireGuard description: WireGuard VPN network metadata query: 'event.dataset:wireguard | groupby -sankey source.ip destination.ip | groupby destination.geo.country_name | groupby source.ip | groupby destination.ip | groupby destination.port' - name: x509 description: x.509 certificates seen by Zeek - query: 'event.dataset:x509 | groupby x509.certificate.key.length | groupby x509.san_dns | groupby x509.certificate.key.type | groupby x509.certificate.subject | groupby x509.certificate.issuer' + query: 'event.dataset:x509 | groupby -sankey x509.certificate.key.length x509.san_dns | groupby x509.certificate.key.length | groupby x509.san_dns | groupby x509.certificate.key.type | groupby x509.certificate.subject | groupby x509.certificate.issuer' - name: ICS Overview description: Overview of ICS (Industrial Control Systems) network metadata query: 'tags:ics | groupby event.dataset | groupby -sankey source.ip destination.ip | groupby source.ip | groupby destination.ip | groupby destination.port | groupby source.mac | groupby destination.mac' From 0889d490254867caba0713dc5a5c50d95a648a41 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 21 Dec 2022 14:00:10 -0500 Subject: [PATCH 02/39] Ensure user/pass values are quoted due to symbol chars appearing in the values --- salt/logstash/init.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index bf4d03984..c8130570e 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -66,8 +66,8 @@ ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}: - template: jinja - defaults: GLOBALS: {{ GLOBALS }} - ES_USER: {{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') }} - ES_PASS: {{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') }} + ES_USER: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') }}" + ES_PASS: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') }}" {% else %} - name: /opt/so/conf/logstash/pipelines/{{PL}}/{{CONFIGFILE.split('/')[1]}} {% endif %} From a4f1f75306172eb54d0ed591fd902852e30c13b3 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Wed, 21 Dec 2022 14:33:01 -0500 Subject: [PATCH 03/39] Improve NIDS Alerts dashboard --- salt/soc/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 90c4c2422..957836ccd 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1386,7 +1386,7 @@ soc: query: 'event.dataset:alert | groupby event.module | groupby rule.name | groupby event.severity | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - name: NIDS Alerts description: NIDS (Network Intrusion Detection System) alerts - query: 'event.category:network AND event.dataset:alert | groupby rule.category | groupby -sankey source.ip destination.ip | groupby rule.gid | groupby rule.uuid | groupby rule.name | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' + query: 'event.category:network AND event.dataset:alert | groupby rule.category | groupby -sankey source.ip destination.ip | groupby rule.name | groupby rule.uuid | groupby rule.gid | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - name: Sysmon Overview description: Overview of all Sysmon data types query: 'event.module:sysmon | groupby -sankey event.dataset winlog.computer_name | groupby -sankey winlog.computer_name user.name | groupby winlog.computer_name | groupby event.dataset | groupby user.name | groupby dns.query.name | groupby process.executable | groupby winlog.event_data.TargetObject | groupby file.target | groupby source.ip | groupby destination.ip | groupby destination.port' From 3a367d69f4e67953827afc92bda4f55b6dea7b5c Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Wed, 21 Dec 2022 14:37:17 -0500 Subject: [PATCH 04/39] Improve FTP dashboard --- salt/soc/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 957836ccd..7bfc33079 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1431,7 +1431,7 @@ soc: query: 'event.dataset:file | groupby file.mime_type | groupby -sankey file.mime_type file.source | groupby file.source | groupby file.bytes.total | groupby source.ip | groupby destination.ip | groupby destination_geo.organization_name' - name: FTP description: FTP (File Transfer Protocol) network metadata - query: 'event.dataset:ftp | groupby ftp.command | groupby ftp.argument | groupby ftp.user | groupby -sankey source.ip destination.ip | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' + query: 'event.dataset:ftp | groupby -sankey ftp.command destination.ip | groupby ftp.command | groupby ftp.argument | groupby ftp.user | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - name: HTTP description: HTTP (Hyper Text Transport Protocol) network metadata query: 'event.dataset:http | groupby http.method | groupby -sankey http.method http.virtual_host | groupby http.virtual_host | groupby http.uri | groupby http.useragent | groupby http.status_code | groupby http.status_message | groupby file.resp_mime_types | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' From 6d6fa4c1e315f53f2243cdc8b5045ac08adf0e6f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Dec 2022 15:06:56 -0500 Subject: [PATCH 05/39] need space between curl.config and -X --- salt/kibana/bin/so-kibana-config-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/bin/so-kibana-config-load index bb7bb1f44..dadb48ffe 100644 --- a/salt/kibana/bin/so-kibana-config-load +++ b/salt/kibana/bin/so-kibana-config-load @@ -63,7 +63,7 @@ update() { IFS=$'\r\n' GLOBIGNORE='*' command eval 'LINES=($(cat $1))' for i in "${LINES[@]}"; do - RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config-X PUT "localhost:5601/api/saved_objects/config/8.5.3" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") + RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.5.3" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi done From 4519c533a22a96c667e291e0cfa3e9b6423aa9fa Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Wed, 21 Dec 2022 15:20:27 -0500 Subject: [PATCH 06/39] Improve Intel dashboard --- salt/soc/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 7bfc33079..ec1e07425 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1437,7 +1437,7 @@ soc: query: 'event.dataset:http | groupby http.method | groupby -sankey http.method http.virtual_host | groupby http.virtual_host | groupby http.uri | groupby http.useragent | groupby http.status_code | groupby http.status_message | groupby file.resp_mime_types | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - name: Intel description: Zeek Intel framework hits - query: 'event.dataset:intel | groupby intel.indicator | groupby intel.indicator_type | groupby intel.seen_where | groupby source.ip | groupby destination.ip | groupby destination.port' + query: 'event.dataset:intel | groupby intel.indicator | groupby -sankey source.ip intel.indicator | groupby intel.indicator_type | groupby intel.seen_where | groupby source.ip | groupby destination.ip | groupby destination.port' - name: IRC description: IRC (Internet Relay Chat) network metadata query: 'event.dataset:irc | groupby irc.command.type | groupby -sankey irc.command.type irc.username | groupby irc.username | groupby irc.nickname | groupby irc.command.value | groupby irc.command.info | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' From d7b2c88201fa838ab8baae0ff2a8c27b0e9a72f2 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Wed, 21 Dec 2022 15:24:58 -0500 Subject: [PATCH 07/39] Improve Software dashboard --- salt/soc/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index ec1e07425..d19041503 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1482,7 +1482,7 @@ soc: query: 'event.dataset:snmp | groupby snmp.community | groupby snmp.version | groupby -sankey source.ip destination.ip | groupby source.ip | groupby destination.ip | groupby destination.port' - name: Software description: Software seen by Zeek via network traffic - query: 'event.dataset:software | groupby software.type | groupby software.name | groupby source.ip' + query: 'event.dataset:software | groupby -sankey software.type source.ip | groupby software.type | groupby software.name | groupby source.ip' - name: SSH description: SSH (Secure Shell) connections seen by Zeek query: 'event.dataset:ssh | groupby ssh.client | groupby ssh.server | groupby -sankey source.ip destination.ip | groupby ssh.direction | groupby ssh.version | groupby ssh.hassh_version | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' From 506556f0d299653983b9e06d1d56cf4f60ae37b4 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Wed, 21 Dec 2022 15:29:09 -0500 Subject: [PATCH 08/39] Improve Firewall dashboard --- salt/soc/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index d19041503..2c7b1a372 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1548,7 +1548,7 @@ soc: query: 'event.dataset:s7* | groupby -sankey event.dataset source.ip destination.ip | groupby event.dataset | groupby source.ip | groupby destination.ip | groupby destination.port' - name: Firewall description: Firewall logs - query: 'event.dataset:firewall | groupby rule.action | groupby interface.name | groupby network.transport | groupby source.ip | groupby destination.ip | groupby destination.port' + query: 'event.dataset:firewall | groupby -sankey rule.action interface.name | groupby rule.action | groupby interface.name | groupby network.transport | groupby source.ip | groupby destination.ip | groupby destination.port' job: alerts: advanced: false From 69415a0d8d6685f7cc15298222d4d85d2de9a2b9 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Wed, 21 Dec 2022 15:34:35 -0500 Subject: [PATCH 09/39] Improve Strelka dashboard --- salt/soc/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 2c7b1a372..ceb94c054 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1407,7 +1407,7 @@ soc: query: 'event.dataset:network_connection | groupby -sankey winlog.computer_name destination.ip destination.port | groupby winlog.computer_name | groupby user.name | groupby process.executable | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - name: Strelka description: Strelka file analysis - query: 'event.module:strelka | groupby file.mime_type | groupby file.name | groupby file.source' + query: 'event.module:strelka | groupby file.mime_type | groupby -sankey file.mime_type file.source | groupby file.source | groupby file.name' - name: Zeek Notice description: Zeek notice logs query: 'event.dataset:notice | groupby -sankey notice.note destination.ip | groupby notice.note | groupby notice.message | groupby notice.sub_message | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' From 3fd210463eaaec3a15837b0f81f6700f0ba19382 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 21 Dec 2022 18:11:39 -0500 Subject: [PATCH 10/39] fix redis defaults to force string keys instead of numeric --- salt/redis/soc_redis.yaml | 6 +++--- salt/soc/soc_soc.yaml | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/salt/redis/soc_redis.yaml b/salt/redis/soc_redis.yaml index dad76bf80..9a86cb4c8 100644 --- a/salt/redis/soc_redis.yaml +++ b/salt/redis/soc_redis.yaml @@ -117,15 +117,15 @@ redis: advanced: True helpLink: redis.html save: - 900: + '900': description: Set the amount of keys that need to change to save after 15 minutes. global: True helpLink: redis.html - 300: + '300: description: Set the amount of keys that need to change to save after 5 minutes. global: True helpLink: redis.html - 60: + '60': description: Set the amount of keys that need to change to save after 1 minute global: True helpLink: redis.html diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index d8dbacc29..8febe0681 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -42,8 +42,9 @@ soc: description: A list of actions a user can take from the SOC UI against a hunt, alert, and other records. The action must be defined in JSON object format, and contain a "name" key and "links" key. The links is a list of URLs, where the most suitable URL in the list will be the selected URL when the user clicks the action. global: True eventFields: - description: A map of event metadata groupings to field lists. The field list specifies all the columns to show when displaying individual events (not groups). The event metadata grouping uses the format ":event.module:event.dataset". - global: True + default: + description: The list of fields to show as columns in the Hunt/Dashboards event table, when no other specific mapping applies. Mappings are defined by the format ":event.module:event.dataset". + global: True server: maxPacketCount: description: Maximum number of packets to show in the PCAP viewer. Larger values can cause more resource utilization on both the SOC server and the browser. @@ -146,6 +147,7 @@ soc: global: True queries: description: List of default queries to show in the query list. Each query is represented in JSON object notation, and must include the "name" key and "query" key. + global: True alerts: *appSettings cases: *appSettings dashboards: *appSettings From 8b6006e9c3d2c310cf82cc21a5ff4e822fa80004 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 21 Dec 2022 18:14:18 -0500 Subject: [PATCH 11/39] fix redis defaults to force string keys instead of numeric --- salt/redis/defaults.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/redis/defaults.yaml b/salt/redis/defaults.yaml index 252082901..a0ff4e685 100644 --- a/salt/redis/defaults.yaml +++ b/salt/redis/defaults.yaml @@ -29,9 +29,9 @@ redis: databases: 16 always-show-logo: 'yes' save: - 900: 1 - 300: 10 - 60: 1000 + '900': 1 + '300': 10 + '60': 1000 stop-writes-on-bgsave-error: 'yes' rdbcompression: 'yes' rdbchecksum: 'yes' From 38634fde17afaa3fc6ca86f5d2df5b3fbffd4b8c Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 21 Dec 2022 18:15:17 -0500 Subject: [PATCH 12/39] fix redis defaults to force string keys instead of numeric --- salt/redis/soc_redis.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/redis/soc_redis.yaml b/salt/redis/soc_redis.yaml index 9a86cb4c8..374b613b5 100644 --- a/salt/redis/soc_redis.yaml +++ b/salt/redis/soc_redis.yaml @@ -121,7 +121,7 @@ redis: description: Set the amount of keys that need to change to save after 15 minutes. global: True helpLink: redis.html - '300: + '300': description: Set the amount of keys that need to change to save after 5 minutes. global: True helpLink: redis.html From dfd5947051d105cb60333f830e737542c0736a7d Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 22 Dec 2022 09:46:27 -0500 Subject: [PATCH 13/39] add influxdb and telegraf to import mode --- salt/allowed_states.map.jinja | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 899a56b23..192a52dd3 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -97,6 +97,8 @@ 'nginx', 'soc', 'kratos', + 'influxdb', + 'telegraf', 'firewall', 'idstools', 'suricata.manager', From 75e16963c8309350ea763f371337eefa11207529 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 22 Dec 2022 09:47:47 -0500 Subject: [PATCH 14/39] add influxdb and telegraf to import mode --- salt/top.sls | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/top.sls b/salt/top.sls index d9653fca1..3997f9725 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -328,6 +328,8 @@ base: - manager - nginx - soc + - influxdb + - telegraf - kratos - firewall - idstools From 9352854fe4075ddd20a09a0c93059d5c2b8917e3 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 22 Dec 2022 09:48:38 -0500 Subject: [PATCH 15/39] enable influxdb for eval and import modes --- salt/influxdb/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index ddcf36195..aa81ece6e 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -4,7 +4,7 @@ {% set GRAFANA = salt['pillar.get']('manager:grafana', '0') %} -{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone'] or (grains.role == 'so-eval' and GRAFANA == 1) %} +{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-eval', 'so-import'] %} {% import_yaml 'influxdb/defaults.yaml' as default_settings %} {% set influxdb = salt['grains.filter_by'](default_settings, default='influxdb', merge=salt['pillar.get']('influxdb', {})) %} {% from 'salt/map.jinja' import PYTHON3INFLUX with context %} From e95034886e9aa251e82c2200f38c2b6531377dd8 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 22 Dec 2022 09:49:57 -0500 Subject: [PATCH 16/39] add influxdb and telegraf to import mode --- salt/common/tools/sbin/so-image-common | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index cfc4ff4ab..9ba66ced8 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -27,6 +27,7 @@ container_list() { "so-elasticsearch" "so-filebeat" "so-idstools" + "so-influxdb" "so-kibana" "so-kratos" "so-nginx" @@ -34,6 +35,7 @@ container_list() { "so-soc" "so-steno" "so-suricata" + "so-telegraf" "so-zeek" ) elif [ $MANAGERCHECK != 'so-helix' ]; then From b6f37f8499fd6dba666315eb38d94cb3e07abcc9 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 22 Dec 2022 10:26:51 -0500 Subject: [PATCH 17/39] Correct indentation of client section --- salt/soc/soc_soc.yaml | 225 +++++++++++++++++++++--------------------- 1 file changed, 113 insertions(+), 112 deletions(-) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 8febe0681..7d96ca46b 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -45,6 +45,7 @@ soc: default: description: The list of fields to show as columns in the Hunt/Dashboards event table, when no other specific mapping applies. Mappings are defined by the format ":event.module:event.dataset". global: True + advanced: True server: maxPacketCount: description: Maximum number of packets to show in the PCAP viewer. Larger values can cause more resource utilization on both the SOC server and the browser. @@ -97,119 +98,119 @@ soc: description: Duration (in milliseconds) that must elapse after a grid node fails to check-in before the node will be marked offline (fault). global: True advanced: True - client: - apiTimeoutMs: - description: Duration (in milliseconds) to wait for a response from the SOC server API before giving up and showing an error on the SOC UI. - global: True - advanced: True - webSocketTimeoutMs: - description: Duration (in milliseconds) to wait for a response from the SOC server websocket before giving up and reconnecting. - global: True - advanced: True - tipTimeoutMs: - description: Duration (in milliseconds) to show the popup tips, which typically indicate a successful operation. - global: True - cacheExpirationMs: - description: Duration (in milliseconds) of cached data within the browser, including users and settings. - global: True - advanced: True - casesEnabled: - description: Set to true to enable case management in SOC. - global: True - inactiveTools: - description: List of external tools to remove from the SOC UI. - global: True - tools: - description: List of available external tools visible in the SOC UI. Each tool is defined in JSON object notation, and must include the "name" key and "link" key, where the link is the tool's URL. - global: True - advanced: True - hunt: &appSettings - groupItemsPerPage: - description: Default number of aggregations to show per page. Larger values consume more vertical area in the SOC UI. - global: True - groupFetchLimit: - description: Default maximum number of aggregations to retrieve per search. Larger values consume more bandwidth and server resources. - global: True - eventItemsPerPage: - description: Default number of items to show per page. Larger values consume more vertical area in the SOC UI. - global: True - eventFetchLimit: - description: Default maximum number of items to retrieve per search. Larger values consume more bandwidth and server resources. - global: True - relativeTimeValue: - description: The duration of time to look backwards when searching for items. Used in combination with the relativeTimeUnit setting. - global: True - relativeTimeUnit: - description: The unit of time for the relativeTimeValue setting. Possible values are 10 (seconds), 20 (minutes), 30 (hours), 40 (days), 50 (weeks), and 60 (months). - global: True - mostRecentlyUsedLimit: - description: Number of items to show in the most recently used queries list. Larger values cause default queries to be located further down the list. - global: True - queries: - description: List of default queries to show in the query list. Each query is represented in JSON object notation, and must include the "name" key and "query" key. - global: True - alerts: *appSettings - cases: *appSettings - dashboards: *appSettings - case: - analyzerNodeId: - description: The node ID on which analyzers will be executed. + client: + apiTimeoutMs: + description: Duration (in milliseconds) to wait for a response from the SOC server API before giving up and showing an error on the SOC UI. global: True advanced: True - mostRecentlyUsedLimit: - description: Number of items to show in the most recently used queries list. Larger values cause default queries to be located further down the list. - global: True - renderAbbreviatedCount: - description: When the number of case related items exceeds this number, the middle section of the results will be hidden from view, avoiding unnecessary scrolling. + webSocketTimeoutMs: + description: Duration (in milliseconds) to wait for a response from the SOC server websocket before giving up and reconnecting. global: True advanced: True - presets: - artifactType: - labels: - description: List of available artifact types. Some of these default types have special characteristics and related functionality, built into SOC. - global: True - customEnabled: - description: Set to true to allow users add their own artifact types directly in the SOC UI. - global: True - category: - labels: - description: List of available case categories. - global: True - customEnabled: - description: Set to true to allow users add their own categories directly in the SOC UI. - global: True - pap: - labels: - description: List of available PAP (Permissible Actions Protocol) values. - global: True - customEnabled: - description: Set to true to allow users add their own PAP values directly in the SOC UI. - global: True - severity: - labels: - description: List of available case severities. - global: True - customEnabled: - description: Set to true to allow users add their own severities directly in the SOC UI. - global: True - status: - labels: - description: List of available case statuses. Some statuses have specifial characteristics and related functionality built into SOC. - global: True - customEnabled: - description: Set to true to allow users add their own case statuses directly in the SOC UI. - global: True - tags: - labels: - description: List of available tags. - global: True - customEnabled: - description: Set to true to allow users add their own tags directly in the SOC UI. - global: True - tlp: - labels: - description: List of available TLP (Traffic Light Protocol) values. - global: True - customEnabled: - description: Set to true to allow users add their own TLP values directly in the SOC UI. - global: True + tipTimeoutMs: + description: Duration (in milliseconds) to show the popup tips, which typically indicate a successful operation. + global: True + cacheExpirationMs: + description: Duration (in milliseconds) of cached data within the browser, including users and settings. + global: True + advanced: True + casesEnabled: + description: Set to true to enable case management in SOC. + global: True + inactiveTools: + description: List of external tools to remove from the SOC UI. + global: True + tools: + description: List of available external tools visible in the SOC UI. Each tool is defined in JSON object notation, and must include the "name" key and "link" key, where the link is the tool's URL. + global: True + advanced: True + hunt: &appSettings + groupItemsPerPage: + description: Default number of aggregations to show per page. Larger values consume more vertical area in the SOC UI. + global: True + groupFetchLimit: + description: Default maximum number of aggregations to retrieve per search. Larger values consume more bandwidth and server resources. + global: True + eventItemsPerPage: + description: Default number of items to show per page. Larger values consume more vertical area in the SOC UI. + global: True + eventFetchLimit: + description: Default maximum number of items to retrieve per search. Larger values consume more bandwidth and server resources. + global: True + relativeTimeValue: + description: The duration of time to look backwards when searching for items. Used in combination with the relativeTimeUnit setting. + global: True + relativeTimeUnit: + description: The unit of time for the relativeTimeValue setting. Possible values are 10 (seconds), 20 (minutes), 30 (hours), 40 (days), 50 (weeks), and 60 (months). + global: True + mostRecentlyUsedLimit: + description: Number of items to show in the most recently used queries list. Larger values cause default queries to be located further down the list. + global: True + queries: + description: List of default queries to show in the query list. Each query is represented in JSON object notation, and must include the "name" key and "query" key. + global: True + alerts: *appSettings + cases: *appSettings + dashboards: *appSettings + case: + analyzerNodeId: + description: The node ID on which analyzers will be executed. + global: True + advanced: True + mostRecentlyUsedLimit: + description: Number of items to show in the most recently used queries list. Larger values cause default queries to be located further down the list. + global: True + renderAbbreviatedCount: + description: When the number of case related items exceeds this number, the middle section of the results will be hidden from view, avoiding unnecessary scrolling. + global: True + advanced: True + presets: + artifactType: + labels: + description: List of available artifact types. Some of these default types have special characteristics and related functionality, built into SOC. + global: True + customEnabled: + description: Set to true to allow users add their own artifact types directly in the SOC UI. + global: True + category: + labels: + description: List of available case categories. + global: True + customEnabled: + description: Set to true to allow users add their own categories directly in the SOC UI. + global: True + pap: + labels: + description: List of available PAP (Permissible Actions Protocol) values. + global: True + customEnabled: + description: Set to true to allow users add their own PAP values directly in the SOC UI. + global: True + severity: + labels: + description: List of available case severities. + global: True + customEnabled: + description: Set to true to allow users add their own severities directly in the SOC UI. + global: True + status: + labels: + description: List of available case statuses. Some statuses have specifial characteristics and related functionality built into SOC. + global: True + customEnabled: + description: Set to true to allow users add their own case statuses directly in the SOC UI. + global: True + tags: + labels: + description: List of available tags. + global: True + customEnabled: + description: Set to true to allow users add their own tags directly in the SOC UI. + global: True + tlp: + labels: + description: List of available TLP (Traffic Light Protocol) values. + global: True + customEnabled: + description: Set to true to allow users add their own TLP values directly in the SOC UI. + global: True From cf02b8e19173d4e3ca641f93d985ecfcf6d95eaa Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 22 Dec 2022 10:34:33 -0500 Subject: [PATCH 18/39] Modify manager for repo --- setup/so-functions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 6447a7ded..97b3fcdd4 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1430,8 +1430,8 @@ create_global() { # Continue adding other details echo " imagerepo: '$IMAGEREPO'" >> $global_pillar_file echo " pipeline: 'redis'" >> $global_pillar_file - echo " repo_host: '$MAINIP'" >> $global_pillar_file - echo " registry_host: '$MAINIP'" >> $global_pillar_file + echo " repo_host: '$HOSTNAME'" >> $global_pillar_file + echo " registry_host: '$HOSTNAME'" >> $global_pillar_file echo " endgamehost: '$ENDGAMEHOST'" >> $global_pillar_file } From 2df4755fef9933e63651f14658c77a6332b0578a Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 22 Dec 2022 12:54:57 -0500 Subject: [PATCH 19/39] Enable Grafana in EVAL mode --- salt/grafana/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/grafana/init.sls b/salt/grafana/init.sls index f20cdffff..b4d762aa3 100644 --- a/salt/grafana/init.sls +++ b/salt/grafana/init.sls @@ -9,7 +9,7 @@ {% import_yaml 'grafana/grafana_defaults.yaml' as default_settings %} {% set GRAFANA_SETTINGS = salt['grains.filter_by'](default_settings, default='grafana', merge=salt['pillar.get']('grafana', {})) %} -{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone'] or (grains.role == 'so-eval' and GRAFANA == 1) %} +{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-eval'] %} {% set ALLOWED_DASHBOARDS = ['overview', 'standalone', 'manager', 'managersearch', 'sensor', 'searchnode', 'heavynode', 'eval', 'receiver'] %} {% set DASHBOARDS = ['overview'] %} From 308228620a55885190e8fe33d46a5219f3e974a5 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 22 Dec 2022 13:05:33 -0500 Subject: [PATCH 20/39] Specify Influxdb host --- salt/grafana/etc/datasources/influxdb.yaml | 2 +- salt/soc/defaults.map.jinja | 2 +- salt/telegraf/etc/telegraf.conf | 4 ++-- setup/so-functions | 1 + 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/salt/grafana/etc/datasources/influxdb.yaml b/salt/grafana/etc/datasources/influxdb.yaml index c95126b14..1c4fd8135 100644 --- a/salt/grafana/etc/datasources/influxdb.yaml +++ b/salt/grafana/etc/datasources/influxdb.yaml @@ -9,7 +9,7 @@ datasources: type: influxdb access: proxy database: telegraf - url: https://{{ GLOBALS.manager_ip }}:8086 + url: https://{{ GLOBALS.influxdb_host }}:8086 jsonData: tlsAuth: false tlsAuthWithCACert: false diff --git a/salt/soc/defaults.map.jinja b/salt/soc/defaults.map.jinja index facba77c8..ecab4d3ce 100644 --- a/salt/soc/defaults.map.jinja +++ b/salt/soc/defaults.map.jinja @@ -15,7 +15,7 @@ {% do SOCDEFAULTS.soc.server.modules.elastic.update({'username': GLOBALS.elasticsearch.auth.users.so_elastic_user.user, 'password': GLOBALS.elasticsearch.auth.users.so_elastic_user.pass}) %} {% if GLOBALS.role != 'so-import' %} -{% do SOCDEFAULTS.soc.server.modules.influxdb.update({'hostUrl': 'https://' ~ GLOBALS.manager_ip ~ ':8086'}) %} +{% do SOCDEFAULTS.soc.server.modules.influxdb.update({'hostUrl': 'https://' ~ GLOBALS.influxdb_host ~ ':8086'}) %} {% endif %} {% do SOCDEFAULTS.soc.server.modules.statickeyauth.update({'anonymousCidr': GLOBALS.docker_range, 'apiKey': pillar.sensoroni.sensoronikey}) %} diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 895a25d6c..c9cbe70e5 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -1,5 +1,5 @@ # Telegraf Configuration -{%- set MANAGER = GLOBALS.manager %} +{%- set INFLUXDBHOST = GLOBALS.influxdb_host %} {%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} {%- set NODEIP = GLOBALS.node_ip %} @@ -72,7 +72,7 @@ # Configuration for sending metrics to InfluxDB [[outputs.influxdb]] - urls = ["https://{{ MANAGER }}:8086"] + urls = ["https://{{ INFLUXDBHOST }}:8086"] ## Optional TLS Config for use on HTTP connections. tls_ca = "/etc/telegraf/ca.crt" diff --git a/setup/so-functions b/setup/so-functions index 97b3fcdd4..4f6af0e7c 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1431,6 +1431,7 @@ create_global() { echo " imagerepo: '$IMAGEREPO'" >> $global_pillar_file echo " pipeline: 'redis'" >> $global_pillar_file echo " repo_host: '$HOSTNAME'" >> $global_pillar_file + echo " influxdb_host: '$HOSTNAME'" >> $global_pillar_file echo " registry_host: '$HOSTNAME'" >> $global_pillar_file echo " endgamehost: '$ENDGAMEHOST'" >> $global_pillar_file } From a924d48408b19044bdd9f5de746bf4847c9caa1f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 22 Dec 2022 13:12:19 -0500 Subject: [PATCH 21/39] Specify Influxdb host --- salt/vars/globals.map.jinja | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/vars/globals.map.jinja b/salt/vars/globals.map.jinja index a88b77e45..0e286754f 100644 --- a/salt/vars/globals.map.jinja +++ b/salt/vars/globals.map.jinja @@ -16,6 +16,7 @@ 'image_repo': INIT.PILLAR.global.imagerepo, 'repo_host': INIT.PILLAR.global.repo_host, 'registry_host': INIT.PILLAR.global.registry_host, + 'influxdb_host': INIT.PILLAR.global.influxdb_host, 'manager_ip': INIT.PILLAR.global.managerip, 'md_engine': INIT.PILLAR.global.mdengine, 'pipeline': INIT.PILLAR.global.pipeline, From 78f851e6c2d4f911aa3c87c676ccaeb2ef325f13 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 22 Dec 2022 13:35:37 -0500 Subject: [PATCH 22/39] Create annotation file for global settings --- salt/global/soc_global.yaml | 49 +++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 salt/global/soc_global.yaml diff --git a/salt/global/soc_global.yaml b/salt/global/soc_global.yaml new file mode 100644 index 000000000..ad4046714 --- /dev/null +++ b/salt/global/soc_global.yaml @@ -0,0 +1,49 @@ +global: + soversion: + description: Current version of Security Onion. + global: True + readonly: True + managerip: + description: The IP address of the grid manager. + global: True + mdengine: + description: What engine to use for meta data generation. Options are ZEEK and SURICATA. + global: True + ids: + description: Which IDS engine to use. Currently only Suricata is supported. + global: True + readonly: True + advanced: True + url_base: + description: Used for handling of authentication cookies. + global: True + airgap: + description: Sets airgap mode. + global: True + readonly: True + imagerepo: + description: Image repo to pull image from. + global: True + advanced: True + pipeline: + description: Sets which pipeline technology for events to use. Currently only Redsi is supported. + global: True + readonly: True + advanced: True + repo_host: + description: Specify the host where operating system packages will be served from. + global: True + advanced: True + registry_host: + description: Specify the host where docker/podman images will be pulled from. + global: True + advanced: True + influxdb_host: + description: Specify the host where influxdb is hosted. + global: True + advanced: True + engame_host: + description: Allows use of Endgame with Security Onion. This feature requires a license from Endgame. + global: True + advanced: True + From cd77e71d8d84d9cde08125ccc441f7aa3b299523 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 22 Dec 2022 13:37:41 -0500 Subject: [PATCH 23/39] Create annotation file for global settings --- salt/global/soc_global.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/global/soc_global.yaml b/salt/global/soc_global.yaml index ad4046714..a80186f05 100644 --- a/salt/global/soc_global.yaml +++ b/salt/global/soc_global.yaml @@ -42,7 +42,7 @@ global: description: Specify the host where influxdb is hosted. global: True advanced: True - engame_host: + engamehost: description: Allows use of Endgame with Security Onion. This feature requires a license from Endgame. global: True advanced: True From 5be074bbea392a0e56c6cb91269b28591b489fad Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 22 Dec 2022 15:45:25 -0500 Subject: [PATCH 24/39] Remove Grafana and Playbook links for Import mode --- setup/so-functions | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/setup/so-functions b/setup/so-functions index 776608b71..7c1ed8895 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1368,6 +1368,13 @@ telegraf_pillar() { manager_pillar() { touch $adv_manager_pillar_file title "Create the manager pillar" + if [[ $is_import ]]; then + GRAFANA=0 + PLAYBOOK=0 + else + GRAFANA=1 + PLAYBOOK=1 + fi printf '%s\n'\ "manager:"\ " proxy: '$so_proxy'"\ From 75ffd1f56b59865974137d1e925de99b08adc023 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 23 Dec 2022 08:55:19 -0500 Subject: [PATCH 25/39] Update soc_global.yaml --- salt/global/soc_global.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/global/soc_global.yaml b/salt/global/soc_global.yaml index a80186f05..b3752938e 100644 --- a/salt/global/soc_global.yaml +++ b/salt/global/soc_global.yaml @@ -42,7 +42,7 @@ global: description: Specify the host where influxdb is hosted. global: True advanced: True - engamehost: + endgamehost: description: Allows use of Endgame with Security Onion. This feature requires a license from Endgame. global: True advanced: True From 136867c96a45a68703faf8390bfe3f06561824dd Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 23 Dec 2022 10:27:49 -0500 Subject: [PATCH 26/39] ensure zombie pipe is destroyed before SOC restarts --- salt/common/tools/sbin/so-soc-restart | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/so-soc-restart b/salt/common/tools/sbin/so-soc-restart index 0a678de96..4d2578c03 100755 --- a/salt/common/tools/sbin/so-soc-restart +++ b/salt/common/tools/sbin/so-soc-restart @@ -9,5 +9,6 @@ . /usr/sbin/so-common +rm -f /opt/so/conf/soc/salt/pipe pkill salt-relay.sh /usr/sbin/so-restart soc $1 From a89976779d70bf32f0650750f32a7b236b814315 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 30 Dec 2022 11:30:09 -0500 Subject: [PATCH 27/39] Ensure create/update dates are both reset when an admin sets a user's password --- salt/common/tools/sbin/so-user | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 3bcc8020f..1e69ee82b 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -230,7 +230,7 @@ function updatePassword() { # Generate password hash passwordHash=$(hashPassword "$password") # Update DB with new hash - echo "update identity_credentials set config=CAST('{\"hashed_password\":\"$passwordHash\"}' as BLOB), updated_at=datetime('now') where identity_id='${identityId}' and identity_credential_type_id=(select id from identity_credential_types where name='password');" | sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath" + echo "update identity_credentials set config=CAST('{\"hashed_password\":\"$passwordHash\"}' as BLOB), created_at=datetime('now'), updated_at=datetime('now') where identity_id='${identityId}' and identity_credential_type_id=(select id from identity_credential_types where name='password');" | sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath" # Deactivate MFA echo "delete from identity_credential_identifiers where identity_credential_id=(select id from identity_credentials where identity_id='${identityId}' and identity_credential_type_id=(select id from identity_credential_types where name='totp'));" | sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath" echo "delete from identity_credentials where identity_id='${identityId}' and identity_credential_type_id=(select id from identity_credential_types where name='totp');" | sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath" From 4e5d1d587ee54243239efad682a6e5cf8980513b Mon Sep 17 00:00:00 2001 From: doug Date: Tue, 3 Jan 2023 09:02:17 -0500 Subject: [PATCH 28/39] update sysmon ingest parser and Sysmon File dashboard --- salt/elasticsearch/files/ingest/sysmon | 181 +++++++++++++++---------- salt/soc/defaults.yaml | 2 +- 2 files changed, 110 insertions(+), 73 deletions(-) diff --git a/salt/elasticsearch/files/ingest/sysmon b/salt/elasticsearch/files/ingest/sysmon index 5fa0e1005..177218108 100644 --- a/salt/elasticsearch/files/ingest/sysmon +++ b/salt/elasticsearch/files/ingest/sysmon @@ -1,78 +1,115 @@ { "description" : "sysmon", "processors" : [ - { "set": { "field": "event.code", "value": "{{winlog.event_id}}", "override": true } }, - { "set": { "field": "event.module", "value": "sysmon", "override": true } }, - { "set": { "if": "ctx.winlog?.computer_name != null", "field": "observer.name", "value": "{{winlog.computer_name}}", "override": true } }, - { "set": { "if": "ctx.event?.code == '3'", "field": "event.category", "value": "host,process,network", "override": true } }, - { "set": { "if": "ctx.event?.code == '1'", "field": "event.category", "value": "host,process", "override": true } }, - { "set": { "if": "ctx.event?.code == '5'", "field": "event.category", "value": "host,process", "override": true } }, - { "set": { "if": "ctx.event?.code == '6'", "field": "event.category", "value": "host,driver", "override": true } }, - { "set": { "if": "ctx.event?.code == '22'", "field": "event.category", "value": "network", "override": true } }, - { "set": { "if": "ctx.event?.code == '1'", "field": "event.dataset", "value": "process_creation", "override": true } }, - { "set": { "if": "ctx.event?.code == '2'", "field": "event.dataset", "value": "process_changed_file", "override": true } }, - { "set": { "if": "ctx.event?.code == '3'", "field": "event.dataset", "value": "network_connection", "override": true } }, - { "set": { "if": "ctx.event?.code == '5'", "field": "event.dataset", "value": "process_terminated", "override": true } }, - { "set": { "if": "ctx.event?.code == '6'", "field": "event.dataset", "value": "driver_loaded", "override": true } }, - { "set": { "if": "ctx.event?.code == '7'", "field": "event.dataset", "value": "image_loaded", "override": true } }, - { "set": { "if": "ctx.event?.code == '8'", "field": "event.dataset", "value": "create_remote_thread", "override": true } }, - { "set": { "if": "ctx.event?.code == '9'", "field": "event.dataset", "value": "raw_file_access_read", "override": true } }, - { "set": { "if": "ctx.event?.code == '10'", "field": "event.dataset", "value": "process_access", "override": true } }, - { "set": { "if": "ctx.event?.code == '11'", "field": "event.dataset", "value": "file_create", "override": true } }, - { "set": { "if": "ctx.event?.code == '12'", "field": "event.dataset", "value": "registry_create_delete", "override": true } }, - { "set": { "if": "ctx.event?.code == '13'", "field": "event.dataset", "value": "registry_value_set", "override": true } }, - { "set": { "if": "ctx.event?.code == '14'", "field": "event.dataset", "value": "registry_key_value_rename", "override": true } }, - { "set": { "if": "ctx.event?.code == '15'", "field": "event.dataset", "value": "file_create_stream_hash", "override": true } }, - { "set": { "if": "ctx.event?.code == '16'", "field": "event.dataset", "value": "config_change", "override": true } }, - { "set": { "if": "ctx.event?.code == '22'", "field": "event.dataset", "value": "dns_query", "override": true } }, - { "kv": {"field": "winlog.event_data.Hashes", "target_field": "file.hash", "field_split": ",", "value_split": "=", "ignore_missing": true } }, - { "kv": {"field": "winlog.event_data.Hash", "target_field": "file.hash", "field_split": ",", "value_split": "=", "ignore_missing": true } }, - { "rename": { "field": "file.hash.IMPHASH", "target_field": "hash.imphash", "ignore_missing":true } }, - { "rename": { "field": "file.hash.MD5", "target_field": "hash.md5", "ignore_missing":true } }, - { "rename": { "field": "file.hash.SHA256", "target_field": "hash.sha256", "ignore_missing":true } }, - { "rename": { "field": "winlog.event_data.SubjectUserName", "target_field": "user.name", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.DestinationHostname", "target_field": "destination.hostname", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.DestinationIp", "target_field": "destination.ip", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.DestinationPort", "target_field": "destination.port", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.image", "target_field": "process.executable", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.Image", "target_field": "process.executable", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.processID", "target_field": "process.pid", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.ProcessId", "target_field": "process.pid", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.processGuid", "target_field": "process.entity_id", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.ProcessGuid", "target_field": "process.entity_id", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.commandLine", "target_field": "process.command_line", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.CommandLine", "target_field": "process.command_line", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.currentDirectory", "target_field": "process.working_directory", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.CurrentDirectory", "target_field": "process.working_directory", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.description", "target_field": "process.pe.description", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.Description", "target_field": "process.pe.description", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.product", "target_field": "process.pe.product", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.Product", "target_field": "process.pe.product", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.company", "target_field": "process.pe.company", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.Company", "target_field": "process.pe.company", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.originalFileName", "target_field": "process.pe.original_file_name", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.OriginalFileName", "target_field": "process.pe.original_file_name", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.fileVersion", "target_field": "process.pe.file_version", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.FileVersion", "target_field": "process.pe.file_version", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.parentCommandLine", "target_field": "process.parent.command_line", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.ParentCommandLine", "target_field": "process.parent.command_line", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.parentImage", "target_field": "process.parent.executable", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.ParentImage", "target_field": "process.parent.executable", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.parentProcessGuid", "target_field": "process.parent.entity_id", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.ParentProcessGuid", "target_field": "process.parent.entity_id", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.parentProcessId", "target_field": "process.ppid", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.ParentProcessId", "target_field": "process.ppid", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.Protocol", "target_field": "network.transport", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.User", "target_field": "user.name", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.SourceHostname", "target_field": "source.hostname", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.SourceIp", "target_field": "source.ip", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.SourcePort", "target_field": "source.port", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.targetFilename", "target_field": "file.target", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.TargetFilename", "target_field": "file.target", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.QueryResults", "target_field": "dns.answers.name", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.QueryName", "target_field": "dns.query.name", "ignore_missing": true } }, - { "remove": { "field": "winlog.event_data.Hash", "ignore_missing": true } }, - { "remove": { "field": "winlog.event_data.Hashes", "ignore_missing": true } }, + { "set": { "field": "event.code", "value": "{{winlog.event_id}}", "override": true } }, + { "set": { "field": "event.module", "value": "sysmon", "override": true } }, + { "set": { "if": "ctx.winlog?.computer_name != null", "field": "observer.name", "value": "{{winlog.computer_name}}", "override": true } }, + { "set": { "if": "ctx.event?.code == '1'", "field": "event.category", "value": "host,process", "override": true } }, + { "set": { "if": "ctx.event?.code == '2'", "field": "event.category", "value": "host,file", "override": true } }, + { "set": { "if": "ctx.event?.code == '3'", "field": "event.category", "value": "host,process,network", "override": true } }, + { "set": { "if": "ctx.event?.code == '4'", "field": "event.category", "value": "host", "override": true } }, + { "set": { "if": "ctx.event?.code == '5'", "field": "event.category", "value": "host,process", "override": true } }, + { "set": { "if": "ctx.event?.code == '6'", "field": "event.category", "value": "host,driver", "override": true } }, + { "set": { "if": "ctx.event?.code == '7'", "field": "event.category", "value": "host,process", "override": true } }, + { "set": { "if": "ctx.event?.code == '8'", "field": "event.category", "value": "host,process", "override": true } }, + { "set": { "if": "ctx.event?.code == '9'", "field": "event.category", "value": "host,file", "override": true } }, + { "set": { "if": "ctx.event?.code == '10'", "field": "event.category", "value": "host,process", "override": true } }, + { "set": { "if": "ctx.event?.code == '11'", "field": "event.category", "value": "host,file", "override": true } }, + { "set": { "if": "ctx.event?.code == '12'", "field": "event.category", "value": "host,registry", "override": true } }, + { "set": { "if": "ctx.event?.code == '13'", "field": "event.category", "value": "host,registry", "override": true } }, + { "set": { "if": "ctx.event?.code == '14'", "field": "event.category", "value": "host,registry", "override": true } }, + { "set": { "if": "ctx.event?.code == '15'", "field": "event.category", "value": "host,file", "override": true } }, + { "set": { "if": "ctx.event?.code == '16'", "field": "event.category", "value": "host", "override": true } }, + { "set": { "if": "ctx.event?.code == '17'", "field": "event.category", "value": "host,pipe", "override": true } }, + { "set": { "if": "ctx.event?.code == '18'", "field": "event.category", "value": "host,pipe", "override": true } }, + { "set": { "if": "ctx.event?.code == '19'", "field": "event.category", "value": "host,wmi", "override": true } }, + { "set": { "if": "ctx.event?.code == '20'", "field": "event.category", "value": "host,wmi", "override": true } }, + { "set": { "if": "ctx.event?.code == '21'", "field": "event.category", "value": "host,wmi", "override": true } }, + { "set": { "if": "ctx.event?.code == '22'", "field": "event.category", "value": "network", "override": true } }, + { "set": { "if": "ctx.event?.code == '23'", "field": "event.category", "value": "host,file", "override": true } }, + { "set": { "if": "ctx.event?.code == '24'", "field": "event.category", "value": "host,clipboard", "override": true } }, + { "set": { "if": "ctx.event?.code == '25'", "field": "event.category", "value": "host,process", "override": true } }, + { "set": { "if": "ctx.event?.code == '26'", "field": "event.category", "value": "host,file", "override": true } }, + { "set": { "if": "ctx.event?.code == '27'", "field": "event.category", "value": "host,file", "override": true } }, + { "set": { "if": "ctx.event?.code == '28'", "field": "event.category", "value": "host,file", "override": true } }, + { "set": { "if": "ctx.event?.code == '255'", "field": "event.category", "value": "host", "override": true } }, + { "set": { "if": "ctx.event?.code == '1'", "field": "event.dataset", "value": "process_creation", "override": true } }, + { "set": { "if": "ctx.event?.code == '2'", "field": "event.dataset", "value": "process_changed_file", "override": true } }, + { "set": { "if": "ctx.event?.code == '3'", "field": "event.dataset", "value": "network_connection", "override": true } }, + { "set": { "if": "ctx.event?.code == '4'", "field": "event.dataset", "value": "service_state_changed", "override": true } }, + { "set": { "if": "ctx.event?.code == '5'", "field": "event.dataset", "value": "process_terminated", "override": true } }, + { "set": { "if": "ctx.event?.code == '6'", "field": "event.dataset", "value": "driver_loaded", "override": true } }, + { "set": { "if": "ctx.event?.code == '7'", "field": "event.dataset", "value": "image_loaded", "override": true } }, + { "set": { "if": "ctx.event?.code == '8'", "field": "event.dataset", "value": "create_remote_thread", "override": true } }, + { "set": { "if": "ctx.event?.code == '9'", "field": "event.dataset", "value": "raw_file_access_read", "override": true } }, + { "set": { "if": "ctx.event?.code == '10'", "field": "event.dataset", "value": "process_access", "override": true } }, + { "set": { "if": "ctx.event?.code == '11'", "field": "event.dataset", "value": "file_create", "override": true } }, + { "set": { "if": "ctx.event?.code == '12'", "field": "event.dataset", "value": "registry_create_delete", "override": true } }, + { "set": { "if": "ctx.event?.code == '13'", "field": "event.dataset", "value": "registry_value_set", "override": true } }, + { "set": { "if": "ctx.event?.code == '14'", "field": "event.dataset", "value": "registry_key_value_rename", "override": true } }, + { "set": { "if": "ctx.event?.code == '15'", "field": "event.dataset", "value": "file_create_stream_hash", "override": true } }, + { "set": { "if": "ctx.event?.code == '16'", "field": "event.dataset", "value": "config_change", "override": true } }, + { "set": { "if": "ctx.event?.code == '17'", "field": "event.dataset", "value": "pipe_created", "override": true } }, + { "set": { "if": "ctx.event?.code == '18'", "field": "event.dataset", "value": "pipe_connected", "override": true } }, + { "set": { "if": "ctx.event?.code == '19'", "field": "event.dataset", "value": "wmi_event_filter", "override": true } }, + { "set": { "if": "ctx.event?.code == '20'", "field": "event.dataset", "value": "wmi_event_consumer", "override": true } }, + { "set": { "if": "ctx.event?.code == '21'", "field": "event.dataset", "value": "wmi_event_consumer_to_filter","override": true } }, + { "set": { "if": "ctx.event?.code == '22'", "field": "event.dataset", "value": "dns_query", "override": true } }, + { "set": { "if": "ctx.event?.code == '23'", "field": "event.dataset", "value": "file_delete_archived", "override": true } }, + { "set": { "if": "ctx.event?.code == '24'", "field": "event.dataset", "value": "clipboard_change", "override": true } }, + { "set": { "if": "ctx.event?.code == '25'", "field": "event.dataset", "value": "process_tampering", "override": true } }, + { "set": { "if": "ctx.event?.code == '26'", "field": "event.dataset", "value": "file_delete", "override": true } }, + { "set": { "if": "ctx.event?.code == '27'", "field": "event.dataset", "value": "file_block_executable", "override": true } }, + { "set": { "if": "ctx.event?.code == '28'", "field": "event.dataset", "value": "file_block_shredding", "override": true } }, + { "set": { "if": "ctx.event?.code == '255'", "field": "event.dataset", "value": "error", "override": true } }, + { "kv": { "field": "winlog.event_data.Hashes", "target_field": "file.hash", "field_split": ",", "value_split": "=", "ignore_missing": true } }, + { "kv": { "field": "winlog.event_data.Hash", "target_field": "file.hash", "field_split": ",", "value_split": "=", "ignore_missing": true } }, + { "rename": { "field": "file.hash.IMPHASH", "target_field": "hash.imphash", "ignore_missing": true } }, + { "rename": { "field": "file.hash.MD5", "target_field": "hash.md5", "ignore_missing": true } }, + { "rename": { "field": "file.hash.SHA256", "target_field": "hash.sha256", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.SubjectUserName", "target_field": "user.name", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.DestinationHostname", "target_field": "destination.hostname", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.DestinationIp", "target_field": "destination.ip", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.DestinationPort", "target_field": "destination.port", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.image", "target_field": "process.executable", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.Image", "target_field": "process.executable", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.processID", "target_field": "process.pid", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.ProcessId", "target_field": "process.pid", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.processGuid", "target_field": "process.entity_id", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.ProcessGuid", "target_field": "process.entity_id", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.commandLine", "target_field": "process.command_line", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.CommandLine", "target_field": "process.command_line", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.currentDirectory", "target_field": "process.working_directory", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.CurrentDirectory", "target_field": "process.working_directory", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.description", "target_field": "process.pe.description", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.Description", "target_field": "process.pe.description", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.product", "target_field": "process.pe.product", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.Product", "target_field": "process.pe.product", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.company", "target_field": "process.pe.company", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.Company", "target_field": "process.pe.company", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.originalFileName", "target_field": "process.pe.original_file_name", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.OriginalFileName", "target_field": "process.pe.original_file_name", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.fileVersion", "target_field": "process.pe.file_version", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.FileVersion", "target_field": "process.pe.file_version", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.parentCommandLine", "target_field": "process.parent.command_line", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.ParentCommandLine", "target_field": "process.parent.command_line", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.parentImage", "target_field": "process.parent.executable", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.ParentImage", "target_field": "process.parent.executable", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.parentProcessGuid", "target_field": "process.parent.entity_id", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.ParentProcessGuid", "target_field": "process.parent.entity_id", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.parentProcessId", "target_field": "process.ppid", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.ParentProcessId", "target_field": "process.ppid", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.Protocol", "target_field": "network.transport", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.User", "target_field": "user.name", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.SourceHostname", "target_field": "source.hostname", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.SourceIp", "target_field": "source.ip", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.SourcePort", "target_field": "source.port", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.targetFilename", "target_field": "file.target", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.TargetFilename", "target_field": "file.target", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.QueryResults", "target_field": "dns.answers.name", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.QueryName", "target_field": "dns.query.name", "ignore_missing": true } }, + { "remove": { "field": "winlog.event_data.Hash", "ignore_missing": true } }, + { "remove": { "field": "winlog.event_data.Hashes", "ignore_missing": true } }, { "community_id": {} } ] } diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index ceb94c054..87ad5c633 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1401,7 +1401,7 @@ soc: query: '(event.dataset:process_creation OR event.dataset:process_terminated OR event.dataset:process_access) | groupby -sankey winlog.computer_name user.name | groupby winlog.computer_name | groupby user.name | groupby event.dataset | groupby process.working_directory | groupby process.executable | groupby process.command_line | groupby process.parent.executable | groupby process.parent.command_line | groupby -sankey process.parent.executable process.executable' - name: Sysmon File description: File activity captured by Sysmon - query: '(event.dataset:file_create OR event.dataset:file_create_stream_hash OR event.dataset:process_changed_file) | groupby -sankey winlog.computer_name process.executable | groupby winlog.computer_name | groupby event.dataset | groupby file.target | groupby process.executable' + query: 'event.module:sysmon AND event.dataset:file_* | groupby -sankey winlog.computer_name process.executable | groupby winlog.computer_name | groupby event.dataset | groupby file.target | groupby process.executable' - name: Sysmon Network description: Network activity captured by Sysmon query: 'event.dataset:network_connection | groupby -sankey winlog.computer_name destination.ip destination.port | groupby winlog.computer_name | groupby user.name | groupby process.executable | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' From c8ff2c7a06ca79db509b912c775acba972257630 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 3 Jan 2023 16:03:49 +0000 Subject: [PATCH 29/39] Update RITA beacon parsing --- salt/elasticsearch/files/ingest/rita.beacon | 107 +++++++------------- 1 file changed, 36 insertions(+), 71 deletions(-) diff --git a/salt/elasticsearch/files/ingest/rita.beacon b/salt/elasticsearch/files/ingest/rita.beacon index ab53be763..2c6138262 100644 --- a/salt/elasticsearch/files/ingest/rita.beacon +++ b/salt/elasticsearch/files/ingest/rita.beacon @@ -17,98 +17,63 @@ "destination.ip", "network.connections", "network.average_bytes", - "beacon.interval.range", - "beacon.size.range", - "beacon.interval.top", - "beacon.size.top", - "beacon.interval.top_count", - "beacon.size.top_count", - "beacon.interval.skew", - "beacon.size.skew", - "beacon.interval.dispersion", - "beacon.size.dispersion", - "network.bytes" + "network.bytes", + "beacon.ts_score", + "beacon.ds_score", + "beacon.duration_score", + "beacon.historical_score", + "beacon.interval.top" ] } }, + { + "convert": { + "field": "beacon.ds_score", + "type": "float" + } + }, + { + "convert": { + "field": "beacon.duration_score", + "type": "float" + } + }, + { + "convert": { + "field": "beacon.historical_score", + "type": "float" + } + }, { "convert": { "field": "beacon.score", "type": "float" } }, + { + "convert": { + "field": "beacon.ts_score", + "type": "float" + } + }, + { + "convert": { + "field": "network.average_bytes", + "type": "float" + } + }, { "convert": { "field": "network.connections", "type": "integer" } }, - { - "convert": { - "field": "network.average_bytes", - "type": "integer" - } - }, - { - "convert": { - "field": "beacon.interval.range", - "type": "integer" - } - }, - { - "convert": { - "field": "beacon.size.range", - "type": "integer" - } - }, { "convert": { "field": "beacon.interval.top", "type": "integer" } }, - { - "convert": { - "field": "beacon.size.top", - "type": "integer" - } - }, - { - "convert": { - "field": "beacon.interval.top_count", - "type": "integer" - } - }, - { - "convert": { - "field": "beacon.size.top_count", - "type": "integer" - } - }, - { - "convert": { - "field": "beacon.interval.skew", - "type": "float" - } - }, - { - "convert": { - "field": "beacon.size.skew", - "type": "float" - } - }, - { - "convert": { - "field": "beacon.interval.dispersion", - "type": "integer" - } - }, - { - "convert": { - "field": "beacon.size.dispersion", - "type": "integer" - } - }, { "convert": { "field": "network.bytes", From df89445ab5731619f3e12da638d5a06b62c03922 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 3 Jan 2023 12:17:14 -0500 Subject: [PATCH 30/39] Update soc_global.yaml --- salt/global/soc_global.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/global/soc_global.yaml b/salt/global/soc_global.yaml index b3752938e..3ce312e7b 100644 --- a/salt/global/soc_global.yaml +++ b/salt/global/soc_global.yaml @@ -26,7 +26,7 @@ global: global: True advanced: True pipeline: - description: Sets which pipeline technology for events to use. Currently only Redsi is supported. + description: Sets which pipeline technology for events to use. Currently only Redia is supported. global: True readonly: True advanced: True From 08d7b24fb4ca91a6521362e00c661b8f6502d153 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 3 Jan 2023 12:17:51 -0500 Subject: [PATCH 31/39] Update soc_global.yaml --- salt/global/soc_global.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/global/soc_global.yaml b/salt/global/soc_global.yaml index 3ce312e7b..6861affd7 100644 --- a/salt/global/soc_global.yaml +++ b/salt/global/soc_global.yaml @@ -26,7 +26,7 @@ global: global: True advanced: True pipeline: - description: Sets which pipeline technology for events to use. Currently only Redia is supported. + description: Sets which pipeline technology for events to use. Currently only Redis is supported. global: True readonly: True advanced: True From 5754365c6d1b5dd35c855e6ca4fcb3c8a826bed1 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Wed, 4 Jan 2023 07:42:24 -0500 Subject: [PATCH 32/39] Improve default sysmon fields and add new network_connection fields --- salt/soc/defaults.yaml | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 87ad5c633..857f245d1 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -496,16 +496,6 @@ soc: - event.severity_label - log.id.uid - network.community_id - ':sysmon:': - - soc_timestamp - - source.ip - - source.port - - destination.ip - - destination.port - - source.hostname - - event.dataset - - process.executable - - user.name ':windows_eventlog:': - soc_timestamp - user.name @@ -570,6 +560,24 @@ soc: - destination.geo.country_iso_code - user.name - source.ip + ':sysmon:': + - soc_timestamp + - event.dataset + - process.executable + - user.name + - file.target + - dns.query.name + - winlog.event_data.TargetObject + '::network_connection': + - soc_timestamp + - source.ip + - source.port + - destination.ip + - destination.port + - source.hostname + - event.dataset + - process.executable + - user.name '::process_terminated': - soc_timestamp - process.executable From 831300b540e3584a543ca2c29ee2c77864642cf8 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 4 Jan 2023 11:02:40 -0500 Subject: [PATCH 33/39] Require password auth for redis access --- pillar/top.sls | 4 ++++ .../pipelines/config/so/0900_input_redis.conf.jinja | 12 +++++++----- .../pipelines/config/so/9999_output_redis.conf.jinja | 3 +++ salt/redis/defaults.yaml | 2 +- salt/redis/soc_redis.yaml | 4 ++++ setup/so-functions | 6 +++++- 6 files changed, 24 insertions(+), 7 deletions(-) diff --git a/pillar/top.sls b/pillar/top.sls index 1e684c682..345f7a689 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -120,6 +120,7 @@ base: '*_heavynode': - elasticsearch.auth - soc_global + - redis.soc_redis - minions.{{ grains.id }} - minions.adv_{{ grains.id }} @@ -136,6 +137,7 @@ base: {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} + - redis.soc_redis - soc_global - adv_global - minions.{{ grains.id }} @@ -148,6 +150,8 @@ base: {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} + - redis.soc_redis + - redis.adv_redis - soc_global - adv_global - minions.{{ grains.id }} diff --git a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja index 2d7a2d4fe..956bbbaa0 100644 --- a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja @@ -1,9 +1,10 @@ -{% set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') -%} -{% set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) -%} -{% from 'logstash/map.jinja' import REDIS_NODES with context -%} +{%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %} +{%- set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %} +{%- from 'logstash/map.jinja' import REDIS_NODES with context %} +{%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %} -{% for index in range(REDIS_NODES|length) -%} -{% for host in REDIS_NODES[index] -%} +{%- for index in range(REDIS_NODES|length) %} +{%- for host in REDIS_NODES[index] %} input { redis { host => '{{ host }}' @@ -14,6 +15,7 @@ input { type => 'redis-input' threads => {{ THREADS }} batch_count => {{ BATCH }} + password => {{ REDIS_PASS }} } } {% endfor %} diff --git a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja index 68b5187f9..d16434148 100644 --- a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja @@ -4,6 +4,8 @@ {%- set HOST = GLOBALS.manager %} {%- endif %} {%- set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %} +{%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %} + output { redis { host => '{{ HOST }}' @@ -14,5 +16,6 @@ output { congestion_threshold => 50000000 batch => true batch_events => {{ BATCH }} + password => {{ REDIS_PASS }} } } diff --git a/salt/redis/defaults.yaml b/salt/redis/defaults.yaml index 252082901..bd1fc74f5 100644 --- a/salt/redis/defaults.yaml +++ b/salt/redis/defaults.yaml @@ -1,7 +1,7 @@ redis: config: bind: '0.0.0.0' - protected-mode: 'no' + protected-mode: 'yes' tls-cert-file: '/certs/redis.crt' tls-key-file: '/certs/redis.key' tls-ca-cert-file: '/certs/ca.crt' diff --git a/salt/redis/soc_redis.yaml b/salt/redis/soc_redis.yaml index dad76bf80..13186ca6f 100644 --- a/salt/redis/soc_redis.yaml +++ b/salt/redis/soc_redis.yaml @@ -10,6 +10,10 @@ redis: global: True advanced: True helpLink: redis.html + requirepass: + description: Password for accessing Redis. + global: True + sensitive: True tls-cert-file: description: TLS cert file location. global: True diff --git a/setup/so-functions b/setup/so-functions index 4f6af0e7c..8545a4c6f 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1209,6 +1209,7 @@ generate_passwords(){ GRAFANAPASS=$(get_random_value) SENSORONIKEY=$(get_random_value) KRATOSKEY=$(get_random_value) + REDISPASS=$(get_random_value) } generate_interface_vars() { @@ -1496,7 +1497,10 @@ docker_pillar() { redis_pillar() { title "Create the redis pillar file" touch $adv_redis_pillar_file - touch $redis_pillar_file + printf '%s\n'\ + "redis:"\ + " config:"\ + " requirepass: '$REDISPASS'" > $redis_pillar_file } influxdb_pillar() { From 7ba4bdd87bb30da68734a65daf6eeda7eb94d037 Mon Sep 17 00:00:00 2001 From: doug Date: Wed, 4 Jan 2023 13:50:25 -0500 Subject: [PATCH 34/39] fix jinja whitespace --- salt/common/tools/sbin/so-elastic-clear | 3 +-- .../sbin/so-elasticsearch-component-templates-list | 3 +-- .../tools/sbin/so-elasticsearch-index-templates-list | 3 +-- salt/common/tools/sbin/so-elasticsearch-indices-list | 2 +- salt/common/tools/sbin/so-elasticsearch-pipeline-stats | 2 +- salt/common/tools/sbin/so-elasticsearch-pipeline-view | 2 +- salt/common/tools/sbin/so-elasticsearch-pipelines-list | 3 +-- salt/common/tools/sbin/so-elasticsearch-shards-list | 2 +- .../common/tools/sbin/so-elasticsearch-template-remove | 2 +- salt/common/tools/sbin/so-elasticsearch-template-view | 2 +- salt/common/tools/sbin/so-elasticsearch-templates-list | 3 +-- salt/common/tools/sbin/so-import-evtx | 8 +++----- salt/common/tools/sbin/so-import-pcap | 4 +--- salt/common/tools/sbin/so-kibana-config-export | 10 ++++------ 14 files changed, 19 insertions(+), 30 deletions(-) diff --git a/salt/common/tools/sbin/so-elastic-clear b/salt/common/tools/sbin/so-elastic-clear index d441e4d65..08da4021c 100755 --- a/salt/common/tools/sbin/so-elastic-clear +++ b/salt/common/tools/sbin/so-elastic-clear @@ -5,8 +5,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. - -{%- set NODEIP = salt['pillar.get']('host:mainip', '') -%} +{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} . /usr/sbin/so-common SKIP=0 diff --git a/salt/common/tools/sbin/so-elasticsearch-component-templates-list b/salt/common/tools/sbin/so-elasticsearch-component-templates-list index f8eab884e..976499574 100755 --- a/salt/common/tools/sbin/so-elasticsearch-component-templates-list +++ b/salt/common/tools/sbin/so-elasticsearch-component-templates-list @@ -5,8 +5,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. - -{%- set NODEIP = salt['pillar.get']('host:mainip', '') -%} +{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} . /usr/sbin/so-common if [ "$1" == "" ]; then curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_component_template | jq '.component_templates[] |.name'| sort diff --git a/salt/common/tools/sbin/so-elasticsearch-index-templates-list b/salt/common/tools/sbin/so-elasticsearch-index-templates-list index 35c478f90..bea975c93 100755 --- a/salt/common/tools/sbin/so-elasticsearch-index-templates-list +++ b/salt/common/tools/sbin/so-elasticsearch-index-templates-list @@ -5,8 +5,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. - -{%- set NODEIP = salt['pillar.get']('host:mainip', '') -%} +{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} . /usr/sbin/so-common if [ "$1" == "" ]; then curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_index_template | jq '.index_templates[] |.name'| sort diff --git a/salt/common/tools/sbin/so-elasticsearch-indices-list b/salt/common/tools/sbin/so-elasticsearch-indices-list index 7267ec1b5..da8ea4cca 100755 --- a/salt/common/tools/sbin/so-elasticsearch-indices-list +++ b/salt/common/tools/sbin/so-elasticsearch-indices-list @@ -5,7 +5,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set NODEIP = salt['pillar.get']('host:mainip', '') -%} +{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} . /usr/sbin/so-common diff --git a/salt/common/tools/sbin/so-elasticsearch-pipeline-stats b/salt/common/tools/sbin/so-elasticsearch-pipeline-stats index 82590d142..952773cda 100755 --- a/salt/common/tools/sbin/so-elasticsearch-pipeline-stats +++ b/salt/common/tools/sbin/so-elasticsearch-pipeline-stats @@ -5,7 +5,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set NODEIP = salt['pillar.get']('host:mainip', '') -%} +{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} . /usr/sbin/so-common diff --git a/salt/common/tools/sbin/so-elasticsearch-pipeline-view b/salt/common/tools/sbin/so-elasticsearch-pipeline-view index 535556f2c..32a26b948 100755 --- a/salt/common/tools/sbin/so-elasticsearch-pipeline-view +++ b/salt/common/tools/sbin/so-elasticsearch-pipeline-view @@ -5,7 +5,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set NODEIP = salt['pillar.get']('host:mainip', '') -%} +{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} . /usr/sbin/so-common diff --git a/salt/common/tools/sbin/so-elasticsearch-pipelines-list b/salt/common/tools/sbin/so-elasticsearch-pipelines-list index 3e6246e31..b690d5846 100755 --- a/salt/common/tools/sbin/so-elasticsearch-pipelines-list +++ b/salt/common/tools/sbin/so-elasticsearch-pipelines-list @@ -5,8 +5,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. - -{%- set NODEIP = salt['pillar.get']('host:mainip', '') -%} +{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} . /usr/sbin/so-common if [ "$1" == "" ]; then curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys' diff --git a/salt/common/tools/sbin/so-elasticsearch-shards-list b/salt/common/tools/sbin/so-elasticsearch-shards-list index 192d2c6f0..8865e05ac 100755 --- a/salt/common/tools/sbin/so-elasticsearch-shards-list +++ b/salt/common/tools/sbin/so-elasticsearch-shards-list @@ -5,7 +5,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set NODEIP = salt['pillar.get']('host:mainip', '') -%} +{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} . /usr/sbin/so-common diff --git a/salt/common/tools/sbin/so-elasticsearch-template-remove b/salt/common/tools/sbin/so-elasticsearch-template-remove index 5028cbb07..f69495152 100755 --- a/salt/common/tools/sbin/so-elasticsearch-template-remove +++ b/salt/common/tools/sbin/so-elasticsearch-template-remove @@ -5,7 +5,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set NODEIP = salt['pillar.get']('host:mainip', '') -%} +{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} . /usr/sbin/so-common diff --git a/salt/common/tools/sbin/so-elasticsearch-template-view b/salt/common/tools/sbin/so-elasticsearch-template-view index 5934de13e..c56127703 100755 --- a/salt/common/tools/sbin/so-elasticsearch-template-view +++ b/salt/common/tools/sbin/so-elasticsearch-template-view @@ -5,7 +5,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set NODEIP = salt['pillar.get']('host:mainip', '') -%} +{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} . /usr/sbin/so-common diff --git a/salt/common/tools/sbin/so-elasticsearch-templates-list b/salt/common/tools/sbin/so-elasticsearch-templates-list index e63c8cf54..7db4fdeff 100755 --- a/salt/common/tools/sbin/so-elasticsearch-templates-list +++ b/salt/common/tools/sbin/so-elasticsearch-templates-list @@ -5,8 +5,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. - -{%- set NODEIP = salt['pillar.get']('host:mainip', '') -%} +{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} . /usr/sbin/so-common if [ "$1" == "" ]; then curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_template/* | jq 'keys' diff --git a/salt/common/tools/sbin/so-import-evtx b/salt/common/tools/sbin/so-import-evtx index a5540f768..0b6bd8383 100755 --- a/salt/common/tools/sbin/so-import-evtx +++ b/salt/common/tools/sbin/so-import-evtx @@ -5,15 +5,13 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. - - {%- set MANAGER = salt['grains.get']('master') %} {%- set VERSION = salt['pillar.get']('global:soversion') %} {%- set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} -{%- set MANAGERIP = salt['pillar.get']('global:managerip') -%} +{%- set MANAGERIP = salt['pillar.get']('global:managerip') %} {%- set URLBASE = salt['pillar.get']('global:url_base') %} -{% set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{% set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} INDEX_DATE=$(date +'%Y.%m.%d') RUNID=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 8 | head -n 1) diff --git a/salt/common/tools/sbin/so-import-pcap b/salt/common/tools/sbin/so-import-pcap index 2c1f27705..b099f39a4 100755 --- a/salt/common/tools/sbin/so-import-pcap +++ b/salt/common/tools/sbin/so-import-pcap @@ -5,12 +5,10 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. - - {%- set MANAGER = salt['grains.get']('master') %} {%- set VERSION = salt['pillar.get']('global:soversion') %} {%- set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} -{%- set MANAGERIP = salt['pillar.get']('global:managerip') -%} +{%- set MANAGERIP = salt['pillar.get']('global:managerip') %} {%- set URLBASE = salt['pillar.get']('global:url_base') %} . /usr/sbin/so-common diff --git a/salt/common/tools/sbin/so-kibana-config-export b/salt/common/tools/sbin/so-kibana-config-export index 1c15fc54c..b313710fe 100755 --- a/salt/common/tools/sbin/so-kibana-config-export +++ b/salt/common/tools/sbin/so-kibana-config-export @@ -1,16 +1,14 @@ #!/bin/bash # -# {%- set FLEET_MANAGER = salt['pillar.get']('global:fleet_manager', False) -%} -# {%- set FLEET_NODE = salt['pillar.get']('global:fleet_node', False) -%} -# {%- set FLEET_IP = salt['pillar.get']('global:fleet_ip', '') %} -# {%- set MANAGER = salt['pillar.get']('global:url_base', '') %} -# # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. - +{%- set FLEET_MANAGER = salt['pillar.get']('global:fleet_manager', False) %} +{%- set FLEET_NODE = salt['pillar.get']('global:fleet_node', False) %} +{%- set FLEET_IP = salt['pillar.get']('global:fleet_ip', '') %} +{%- set MANAGER = salt['pillar.get']('global:url_base', '') %} KIBANA_HOST={{ MANAGER }} KSO_PORT=5601 From 48a3f4e2619fa08b30a57587ec8c143f0ae53d81 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 4 Jan 2023 14:14:10 -0500 Subject: [PATCH 35/39] Allow auth for redis check for tgraf --- salt/telegraf/scripts/redis.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/salt/telegraf/scripts/redis.sh b/salt/telegraf/scripts/redis.sh index f0c361037..3dffa21d6 100644 --- a/salt/telegraf/scripts/redis.sh +++ b/salt/telegraf/scripts/redis.sh @@ -4,14 +4,13 @@ # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. - - +{%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass', '0') %} # if this script isn't already running if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then - UNPARSED=$(redis-cli llen logstash:unparsed | awk '{print $1}') - PARSED=$(redis-cli llen logstash:parsed | awk '{print $1}') + UNPARSED=$(redis-cli -a {{ REDIS_PASS }} llen logstash:unparsed | awk '{print $1}') + PARSED=$(redis-cli -a {{ REDIS_PASS }} llen logstash:parsed | awk '{print $1}') echo "redisqueue unparsed=$UNPARSED,parsed=$PARSED" From 9bc08661c5031e0e480791c0dc4349722ac9df25 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 4 Jan 2023 14:15:53 -0500 Subject: [PATCH 36/39] Allow auth for redis check for tgraf --- salt/common/tools/sbin/so-redis-count | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/so-redis-count b/salt/common/tools/sbin/so-redis-count index e90e0754e..e37526013 100755 --- a/salt/common/tools/sbin/so-redis-count +++ b/salt/common/tools/sbin/so-redis-count @@ -4,9 +4,8 @@ # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. - - +{%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass', '0') %} . /usr/sbin/so-common -docker exec so-redis redis-cli llen logstash:unparsed +docker exec so-redis redis-cli -a {{ REDIS_PASS }} llen logstash:unparsed From 5afad52b3f051590a89b74ddb38cee3c220667f9 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 4 Jan 2023 14:18:08 -0500 Subject: [PATCH 37/39] Allow auth for redis check for tgraf --- salt/common/tools/sbin/so-redis-count | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-redis-count b/salt/common/tools/sbin/so-redis-count index e37526013..787c1b7e6 100755 --- a/salt/common/tools/sbin/so-redis-count +++ b/salt/common/tools/sbin/so-redis-count @@ -7,5 +7,5 @@ {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass', '0') %} . /usr/sbin/so-common - -docker exec so-redis redis-cli -a {{ REDIS_PASS }} llen logstash:unparsed +REDISCLI_AUTH={{ REDIS_PASS }} +docker exec so-redis redis-cli llen logstash:unparsed From e52b54720ac7508ca4bd9b88adefb31142c2c858 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 4 Jan 2023 14:26:24 -0500 Subject: [PATCH 38/39] Allow auth for redis check for tgraf --- salt/common/tools/sbin/so-redis-count | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-redis-count b/salt/common/tools/sbin/so-redis-count index 787c1b7e6..7f87b6a9f 100755 --- a/salt/common/tools/sbin/so-redis-count +++ b/salt/common/tools/sbin/so-redis-count @@ -7,5 +7,4 @@ {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass', '0') %} . /usr/sbin/so-common -REDISCLI_AUTH={{ REDIS_PASS }} -docker exec so-redis redis-cli llen logstash:unparsed +docker exec so-redis /bin/sh -c "export REDISCLI_AUTH={{ REDIS_PASS }} && redis-cli llen logstash:unparsed" \ No newline at end of file From 275aead5b9a07e1a85e7c0def8768c5249c115af Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 4 Jan 2023 14:30:28 -0500 Subject: [PATCH 39/39] Allow auth for redis check for tgraf --- salt/telegraf/scripts/redis.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/telegraf/scripts/redis.sh b/salt/telegraf/scripts/redis.sh index 3dffa21d6..c730885d4 100644 --- a/salt/telegraf/scripts/redis.sh +++ b/salt/telegraf/scripts/redis.sh @@ -5,12 +5,12 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass', '0') %} - +export REDISCLI_AUTH={{ REDIS_PASS }} # if this script isn't already running if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then - UNPARSED=$(redis-cli -a {{ REDIS_PASS }} llen logstash:unparsed | awk '{print $1}') - PARSED=$(redis-cli -a {{ REDIS_PASS }} llen logstash:parsed | awk '{print $1}') + UNPARSED=$(redis-cli llen logstash:unparsed | awk '{print $1}') + PARSED=$(redis-cli llen logstash:parsed | awk '{print $1}') echo "redisqueue unparsed=$UNPARSED,parsed=$PARSED"