CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2025-12-06 09:12:45 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #8738 from Security-Onion-Solutions/fix/remove_old_pipeline_config

Browse Source 
Remove old Logstash pipeline configuration - initial cleanup
This commit is contained in:
weslambert
2022-09-14 10:30:37 -04:00
committed by GitHub
parent 0d32cc38d6 926a1e0189
commit 39c7c8cf80
71 changed files with 0 additions and 4205 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
salt/logstash/pipelines/config/so/1000_preprocess_log_elapsed.conf
View File

@@ -1,13 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
filter {
ruby {
code => "event.set('task_start', Time.now.to_f)"
}
mutate {
#add_tag => [ "conf_file_1000"]
}
}

33
salt/logstash/pipelines/config/so/1001_preprocess_syslogng.conf
View File

@@ -1,33 +0,0 @@
# Updated by: Doug Burks and Wes Lambert
# Last Update: 10/30/2018
filter {
if "syslogng" in [tags] {
mutate {
rename => { "MESSAGE" => "message" }
rename => { "PROGRAM" => "type" }
rename => { "FACILITY" => "syslog-facility" }
rename => { "FILE_NAME" => "syslog-file_name" }
rename => { "HOST" => "syslog-host" }
rename => { "HOST_FROM" => "syslog-host_from" }
rename => { "LEGACY_MSGHDR" => "syslog-legacy_msghdr" }
rename => { "PID" => "syslog-pid" }
rename => { "PRIORITY" => "syslog-priority" }
rename => { "SOURCEIP" => "syslog-sourceip" }
rename => { "TAGS" => "syslog-tags" }
lowercase => [ "syslog-host_from" ]
remove_field => [ "ISODATE" ]
remove_field => [ "SEQNUM" ]
#add_tag => [ "conf_file_1001"]
}
if "bro_" in [type] {
mutate {
add_tag => [ "bro" ]
}
} else if [type] !~ /ossec.*|snort/ and "firewall" not in [tags] {
mutate {
add_tag => [ "syslog" ]
}
}
}
}

18
salt/logstash/pipelines/config/so/1002_preprocess_json.conf
View File

@@ -1,18 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
filter {
if "json" in [tags]{
json {
source => "message"
}
mutate {
remove_tag => [ "json" ]
}
mutate {
#add_tag => [ "conf_file_1002"]
}
}
}

19
salt/logstash/pipelines/config/so/1004_preprocess_syslog_types.conf
View File

@@ -1,19 +0,0 @@
filter {
if "syslog" in [tags] {
if [host] == "172.16.1.1" {
mutate {
add_field => { "type" => "fortinet" }
add_tag => [ "firewall" ]
}
}
if [host] == "10.0.0.101" {
mutate {
add_field => { "type" => "brocade" }
add_tag => [ "switch" ]
}
}
mutate {
#add_tag => [ "conf_file_1004"]
}
}
}

140
salt/logstash/pipelines/config/so/1026_preprocess_dhcp.conf
View File

@@ -1,140 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolutions.com
# Last Update: 12/9/2016
# This conf file is based on accepting logs for DHCP. It is currently based on Windows DHCP only.
filter {
if [type] == "dhcp" {
mutate {
add_field => { "Hostname" => "%{host}" }
}
mutate {
strip => "message"
}
# This is the initial parsing of the log
grok {
# Server 2008+
match => { "message" => "%{DATA:id},%{DATE_US:date},(?<time>%{HOUR}:%{MINUTE}:%{SECOND}),%{DATA:description},%{IPV4:ip},%{DATA:Hostname},%{DATA:mac},%{DATA:Username},%{INT:TransactionID},%{INT:QResult},%{DATA:ProbationTime},%{DATA:CorrelationID}"}
# Server 2003
match => { "message" => "%{DATA:id},%{DATE_US:date},(?<time>%{HOUR}:%{MINUTE}:%{SECOND}),%{DATA:description},%{IPV4:ip},%{DATA:Hostname},%{DATA:mac},"}
match => { "message" => "%{DATA:id},%{DATA:date},(?<time>%{HOUR}:%{MINUTE}:%{SECOND}),%{DATA:description},%{DATA:ip},%{DATA:Hostname},%{DATA:mac},"}
}
# This section below translates the message ID into something humans can understand.
if [id] == "00" {
mutate {
add_field => [ "event", "The log was started"]
}
}
if [id] == "01" {
mutate {
add_field => [ "event", "The log was stopped"]
}
}
if [id] == "02" {
mutate {
add_field => [ "event", "The log was temporarily paused due to low disk space"]
}
}
if [id] == "10" {
mutate {
add_field => [ "event", "A new IP address was leased to a client"]
}
}
if [id] == "11" {
mutate {
add_field => [ "event", "A lease was renewed by a client"]
}
}
if [id] == "12" {
mutate {
add_field => [ "event", "A lease was released by a client"]
}
}
if [id] == "13" {
mutate {
add_field => [ "event", "An IP address was found to be in use on the network"]
}
}
if [id] == "14" {
mutate {
add_field => [ "event", "A lease request could not be satisfied because the scope's address pool was exhausted"]
}
}
if [id] == "15" {
mutate {
add_field => [ "event", "A lease was denied"]
}
}
if [id] == "16" {
mutate {
add_field => [ "event", "A lease was deleted"]
}
}
if [id] == "17" {
mutate {
add_field => [ "event", "A lease was expired and DNS records for an expired leases have not been deleted"]
}
}
if [id] == "18" {
mutate {
add_field => [ "event", "A lease was expired and DNS records were deleted"]
}
}
if [id] == "20" {
mutate {
add_field => [ "event", "A BOOTP address was leased to a client"]
}
}
if [id] == "21" {
mutate {
add_field => [ "event", "A dynamic BOOTP address was leased to a client"]
}
}
if [id] == "22" {
mutate {
add_field => [ "event", "A BOOTP request could not be satisfied because the scope's address pool for BOOTP was exhausted"]
}
}
if [id] == "23" {
mutate {
add_field => [ "event", "A BOOTP IP address was deleted after checking to see it was not in use"]
}
}
if [id] == "24" {
mutate {
add_field => [ "event", "IP address cleanup operation has began"]
}
}
if [id] == "25" {
mutate {
add_field => [ "event", "IP address cleanup statistics"]
}
}
if [id] == "30" {
mutate {
add_field => [ "event", "DNS update request to the named DNS server"]
}
}
if [id] == "31" {
mutate {
add_field => [ "event", "DNS update failed"]
}
}
if [id] == "32" {
mutate {
add_field => [ "event", "DNS update successful"]
}
}
if [id] == "33" {
mutate {
add_field => [ "event", "Packet dropped due to NAP policy"]
}
}
# If the message failed to parse correctly keep the message for debugging. Otherwise, drop it.
#if "_grokparsefailure" not in [tags] {
# mutate {
# remove_field => [ "message"]
# }
#}
}
}

31
salt/logstash/pipelines/config/so/1029_preprocess_esxi.conf
View File

@@ -1,31 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
#
# This configuration file takes ESXi syslog messages and filters them. There is no input as the logs would have came in via syslog
filter {
# This is an example of using an IP address range to classify a syslog message to a specific type of log
# This is helpful as so many devices only send logs via syslog
if [host] =~ "10\.[0-1]\.9\." {
mutate {
replace => ["type", "esxi"]
}
}
if [host] =~ "\.234$" {
mutate {
replace => ["type", "esxi"]
}
}
if [type] == "esxi" {
grok {
match => { "message" => "(?:%{SYSLOGTIMESTAMP:timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) (?:%{SYSLOGHOST:logsource}) (?:%{SYSLOGPROG}): (?<messagebody>(?:\[(?<esxi_thread_id>[0-9A-Z]{8,8}) %{DATA:esxi_loglevel} \'%{DATA:esxi_service}\'\] %{GREEDYDATA:esxi_message}|%{GREEDYDATA}))"}
# pattern => ['(?:%{SYSLOGTIMESTAMP:timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) (?:%{SYSLOGHOST:logsource}) (?:%{SYSLOGPROG}): (?<messagebody>(?:\[(?<esxi_thread_id>[0-9A-Z]{8,8}) %{DATA:esxi_loglevel} \'%{DATA:esxi_service}\'\] %{GREEDYDATA:esxi_message}|%{GREEDYDATA}))']
}
mutate {
#add_tag => [ "conf_file_1029"]
}
}
}

21
salt/logstash/pipelines/config/so/1030_preprocess_greensql.conf
View File

@@ -1,21 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
filter {
if [type] == "greensql" {
# This section is parsing out the fields for GreenSQL syslog data
grok {
match => { "message" => "<%{INT:Code}>%{DATA:Category}\[%{INT:Transcation}\]:\s*Database=%{DATA:Database}\sUser=%{DATA:UserName}\sApplication Name=%{DATA:Application}\sSource IP=%{IPV4:SrcIp}\sSource Port=%{INT:SrcPort}\sTarget IP=?%{IPV4:DstIp}\sTarget Port=%{DATA:DstPort}\sQuery=%{GREEDYDATA:Query}"}
match => { "message" => "<%{INT:Code}>%{DATA:Category}\[%{INT:Transcation}\]:\sAdmin_Name=%{DATA:UserName}\sIP_Address=%{IPV4:SrcIp}\sUser_Agent=%{DATA:UserAgent}\sMessage=%{DATA:StatusMessage}\sDescription=%{DATA:Description}\sSeverity=%{GREEDYDATA:Severity}"}
}
# Remove the message field as it is unnecessary
#mutate {
# remove_field => [ "message"]
#}
mutate {
#add_tag => [ "conf_file_1030"]
}
}
}

21
salt/logstash/pipelines/config/so/1031_preprocess_iis.conf
View File

@@ -1,21 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
filter {
if [type] == "iis" {
# The log is expected to have come from NXLog and in JSON format. This allows for automatic parsing of fields
json {
source => "message"
}
# This removes the message field as it is unneccesary and tags the packet as web
mutate {
# remove_field => [ "message"]
add_tag => [ "web" ]
}
mutate {
#add_tag => [ "conf_file_1031"]
}
}
}

26
salt/logstash/pipelines/config/so/1032_preprocess_mcafee.conf
View File

@@ -1,26 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
#
# This file looks for McAfee EPO logs
filter {
if [type] == "mcafee" {
# NXLog should be sending the logs in JSON format so they auto parse
json {
source => "message"
}
# This section converts the UTC fields to the proper time format
date {
match => [ "ReceivedUTC", "YYYY-MM-dd HH:mm:ss" ]
target => [ "ReceivedUTC" ]
}
date {
match => [ "DetectedUTC", "YYYY-MM-dd HH:mm:ss" ]
target => [ "DetectedUTC" ]
}
mutate {
#add_tag => [ "conf_file_1032"]
}
}
}

125
salt/logstash/pipelines/config/so/1033_preprocess_snort.conf
View File

@@ -1,125 +0,0 @@
filter {
if [engine] == "suricata" {
json {
source => "message"
}
mutate {
# Make this compatible with event.id as a string
convert => { "[flow_id]" => "string" }
rename => {
"proto" => "[network][transport]"
"event_type" => "[event][dataset]"
"flow_id" => "[event][id]"
"community_id" => "[network][community_id]"
}
lowercase => [ "[network][transport]" ]
merge => {"[event][id]" => "[related][id]" }
add_field => {
"[related][domain]" => []
"[related][ip]" => []
"[related][id]" => []
"[event][module]" => "suricata"
"[event][created]" => "%{[@timestamp]}"
"[event][version]" => "1.0.0"
"[event][category]" => "network"
}
}
# Set the timestamp from the event
date {
match => [ "timestamp", "ISO8601" ]
tag_on_failure => [ "_dateparsefailure", "_parsefailure", "_suricata_dateparsefailure" ]
remove_field => [ "timestamp" ]
}
# Suricata uses top-level src/dest to track flow
if [src_ip] {
mutate {
rename => {
"[src_ip]" => "[source][ip]"
"[src_port]" => "[source][port]"
}
merge => { "[related][ip]" => "[source][ip]" }
}
}
if [dest_ip] {
mutate {
rename => {
"[dest_ip]" => "[destination][ip]"
"[dest_port]" => "[destination][port]"
}
merge => { "[related][ip]" => "[destination][ip]" }
}
}
if [vlan] {
mutate {
rename => { "[vlan]" => "[vlan][id]" }
}
}
if [app_proto] {
if [app_proto] == "failed" {
# delete failed detections to be consistent with zeek
mutate { rename => { "app_proto" => "[error][message]" } }
}
else {
mutate { rename => {"app_proto" => "[network][protocol]"}}
}
}
if [event_type] == "alert" {
if [alert][severity] == 1 {
mutate {
add_field => { "severity" => "High" }
}
}
if [alert][severity] == 2 {
mutate {
add_field => { "severity" => "Medium" }
}
}
if [alert][severity] == 3 {
mutate {
add_field => { "severity" => "Low" }
}
}
# If the alert is a Snort GPL alert break it apart for easier reading and categorization
if [alert][signature] =~ "GPL " {
# This will parse out the category type from the alert
grok {
match => { "[alert][signature]" => "GPL\s+%{DATA:category}\s" }
}
# This will store the category
mutate {
add_field => { "rule_type" => "Snort GPL" }
lowercase => [ "category" ]
}
}
# If the alert is an Emerging Threat alert break it apart for easier reading and categorization
if [alert][signature] =~ "ET " {
# This will parse out the category type from the alert
grok {
match => { "[alert][signature]" => "ET\s+%{DATA:category}\s" }
}
# This will store the category
mutate {
add_field => { "rule_type" => "Emerging Threats" }
lowercase => [ "category" ]
}
}
# This section adds URLs to lookup information about a rule online
if [rule_type] == "Snort GPL" {
mutate {
add_field => [ "signature_info", "https://www.snort.org/search?query=%{[alert][gid]}-%{[alert][signature_id]}" ]
}
}
if [rule_type] == "Emerging Threats" {
mutate {
add_field => [ "signature_info", "http://doc.emergingthreats.net/%{[alert][signature_id]}" ]
}
}
}
mutate {
remove_field => [ "alert" ]
}
}
}

16
salt/logstash/pipelines/config/so/1034_preprocess_syslog.conf
View File

@@ -1,16 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 5/22/2017
filter {
if [type] == "syslog" {
# This drops syslog messages regarding license messages. You may want to comment it out.
#if [message] =~ "license" {
# drop { }
#}
mutate {
#convert => [ "status_code", "integer" ]
}
}
}

77
salt/logstash/pipelines/config/so/1100_preprocess_bro_conn.conf
View File

@@ -1,77 +0,0 @@
# Original Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
#
# This conf file is based on accepting logs for conn.log from Bro systems
filter {
if [type] == "bro_conn" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
rename => { "proto" => "protocol" }
#service
#duration
rename => { "orig_bytes" => "original_bytes" }
rename => { "resp_bytes" => "respond_bytes" }
rename => { "conn_state" => "connection_state" }
#local_orig
rename => { "local_resp" => "local_respond" }
#missed_bytes
#history
rename => { "orig_pkts" => "original_packets" }
rename => { "orig_ip_bytes" => "original_ip_bytes" }
rename => { "resp_pkts" => "respond_packets" }
rename => { "resp_ip_bytes" => "respond_ip_bytes" }
#tunnel_parents
rename => { "orig_cc" => "original_country_code" }
rename => { "resp_cc" => "respond_country_code" }
rename => { "sensorname" => "sensor_name" }
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","protocol","service","duration","original_bytes","respond_bytes","connection_state","local_orig","local_respond","missed_bytes","history","original_packets","original_ip_bytes","respond_packets","respond_ip_bytes","tunnel_parents","sensor_name"]
# If you use a custom delimiter, change the following value in between the quotes to your delimiter. Otherwise, insert a literal <tab> in between the two quotes on your logstash system, use a text editor like nano that doesn't convert tabs to spaces.
separator => " "
}
}
translate {
field => "connection_state"
destination => "connection_state_description"
dictionary => [
"S0", "Connection attempt seen, no reply",
"S1", "Connection established, not terminated",
"S2", "Connection established and close attempt by originator seen (but no reply from responder)",
"S3", "Connection established and close attempt by responder seen (but no reply from originator)",
"SF", "Normal SYN/FIN completion",
"REJ", "Connection attempt rejected",
"RSTO", "Connection established, originator aborted (sent a RST)",
"RSTR", "Established, responder aborted",
"RSTOS0", "Originator sent a SYN followed by a RST, we never saw a SYN-ACK from the responder",
"RSTRH", "Responder sent a SYN ACK followed by a RST, we never saw a SYN from the (purported) originator",
"SH", "Originator sent a SYN followed by a FIN, we never saw a SYN ACK from the responder (hence the connection was 'half' open)",
"SHR", "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator",
"OTH", "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed)"
]
}
mutate {
#add_tag => [ "conf_file_1100"]
}
}
}

56
salt/logstash/pipelines/config/so/1101_preprocess_bro_dhcp.conf
View File

@@ -1,56 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks and Wes Lambert
# Last Update: 1/3/2019
#
# This conf file is based on accepting logs for dhcp.log from Bro systems
filter {
if [type] == "bro_dhcp" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#mac
#assigned_ip
#lease_time
rename => { "trans_id" => "transaction_id" }
# new dhcp log format
rename => { "assigned_addr" => "assigned_ip" }
rename => { "client_addr" => "source_ip" }
rename => { "server_addr" => "destination_ip" }
rename => { "requested_addr" => "requested_ip" }
rename => { "domain" => "domain_name" }
rename => { "host_name" => "hostname" }
rename => { "msg_types" => "message_types" }
rename => { "uids" => "uid" }
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
# Bro logs in TSV format
csv {
columns => [ "timestamp", "uid", "source_ip", "destination_ip", "mac", "hostname", "client_fqdn", "domain_name", "requested_ip", "assigned_ip", "lease_time","client_message", "server_message", "message_types", "duration" ]
separator => " "
}
# Remove fields with empty values (-) to prevent field data type conflict
ruby {
code =>"
hash = event.to_hash.each do |key,value|
if value == '-'
event.remove(key)
end
end"
}
}
}
}

74
salt/logstash/pipelines/config/so/1102_preprocess_bro_dns.conf
View File

@@ -1,74 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for dns.log from Bro systems
filter {
if [type] == "bro_dns" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
rename => { "proto" => "protocol" }
rename => { "trans_id" => "transaction_id" }
#rtt field
#query field
rename => { "qclass" => "query_class" }
rename => { "qclass_name" => "query_class_name" }
rename => { "qtype" => "query_type" }
rename => { "qtype_name" => "query_type_name" }
#rcode
#rcode_name
rename => { "AA" => "aa" }
rename => { "TC" => "tc" }
rename => { "RD" => "rd" }
rename => { "RA" => "ra" }
rename => { "Z" => "z" }
#answers
rename => { "TTLs" => "ttls" }
#rejected
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","protocol","transaction_id","rtt","query","query_class","query_class_name","query_type","query_type_name","rcode","rcode_name","aa","tc","rd","ra","z","answers","ttls","rejected"]
#If you use a custom delimiter, change the following value in between the quotes to your delimiter. Otherwise, insert a literal <tab> in between the two quotes on your logstash system, use a text editor like nano that doesn't convert tabs to spaces.
separator => " "
}
}
mutate {
add_tag => [ "dns" ]
}
if [ttls] == "-" {
mutate {
remove_field => [ "ttls" ]
}
}
if [rtt] == "-" {
mutate {
remove_field => [ "rtt" ]
}
}
#mutate {
#convert => [ "rtt", "float" ]
#}
mutate {
#add_tag => [ "conf_file_1102"]
}
}
}

42
salt/logstash/pipelines/config/so/1103_preprocess_bro_dpd.conf
View File

@@ -1,42 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for dpd.log from Bro systems
filter {
if [type] == "bro_dpd" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
rename => { "proto" => "protocol" }
#analyzer
#failure_reason
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","protocol","analyzer","failure_reason"]
separator => " "
}
}
mutate {
#add_tag => [ "conf_file_1103"]
}
}
}

64
salt/logstash/pipelines/config/so/1104_preprocess_bro_files.conf
View File

@@ -1,64 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for files.log from Bro systems
filter {
if [type] == "bro_files" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#fuid
rename => { "tx_hosts" => "file_ip" }
rename => { "rx_hosts" => "destination_ip" }
rename => { "conn_uids" => "connection_uids" }
#source field
#depth field
rename => { "analyzers" => "analyzer" }
rename => { "mime_type" => "mimetype" }
rename => { "filename" => "file_name" }
#duration
#local_orig
#is_orig
#seen_bytes
#total_bytes
#missing_bytes
#overflow_bytes
rename => { "timedout" => "timed_out" }
#parent_fuid
#md5
#sha1
#sha256
#extracted
#extracted_cutoff
#extracted_size
}
} else {
csv {
columns => ["timestamp","fuid","file_ip","destination_ip","connection_uids","source","depth","analyzer","mimetype","file_name","duration","local_orig","is_orig","seen_bytes","total_bytes","missing_bytes","overflow_bytes","timed_out","parent_fuid","md5","sha1","sha256","extracted","extracted_cutoff","extracted_size"]
separator => " "
}
if [destination_ip] =~ /,/ {
mutate {
split => { "destination_ip" => "," }
}
}
if [file_ip] =~ /,/ {
mutate {
split => { "file_ip" => "," }
}
}
}
mutate {
#add_tag => [ "conf_file_1104"]
}
}
}

56
salt/logstash/pipelines/config/so/1105_preprocess_bro_ftp.conf
View File

@@ -1,56 +0,0 @@
# Original Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for ftp.log from Bro systems
filter {
if [type] == "bro_ftp" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
rename => { "user" => "username" }
#password
rename => { "command" => "ftp_command" }
rename => { "arg" => "ftp_argument" }
rename => { "mime_type" => "mimetype" }
#file_size
#reply_code
rename => { "reply_msg" => "reply_message" }
rename => { "data_channel.passive" => "data_channel_passive" }
rename => { "data_channel.orig_h" => "data_channel_source_ip" }
rename => { "data_channel.resp_h" => "data_channel_destination_ip" }
rename => { "data_channel.resp_p" => "data_channel_destination_port" }
#fuid
}
mutate {
convert => { "reply" => "string" }
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","username","password","ftp_command","ftp_argument","mimetype","file_size","reply_code","reply_message","data_channel_passive","data_channel_source_ip","data_channel_destination_ip","data_channel_destination_port","fuid"]
separator => " "
}
}
mutate {
#add_tag => [ "conf_file_1105"]
}
}
}

77
salt/logstash/pipelines/config/so/1106_preprocess_bro_http.conf
View File

@@ -1,77 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
#
# This conf file is based on accepting logs for http.log from Bro systems
filter {
if [type] == "bro_http" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
# Rename logstash tags field to avoid being overwritten by Bro's http tags field
mutate {
rename => { "tags" => "tags-orig" }
}
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#trans_depth
#method
rename => { "host" => "virtual_host" }
#uri
#referrer
#version
#convert => { "version" => "string" }
rename => { "user_agent" => "useragent" }
#origin
rename => { "request_body_len" => "request_body_length" }
rename => { "response_body_len" => "response_body_length" }
#status_code
#status_message
rename => { "status_msg" => "status_message" }
#info_code
rename => { "info_msg" => "info_message" }
#tags
# Rename http tags field to http-tags
rename => { "tags" => "http-tags" }
# Rename logstash tags field to tags
rename => { "tags-orig" => "tags" }
#username
#password
#proxied
#orig_fuids
#orig_filenames
#orig_mime_types
#resp_fuids
#resp_filenames
#resp_mime_types
}
if [http-tags] {
mutate {
remove_field => [ "http-tags" ]
}
}
} else {
grok {
match => [ "message", "(?<timestamp>(.*?))\t(?<uid>(.*?))\t(?<source_ip>(.*?))\t(?<source_port>(.*?))\t(?<destination_ip>(.*?))\t(?<destination_port>(.*?))\t(?<trans_depth>(.*?))\t(?<method>(.*?))\t(?<virtual_host>(.*?))\t(?<uri>(.*?))\t(?<referrer>(.*?))\t(?<version>(.*?))\t(?<useragent>(.*?))\t(?<origin>(.*?))\t(?<request_body_length>(.*?))\t(?<response_body_length>(.*?))\t(?<status_code>(.*?))\t(?<status_message>(.*?))\t(?<info_code>(.*?))\t(?<info_message>(.*?))\t(?<tags>(.*))\t(?<username>(.*))\t(?<password>(.*))\t(?<proxied>(.*))\t(?<orig_fuids>(.*))\t(?<orig_filenames>(.*?))\t(?<orig_mime_types>(.*))\t(?<resp_fuids>(.*))\t(?<resp_filenames>(.*?))\t(?<resp_mime_types>(.*))" ]
}
}
if [useragent] == "-" {
mutate {
remove_field => [ "useragent" ]
}
}
mutate {
#add_tag => [ "conf_file_1106"]
}
}
}

46
salt/logstash/pipelines/config/so/1107_preprocess_bro_irc.conf
View File

@@ -1,46 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for irc.log from Bro systems
filter {
if [type] == "bro_irc" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#nick
rename => { "user" => "irc_username" }
rename => { "command" => "irc_command" }
#value
rename => { "addl" => "additional_info" }
#dcc_file_name
#dcc_file_size
#dcc_mime_type
#fuid
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","nick","irc_username","irc_command","value","additional_info","dcc_file_name","dcc_file_size","dcc_mime_type","fuid"]
separator => " "
}
}
mutate {
#add_tag => [ "conf_file_1107"]
}
}
}

56
salt/logstash/pipelines/config/so/1108_preprocess_bro_kerberos.conf
View File

@@ -1,56 +0,0 @@
# Original Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for kerberos.log from Bro systems
filter {
if [type] == "bro_kerberos" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#request_type
#client
#service
rename => { "success" => "kerberos_success" }
rename => { "error_msg" => "error_message" }
rename => { "from" => "valid_from" }
rename => { "till" => "valid_till" }
#cipher
#forwardable
#renewable
rename => { "client_cert_subject" => "client_certificate_subject" }
rename => { "client_cert_fuid" => "client_certificate_fuid" }
rename => { "server_cert_subject" => "server_certificate_subject" }
rename => { "server_cert_fuid" => "server_certificate_fuid" }
}
mutate {
convert => { "kerberos_success" => "string" }
convert => { "renewable" => "string" }
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","request_type","client","service","kerberos_success","error_message","valid_from","valid_till","cipher","forwardable","renewable","client_certificate_subject","client_certificate_fuid","server_certificate_subject","server_certificate_fuid"]
separator => " "
}
}
mutate {
#add_tag => [ "conf_file_1108"]
}
}
}

56
salt/logstash/pipelines/config/so/1109_preprocess_bro_notice.conf
View File

@@ -1,56 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for notice.log from Bro systems
filter {
if [type] == "bro_notice" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#fuid
rename => { "mime" => "file_mime_type" }
rename => { "desc" => "file_description" }
rename => { "proto" => "protocol" }
rename => { "note" => "note" }
rename => { "msg" => "msg" }
rename => { "sub" => "sub_msg" }
rename => { "src" => "source_ip" }
rename => { "dst" => "destination_ip" }
#p
#n
rename => { "peer_descr" => "peer_description" }
rename => { "actions" => "action" }
#suppress_for
#destination_country_code
#destination_region
#destination_city
#destination_latitude
#destination_longitude
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","fuid","file_mime_type","file_description","protocol","note","msg","sub_msg","source_ip","destination_ip","p","n","peer_description","action","suppress_for","destination_country_code","destination_region","destination_city","destination_latitude","destination_longitude"]
separator => " "
}
}
mutate {
#add_tag => [ "conf_file_1109"]
}
}
}

52
salt/logstash/pipelines/config/so/1110_preprocess_bro_rdp.conf
View File

@@ -1,52 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
#
# This conf file is based on accepting logs for rdp.log from Bro systems
filter {
if [type] == "bro_rdp" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#cookie
#result
#security_protocol
#client_channels
#keyboard_layout
#client_build
#client_name
rename => { "client_dig_product_id" => "client_digital_product_id" }
#desktop_width
#desktop_height
#requested_color_depth
rename => { "cert_type" => "certificate_type" }
rename => { "cert_count" => "certificate_count" }
rename => { "cert_permanent" => "certificate_permanent" }
#encryption_level
#encryption_method
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","cookie","result","security_protocol","client_channels","keyboard_layout","client_build","client_name","client_digital_product_id","desktop_width","desktop_height","requested_color_depth","certificate_type","certificate_count","certificate_permanent","encryption_level","encryption_method"]
separator => " "
}
}
mutate {
#add_tag => [ "conf_file_1110"]
}
}
}

43
salt/logstash/pipelines/config/so/1111_preprocess_bro_signatures.conf
View File

@@ -1,43 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for signatures.log from Bro systems
filter {
if [type] == "bro_signatures" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#note
rename => { "sig_id" => "signature_id" }
rename => { "event_msg" => "event_message" }
rename => { "sub_msg" => "sub_message" }
rename => { "sig_count" => "signature_count" }
#host_count
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","note","signature_id","event_message","sub_message","signature_count","host_count"]
separator => " "
}
}
mutate {
#add_tag => [ "conf_file_1111"]
}
}
}

65
salt/logstash/pipelines/config/so/1112_preprocess_bro_smtp.conf
View File

@@ -1,65 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for smtp.log from Bro systems
filter {
if [type] == "bro_smtp" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#trans_depth
#helo
rename => { "mailfrom" => "mail_from" }
rename => { "rcptto" => "recipient_to" }
rename => { "date" => "mail_date" }
#from
#to
#cc
#reply_to
rename => { "msg_id" => "message_id" }
#in_reply_to
#subject
#x_originating_ip
#first_received
#second_received
#last_reply
#path
rename => { "user_agent" => "useragent" }
#tls
#fuids
#is_webmail
}
mutate {
convert => { "tls" => "string" }
convert => { "is_webmail" => "string" }
}
} else {
grok {
match => [ "message", "(?<timestamp>(.*?))\t(?<uid>(.*?))\t(?<source_ip>(.*?))\t(?<source_port>(.*?))\t(?<destination_ip>(.*?))\t(?<destination_port>(.*?))\t(?<trans_depth>(.*?))\t(?<helo>(.*?))\t(?<mail_from>(.*?))\t(?<recipient_to>(.*?))\t(?<mail_date>(.*?))\t(?<from>(.*?))\t(?<to>(.*?))\t(?<cc>(.*?))\t(?<reply_to>(.*?))\t(?<message_id>(.*?))\t(?<in_reply_to>(.*?))\t(?<subject>(.*?))\t(?<x_originating_ip>(.*?))\t(?<first_received>(.*))\t(?<second_received>(.*))\t(?<last_reply>(.*))\t(?<path>(.*))\t(?<useragent>(.*))\t(?<tls>(.*))\t(?<fuids>(.*))\t(?<is_webmail>(.*))" ]
}
}
if [useragent] == "-" {
mutate {
remove_field => [ "useragent" ]
}
}
mutate {
#add_tag => [ "conf_file_1112"]
}
}
}

47
salt/logstash/pipelines/config/so/1113_preprocess_bro_snmp.conf
View File

@@ -1,47 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for snmp.log from Bro systems
filter {
if [type] == "bro_snmp" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#duration
#version
#convert => { "version" => "string" }
#community
#get_requests
#get_bulk_requests
#get_responses
#set_requests
#display_string
#up_since
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","duration","version","community","get_requests","get_bulk_requests","get_responses","set_requests","display_string","up_since"]
separator => " "
}
}
mutate {
#add_tag => [ "conf_file_1113"]
}
}
}

49
salt/logstash/pipelines/config/so/1114_preprocess_bro_software.conf
View File

@@ -1,49 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for software.log from Bro systems
filter {
if [type] == "bro_software" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "host" => "source_ip" }
rename => { "host_p" => "source_port" }
#software_type
#name
rename => { "version.major" => "version_major" }
rename => { "version.minor" => "version_minor" }
rename => { "version.minor2" => "version_minor2" }
rename => { "version.minor3" => "version_minor3" }
rename => { "version.addl" => "version_additional_info" }
#unparsed_version
}
mutate {
convert => { "version_major" => "string" }
convert => { "version_minor" => "string" }
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","source_ip","source_port","software_type","name","version_major","version_minor","version_minor2","version_minor3","version_additional_info","unparsed_version"]
separator => " "
}
}
mutate {
#add_tag => [ "conf_file_1114"]
}
}
}

66
salt/logstash/pipelines/config/so/1115_preprocess_bro_ssh.conf
View File

@@ -1,66 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks and Wes Lambert
# Last Update: 10/30/2018
#
# This conf file is based on accepting logs for ssh.log from Bro systems
filter {
if [type] == "bro_ssh" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#version
#convert => { "version" => "string" }
rename => { "auth_success" => "authentication_success" }
rename => { "auth_attempts" => "authentication_attempts" }
#direction
#client
#server
rename => { "cipher_alg" => "cipher_algorithm" }
rename => { "compression_alg" => "compression_algorithm" }
rename => { "cshka" => "client_host_key_algorithms" }
rename => { "host_key_alg" => "host_key_algorithm" }
rename => { "hasshAlgorithms" => "hassh_algorithms" }
rename => { "hasshServer" => "hassh_server" }
rename => { "hasshServerAlgorithms" => "hassh_server_algorithms" }
rename => { "hasshVersion" => "hassh_version" }
rename => { "kex_alg" => "kex_algorithm" }
rename => { "mac_alg" => "mac_algorithm" }
rename => { "sshka" => "server_host_key_algorithms" }
#host_key
#destination_country_code
#destination_region
#destination_city
#destination_latitude
#destination_longitude
}
mutate {
convert => { "authentication_success" => "string" }
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","version","authentication_success","authentication_attempts","direction","client","server","cipher_algorithm","mac_algorithm","compression_algorithm","kex_algorithm","host_key_algorithm","host_key","destination_country_code","destination_region","destination_city","destination_latitude","destination_longitude","hassh_version","hassh","hassh_server","client_host_key_algorithms","hassh_algorithms","server_host_key_algorithms","hassh_server_algorithms"]
separator => " "
}
}
mutate {
#add_tag => [ "conf_file_1115"]
}
}
}

186
salt/logstash/pipelines/config/so/1116_preprocess_bro_ssl.conf
View File

@@ -1,186 +0,0 @@
# Original Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 10/30/2018
#
# This conf file is based on accepting logs for ssl.log from Bro systems
filter {
if [type] == "bro_ssl" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#version
#convert => { "version" => "string" }
#cipher
#curve
#server_name
#resumed
#last_alert
#next_protocol
#established
rename => { "cert_chain_fuids" => "certificate_chain_fuids" }
rename => { "client_cert_chain_fuids" => "client_certificate_chain_fuids" }
rename => { "subject" => "certificate_subject" }
rename => { "issuer" => "certificate_issuer" }
#client_subject
#client_issuer
#validation_status
#ja3
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","version","cipher","curve","server_name","resumed","last_alert","next_protocol","established","certificate_chain_fuids","client_certificate_chain_fuids","certificate_subject","certificate_issuer","client_subject","client_issuer","validation_status","ja3","ja3s"]
separator => " "
}
}
mutate {
gsub => [ "subject", "\\\\,", "|" ]
}
kv {
include_keys => [ "CN", "C", "O", "OU", "ST", "SN", "L", "DC", "GN", "pseudonym", "serialNumber", "title", "initials" ]
field_split => ","
source => "certificate_issuer"
}
mutate {
rename => { "CN" => "issuer_common_name"}
rename => { "C" => "issuer_country_code"}
rename => { "O" => "issuer_organization"}
rename => { "OU" => "issuer_organization_unit"}
rename => { "ST" => "issuer_state"}
rename => { "SN" => "issuer_surname"}
rename => { "L" => "issuer_locality"}
rename => { "DC" => "issuer_distinguished_name"}
rename => { "GN" => "issuer_given_name"}
rename => { "pseudonym" => "issuer_pseudonym"}
rename => { "serialNumber" => "issuer_serial_number"}
rename => { "title" => "issuer_title"}
rename => { "initials" => "issuer_initials"}
}
kv {
include_keys => [ "CN", "C", "O", "OU", "ST", "SN", "L", "GN", "pseudonym", "serialNumber", "title", "initials" ]
field_split => ","
source => "certificate_subject"
}
mutate {
rename => { "CN" => "certificate_common_name"}
rename => { "C" => "certificate_country_code"}
rename => { "O" => "certificate_organization"}
rename => { "OU" => "certificate_organization_unit"}
rename => { "ST" => "certificate_state"}
rename => { "SN" => "certificate_surname"}
rename => { "L" => "certificate_locality"}
rename => { "GN" => "certificate_given_name"}
rename => { "pseudonym" => "certificate_pseudonym"}
rename => { "serialNumber" => "certificate_serial_number"}
rename => { "title" => "certificate_title"}
rename => { "initials" => "certificate_initials"}
}
if [certificate_subject] == "-" {
mutate {
remove_field => [ "certificate_subject" ]
}
}
if [certificate_issuer] == "-" {
mutate {
remove_field => [ "certificate_issuer" ]
}
}
if [certificate_common_name] {
ruby {
code => "event.set('certificate_common_name_length', event.get('certificate_common_name').length)"
}
}
if [issuer_common_name] {
ruby {
code => "event.set('issuer_common_name_length', event.get('issuer_common_name').length)"
}
}
if [server_name] {
if [server_name] == "-" {
mutate {
remove_field => [ "server_name" ]
}
} else {
ruby {
code => "event.set('server_name_length', event.get('server_name').length)"
}
}
}
if [certificate_chain_fuids] {
if [certificate_chain_fuids] == "-" {
mutate {
remove_field => [ "certificate_chain_fuids" ]
}
} else {
ruby {
code => "event.set('certificate_chain_count', event.get('certificate_chain_fuids').count(',') + 1)"
}
mutate {
convert => [ "certificate_chain_length", "integer" ]
}
}
}
if [client_certificate_chain_fuids] == "-" {
mutate {
remove_field => [ "client_certificate_chain_fuids" ]
}
}
if [client_issuer] == "-" {
mutate {
remove_field => [ "client_issuer" ]
}
}
if [client_subject] == "-" {
mutate {
remove_field => [ "client_subject" ]
}
}
if [curve] == "-" {
mutate {
remove_field => [ "curve" ]
}
}
if [issuer] == "-" {
mutate {
remove_field => [ "issuer" ]
}
}
if [query] == "-" {
mutate {
remove_field => [ "query" ]
}
}
if [subject] == "-" {
mutate {
remove_field => [ "subject" ]
}
}
if [validation_status] == "-" {
mutate {
remove_field => [ "validation_status" ]
}
}
if [ja3] == "-" {
mutate {
remove_field => [ "ja3" ]
}
}
mutate {
#add_tag => [ "conf_file_1116"]
}
}
}

41
salt/logstash/pipelines/config/so/1117_preprocess_bro_syslog.conf
View File

@@ -1,41 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for syslog.log from Bro systems
filter {
if [type] == "bro_syslog" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
rename => { "proto" => "protocol" }
#facility
#severity
#message
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","protocol","facility","severity","message"]
separator => " "
}
}
mutate {
#add_tag => [ "conf_file_1117"]
}
}
}

40
salt/logstash/pipelines/config/so/1118_preprocess_bro_tunnel.conf
View File

@@ -1,40 +0,0 @@
# Original Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for tunnel.log from Bro systems
# Security Onion syslog-ng.conf sets type to "bro_tunnels"
filter {
if [type] == "bro_tunnels" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#tunnel_type
#action
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","tunnel_type","action"]
separator => " "
}
}
mutate {
#add_tag => [ "conf_file_1118"]
}
}
}

42
salt/logstash/pipelines/config/so/1119_preprocess_bro_weird.conf
View File

@@ -1,42 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for weird.log from Bro systems
filter {
if [type] == "bro_weird" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#name
rename => { "addl" => "additional_info" }
#notice
#peer
}
mutate {
convert => { "notice" => "string" }
}
} else {
grok {
match => [ "message", "(?<timestamp>(.*?))\t(?<uid>(.*?))\t(?<source_ip>(.*?))\t(?<source_port>(.*?))\t(?<destination_ip>(.*?))\t(?<destination_port>(.*?))\t(?<name>(.*?))\t(?<additional_info>(.*?))\t(?<notice>(.*?))\t(?<peer>(.*))" ]
}
}
mutate {
#add_tag => [ "conf_file_1119"]
}
}
}

57
salt/logstash/pipelines/config/so/1121_preprocess_bro_mysql.conf
View File

@@ -1,57 +0,0 @@
# Original Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for mysql.log from Bro systems
#
# Parse using grok
filter {
if [type] == "bro_mysql" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
rename => { "cmd" => "mysql_command" }
rename => { "arg" => "mysql_argument" }
rename => { "success" => "mysql_success" }
#rows
#response
}
mutate {
convert => { "mysql_success" => "string" }
}
} else {
grok {
match => [ "message", "(?<timestamp>(.*?))\t(?<uid>(.*?))\t(?<source_ip>(.*?))\t(?<source_port>(.*?))\t(?<destination_ip>(.*?))\t(?<destination_port>(.*?))\t(?<mysql_command>(.*?))\t(?<mysql_argument>(.*?))\t(?<mysql_success>(.*?))\t(?<rows>(.*?))\t(?<response>(.*))" ]
}
}
mutate {
#add_tag => [ "conf_file_1121"]
}
}
}
# Reverting to grok for now, due to double-quoted values in log file
# Parse using csv filter
#filter {
# if [type] == "bro_mysql" {
# csv {
# columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","mysql_command","mysql_argument","mysql_success","rows","response"]
# separator => " "
# quote_char=
# }
# }
#}

62
salt/logstash/pipelines/config/so/1122_preprocess_bro_socks.conf
View File

@@ -1,62 +0,0 @@
# Original Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for socks.log from Bro systems
# Parse using csv
filter {
if [type] == "bro_socks" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#version
#convert => { "version" => "string" }
rename => { "user" => "username" }
#password
rename => { "status" => "server_status" }
rename => { "request.host" => "request_host" }
rename => { "request.name" => "request_name" }
rename => { "request_p" => "request_port" }
rename => { "bound.host" => "bound_host" }
rename => { "bound.name" => "bound_name" }
rename => { "bound_p" => "bound_port" }
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","version","username","password","server_status","request_host","request_name","request_port","bound_host","bound_name","bound_port"]
separator => " "
}
}
mutate {
#add_tag => [ "conf_file_1122"]
}
}
}
# Parse using grok
#filter {
# if [type] == "bro_socks" {
# # This is the initial parsing of the log
# grok {
# match => [ "message", "(?<timestamp>(.*?))\t(?<uid>(.*?))\t(?<source_ip>(.*?))\t(?<source_port>(.*?))\t(?<destination_ip>(.*?))\t(?<destination_port>(.*?))\t(?<version>(.*?))\t(?<user>(.*?))\t(?<password>(.*?))\t(?<status>(.*))\t(?<request_host>(.*))\t(?<request_name>(.*))\t(?<request_port>(.*))\t(?<bound_host>(.*))\t(?<bound_name>(.*))\t(?<bound_port>(.*))" ]
# }
# mutate {
# #add_tag => [ "conf_file_1122"]
# }
# }
#}

154
salt/logstash/pipelines/config/so/1123_preprocess_bro_x509.conf
View File

@@ -1,154 +0,0 @@
# Original Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for x509.log from Bro systems
filter {
if [type] == "bro_x509" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#id
rename => { "certificate.version" => "certificate_version" }
rename => { "certificate.serial" => "certificate_serial" }
rename => { "certificate.subject" => "certificate_subject" }
rename => { "certificate.issuer" => "certificate_issuer" }
rename => { "certificate.not_valid_before" => "certificate_not_valid_before" }
rename => { "certificate.not_valid_after" => "certificate_not_valid_after" }
rename => { "certificate.key_alg" => "certificate_key_algorithm" }
rename => { "certificate.sig_alg" => "certificate_signing_algorithm" }
rename => { "certificate.key_type" => "certificate_key_type" }
rename => { "certificate.key_length" => "certificate_key_length" }
rename => { "certificate.exponent" => "certificate_exponent" }
rename => { "certificate.curve" => "certificate_curve" }
rename => { "id" => "fuid" }
rename => { "san.dns" => "san_dns" }
rename => { "san.uri" => "san_uri" }
rename => { "san.email" => "san_email" }
rename => { "san.ip" => "san_ip" }
rename => { "basic_constraints.ca" => "basic_constraints_ca" }
rename => { "basic_constraints.path_length" => "basic_constraints_path_length" }
}
} else {
grok {
match => [ "message", "(?<timestamp>(.*?))\t(?<fuid>(.*?))\t(?<certificate_version>(.*?))\t(?<certificate_serial>(.*?))\t(?<certificate_subject>(.*?))\t(?<certificate_issuer>(.*?))\t(?<certificate_not_valid_before>(.*?))\t(?<certificate_not_valid_after>(.*?))\t(?<certificate_key_algorithm>(.*?))\t(?<certificate_signing_algorithm>(.*))\t(?<certificate_key_type>(.*))\t(?<certificate_key_length>(.*))\t(?<certificate_exponent>(.*))\t(?<certificate_curve>(.*))\t(?<san_dns>(.*))\t(?<san_uri>(.*))\t(?<san_email>(.*))\t(?<san_ip>(.*))\t(?<basic_constraints_ca>(.*))\t(?<basic_constraints_path_length>(.*))" ]
}
}
mutate {
gsub => [ "certificate_issuer", "\\\\,", "|" ]
gsub => [ "certificate_subject", "\\\\,", "|" ]
}
kv {
include_keys => [ "CN", "C", "O", "OU", "ST", "SN", "L", "DC", "GN", "pseudonym", "serialNumber", "title", "initials" ]
field_split => ","
source => "certificate_issuer"
}
mutate {
rename => { "CN" => "issuer_common_name"}
rename => { "C" => "issuer_country_code"}
rename => { "O" => "issuer_organization"}
rename => { "OU" => "issuer_organization_unit"}
rename => { "ST" => "issuer_state"}
rename => { "SN" => "issuer_surname"}
rename => { "L" => "issuer_locality"}
rename => { "DC" => "issuer_distinguished_name"}
rename => { "GN" => "issuer_given_name"}
rename => { "pseudonym" => "issuer_pseudonym"}
rename => { "serialNumber" => "issuer_serial_number"}
rename => { "title" => "issuer_title"}
rename => { "initials" => "issuer_initials"}
}
kv {
include_keys => [ "CN", "C", "O", "OU", "ST", "SN", "L", "GN", "pseudonym", "serialNumber", "title", "initials" ]
field_split => ","
source => "certificate_subject"
}
mutate {
rename => { "CN" => "certificate_common_name"}
rename => { "C" => "certificate_country_code"}
rename => { "O" => "certificate_organization"}
rename => { "OU" => "certificate_organization_unit"}
rename => { "ST" => "certificate_state"}
rename => { "SN" => "certificate_surname"}
rename => { "L" => "certificate_locality"}
rename => { "GN" => "certificate_given_name"}
rename => { "pseudonym" => "certificate_pseudonym"}
rename => { "serialNumber" => "certificate_serial_number"}
rename => { "title" => "certificate_title"}
rename => { "initials" => "certificate_initials"}
convert => [ "certificate_key_length", "integer" ]
convert => [ "certificate_not_valid_after", "integer" ]
convert => [ "certificate_not_valid_before", "integer" ]
}
if [query] == "-" {
mutate {
remove_field => [ "query" ]
}
}
if [san_dns] == "-" {
mutate {
remove_field => [ "san_dns" ]
}
}
if [san_email] == "-" {
mutate {
remove_field => [ "san_email" ]
}
}
if [san_uri] == "-" {
mutate {
remove_field => [ "san_uri" ]
}
}
if [san_ip] == "-" {
mutate {
remove_field => [ "san_ip" ]
}
}
if [certificate_common_name] {
ruby {
code => "event.set('certificate_common_name_length', event.get('certificate_common_name').length)"
}
}
if [issuer_common_name] {
ruby {
code => "event.set('issuer_common_name_length', event.get('issuer_common_name').length)"
}
}
if [certificate_not_valid_after] == "-" {
mutate {
remove_field => [ "certificate_not_valid_after" ]
}
}
if [certificate_not_valid_before] == "-" {
mutate {
remove_field => [ "certificate_not_valid_before" ]
}
}
if [certificate_not_valid_after] and [certificate_not_valid_before] {
ruby {
code => "event.set('certificate_number_days_valid', ((event.get('certificate_not_valid_after') - event.get('certificate_not_valid_before')) / 86400).ceil)"
}
date {
match => [ "certificate_not_valid_after", "UNIX" ]
target => "certificate_not_valid_after"
}
date {
match => [ "certificate_not_valid_before", "UNIX" ]
target => "certificate_not_valid_before"
}
}
mutate {
#add_tag => [ "conf_file_1123"]
}
}
}

46
salt/logstash/pipelines/config/so/1124_preprocess_bro_intel.conf
View File

@@ -1,46 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for intel.log from Bro systems
filter {
if [type] == "bro_intel" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
rename => { "seen.indicator" => "indicator" }
rename => { "seen.indicator_type" => "indicator_type" }
rename => { "seen.where" => "seen_where" }
rename => { "seen.node" => "seen_node" }
#matched
#sources
#fuid
rename => { "file_mime_type" => "mimetype" }
rename => { "file_desc" => "file_description" }
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","indicator","indicator_type","seen_where","seen_node","matched","sources","fuid","mimetype","file_description"]
separator => " "
}
}
mutate {
#add_tag => [ "conf_file_1124"]
}
}
}

49
salt/logstash/pipelines/config/so/1125_preprocess_bro_modbus.conf
View File

@@ -1,49 +0,0 @@
# Author: Wes Lambert
# Adapted from existing filters provided by Justin Henderson
#
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for modbus.log from Bro systems
#
filter {
if [type] == "bro_modbus" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
rename => { "func" => "function" }
#exception
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","function","exception"]
separator => " "
}
}
}
}
# Parse using grok
#filter {
# if [type] == "bro_modbus" {
# grok {
# match => [ "message", "(?<timestamp>(.*?))\t(?<uid>(.*?))\t(?<source_ip>(.*?))\t(?<source_port>(.*?))\t(?<destination_ip>(.*?))\t(?<destination_port>(.*?))\t(?<func>(.*?))\t(?<exception>(.*?))$" ]
# }
#mutate {
#add_tag => [ "conf_file_1125"]
#}
# }
#}

66
salt/logstash/pipelines/config/so/1126_preprocess_bro_sip.conf
View File

@@ -1,66 +0,0 @@
# Author: Wes Lambert
#
# Adapted from existing filters provided by Justin Henderson
#
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for sip.log from Bro systems
#
filter {
if [type] == "bro_sip" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#trans_depth
#method
#uri
#date
#request_from
#request_to
#response_from
#response_to
#reply_to
#call_id
#seq
#subject
#request_path
#response_path
#user_agent
#status_code
#status_msg
#warning
rename => { "request_body_len" => "request_body_length" }
rename => { "response_body_len" => "response_body_length" }
#content_type
}
} else {
grok {
match => [ "message", "(?<timestamp>(.*?))\t(?<uid>(.*?))\t(?<source_ip>(.*?))\t(?<source_port>(.*?))\t(?<destination_ip>(.*?))\t(?<destination_port>(.*?))\t(?<trans_depth>(.*?))\t(?<method>(.*?))\t(?<uri>(.*?))\t(?<date>(.*?))\t(?<request_from>(.*?))\t(?<request_to>(.*?))\t(?<response_from>(.*?))\t(?<response_to>(.*?))\t(?<reply_to>(.*?))\t(?<call_id>(.*?))\t(?<seq>(.*?))\t(?<subject>(.*?))\t(?<request_path>(.*?))\t(?<response_path>(.*?))\t(?<user_agent>(.*?))\t(?<status_code>(.*?))\t(?<status_msg>(.*?))\t(?<warning>(.*?))\t(?<request_body_length>(.*?))\t(?<response_body_length>(.*?))\t(?<content_type>(.*?))$" ]
}
}
mutate {
add_tag => [ "conf_file_1126"]
}
}
}
# Parse using csv filter
#filter {
# if [type] == "bro_sip" {
# csv {
# columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","trans_depth","method","uri","date","request_from","request_to","response_from","response_to","reply_to","call_id","seq","subject","request_path","response_path","user_agent","status_code","status_msg","warning","request_body_len","response_body_len","content_type"]
# separator => " "
# }
# }
#}

73
salt/logstash/pipelines/config/so/1127_preprocess_bro_radius.conf
View File

@@ -1,73 +0,0 @@
# Author: Wes Lambert
#
# Adapted from existing filters provided by Justin Henderson
#
# Updated by: Doug Burks
#
# This conf file is based on accepting logs for radius.log from Bro systems
#
filter {
if [type] == "bro_radius" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#username
#mac
#framed_addr
#tunnel_client
#connect_info
rename => { "reply_msg" => "reply_message" }
#result
#ttl
#logged
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","username","mac","framed_addr","tunnel_client","connect_info","reply_message","result","ttl","logged"]
separator => " "
}
if [tunnel_client] == "-" {
mutate {
remove_field => [ "tunnel_client" ]
}
}
}
# Remove the ttl and framed_addr fields
if [ttl] {
mutate {
remove_field => [ "ttl" ]
}
}
if [framed_addr] {
mutate {
remove_field => [ "framed_addr" ]
}
}
}
}
# Parse using grok
#filter {
# if [type] == "bro_radius" {
# grok {
# match => [ "message", "(?<timestamp>(.*?))\t(?<uid>(.*?))\t(?<source_ip>(.*?))\t(?<source_port>(.*?))\t(?<destination_ip>(.*?))\t(?<destination_port>(.*?))\t(?<username>(.*?))\t(?<mac>(.*?))\t(?<tunnel_client>(.*?))\t(?<logged>(.*?))\t(?<connect_info>(.*?))$" ]
# }
# mutate {
# #add_tag => [ "conf_file_1127"]
# }
# }
#}

46
salt/logstash/pipelines/config/so/1128_preprocess_bro_pe.conf
View File

@@ -1,46 +0,0 @@
# Author: Wes Lambert
#
# Adapted from existing filters provided by Justin Henderson
#
# Updated by: Doug Burks
#
# This conf file is based on accepting logs for pe.log from Bro systems
#
filter {
if [type] == "bro_pe" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
rename => { "id" => "fuid" }
#machine
#compile_ts
#os
#subsystem
#is_exe
#is_64bit
#uses_aslr
#uses_dep
#uses_code_integrity
#uses_seh
#has_import_table
#has_export_table
#has_cert_table
#has_debug_data
#section_names
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","fuid","machine","compile_ts","os","subsystem","is_exe","is_64bit","uses_aslr","uses_dep","uses_code_integrity","uses_seh","has_import_table","has_export_table","has_cert_table","has_debug_data","section_names"]
separator => " "
}
}
}
}

65
salt/logstash/pipelines/config/so/1129_preprocess_bro_rfb.conf
View File

@@ -1,65 +0,0 @@
# Author: Wes Lambert
#
# Adapted from existing filters provided by Justin Henderson
#
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for rfb.log from Bro systems
#
# Parse using csv filter
filter {
if [type] == "bro_rfb" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#client_major_version
#client_minor_version
#server_major_version
#server_minor_version
#authentication_method
#auth
#share_flag
#desktop_name
#width
#height
}
mutate {
convert => { "auth" => "string" }
convert => { "share_flag" => "string" }
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","client_major_version","client_minor_version","server_major_version","server_minor_version","authentication_method","auth","share_flag","desktop_name","width","height"]
separator => " "
}
}
}
}
# Parse using grok
#filter {
# if [type] == "bro_rfb" {
# grok {
# match => [ "message", "(?<timestamp>(.*?))\t(?<uid>(.*?))\t(?<source_ip>(.*?))\t(?<source_port>(.*?))\t(?<destination_ip>(.*?))\t(?<destination_port>(.*?))\t(?<client_major_version>(.*?))\t(?<client_minor_version>(.*?))\t(?<server_major_version>(.*?))\t(?<server_minor_version>(.*?))\t(?<authentication_method>(.*?))\t(?<auth>(.*?))\t(?<share_flag>(.*?))\t(?<desktop_name>(.*?))\t(?<width>(.*?))\t(?<height>(.*?))$" ]
# }
# mutate {
# #add_tag => [ "conf_file_1129"]
# }
# }
#}

51
salt/logstash/pipelines/config/so/1130_preprocess_bro_dnp3.conf
View File

@@ -1,51 +0,0 @@
# Author: Wes Lambert
#
# Adapted from existing filters provided by Justin Henderson
#
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for dnp3.log from Bro systems
#
filter {
if [type] == "bro_dnp3" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#fc_request
#fc_reply
#iin
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","fc_request","fc_reply","iin"]
separator => " "
}
}
}
}
# Parse using grok
#filter {
# if [type] == "bro_dnp3" {
# grok {
# match => [ "message", "(?<timestamp>(.*?))\t(?<uid>(.*?))\t(?<source_ip>(.*?))\t(?<source_port>(.*?))\t(?<destination_ip>(.*?))\t(?<destination_port>(.*?))\t(?<fc_request>(.*?))\t(?<fc_reply>(.*?))\t(?<iin>(.*?))$" ]
# }
# mutate {
# #add_tag => [ "conf_file_1130"]
# }
# }
#}

46
salt/logstash/pipelines/config/so/1131_preprocess_bro_smb_files.conf
View File

@@ -1,46 +0,0 @@
# Author: Wes Lambert
#
# Adapted from existing filters provided by Justin Henderson
#
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for smb_files.log from Bro systems
#
filter {
if [type] == "bro_smb_files" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#fuid
#action
#path
#name
#size
#prev_name
rename => { "times.modified" => "times_modified" }
rename => { "times.accessed" => "times_accessed" }
rename => { "times.created" => "times_created" }
rename => { "times.changed" => "times_changed" }
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","fuid","action","path","name","size","prev_name","times_modified","times_accessed","times_created","times_changed"]
separator => " "
}
}
}
}

40
salt/logstash/pipelines/config/so/1132_preprocess_bro_smb_mapping.conf
View File

@@ -1,40 +0,0 @@
# Author: Wes Lambert
#
# Adapted from existing filters provided by Justin Henderson
#
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for smb_mapping.log from Bro systems
#
filter {
if [type] == "bro_smb_mapping" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#path
#service
#native_file_system
#share_type
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","path","service","native_file_system","share_type"]
separator => " "
}
}
}
}

50
salt/logstash/pipelines/config/so/1133_preprocess_bro_ntlm.conf
View File

@@ -1,50 +0,0 @@
# Author: Wes Lambert
#
# Adapted from existing filters provided by Justin Henderson
#
# Updated by: Doug Burks and Wes Lambert
# Last Update: 1/2/2019
#
# This conf file is based on accepting logs for ntlm.log from Bro systems
#
filter {
if [type] == "bro_ntlm" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#hostname
rename => { "domainname" => "domain_name" }
rename => { "success" => "ntlm_success" }
#status
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => [ "timestamp", "uid", "source_ip", "source_port", "destination_ip", "destination_port", "username", "hostname", "domain_name", "server_nb_computer_name", "server_dns_computer_name", "server_tree_name", "ntlm_success"]
separator => " "
}
ruby {
code =>"
hash = event.to_hash.each do |key,value|
if value == '-'
event.remove(key)
end
end"
}
}
}
}

54
salt/logstash/pipelines/config/so/1134_preprocess_bro_dce_rpc.conf
View File

@@ -1,54 +0,0 @@
# Author: Wes Lambert
#
# Adapted from existing filters provided by Justin Henderson
#
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for dce_rpc.log from Bro systems
#
filter {
if [type] == "bro_dce_rpc" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#rtt
#named_pipe
#endpoint
#operation
}
#mutate {
#convert => { "rtt" => "float" }
#}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","rtt","named_pipe","endpoint","operation"]
separator => " "
}
if [rtt] == "-" {
mutate {
remove_field => [ "rtt" ]
}
}
#mutate {
#convert => [ "rtt", "float" ]
#}
}
}
}

59
salt/logstash/pipelines/config/so/2000_network_flow.conf
View File

@@ -1,59 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
filter {
if [type] == "sflow" {
if [message] =~ /CNTR/ {
drop { }
}
grok {
match => { "message" => "%{WORD:sample_type},%{IP:sflow_source_ip},%{WORD:in_port:int},%{WORD:out_port:int},%{WORD:source_mac},%{WORD:destination_mac},%{WORD:ether_type},%{NUMBER:in_vlan:int},%{NUMBER:out_vlan:int},%{IP:source_ip},%{IP:destination_ip},%{NUMBER:protocol:int},%{WORD:type_of_service},%{WORD:ttl:int},%{NUMBER:source_port:int},%{NUMBER:destination_port:int},%{DATA:tcp_flags},%{NUMBER:packet_size:int},%{NUMBER:ip_size:int},%{NUMBER:sample_rate:int}" }
}
if "_grokparsefailure" in [tags] {
drop { }
}
mutate {
add_field => {
"[source_hostname]" => "%{source_ip}"
"[destination_hostname]" => "%{destination_ip}"
"[sflow_source_hostname]" => "%{sflow_source_ip}"
}
}
translate {
field => "[source_port]"
destination => "[source_service]"
dictionary_path => "/lib/dictionaries/iana_services.yaml"
}
translate {
field => "[destination_port]"
destination => "[destination_service]"
dictionary_path => "/lib/dictionaries/iana_services.yaml"
}
translate {
field => "[protocol]"
destination => "[protocol_name]"
dictionary_path => "/lib/dictionaries/iana_protocols.yaml"
}
translate {
field => "[tcp_flags]"
destination => "[tcp_flag]"
dictionary_path => "/lib/dictionaries/tcp_flags.yaml"
}
mutate {
add_field => { "ips" => [ "%{sflow_source_ip}" ] }
}
mutate {
#add_tag => [ "conf_file_2000"]
}
}
}

228
salt/logstash/pipelines/config/so/6000_bro.conf
View File

@@ -1,228 +0,0 @@
# Original Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/10/2018
#
filter {
if "bro" in [tags] {
# Bro logs have a high quality timestamp, so let's copy that to @timestamp.
# Before we do, let's copy the existing logstash @timestamp to timestamp.
mutate {
add_field => { "logstash_timestamp" => "%{@timestamp}" }
}
mutate {
convert => { "logstash_timestamp" => "string" }
}
mutate {
convert => { "timestamp" => "string" }
}
# New Bro JSON logs use ISO8601 timestamps.
# Old Bro TSV logs use UNIX timestamps.
date {
match => [ "timestamp", "ISO8601", "UNIX" ]
}
mutate {
rename => { "logstash_timestamp" => "timestamp" }
}
if [duration] == "-" {
mutate {
replace => [ "duration", "0" ]
}
}
if [original_bytes] == "-" {
mutate {
replace => [ "original_bytes", "0" ]
}
}
# If MissedBytes is unspecified set it to zero so it is an integer
if [missed_bytes] == "-" {
mutate {
replace => [ "missed_bytes", "0" ]
}
}
# If OriginalIPBytes is unspecified set it to zero so it is an integer
if [original_ip_bytes] == "-" {
mutate {
replace => [ "original_ip_bytes", "0" ]
}
}
# If RespondBytes is unspecified set it to zero so it is an integer
if [respond_bytes] == "-" {
mutate {
replace => [ "respond_bytes", "0" ]
}
}
# If RespondIPBytes is unspecified set it to zero so it is an integer
if [respond_ip_bytes] == "-" {
mutate {
replace => [ "respond_ip_bytes", "0" ]
}
}
if [request_body_length] == "-" {
mutate {
replace => [ "request_body_length", "0" ]
}
}
if [response_body_length] == "-" {
mutate {
replace => [ "response_body_length", "0" ]
}
}
if [source_port] == "-" {
mutate {
remove_field => ["source_port"]
}
}
if [destination_port] == "-" {
mutate {
remove_field => ["destination_port"]
}
}
if [virtual_host] == "-" {
mutate {
remove_field => ["virtual_host"]
}
}
if [x_originating_ip] == "-" {
mutate {
remove_field => ["x_originating_ip"]
}
}
if [basic_constraints_path_length] == "-" {
mutate {
remove_field => ["basic_constraints_path_length"]
}
}
if [data_channel_source_ip] == "-" {
mutate {
remove_field => ["data_channel_source_ip"]
}
}
if [data_channel_destination_ip] == "-" {
mutate {
remove_field => ["data_channel_destination_ip"]
}
}
if [desktop_width] == "-" {
mutate {
remove_field => ["desktop_width"]
}
}
if [desktop_height] == "-" {
mutate {
remove_field => ["desktop_height"]
}
}
if [height] == "-" {
mutate {
remove_field => ["height"]
}
}
# I renamed conn_uids to uid so that it is easy to pivot to all things tied to a connection
mutate {
rename => [ "connection_uids", "uid" ]
}
# If total_bytes is set to "-" change it to 0 so it is an integer
if [total_bytes] == "-" {
mutate {
replace => [ "total_bytes", "0" ]
}
}
# If seen_bytes is set to "-" change it to 0 so it is an integer
if [seen_bytes] == "-" {
mutate {
replace => [ "seen_bytes", "0" ]
}
}
# If missing_bytes is set to "-" change it to 0 so it is an integer
if [missing_bytes] == "-" {
mutate {
replace => [ "missing_bytes", "0" ]
}
}
# If overflow_bytes is set to "-" change it to 0 so it is an integer
if [overflow_bytes] == "-" {
mutate {
replace => [ "overflow_bytes", "0" ]
}
}
if [dcc_file_size] == "-" {
mutate {
replace => [ "dcc_file_size", "0" ]
}
}
if [authentication_attempts] == "-" {
mutate {
replace => [ "authentication_attempts", "0" ]
}
}
if [file_size] == "-" {
mutate {
replace => [ "file_size", "0" ]
}
}
if [original_ip_bytes] == "-" {
mutate {
replace => [ "original_ip_bytes", "0" ]
}
}
# I recommend changing the field types below to integer or floats so searches can do greater than or less than
# and also so math functions can be ran against them
mutate {
convert => [ "bound_port", "integer" ]
convert => [ "data_channel_destination_port", "integer" ]
convert => [ "destination_port", "integer" ]
convert => [ "depth", "integer" ]
#convert => [ "duration", "float" ]
convert => [ "info_code", "integer" ]
convert => [ "missed_bytes", "integer" ]
convert => [ "missing_bytes", "integer" ]
convert => [ "n", "integer" ]
convert => [ "original_bytes", "integer" ]
convert => [ "original_packets", "integer" ]
convert => [ "original_ip_bytes", "integer" ]
convert => [ "overflow_bytes", "integer" ]
convert => [ "p", "integer" ]
convert => [ "query_class", "integer" ]
convert => [ "query_type", "integer" ]
convert => [ "rcode", "integer" ]
convert => [ "request_body_length", "integer" ]
convert => [ "request_port", "integer" ]
convert => [ "respond_bytes", "integer" ]
convert => [ "respond_packets", "integer" ]
convert => [ "respond_ip_bytes", "integer" ]
convert => [ "response_body_length", "integer" ]
convert => [ "seen_bytes", "integer" ]
convert => [ "source_port", "integer" ]
convert => [ "status_code", "integer" ]
#convert => [ "suppress_for", "float" ]
convert => [ "total_bytes", "integer" ]
convert => [ "trans_depth", "integer" ]
convert => [ "transaction_id", "integer" ]
# convert the following boolean to text for now
convert => [ "local_respond", "string" ]
convert => [ "tc", "string" ]
convert => [ "is_orig", "string" ]
convert => [ "local_orig", "string" ]
lowercase => [ "query" ]
#remove_field => [ "timestamp" ]
}
# Combine OriginalBytes and RespondBytes and save the value to total_bytes
if [original_bytes] {
if [respond_bytes] {
ruby {
code => "event.set('total_bytes', event.get('original_bytes') + event.get('respond_bytes'))"
}
}
}
mutate {
#add_tag => [ "conf_file_6000"]
}
}
}

16
salt/logstash/pipelines/config/so/6001_bro_import.conf
View File

@@ -1,16 +0,0 @@
# Updated by: Doug Burks
# Last Update: 2/10/2018
#
filter {
if "import" in [tags] and "bro" in [tags] {
# we're setting timestamp in 6000 now
#date {
# match => [ "timestamp", "UNIX" ]
#}
mutate {
#add_tag => [ "conf_file_6001"]
}
}
}

11
salt/logstash/pipelines/config/so/6002_syslog.conf
View File

@@ -1,11 +0,0 @@
# Updated by: Doug Burks
# Last Update: 5/16/2017
#
filter {
if "syslog" in [tags] {
mutate {
#convert => [ "status_code", "integer" ]
#add_tag => [ "conf_file_6002"]
}
}
}

33
salt/logstash/pipelines/config/so/6101_switch_brocade.conf
View File

@@ -1,33 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
filter {
if [type] == "brocade" {
grok {
match => ["message", "<%{DATA}>%{GREEDYDATA:sys_message}"]
}
grok {
match => { "sys_message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid:int}\])?: %{GREEDYDATA:syslog_message}" }
add_field => [ "received_at", "%{@timestamp}" ]
}
if [syslog_message] =~ "Interface ethernet" or [syslog_program] == "PORT" {
grok {
match => { "syslog_message" => "%{DATA}%{INT:unit}\/%{INT:interface_type}\/%{INT:interface:int}" }
}
mutate {
add_field => { "interface_port" => "%{unit}/%{interface_type}/%{interface}" }
}
}
date {
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
timezone => "America/Chicago"
remove_field => "syslog_timestamp"
remove_field => "received_at"
}
mutate {
#add_tag => [ "conf_file_6101"]
}
}
}

281
salt/logstash/pipelines/config/so/6200_firewall_fortinet.conf
View File

@@ -1,281 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
filter {
if [type] == "fortinet" {
mutate {
gsub => [ "message", "= ", "=NA " ]
}
grok {
match => ["message", "type=%{DATA:event_type}\s+"]
tag_on_failure => []
}
grok {
match => ["message", "<%{DATA}>%{GREEDYDATA:kv}"]
tag_on_failure => []
}
kv {
source => "kv"
exclude_keys => [ "type" ]
}
mutate {
gsub => [ "log", "= ", "=NA " ]
}
kv {
source => "log"
target => "SubLog"
}
grok {
match => ["message", "custom: DOM-ALL, dns_query=%{DATA:dns_query};"]
tag_on_failure => [ "" ]
}
mutate {
rename => { "action" => "action" }
rename => { "addr" => "addr_ip" }
rename => { "age" => "age" }
rename => { "assigned" => "assigned_ip" }
rename => { "assignip" => "assign_ip" }
rename => { "ap" => "access_point" }
rename => { "app" => "application" }
rename => { "appcat" => "application_category" }
rename => { "applist" => "application_list" }
rename => { "apprisk" => "application_risk" }
rename => { "approfile" => "accessPoint_profile" }
rename => { "apscan" => "access_point_scan" }
rename => { "apstatus" => "acces_point_status" }
rename => { "aptype" => "access_point_type" }
rename => { "authproto" => "authentication_protocol" }
rename => { "bandwidth" => "bandwidth" }
rename => { "banned_src" => "banned_source" }
rename => { "cat" => "category" }
rename => { "catdesc" => "category_description" }
rename => { "cfgattr" => "configuration_attribute" }
rename => { "cfgobj" => "configuration_object" }
rename => { "cfgpath" => "configuration_path" }
rename => { "cfgtid" => "configuration_transaction_id" }
rename => { "channel" => "channel" }
rename => { "community" => "community" }
rename => { "cookies" => "cookies" }
rename => { "craction" => "cr_action" }
rename => { "crlevel" => "cr_level" }
rename => { "crscore" => "cr_score" }
rename => { "datarange" => "data_range" }
rename => { "desc" => "description" }
rename => { "detectionmethod" => "detection_method" }
rename => { "devid" => "device_id" }
rename => { "devname" => "device_name" }
rename => { "devtype" => "device_type" }
rename => { "dhcp_msg" => "dhcp_message" }
rename => { "disklograte" => "disk_lograte" }
rename => { "dstcountry" => "destination_country" }
rename => { "dstintf" => "destination_interface" }
rename => { "dstip" => "destination_ip" }
rename => { "dstport" => "destination_port" }
rename => { "duration" => "elapsed_time" }
rename => { "error_num" => "error_number" }
rename => { "espauth" => "esp_authentication" }
rename => { "esptransform" => "esp_transform" }
rename => { "eventid" => "event_id" }
rename => { "eventtype" => "event_type" }
rename => { "fazlograte" => "faz_lograte" }
rename => { "filename" => "file_name" }
rename => { "filesize" => "file_size" }
rename => { "filetype" => "file_type" }
rename => { "hostname" => "hostname" }
rename => { "ip" => "source_ip" }
rename => { "localip" => "source_ip" }
rename => { "locip" => "local_ip" }
rename => { "locport" => "source_port" }
rename => { "logid" => "log_id" }
rename => { "logver" => "log_version" }
rename => { "manuf" => "manufacturer" }
rename => { "mem" => "memory" }
rename => { "meshmode" => "mesh_mode" }
rename => { "msg" => "message" }
rename => { "nextstat" => "next_stat" }
rename => { "onwire" => "on_wire" }
rename => { "osname" => "os_name" }
rename => { "osversion" => "unauthenticated_user" }
rename => { "outintf" => "outbound_interface" }
rename => { "peer_notif" => "peer_notification" }
rename => { "phase2_name" => "phase2_name" }
rename => { "policyid" => "policy_id" }
rename => { "policytype" => "policy_type" }
rename => { "port" => "port" }
rename => { "probeproto" => "probe_protocol" }
rename => { "proto" => "protocol_number" }
rename => { "radioband" => "radio_band" }
rename => { "radioidclosest" => "radio_id_closest" }
rename => { "radioiddetected" => "radio_id_detected" }
rename => { "rcvd" => "bytes_received" }
rename => { "rcvdbyte" => "bytes_received" }
rename => { "rcvdpkt" => "packets_received" }
rename => { "remip" => "destination_ip" }
rename => { "remport" => "remote_port" }
rename => { "reqtype" => "request_type" }
rename => { "scantime" => "scan_time" }
rename => { "securitymode" => "security_mode" }
rename => { "sent" => "bytes_sent" }
rename => { "sentbyte" => "bytes_sent" }
rename => { "sentpkt" => "packets_sent" }
rename => { "session_id" => "session_id" }
rename => { "setuprate" => "setup_rate" }
rename => { "sn" => "serial" }
rename => { "snclosest" => "serial_closest_access_point" }
rename => { "sndetected" => "serial_access_point_that_detected_rogue_ap" }
rename => { "snmeshparent" => "serial_mesh_parent" }
rename => { "srccountry" => "source_country" }
rename => { "srcip" => "source_ip" }
rename => { "srcmac" => "source_mac" }
rename => { "srcname" => "source_name" }
rename => { "srcintf" => "source_interface" }
rename => { "srcport" => "source_port" }
rename => { "stacount" => "station_count" }
rename => { "stamac" => "static_mac" }
rename => { "srccountry" => "source_country" }
rename => { "srcip" => "source_ip" }
rename => { "srcmac" => "source_mac" }
rename => { "srcname" => "source_name" }
rename => { "sn" => "serial" }
rename => { "srcintf" => "source_interface" }
rename => { "srcport" => "source_port" }
rename => { "total" => "total_bytes" }
rename => { "totalsession" => "total_sessions" }
rename => { "trandisp" => "nat_translation_type" }
rename => { "tranip" => "nat_destination_ip" }
rename => { "tranport" => "nat_destination_port" }
rename => { "transip" => "nat_source_ip" }
rename => { "transport" => "nat_source_port" }
rename => { "tunnelid" => "tunnel_id" }
rename => { "tunnelip" => "tunnel_ip" }
rename => { "tunneltype" => "tunnel_type" }
rename => { "unauthuser" => "unauthenticated_user_source" }
rename => { "unauthusersource" => "os_version" }
rename => { "vendorurl" => "vendor_url" }
rename => { "vpntunnel" => "vpn_tunnel" }
rename => { "vulncat" => "vulnerability_category" }
rename => { "vulncmt" => "vulnerability_count" }
rename => { "vulnid" => "vulnerability_id" }
rename => { "vulnname" => "vulnerability_name" }
rename => { "vulnref" => "vulnerability_reference" }
rename => { "vulnscore" => "vulnerability_score" }
rename => { "xauthgroup" => "x_authentication_group" }
rename => { "xauthuser" => "x_authentication_user" }
rename => { "[SubLog][appid]" => "sub_application_id" }
rename => { "[SubLog][devid]" => "sub_device_id" }
rename => { "[SubLog][dstip]" => "sub_destination_ip" }
rename => { "[SubLog][srcip]" => "sub_source_ip" }
rename => { "[SubLog][dstport]" => "sub_destination_port" }
rename => { "[SubLog][eventtype]" => "sub_event_type" }
rename => { "[SubLog][proto]" => "sub_protocol_number" }
rename => { "[SubLog][date]" => "sub_date" }
rename => { "[SubLog][time]" => "sub_time" }
rename => { "[SubLog][srcport]" => "sub_source_port" }
rename => { "[SubLog][subtype]" => "sub_subtype" }
rename => { "[SubLog][devname]" => "sub_device_name" }
rename => { "[SubLog][itime]" => "sub_itime" }
rename => { "[SubLog][level]" => "sub_level" }
rename => { "[SubLog][logid]" => "sub_log_id" }
rename => { "[SubLog][logver]" => "sub_log_version" }
rename => { "[SubLog][type]" => "sub_event_type" }
rename => { "[SubLog][vd]" => "sub_vd" }
rename => { "[SubLog][action]" => "sub_action" }
rename => { "[SubLog][logdesc]" => "sub_destination_ip" }
rename => { "[SubLog][policyid]" => "sub_olicy_id" }
rename => { "[SubLog][reason]" => "sub_reason" }
rename => { "[SubLog][service]" => "sub_service" }
rename => { "[SubLog][sessionid]" => "sub_session_id" }
rename => { "[SubLog][src]" => "sub_source_ip" }
rename => { "[SubLog][status]" => "sub_status" }
rename => { "[SubLog][ui]" => "sub_ui" }
rename => { "[SubLog][urlfilteridx]" => "sub_url_filter_idx" }
strip => [ "bytes_sent", "bytes_received" ]
convert => [ "bytes_sent", "integer" ]
convert => [ "bytes_received", "integer" ]
convert => [ "cr_score", "integer" ]
convert => [ "cr_action", "integer" ]
convert => [ "elapsed_time", "integer" ]
convert => [ "destination_port", "integer" ]
convert => [ "source_port", "integer" ]
convert => [ "local_port", "integer" ]
convert => [ "remote_port", "integer" ]
convert => [ "packets_sent", "integer" ]
convert => [ "packets_received", "integer" ]
convert => [ "port", "integer" ]
convert => [ "ProtocolNumber", "integer" ]
convert => [ "XAuthUser", "string" ]
remove_field => [ "kv", "log" ]
}
if [tunnel_ip] == "N/A" {
mutate {
remove_field => [ "tunnel_ip" ]
}
}
if [nat_destination_ip] {
mutate {
add_field => { "ips" => [ "%{nat_destination_ip}" ] }
add_field => { "destination_ips" => [ "%{nat_destination_ip}" ] }
}
}
if [sub_destination_ip] {
mutate {
add_field => { "ips" => [ "%{sub_destination_ip}" ] }
add_field => { "destination_ips" => [ "%{sub_destination_ip}" ] }
}
}
if [nat_source_ip] {
mutate {
add_field => { "ips" => [ "%{nat_source_ip}" ] }
add_field => { "source_ips" => [ "%{nat_source_ip}" ] }
}
}
if [sub_source_ip] {
mutate {
add_field => { "ips" => [ "%{sub_source_ip}" ] }
add_field => { "source_ips" => [ "%{sub_source_ip}" ] }
}
}
if [addr_ip] {
mutate {
add_field => { "ips" => [ "%{addr_ip}" ] }
}
}
if [assign_ip] {
mutate {
add_field => { "ips" => [ "%{assign_ip}" ] }
}
}
if [assigned_ip] {
mutate {
add_field => { "ips" => [ "%{assigned_ip}" ] }
}
}
grok {
match => ["message", "type=%{DATA:event_type}\s+"]
}
if [date] and [time] {
mutate {
add_field => { "receive_time" => "%{date} %{time}" }
remove_field => [ "date", "time" ]
}
date {
timezone => "America/Chicago"
match => [ "receive_time", "YYYY-MM-dd HH:mm:ss" ]
target => "receive_time"
}
mutate {
rename => { "receive_time" => "@timestamp" }
}
} else {
mutate {
add_tag => [ "missing_date" ]
}
}
mutate {
#add_tag => [ "conf_file_6200"]
}
}
}

56
salt/logstash/pipelines/config/so/6201_firewall_pfsense.conf
View File

@@ -1,56 +0,0 @@
# Author: Wes Lambert
# Updated by: Doug Burks
filter {
if [type] == "filterlog" {
dissect {
mapping => {
"message" => "%{rule_number},%{sub_rule_number},%{anchor},%{tracker_id},%{interface},%{reason},%{action},%{direction},%{ip_version},%{sub_msg}"
}
}
if [ip_version] == "4" {
dissect {
mapping => {
"sub_msg" => "%{ipv4_tos},%{ipv4_ecn},%{ipv4_ttl},%{ipv4_id},%{ipv4_offset},%{ipv4_flags},%{protocol_id},%{protocol},%{protocol_length},%{source_ip},%{destination_ip},%{ip_sub_msg}"
}
}
}
if [ip_version] == "6" {
dissect {
mapping => {
"sub_msg" => "%{class},%{flow_label},%{hop_limit},%{protocol},%{protocol_id},%{length},%{source_ip},%{destination_ip},%{ip_sub_msg}"
}
}
}
if [protocol] == "tcp" {
dissect {
mapping => {
"ip_sub_msg" => "%{source_port},%{destination_port},%{data_length},%{tcp_flags},"
}
}
}
if [protocol] == "udp" {
dissect {
mapping => {
"ip_sub_msg" => "%{source_port},%{destination_port},%{data_length}"
}
}
}
if [protocol] == "Options" {
mutate {
copy => { "ip_sub_msg" => "options" }
}
mutate {
split => { "options" => "," }
}
}
mutate {
convert => [ "destination_port", "integer" ]
convert => [ "source_port", "integer" ]
convert => [ "ip_version", "integer" ]
replace => { "type" => "firewall" }
add_tag => [ "pfsense","firewall" ]
remove_field => [ "sub_msg", "ip_sub_msg" ]
}
}
}

161
salt/logstash/pipelines/config/so/6300_windows.conf
View File

@@ -1,161 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
filter {
if [type] == "windows" {
# json {
# source => "message"
# }
date {
match => ["EventTime", "YYYY-MM-dd HH:mm:ss"]
remove_field => [ "EventTime" ]
}
if [EventID] == 4634 {
mutate {
add_tag => [ "logoff" ]
}
}
if [EventID] == 4624 or [EventID] == 528 or [EventID] == 540 or [EventID] == 552 or [EventID] == 682 or [EventID] == 4648 or [EventID] == 4778 {
mutate {
add_tag => [ "logon" ]
add_tag => [ "alert_data" ]
}
}
if [EventID] == 529 or [EventID] == 4625 or [EventID] == 530 or [EventID] == 531 or [EventID] == 532 or [EventID] == 533 or [EventID] == 534 or [EventID] == 535 or [EventID] == 536 or [EventID] == 536 or [EventID] == 537 or [EventID] == 538 or [EventID] == 539 or [EventID] == 4625 or [EventID] == 4771 {
mutate {
add_tag => [ "logon_failure" ]
add_tag => [ "alert_data" ]
}
}
# Critical event IDs to monitor
if [EventID] == 7030 or [EventID] == 4720 or [EventID] == 4722 or [EventID] == 4724 or [EventID] == 4738 or [EventID] == 4732 or [EventID] == 1102 or [EventID] == 1056 or [EventID] == 2003 or [EventID] == 2005 or [EventID] == 8003 or [EventID] == 8004 or [EventID] == 8006 or [EventID] == 8007 {
mutate {
add_tag => [ "alert_data" ]
}
}
# Critical event IDs to monitor
if [EventID] == 5152 { drop {} }
if [EventID] == 4688 { drop {} }
if [EventID] == 4689 { drop {} } # Process Termination:Not needed due to Sysmon
if [Channel] == "Microsoft-Windows-Known Folders API Service" { drop {} }
if [EventID] == 3 and [SourceIp] =~ "255$" { drop {} }
if [EventID] == 3 and [DestinationIp] =~ "255$" { drop {} }
# Whitelist/Blacklist check
if [EventID] == 7045 {
translate {
field => "ServiceName"
destination => "ServiceCheck"
dictionary_path => "/lib/dictionaries/services.yaml"
}
}
if [EventID] == 7045 and !([ServiceCheck]) {
mutate {
add_tag => [ "alert_data","new_service" ]
}
}
if [ServiceCheck] == 'whitelist' {
mutate {
remove_field => [ "ServiceCheck" ]
add_tag => [ "whitelist" ]
}
}
if [ServiceCheck] == 'blacklist' {
mutate {
remove_field => [ "ServiceCheck" ]
add_tag => [ "blacklist" ]
}
}
if [EventID] == 5158 {
if [Application] == "System" { drop {} }
if [Application] =~ "\\windows\\system32\\spoolsv\.exe" { drop {} }
if [Application] =~ "\\windows\\system32\\wbem\\wmiprvse\.exe" { drop {} }
if [Application] =~ "mcafee" { drop {} }
if [Application] =~ "carestream" { drop {} }
if [Application] =~ "Softdent" { drop {} }
}
if [ProcessName] == "C:\\Windows\\System32\\wbem\\WmiPrvSE\.exe" and [SubjectUserName] == "SolarwindsHO" { drop {} }
if [EventID] == 4690 { drop {} }
if [EventID] == 861 and [AccountName] == "ntp" { drop {} }
if [EventID] == 5158 and [Application] =~ "\\windows\\system32\\lsass\.exe$" { drop {} }
if [EventID] == 5158 and [Application] =~ "\\windows\\system32\\svchost\.exe$" { drop {} }
if [EventID] == 5158 and [Application] =~ "\\windows\\system32\\dfsrs\.exe$" { drop {} }
if [EventID] == 5447 { drop {} }
mutate {
rename => [ "AccountName", "user" ]
rename => [ "AccountType", "account_type" ]
rename => [ "ActivityID", "activity_id" ]
rename => [ "Category", "category" ]
rename => [ "ClientAddress", "client_ip" ]
rename => [ "Channel", "channel" ]
rename => [ "DCIPAddress", "domain_controller_ip" ]
rename => [ "DCName", "domain_controller_name" ]
rename => [ "EventID", "event_id" ]
rename => [ "EventReceivedTime", "event_received_time" ]
rename => [ "EventType", "event_type" ]
rename => [ "GatewayIPAddress", "gateway_ip" ]
rename => [ "IPAddress", "client_ip" ]
rename => [ "Ipaddress", "client_ip" ]
rename => [ "IpAddress", "client_ip" ]
rename => [ "IPPort", "source_port" ]
rename => [ "OpcodeValue", "opcode_value" ]
rename => [ "PreAuthType", "preauthentication_type" ]
rename => [ "PrincipleSAMName", "user" ]
rename => [ "ProcessID", "process_id" ]
rename => [ "ProviderGUID", "providerguid" ]
rename => [ "RecordNumber", "record_number" ]
rename => [ "RemoteAddress", "destination_ip" ]
rename => [ "ServiceName", "service_name" ]
rename => [ "ServiceID", "service_id" ]
rename => [ "SeverityValue", "severity_value" ]
rename => [ "SourceAddress", "client_ip" ]
rename => [ "SourceModuleName", "source_module_name" ]
rename => [ "SourceModuleType", "source_module_type" ]
rename => [ "SourceName", "source_name" ]
rename => [ "SubjectUserName", "user" ]
rename => [ "TaskName", "task_name" ]
rename => [ "TargetDomainName", "target_domain_name" ]
rename => [ "TargetUserName", "user" ]
rename => [ "ThreadID", "thread_id" ]
rename => [ "User_ID", "user" ]
rename => [ "UserID", "user" ]
rename => [ "username", "user" ]
}
# For any accounts that are service accounts or special accounts add the tag of service_account
# This example applies the tag to any username that starts with SVC_. If you use a different
# standard change this.
if [user] =~ "^DWM-*" or [user] == "SYSTEM" or [user] == "NETWORK SERVICE" or [user] == "LOCAL SERVICE" or [user] =~ "^SVC_*" {
mutate {
add_tag => [ "service_account" ]
}
}
# This looks for events that are typically noisy but may be of use for deep dive investigations
# A tag of noise is added to quickly filter out noise
if [event_id] == 7036 or [source_name] == "Desktop Window Manager" or [category] == "Engine Lifecycle" or [category] == "Provider Lifecycle" {
mutate {
add_tag => [ "noise" ]
}
}
#Identify machine accounts
if [user] =~ /\$/ {
mutate {
add_tag => [ "machine", "noise" ]
}
}
# Lower case all field names
ruby {
code => "
event_hash = event.to_hash
new_event = {}
event_hash.keys.each do |key|
new_event[key.downcase] = event[key]
end
event.instance_variable_set(:@data, new_event)"
}
mutate {
#add_tag => [ "conf_file_6300"]
}
}
}

49
salt/logstash/pipelines/config/so/6301_dns_windows.conf
View File

@@ -1,49 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
filter {
if [type] == "dns" and "bro" not in [tags] {
json {
source => "message"
}
# strip whitespace from message field
mutate {
strip => "message"
}
# If the message is blank, drop the log
if [Message] =~ /^$/ {
drop { }
} else {
if [type] == "dns" {
# This section is lookup for a match against the log and parsing out the fields
grok {
match => { "Message" => "(?<timestamp>%{DATE_US} %{TIME} (?:AM|PM))\s+%{DATA:thread_id}\s+%{WORD:dns_type}\s+ %{BASE16NUM:packet_id}\s+%{WORD:dns_protocol}\s+%{WORD:dns_direction}\s+%{IP:dns_ip}\s+ %{BASE16NUM:xid}\s+%{DATA:response}\s+%{WORD:dns_query_type}\s+\[%{BASE16NUM:hex_flags}\s+%{WORD:flags}\s+ %{WORD:rcode_name}\]\s+%{WORD:query_type_name}\s+%{GREEDYDATA:dns_domain}"}
match => { "Message" => "(?<timestamp>%{DATE_US} %{TIME} (?:AM|PM))\s+%{DATA:thread_id}\s+%{WORD:dns_type}\s+ %{BASE16NUM:packet_id}\s+%{WORD:dns_protocol}\s+%{WORD:dns_direction}\s+%{IP:dns_ip}\s+ %{BASE16NUM:xid}\s+%{DATA:response}\s+%{WORD:dns_query_type}\s+\[%{BASE16NUM:hex_flags}\s+%{WORD:flags}\s+%{WORD:recursion}\s+%{WORD:rcode_name}\]\s+%{WORD:query_type_name}\s+%{GREEDYDATA:dns_domain}"}
match => { "Message" => "(?<timestamp>%{DATE_US} %{TIME} (?:AM|PM))\s+%{DATA:thread_id}\s+%{WORD:dns_type}\s+ %{BASE16NUM:packet_id}\s+%{WORD:dns_protocol}\s+%{WORD:dns_direction}\s+%{IP:dns_ip}\s+ %{BASE16NUM:xid}\s+%{DATA:response}\s+%{WORD:dns_query_type}\s+\[%{BASE16NUM:hex_flags}\s+%{WORD:recursion}\s+%{WORD:rcode_name}\]\s+%{WORD:query_type_name}\s+%{GREEDYDATA:dns_domain}"}
match => { "Message" => "(?<timestamp>%{DATE_US} %{TIME} (?:AM|PM))\s+%{DATA:thread_id}\s+%{WORD:dns_type}\s+ %{BASE16NUM:packet_id}\s+%{WORD:dns_protocol}\s+%{WORD:dns_direction}\s+%{IP:dns_ip}\s+ %{BASE16NUM:xid}\s+%{DATA:response}\s+%{WORD:dns_query_type}\s+\[%{BASE16NUM:hex_flags}\s+%{WORD:rcode_name}\]\s+%{WORD:query_type_name}\s+%{GREEDYDATA:dns_domain}"}
match => { "Message" => "(?<timestamp>%{DATE_US} %{TIME} (?:AM|PM))\s+%{DATA:thread_id}\s+%{WORD:dns_type}\s+ %{BASE16NUM:packet_id}\s+%{WORD:dns_protocol}\s+%{WORD:dns_direction}\s+%{IP:dns_ip}\s+ %{BASE16NUM:xid}\s+%{WORD:dns_query_type}\s+\[%{BASE16NUM:hex_flags}\s+%{WORD:flags}\s+ %{WORD:rcode_name}\]\s+%{WORD:query_type_name}\s+%{GREEDYDATA:dns_domain}"}
# Server 2003 DNS logs do not include slashes or AM/PM in timestamp
match => { "Message" => "(?<timestamp>%{YEAR}%{MONTHNUM}%{MONTHDAY} %{TIME})\s+%{DATA:thread_id}\s+%{WORD:dns_type}\s+ %{BASE16NUM:packet_id}\s+%{WORD:dns_protocol}\s+%{WORD:dns_direction}\s+%{IP:dns_ip}\s+ %{BASE16NUM:xid}\s+%{DATA:response}\s+%{WORD:dns_query_type}\s+\[%{BASE16NUM:hex_flags}\s+%{WORD:flags}\s+ %{WORD:rcode_name}\]\s+%{WORD:query_type_name}\s+%{GREEDYDATA:dns_domain}"}
match => { "Message" => "(?<timestamp>%{YEAR}%{MONTHNUM}%{MONTHDAY} %{TIME})\s+%{DATA:thread_id}\s+%{WORD:dns_type}\s+ %{BASE16NUM:packet_id}\s+%{WORD:dns_protocol}\s+%{WORD:dns_direction}\s+%{IP:dns_ip}\s+ %{BASE16NUM:xid}\s+%{DATA:response}\s+%{WORD:dns_query_type}\s+\[%{BASE16NUM:hex_flags}\s+%{WORD:flags}\s+%{WORD:recursion}\s+%{WORD:rcode_name}\]\s+%{WORD:query_type_name}\s+%{GREEDYDATA:dns_domain}"}
match => { "Message" => "(?<timestamp>%{YEAR}%{MONTHNUM}%{MONTHDAY} %{TIME})\s+%{DATA:thread_id}\s+%{WORD:dns_type}\s+ %{BASE16NUM:packet_id}\s+%{WORD:dns_protocol}\s+%{WORD:dns_direction}\s+%{IP:dns_ip}\s+ %{BASE16NUM:xid}\s+%{DATA:response}\s+%{WORD:dns_query_type}\s+\[%{BASE16NUM:hex_flags}\s+%{WORD:recursion}\s+%{WORD:rcode_name}\]\s+%{WORD:query_type_name}\s+%{GREEDYDATA:dns_domain}"}
match => { "Message" => "(?<timestamp>%{YEAR}%{MONTHNUM}%{MONTHDAY} %{TIME})\s+%{DATA:thread_id}\s+%{WORD:dns_type}\s+ %{BASE16NUM:packet_id}\s+%{WORD:dns_protocol}\s+%{WORD:dns_direction}\s+%{IP:dns_ip}\s+ %{BASE16NUM:xid}\s+%{DATA:response}\s+%{WORD:dns_query_type}\s+\[%{BASE16NUM:hex_flags}\s+%{WORD:rcode_name}\]\s+%{WORD:query_type_name}\s+%{GREEDYDATA:dns_domain}"}
match => { "Message" => "(?<timestamp>%{YEAR}%{MONTHNUM}%{MONTHDAY} %{TIME})\s+%{DATA:thread_id}\s+%{WORD:dns_type}\s+ %{BASE16NUM:packet_id}\s+%{WORD:dns_protocol}\s+%{WORD:dns_direction}\s+%{IP:dns_ip}\s+ %{BASE16NUM:xid}\s+%{WORD:dns_query_type}\s+\[%{BASE16NUM:hex_flags}\s+%{WORD:flags}\s+ %{WORD:rcode_name}\]\s+%{WORD:query_type_name}\s+%{GREEDYDATA:dns_domain}"}
remove_field => [ "Message" ]
}
# This section attempts to convert the dns_domain into the traditional domain.com format
mutate {
gsub => [ "dns_domain", "(\(\d+\))", "." ]
}
grok {
match => { "dns_domain" => "\.%{DATA:query}\.$" }
remove_field => [ "dns_domain" ]
}
}
}
mutate {
#add_tag => [ "conf_file_6301"]
}
}
}

92
salt/logstash/pipelines/config/so/6400_suricata.conf
View File

@@ -1,92 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
#
# This conf file is based on accepting logs for suricata json events
filter {
if [type] == "suricata" {
if "test_data" not in [tags] {
date {
match => [ "timestamp", "ISO8601" ]
}
} else {
mutate {
remove_field => [ "netflow.start","netflow.end","timestamp" ]
}
}
if [event_type] == "fileinfo" {
ruby {
code => "if event['event_type'] == 'fileinfo'; event['fileinfo']['type']=event['fileinfo']['magic'].to_s.split(',')[0]; end;"
}
}
# I recommend renaming the fields below to be consistent with other log sources. This makes it easy to "pivot" between logs
mutate {
rename => [ "src_ip", "source_ip" ]
rename => [ "dest_ip", "destination_ip" ]
rename => [ "src_port", "source_port" ]
rename => [ "dest_port", "destination_port" ]
}
# This will translate the alert.severity field into a severity field of either High, Medium, or Low
if [event_type] == "alert" {
if [alert][severity] == 1 {
mutate {
add_field => { "severity" => "High" }
}
}
if [alert][severity] == 2 {
mutate {
add_field => { "severity" => "Medium" }
}
}
if [alert][severity] == 3 {
mutate {
add_field => { "severity" => "Low" }
}
}
# If the alert is a Snort GPL alert break it apart for easier reading and categorization
if [alert][signature] =~ "GPL " {
# This will parse out the category type from the alert
grok {
match => { "[alert][signature]" => "GPL\s+%{DATA:category}\s" }
}
# This will store the category
mutate {
add_field => { "rule_type" => "Snort GPL" }
lowercase => [ "category" ]
}
}
# If the alert is an Emerging Threat alert break it apart for easier reading and categorization
if [alert][signature] =~ "ET " {
# This will parse out the category type from the alert
grok {
match => { "[alert][signature]" => "ET\s+%{DATA:category}\s" }
}
# This will store the category
mutate {
add_field => { "rule_type" => "Emerging Threats" }
lowercase => [ "category" ]
}
}
# This section adds URLs to lookup information about a rule online
if [rule_type] == "Snort GPL" {
mutate {
add_field => [ "signature_info", "https://www.snort.org/search?query=%{[alert][gid]}-%{[alert][signature_id]}" ]
}
}
if [rule_type] == "Emerging Threats" {
mutate {
add_field => [ "signature_info", "http://doc.emergingthreats.net/%{[alert][signature_id]}" ]
}
}
}
if "_grokparsefailure" not in [tags] and "_csvparsefailure" not in [tags] and "_jsonparsefailure" not in [tags] {
# mutate {
# remove_field => [ "message" ]
# }
}
mutate {
#add_tag => [ "conf_file_6400"]
}
}
}

160
salt/logstash/pipelines/config/so/6500_ossec.conf
View File

@@ -1,160 +0,0 @@
# Author: Wes Lambert
#
# Last Update: 09/19/2018
#
# This conf file is based on accepting logs from OSSEC
filter {
# OSSEC Alerts
if [type] == "ossec" {
# Sysmon/Autoruns logs transported by OSSEC
if [message] =~ "Microsoft-Windows-Sysmon" {
mutate {
replace => { "type" => "sysmon" }
add_tag => [ "ossec" ]
}
}
if [message] =~ "AR-LOG" {
mutate {
replace => { "type" => "autoruns" }
add_tag => [ "ossec" ]
}
}
# If message looks like json, try to parse it as such. Otherwise, grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "rule" => "wazuh-rule" }
rename => { "[wazuh-rule][level]" => "alert_level" }
rename => { "[wazuh-rule][description]" => "description" }
rename => { "[data][srcuser]" => "username" }
rename => { "[data][dstuser]" => "escalated_user" }
rename => { "[data][command]" => "command" }
rename => { "[predecoder][program_name]" => "process" }
}
# Wazuh 3.8.2
if [data][EventChannel] {
mutate {
rename => { "[data][EventChannel][EventData][User]" => "username" }
rename => { "[data][EventChannel][System][EventID]" => "event_id" }
rename => { "[data][EventChannel][EventData][DestinationPort]" => "destination_port" }
rename => { "[data][EventChannel][EventData][DestinationIp]" => "destination_ip" }
rename => { "[data][EventChannel][EventData][SourcePort]" => "source_port" }
rename => { "[data][EventChannel][EventData][SourceIp]" => "source_ip" }
rename => { "[data][EventChannel][EventData][SourceHostname]" => "source_hostname" }
rename => { "[data][EventChannel][EventData][DestinationHostname]" => "destination_hostname" }
}
}
# Wazuh 3.9.2
if [data][win] {
mutate {
rename => { "[data][win][eventdata][user]" => "username" }
rename => { "[data][win][system][eventID]" => "event_id" }
rename => { "[data][win][eventdata][destinationPort]" => "destination_port" }
rename => { "[data][win][eventdata][destinationIp]" => "destination_ip" }
rename => { "[data][win][eventdata][sourcePort]" => "source_port" }
rename => { "[data][win][eventdata][sourceIp]" => "source_ip" }
rename => { "[data][win][eventdata][sourceHostname]" => "source_hostname" }
rename => { "[data][win][eventdata][destinationHostname]" => "destination_hostname" }
}
}
} else {
grok {
match => ["message", "Alert Level: %{NONNEGINT;alert_level}; Rule: %{NONNEGINT:rule} - %{DATA:description}; Location: %{DATA:location}; user: +%{DATA:username}; %{SYSLOGTIMESTAMP} %{DATA:host} %{DATA:process}\[%{INT:pid}]: %{GREEDYDATA:details}",
"message", "Alert Level: %{NONNEGINT:alert_level}; Rule: %{NONNEGINT:rule} - %{DATA:description}; Location: %{DATA:location}; %{SYSLOGTIMESTAMP:timestamp} %{DATA:host} %{DATA:process}\[%{NONNEGINT:pid}]: %{GREEDYDATA:details}",
"message", "Alert Level: %{NONNEGINT:alert_level}; Rule: %{NONNEGINT:rule} - %{DATA:description}; Location: %{DATA:location}; %{SYSLOGTIMESTAMP} %{DATA:host} %{DATA:process}\[%{NONNEGINT:pid}]: %{GREEDYDATA:details}",
"message", "Alert Level: %{NONNEGINT:alert_level}; Rule: %{NONNEGINT:rule} - %{DATA:description}; Location: %{DATA:location}; %{SYSLOGTIMESTAMP:timestamp} %{DATA:host} %{DATA:program}: +%{DATA:username} : TTY=%{DATA:tty} ; PWD=%{DATA:dir} ; USER=%{DATA:escalated_user} ; COMMAND=%{GREEDYDATA:command}",
"message", "Alert Level: %{NONNEGINT:alert_level}; Rule: %{NONNEGINT:rule} - %{DATA:description}; Location: %{DATA:location}; %{SYSLOGTIMESTAMP:timestamp} %{DATA:host} %{DATA:program}: %{GREEDYDATA:details}",
"message", "Alert Level: %{NONNEGINT:alert_level}; Rule: %{NONNEGINT:rule} - %{DATA:description}; Location: %{DATA:location}; %{SYSLOGTIMESTAMP:timestamp} %{DATA:host} %{DATA:program}: +%{DATA:username} : %{GREEDYDATA:details}",
"message", "Alert Level: %{NONNEGINT:alert_level}; Rule: %{NONNEGINT:rule} - %{DATA:description}; Location: %{DATA:location}; srcip: %{IP:source_ip};%{GREEDYDATA:details}",
"message", "Alert Level: %{NONNEGINT:alert_level}; Rule: %{NONNEGINT:rule} - %{DATA:description}; Location: %{DATA:location}; %{DATA:username}: %{DATA}: \'%{DATA}': %{DATA:interface}: %{INT:num_packets}",
"message", "Alert Level: %{NONNEGINT:alert_level}; Rule: %{NONNEGINT:rule} - %{DATA:description}; Location: %{DATA:location}; %{DATA:username}: %{GREEDYDATA:details}.",
"message", "Alert Level: %{NONNEGINT:alert_Level}; Rule: %{NONNEGINT:Rule} - %{DATA:Description}; Location: %{DATA:location}; user: +%{DATA:username};",
"message", "Alert Level: %{NONNEGINT:alert_level}; Rule: %{NONNEGINT:rule} - %{DATA:description}; Location: %{DATA:location}; %{DATA}: %{DATA}: \'%{DATA}': %{DATA:interface}: %{NONNEGINT:num_packets}",
"message", "Alert Level: %{NONNEGINT:alert_level}; Rule: %{NONNEGINT:rule} - %{DATA:description}; Location: %{DATA:location}; %{GREEDYDATA:details}"]
}
}
# Add tag for OSSEC alerts
if [alert_level] {
mutate {
add_tag => [ "alert" ]
}
}
translate {
field => "alert_level"
destination => "classification"
dictionary => [
"1", "None",
"2", "System low priority notification",
"3", "Successful/authorized event",
"4", "System low priority error",
"5", "User generated error",
"6", "Low relevance attack",
"7", '"Bad word" matching',
"8", "First time seen",
"9", "Error from invalid source",
"10", "Multiple user generated errors",
"11", "Integrity checking warning",
"12", "High importance event",
"13", "Unusal error (high importance)",
"14", "High importance security event",
"15", "Severe attack"
]
}
}
# OSSEC Archive Logs
if [type] == "ossec_archive" {
# Sysmon/Autoruns logs transported by OSSEC
if [message] =~ "Microsoft-Windows-Sysmon" {
mutate {
replace => { "type" => "sysmon" }
add_tag => [ "ossec" ]
}
}
if [message] =~ "AR-LOG" {
mutate {
replace => { "type" => "autoruns" }
add_tag => [ "ossec" ]
}
}
# If message looks like json, try to parse it as such. Otherwise, grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => [ "rule", "wazuh-rule" ]
rename => [ "[wazuh-rule][level]", "alert_level" ]
rename => [ "[wazuh-rule][description]", "description" ]
rename => [ "[data][srcuser]", "username" ]
rename => [ "[data][dstuser]", "escalated_user" ]
rename => [ "[data][command]", "command" ]
rename => [ "[predecoder][program_name]", "process" ]
}
} else {
grok {
match => ["message",'%{YEAR:year} %{SYSLOGTIMESTAMP:timestamp} %{DATA:location} %{IP:source_ip} - %{DATA:username} \[%{DATA:request_timestamp}] "%{DATA:method} %{DATA:requested_resource} %{DATA:protocol}\/%{DATA:protocol_version}" %{NONNEGINT:status_code} %{NONNEGINT:object_size} "%{DATA:referrer}" "%{DATA:user_agent}"',
"message","%{YEAR:year} %{SYSLOGTIMESTAMP:timestamp} %{DATA:location} %{SYSLOGTIMESTAMP:ossec_timestamp} %{DATA:host} %{DATA:process}\[%{NONNEGINT:pid}]: \(%{DATA:username}\) CMD \(%{DATA:command}\)",
"message", "%{YEAR:year} %{SYSLOGTIMESTAMP:timestamp} %{DATA:location} %{GREEDYDATA:details}","message","%{YEAR:year} %{SYSLOGTIMESTAMP:timestamp} %{DATA:location} %{SYSLOGTIMESTAMP:ossec_timestamp} %{DATA:ossec_host} %{DATA:process}\[%{NONNEGINT:pid}]: %{GREEDYDATA:details}",
"message","%{DATA:age} %{DATA:program} %{DATA} '%{DATA:checksum}'",
"message", "%{DATA:username} : TTY=%{DATA:tty} ; PWD=%{DATA:dir} ; USER=%{DATA:escalated_user} ; COMMAND=%{GREEDYDATA:command}"]
remove_field => [ "ossec_timestamp" ]
}
mutate {
convert => [ "status_code", "integer" ]
}
}
}
}

118
salt/logstash/pipelines/config/so/6501_ossec_sysmon.conf
View File

@@ -1,118 +0,0 @@
# Author: Wes Lambert
# wlambertts@gmail.com
#
# This conf file is based on accepting Sysmon logs from OSSEC
#
# Parse using grok
filter {
# OSSEC Logs and Alerts
if [type] == "sysmon" or "sysmon" in [tags] {
if [message] !~ /^{.*}$/ {
#mutate { replace => { "type" => "sysmon" } }
grok {
# match => ["message","%{YEAR:year} %{SYSLOGTIMESTAMP:timestamp} %{DATA:location} %{IP:source_ip}->WinEvtLog %{YEAR:year} %{SYSLOGTIMESTAMP:ossec_timestamp} WinEvtLog: Microsoft-Windows-Sysmon/Operational: INFORMATION\(%{INT:sysmon_event_id}\):"]
match => ["message", "%{YEAR:year}%{SPACE}%{SYSLOGTIMESTAMP:timestamp}%{SPACE}%{DATA:location}%{SPACE}(any|%{IP:source_ip})->WinEvtLog%{SPACE}%{YEAR:year}%{SPACE}%{SYSLOGTIMESTAMP:ossec_timestamp}%{SPACE}WinEvtLog:%{SPACE}Microsoft-Windows-Sysmon/Operational:%{SPACE}INFORMATION\(%{INT:event_id}\):%{SPACE}%{GREEDYDATA:rest_of_msg}"]
}
mutate {
convert => ["event_id", "integer"]
remove_field => ["timestamp"]
remove_field => ["year"]
}
if [event_id] == 1 {
grok {
match => ["rest_of_msg", "Microsoft-Windows-Sysmon:%{SPACE}SYSTEM:%{SPACE}NT%{SPACE}AUTHORITY:%{SPACE}%{DATA:hostname}:%{SPACE}%{DATA:event_type}:%{SPACE}UtcTime:%{SPACE}%{DATA:sysmon_timestamp}%{SPACE}ProcessGuid:%{SPACE}\{%{DATA:process_guid}\}%{SPACE}ProcessId:%{SPACE}%{INT:process_id}%{SPACE}Image:%{SPACE}%{DATA:image_path}%{SPACE}CommandLine:%{SPACE}%{DATA:process_name} %{DATA:process_arguments}%{SPACE}CurrentDirectory:%{SPACE}%{DATA:current_directory}%{SPACE}User:%{SPACE}%{DATA:username}%{SPACE}LogonGuid:%{SPACE}\{%{DATA:logon_guid}\}%{SPACE}LogonId:%{SPACE}%{DATA:logon_id}%{SPACE}TerminalSessionId:%{SPACE}%{INT:terminal_id}%{SPACE}IntegrityLevel:%{SPACE}%{DATA:integrity_level}%{SPACE}Hashes:%{SPACE}MD5=%{DATA:md5},SHA256=%{DATA:sha256}%{SPACE}ParentProcessGuid:%{SPACE}\{%{DATA:parent_process_guid}\}%{SPACE}ParentProcessId:%{SPACE}%{NONNEGINT:parent_process_id}%{SPACE}ParentImage:%{SPACE}%{DATA:parent_image_path}%{SPACE}ParentCommandLine:%{SPACE}%{GREEDYDATA:parent_process_name}",
"rest_of_msg", 'Microsoft-Windows-Sysmon:%{SPACE}SYSTEM:%{SPACE}NT%{SPACE}AUTHORITY:%{SPACE}%{DATA:hostname}:%{SPACE}%{DATA:event_type}:%{SPACE}UtcTime:%{SPACE}%{DATA:sysmon_timestamp}%{SPACE}ProcessGuid:%{SPACE}\{%{DATA:process_guid}\}%{SPACE}ProcessId:%{SPACE}%{INT:process_id}%{SPACE}Image:%{SPACE}%{DATA:image_path}%{SPACE}CommandLine:%{SPACE}"%{DATA:process_name}"%{SPACE}%{DATA:process_arguments}%{SPACE}CurrentDirectory:%{SPACE}%{DATA:current_directory}%{SPACE}User:%{SPACE}%{DATA:username}%{SPACE}LogonGuid:%{SPACE}\{%{DATA:logon_guid}\}%{SPACE}LogonId:%{SPACE}%{DATA:logon_id}%{SPACE}TerminalSessionId:%{SPACE}%{INT:terminal_id}%{SPACE}IntegrityLevel:%{DATA:integrity_level}',
"rest_of_msg", "Microsoft-Windows-Sysmon/Operational:%{SPACE}INFORMATION(%{INT:event_id}):%{SPACE}Microsoft-Windows-Sysmon:%{SPACE}SYSTEM:%{SPACE}NT AUTHORITY:%{SPACE}%{DATA:hostname}:%{SPACE}%{DATA:event_type}:%{SPACE}UtcTime:%{SPACE}%{DATA:sysmon_timestamp}%{SPACE}ProcessGuid:%{SPACE}{%{DATA:process_guid}}%{SPACE}ProcessId:%{SPACE}%{INT:process_id}%{SPACE}Image:%{SPACE}%{DATA:image_path}%{SPACE}CommandLine:%{SPACE}%{DATA:process_name}%{SPACE}%{DATA:process_arguments}CurrentDirectory:%{SPACE}%{DATA:current_directory}%{SPACE}User:%{SPACE}%{DATA:username}%{SPACE}LogonGuid:%{SPACE}{%{DATA:logon_guid}}%{SPACE}LogonId:%{SPACE}%{DATA:logon_id}%{SPACE}TerminalSessionId:%{SPACE}%{INT:terminal_id}%{SPACE}IntegrityLevel:%{SPACE}%{DATA:integrity_level}%{SPACE}Hashes:%{SPACE}MD5=%{DATA:md5},SHA256=%{DATA:sha256}%{SPACE}ParentProcessGuid:%{SPACE}{%{DATA:parent_process_guid}}%{SPACE}ParentProcessId:%{SPACE}%{NONNEGINT:parent_process_id}%{SPACE}ParentImage:%{SPACE}%{DATA:parent_image_path}%{SPACE}ParentCommandLine:%{SPACE}%{GREEDYDATA:parent_process_name}"]
}
mutate {
convert => ["process_guid", "integer"]
convert => ["process_id", "integer"]
add_tag => ["process_creation"]
}
}
if [event_id] == 3 {
mutate {
remove_field => ["source_ip"]
}
grok {
match => ["rest_of_msg", "Microsoft-Windows-Sysmon:%{SPACE}SYSTEM:%{SPACE}NT%{SPACE}AUTHORITY:%{SPACE}%{DATA:hostname}:%{SPACE}%{DATA:event_type}:%{SPACE}UtcTime:%{SPACE}%{DATA:sysmon_timestamp}%{SPACE}ProcessGuid:%{SPACE}\{%{DATA:process_guid}\}%{SPACE}ProcessId:%{SPACE}%{NONNEGINT:process_id}%{SPACE}Image:%{SPACE}%{DATA:image_path}%{SPACE}User:%{SPACE}%{DATA:username}%{SPACE}Protocol:%{SPACE}%{DATA:protocol}%{SPACE}Initiated:%{SPACE}%{DATA:initiated}%{SPACE}SourceIsIpv6:%{SPACE}%{DATA:is_source_ipv6}%{SPACE}SourceIp:%{SPACE}%{IP:source_ip}%{SPACE}SourceHostname:%{SPACE}%{DATA:source_hostname}%{SPACE}SourcePort:%{SPACE}%{NONNEGINT:source_port}%{SPACE}SourcePortName:%{SPACE}%{DATA:source_port_name}%{SPACE}DestinationIsIpv6:%{SPACE}%{DATA:dest_is_ipv6}%{SPACE}DestinationIp:%{SPACE}%{IP:destination_ip}%{SPACE}DestinationHostname:%{SPACE}%{DATA:destination_hostname}%{SPACE}DestinationPort:%{SPACE}%{NONNEGINT:destination_port}%{SPACE}DestinationPortName:%{SPACE}%{GREEDYDATA:destination_port_name}"]
}
mutate {
convert => ["process_guid", "integer"]
convert => ["process_id", "integer"]
convert => ["source_port", "integer"]
convert => ["destination_port", "integer"]
add_tag => ["network_connection"]
}
}
if [event_id] == 5 {
grok {
match => ["rest_of_msg", "Microsoft-Windows-Sysmon:%{SPACE}SYSTEM:%{SPACE}NT%{SPACE}AUTHORITY:%{SPACE}%{DATA:hostname}:%{SPACE}%{DATA:event_type}:%{SPACE}UtcTime:%{SPACE}%{DATA:sysmon_timestamp}%{SPACE}ProcessGuid:%{SPACE}\{%{DATA:process_guid}\}%{SPACE}ProcessId:%{SPACE}%{NONNEGINT:process_id}%{SPACE}Image:%{SPACE}%{GREEDYDATA:image_path}"]
}
mutate {
convert => ["process_guid", "integer"]
convert => ["process_id", "integer"]
add_tag => ["process_termination"]
}
}
if [event_id] == 11 {
grok {
match => ["rest_of_msg","Microsoft-Windows-Sysmon:%{SPACE}SYSTEM:%{SPACE}NT%{SPACE}AUTHORITY:%{SPACE}%{DATA:hostname}:%{SPACE}%{DATA:event_type}:%{SPACE}UtcTime:%{SPACE}%{DATA:sysmon_timestamp}%{SPACE}ProcessGuid:%{SPACE}\{%{DATA:process_guid}\}%{SPACE}ProcessId:%{SPACE}%{NONNEGINT:process_id}%{SPACE}Image:%{SPACE}%{DATA:image_path}%{SPACE}TargetFilename:%{SPACE}%{DATA:target_filename}%{SPACE}CreationUtcTime:%{SPACE}%{DATA:creation_time}%{SPACE}"]
}
mutate {
convert => ["process_guid", "integer"]
convert => ["process_id", "integer"]
add_tag => ["file_created"]
}
}
mutate {
remove_field => ["rest_of_msg"]
}
} else {
mutate {
rename => { "[data][srcuser]" => "username" }
rename => { "[data][id]" => "event_id" }
rename => { "[data][dstport]" => "destination_port" }
rename => { "[data][dstip]" => "destination_ip" }
rename => { "[data][srcip]" => "source_ip" }
rename => { "[data][sysmon][image]" => "image_path" }
rename => { "[data][sysmon][parentImage]" => "parent_image_path" }
rename => { "[data][sysmon][targetfilename]" => "target_filename" }
rename => { "[data][sysmon][sourceHostname]" => "source_hostname" }
rename => { "[data][sysmon][destinationHostname]" => "destination_hostname" }
}
# Wazuh 3.8.2
if [data][EventChannel] {
mutate {
rename => { "[data][EventChannel][EventData][User]" => "username" }
rename => { "[data][EventChannel][System][EventID]" => "event_id" }
rename => { "[data][EventChannel][EventData][DestinationPort]" => "destination_port" }
rename => { "[data][EventChannel][EventData][DestinationIp]" => "destination_ip" }
rename => { "[data][EventChannel][EventData][SourcePort]" => "source_port" }
rename => { "[data][EventChannel][EventData][SourceIp]" => "source_ip" }
rename => { "[data][EventChannel][EventData][Image]" => "image_path" }
rename => { "[data][EventChannel][EventData][ParentImage]" => "parent_image_path" }
rename => { "[data][EventChannel][EventData][TargetFilename]" => "target_filename" }
rename => { "[data][EventChannel][EventData][SourceHostname]" => "source_hostname" }
rename => { "[data][EventChannel][EventData][DestinationHostname]" => "destination_hostname" }
}
}
# Wazuh 3.9.2
if [data][win] {
mutate {
rename => { "[data][win][eventdata][user]" => "username" }
rename => { "[data][win][system][eventID]" => "event_id" }
rename => { "[data][win][eventdata][destinationPort]" => "destination_port" }
rename => { "[data][win][eventdata][destinationIp]" => "destination_ip" }
rename => { "[data][win][eventdata][sourcePort]" => "source_port" }
rename => { "[data][win][eventdata][sourceIp]" => "source_ip" }
rename => { "[data][win][eventdata][image]" => "image_path" }
rename => { "[data][win][eventdata][parentImage]" => "parent_image_path" }
rename => { "[data][win][eventdata][targetFilename]" => "target_filename" }
rename => { "[data][win][eventdata][sourceHostname]" => "source_hostname" }
rename => { "[data][win][eventdata][destinationHostname]" => "destination_hostname" }
}
}
}
}
}

43
salt/logstash/pipelines/config/so/6502_ossec_autoruns.conf
View File

@@ -1,43 +0,0 @@
# Author: Wes Lambert
# wlambertts@gmail.com
#
# Updated by: Dustin Lee
# Last Update: 06/13/2019
#
# This conf file is based on accepting Autoruns logs from OSSEC
#
# Parse using grok
filter {
if [type] == "autoruns" or "autoruns" in [tags] {
if [message] !~ /^{.*}$/ {
grok {
match => [
"message", "%{YEAR:year} %{SYSLOGTIMESTAMP:ossec_timestamp} \(%{DATA:ossec_agent_name}\) %{IP:source_ip}->%{DATA:location} %{DATA:log_name}\|%{DATA:hostname}\|%{DATESTAMP:log_timestamp}\|%{DATA:event_timestamp}\|%{DATA:image_timestamp}\|%{DATA:entry_location}\|%{DATA:entry}\|%{DATA:enabled}\|%{DATA:category}\|%{DATA:profile}\|%{DATA:description}\|%{DATA:signer}\|%{DATA:company}\|%{DATA:image_path}\|%{DATA:version}\|%{DATA:launch_string}\|%{DATA:md5}\|%{DATA:sha1}\|%{DATA:pesha1}\|%{DATA:pesha256}\|%{DATA:sha256}\|%{DATA:imphash}",
"message", "%{YEAR:year} %{SYSLOGTIMESTAMP:ossec_timestamp} \(%{DATA:ossec_agent_name}\) %{IP:source_ip}->%{DATA:location} %{DATA:log_name}\|%{DATA:hostname}\|%{DATESTAMP:log_timestamp}\|%{DATA:event_timestamp}\|%{DATA:entry_location}\|%{DATA:entry}\|%{DATA:enabled}\|%{DATA:category}\|%{DATA:profile}\|%{DATA:description}\|%{DATA:signer}\|%{DATA:company}\|%{DATA:image_path}\|%{DATA:version}\|%{DATA:launch_string}\|%{DATA:md5}\|%{DATA:sha1}\|%{DATA:pesha1}\|%{DATA:pesha256}\|%{DATA:sha256}\|%{DATA:imphash}"
]
}
#csv {
# columns => ["log_name","entry_location","entry","enabled","category","autoruns_description","signer","company","image_path","version","launch_string","md5","sha1","pesha1","pesha256","sha256","imphash"]
# separator => "|"
# }
mutate {
remove_field => [ "year" ]
remove_field => [ "timestamp" ]
}
} else {
grok {
match => [
"full_log", "AR-LOG\|%{DATA:hostname}\|%{DATA:event_timestamp}\|%{DATA:image_timestamp}\|%{DATA:entry_location}\|%{DATA:entry}\|%{DATA:enabled}\|%{DATA:category}\|%{DATA:profile}\|%{DATA:description}\|%{DATA:signer}\|%{DATA:company}\|%{DATA:image_path}\|%{DATA:version}\|%{DATA:launch_string}\|%{DATA:md5}\|%{DATA:sha1}\|%{DATA:pesha1}\|%{DATA:pesha256}\|%{DATA:sha256}\|%{DATA:imphash}",
"full_log", "AR-LOG\|%{DATA:hostname}\|%{DATA:event_timestamp}\|%{DATA:entry_location}\|%{DATA:entry}\|%{DATA:enabled}\|%{DATA:category}\|%{DATA:profile}\|%{DATA:description}\|%{DATA:signer}\|%{DATA:company}\|%{DATA:image_path}\|%{DATA:version}\|%{DATA:launch_string}\|%{DATA:md5}\|%{DATA:sha1}\|%{DATA:pesha1}\|%{DATA:pesha256}\|%{DATA:sha256}\|%{DATA:imphash}"
]
}
mutate {
# Rename fields
}
}
date {
match => [ "image_timestamp", "yyyyMMdd-HHmmss" ]
target => "image_timestamp"
}
}
}

23
salt/logstash/pipelines/config/so/6600_winlogbeat_sysmon.conf
View File

@@ -1,23 +0,0 @@
# Author: Wes Lambert
#
# Last Update: 09/24/2018
#
# This conf file is based on accepting Sysmon logs from winlogbeat
filter {
if "beat" in [tags] and [source_name] =~ "Microsoft-Windows-Sysmon" {
mutate {
replace => { "type" => "sysmon" }
rename => { "[event_data][User]" => "username" }
rename => { "[event_data][DestinationPort]" => "destination_port" }
rename => { "[event_data][DestinationIp]" => "destination_ip" }
rename => { "[event_data][SourceIp]" => "source_ip" }
rename => { "[event_data][Image]" => "image_path" }
rename => { "[event_data][ParentImage]" => "parent_image_path" }
rename => { "[data][sysmon][targetfilename]" => "target_filename" }
rename => { "[event_data][SourceHostname]" => "source_hostname" }
rename => { "[event_data][DestinationHostname]" => "destination_hostname" }
rename => { "[event_data][TargetFilename]" => "target_filename" }
}
}
}

17
salt/logstash/pipelines/config/so/6700_winlogbeat.conf
View File

@@ -1,17 +0,0 @@
# Author: Doug Burks
#
# Last Update: 09/24/2018
#
# This conf file is for beat data
filter {
if "beat" in [tags] {
mutate {
# As of beats 6.3.0, host is now an object:
# https://www.elastic.co/guide/en/beats/libbeat/current/release-notes-6.3.0.html
# This creates a conflict with our existing host string.
# So let's rename the host object to beat_host.
rename => { "host" => "beat_host" }
}
}
}

23
salt/logstash/pipelines/config/so/7100_osquery_wel.conf
View File

@@ -1,23 +0,0 @@
# Author: Josh Brower
# Last Update: 12/28/2018
# If log is tagged osquery and there is an eventid column - then cleanup and parse out the EventData column
filter {
if "osquery" in [tags] and [osquery][columns][eventid] {
mutate {
gsub => ["[osquery][columns][data]", "\\x0A", ""]
}
json {
source => "[osquery][columns][data]"
target => "[osquery][columns][data]"
}
mutate {
merge => { "[osquery][columns]" => "[osquery][columns][data]" }
remove_field => ["[osquery][columns][data]"]
}
}
}

8
salt/logstash/pipelines/config/so/7200_strelka.conf
View File

@@ -1,8 +0,0 @@
filter {
if [type] =~ "strelka" {
json {
source => "message"
}
}
}

17
salt/logstash/pipelines/config/so/8000_postprocess_bro_cleanup.conf
View File

@@ -1,17 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
filter {
if "bro" in [tags] {
if "_grokparsefailure" not in [tags] and "_csvparsefailure" not in [tags] and "_jsonparsefailure" not in [tags] {
#mutate {
# remove_field => [ "message" ]
#}
}
mutate {
#add_tag => [ "conf_file_8000"]
}
}
}

58
salt/logstash/pipelines/config/so/8001_postprocess_common_ip_augmentation.conf
View File

@@ -1,58 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 5/20/2017
filter {
if [source_ip] {
if [source_ip] == "-" {
mutate {
replace => { "source_ip" => "0.0.0.0" }
}
}
if [source_ip] =~ "10\." or [source_ip] =~ "192\.168\." or [source_ip] =~ "172\.(1[6-9]|2[0-9]|3[0-1])\." or [source_ip] =~ "fe80::20c:29ff:fe19:f7d" or [source_ip] =~ "::1" {
mutate {
}
} else {
geoip {
source => "[source_ip]"
target => "source_geo"
}
}
if [source_ip] {
mutate {
add_field => { "ips" => "%{source_ip}" }
add_field => { "source_ips" => [ "%{source_ip}" ] }
}
}
}
if [destination_ip] {
if [destination_ip] == "-" {
mutate {
replace => { "destination_ip" => "0.0.0.0" }
}
}
if [destination_ip] =~ "10\." or [destination_ip] =~ "192\.168\." or [destination_ip] =~ "172\.(1[6-9]|2[0-9]|3[0-1])\." or [destination_ip] =~ "239.255.255.250" or [destination_ip] =~ "224\.0\.0\." or [destination_ip] =~ "255.255.255.255" or [destination_ip] =~ "ff02::fb" or [destination_ip] =~ "fe80::20c:29ff:fe19:f7d" or [destination_ip] =~ "224\.0\.1\." {
mutate {
}
}
else {
geoip {
source => "[destination_ip]"
target => "destination_geo"
}
}
}
if [destination_ip] {
mutate {
add_field => { "ips" => "%{destination_ip}" }
add_field => { "destination_ips" => [ "%{destination_ip}" ] }
}
}
}
#if [source_ip] or [destination_ip] {
# mutate {
#add_tag => [ "conf_file_8001"]
# }
#}

47
salt/logstash/pipelines/config/so/8006_postprocess_dns.conf
View File

@@ -1,47 +0,0 @@
# Original Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 5/13/2017
filter {
if [type] == "bro_dns" or "dns" in [tags] {
# Used for whois lookups - can create log loop
if [query] =~ "^whois\." {
drop { }
}
# REPLACE test.int with your internal domain
if [query] and [query] !~ "\.test\.int$" {
mutate {
lowercase => [ "query" ]
}
if [query_type_name] != "NB" and [query_type_name] != "TKEY" and [query_type_name] != "NBSTAT" and [query_type_name] != "PTR" {
tld {
source => "query"
}
ruby {
code => "event.set('query_length', event.get('query').length)"
}
mutate {
rename => { "[SubLog][sessionid]" => "sub_session_id" }
rename => { "[tld][domain]" => "highest_registered_domain" }
rename => { "[tld][trd]" => "subdomain" }
rename => { "[tld][tld]" => "top_level_domain" }
rename => { "[tld][sld]" => "parent_domain" }
}
if [parent_domain] {
ruby {
code => "event.set('parent_domain_length', event.get('parent_domain').length)"
}
}
if [subdomain] {
ruby {
code => "event.set('subdomain_length', event.get('subdomain').length)"
}
}
}
}
mutate {
#add_tag => [ "conf_file_8006"]
}
}
}

27
salt/logstash/pipelines/config/so/8007_postprocess_http.conf
View File

@@ -1,27 +0,0 @@
# Original Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 5/13/2017
filter {
if [type] == "bro_http" {
if [uri] {
ruby {
code => "event.set('uri_length', event.get('uri').length)"
}
}
if [virtual_host] {
ruby {
code => "event.set('virtual_host_length', event.get('virtual_host').length)"
}
}
if [useragent] {
ruby {
code => "event.set('useragent_length', event.get('useragent').length)"
}
}
mutate {
##add_tag => [ "conf_file_8007"]
}
}
}

63
salt/logstash/pipelines/config/so/8200_postprocess_tagging.conf
View File

@@ -1,63 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
filter {
if [destination_ip] {
if [destination_ip] =~ "10\." or [destination_ip] =~ "192\.168\." or [destination_ip] =~ "172\.(1[6-9]|2[0-9]|3[0-1])\." {
mutate {
add_tag => [ "internal_destination" ]
}
} else {
mutate {
add_tag => [ "external_destination" ]
}
}
if "internal_destination" not in [tags] {
if [destination_ip] == "198.41.0.4" or [destination_ip] == "192.228.79.201" or [destination_ip] == "192.33.4.12" or [destination_ip] == "199.7.91.13" or [destination_ip] == "192.203.230.10" or [destination_ip] == "192.5.5.241" or [destination_ip] == "192.112.36.4" or [destination_ip] == "198.97.190.53" or [destination_ip] == "192.36.148.17" or [destination_ip] == "192.58.128.30" or [destination_ip] == "193.0.14.129" or [destination_ip] == "199.7.83.42" or [destination_ip] == "202.12.27.33" {
mutate {
add_tag => [ "root_dns_server" ]
}
}
}
# Customize this section to your environment
if [destination_ip] == "74.40.74.40" or [destination_ip] == "74.40.74.41" {
mutate {
add_tag => [ "authorized_dns_server" ]
}
}
}
if [source_ip] {
if [source_ip] =~ "10\." or [source_ip] =~ "192\.168\." or [source_ip] =~ "172\.(1[6-9]|2[0-9]|3[0-1])\." {
mutate {
add_tag => [ "internal_source" ]
}
} else {
mutate {
add_tag => [ "external_source" ]
}
}
if "internal_source" not in [tags] {
if [source_ip] == "198.41.0.4" or [source_ip] == "192.228.79.201" or [source_ip] == "192.33.4.12" or [source_ip] == "199.7.91.13" or [source_ip] == "192.203.230.10" or [source_ip] == "192.5.5.241" or [source_ip] == "192.112.36.4" or [source_ip] == "198.97.190.53" or [source_ip] == "192.36.148.17" or [source_ip] == "192.58.128.30" or [source_ip] == "193.0.14.129" or [source_ip] == "199.7.83.42" or [source_ip] == "202.12.27.33" {
mutate {
add_tag => [ "root_dns_server" ]
}
}
}
# Customize this section to your environment
if [destination_ip] == "74.40.74.40" and "authorized_dns_server" not in [tags] or [destination_ip] == "74.40.74.41" and "authorized_dns_server" not in [tags] {
mutate {
add_tag => [ "authorized_dns_server" ]
}
}
mutate {
##add_tag => [ "conf_file_8200"]
}
}
if [type] =~ /ossec|snort|firewall/ or "firewall" in [tags] {
mutate {
remove_tag => [ "syslog" ]
}
}
}

19
salt/logstash/pipelines/config/so/8998_postprocess_log_elapsed.conf
View File

@@ -1,19 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
filter {
ruby {
code => "event.set('task_end', Time.now.to_f)"
}
ruby {
code => "event.set('logstash_time', event.get('task_end') - event.get('task_start'))"
}
mutate {
remove_field => [ 'task_start', 'task_end' ]
}
mutate {
#add_tag => [ "conf_file_8998"]
}
}

9
salt/logstash/pipelines/config/so/8999_postprocess_rename_type.conf
View File

@@ -1,9 +0,0 @@
# Author: Doug Burks
# Last Update: 12/10/2017
filter {
mutate {
rename => [ "type", "event_type" ]
remove_field => [ "host" ]
}
}

15
salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja
View File

@@ -1,15 +0,0 @@
{%- set ES = salt['grains.get']('master') -%}
{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
output {
if [event_type] == "sflow" {
elasticsearch {
hosts => "{{ ES }}"
user => "{{ ES_USER }}"
password => "{{ ES_PASS }}"
index => "so-flow"
ssl => true
ssl_certificate_verification => false
}
}
}

15
salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja
View File

@@ -1,15 +0,0 @@
{%- set ES = salt['grains.get']('master') -%}
{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
output {
if [event_type] == "ids" and "import" not in [tags] {
elasticsearch {
hosts => "{{ ES }}"
user => "{{ ES_USER }}"
password => "{{ ES_PASS }}"
index => "so-ids"
ssl => true
ssl_certificate_verification => false
}
}
}