CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2025-12-06 17:22:49 +01:00
Code Issues Packages Projects Releases Wiki Activity

Remove preprocess configuration

Browse Source
This commit is contained in:
Wes
2022-09-14 14:19:07 +00:00
parent 0d32cc38d6
commit 43f89adbd4
45 changed files with 0 additions and 2557 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
salt/logstash/pipelines/config/so/1000_preprocess_log_elapsed.conf
View File

@@ -1,13 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
filter {
ruby {
code => "event.set('task_start', Time.now.to_f)"
}
mutate {
#add_tag => [ "conf_file_1000"]
}
}

33
salt/logstash/pipelines/config/so/1001_preprocess_syslogng.conf
View File

@@ -1,33 +0,0 @@
# Updated by: Doug Burks and Wes Lambert
# Last Update: 10/30/2018
filter {
if "syslogng" in [tags] {
mutate {
rename => { "MESSAGE" => "message" }
rename => { "PROGRAM" => "type" }
rename => { "FACILITY" => "syslog-facility" }
rename => { "FILE_NAME" => "syslog-file_name" }
rename => { "HOST" => "syslog-host" }
rename => { "HOST_FROM" => "syslog-host_from" }
rename => { "LEGACY_MSGHDR" => "syslog-legacy_msghdr" }
rename => { "PID" => "syslog-pid" }
rename => { "PRIORITY" => "syslog-priority" }
rename => { "SOURCEIP" => "syslog-sourceip" }
rename => { "TAGS" => "syslog-tags" }
lowercase => [ "syslog-host_from" ]
remove_field => [ "ISODATE" ]
remove_field => [ "SEQNUM" ]
#add_tag => [ "conf_file_1001"]
}
if "bro_" in [type] {
mutate {
add_tag => [ "bro" ]
}
} else if [type] !~ /ossec.*|snort/ and "firewall" not in [tags] {
mutate {
add_tag => [ "syslog" ]
}
}
}
}

18
salt/logstash/pipelines/config/so/1002_preprocess_json.conf
View File

@@ -1,18 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
filter {
if "json" in [tags]{
json {
source => "message"
}
mutate {
remove_tag => [ "json" ]
}
mutate {
#add_tag => [ "conf_file_1002"]
}
}
}

19
salt/logstash/pipelines/config/so/1004_preprocess_syslog_types.conf
View File

@@ -1,19 +0,0 @@
filter {
if "syslog" in [tags] {
if [host] == "172.16.1.1" {
mutate {
add_field => { "type" => "fortinet" }
add_tag => [ "firewall" ]
}
}
if [host] == "10.0.0.101" {
mutate {
add_field => { "type" => "brocade" }
add_tag => [ "switch" ]
}
}
mutate {
#add_tag => [ "conf_file_1004"]
}
}
}

140
salt/logstash/pipelines/config/so/1026_preprocess_dhcp.conf
View File

@@ -1,140 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolutions.com
# Last Update: 12/9/2016
# This conf file is based on accepting logs for DHCP. It is currently based on Windows DHCP only.
filter {
if [type] == "dhcp" {
mutate {
add_field => { "Hostname" => "%{host}" }
}
mutate {
strip => "message"
}
# This is the initial parsing of the log
grok {
# Server 2008+
match => { "message" => "%{DATA:id},%{DATE_US:date},(?<time>%{HOUR}:%{MINUTE}:%{SECOND}),%{DATA:description},%{IPV4:ip},%{DATA:Hostname},%{DATA:mac},%{DATA:Username},%{INT:TransactionID},%{INT:QResult},%{DATA:ProbationTime},%{DATA:CorrelationID}"}
# Server 2003
match => { "message" => "%{DATA:id},%{DATE_US:date},(?<time>%{HOUR}:%{MINUTE}:%{SECOND}),%{DATA:description},%{IPV4:ip},%{DATA:Hostname},%{DATA:mac},"}
match => { "message" => "%{DATA:id},%{DATA:date},(?<time>%{HOUR}:%{MINUTE}:%{SECOND}),%{DATA:description},%{DATA:ip},%{DATA:Hostname},%{DATA:mac},"}
}
# This section below translates the message ID into something humans can understand.
if [id] == "00" {
mutate {
add_field => [ "event", "The log was started"]
}
}
if [id] == "01" {
mutate {
add_field => [ "event", "The log was stopped"]
}
}
if [id] == "02" {
mutate {
add_field => [ "event", "The log was temporarily paused due to low disk space"]
}
}
if [id] == "10" {
mutate {
add_field => [ "event", "A new IP address was leased to a client"]
}
}
if [id] == "11" {
mutate {
add_field => [ "event", "A lease was renewed by a client"]
}
}
if [id] == "12" {
mutate {
add_field => [ "event", "A lease was released by a client"]
}
}
if [id] == "13" {
mutate {
add_field => [ "event", "An IP address was found to be in use on the network"]
}
}
if [id] == "14" {
mutate {
add_field => [ "event", "A lease request could not be satisfied because the scope's address pool was exhausted"]
}
}
if [id] == "15" {
mutate {
add_field => [ "event", "A lease was denied"]
}
}
if [id] == "16" {
mutate {
add_field => [ "event", "A lease was deleted"]
}
}
if [id] == "17" {
mutate {
add_field => [ "event", "A lease was expired and DNS records for an expired leases have not been deleted"]
}
}
if [id] == "18" {
mutate {
add_field => [ "event", "A lease was expired and DNS records were deleted"]
}
}
if [id] == "20" {
mutate {
add_field => [ "event", "A BOOTP address was leased to a client"]
}
}
if [id] == "21" {
mutate {
add_field => [ "event", "A dynamic BOOTP address was leased to a client"]
}
}
if [id] == "22" {
mutate {
add_field => [ "event", "A BOOTP request could not be satisfied because the scope's address pool for BOOTP was exhausted"]
}
}
if [id] == "23" {
mutate {
add_field => [ "event", "A BOOTP IP address was deleted after checking to see it was not in use"]
}
}
if [id] == "24" {
mutate {
add_field => [ "event", "IP address cleanup operation has began"]
}
}
if [id] == "25" {
mutate {
add_field => [ "event", "IP address cleanup statistics"]
}
}
if [id] == "30" {
mutate {
add_field => [ "event", "DNS update request to the named DNS server"]
}
}
if [id] == "31" {
mutate {
add_field => [ "event", "DNS update failed"]
}
}
if [id] == "32" {
mutate {
add_field => [ "event", "DNS update successful"]
}
}
if [id] == "33" {
mutate {
add_field => [ "event", "Packet dropped due to NAP policy"]
}
}
# If the message failed to parse correctly keep the message for debugging. Otherwise, drop it.
#if "_grokparsefailure" not in [tags] {
# mutate {
# remove_field => [ "message"]
# }
#}
}
}

31
salt/logstash/pipelines/config/so/1029_preprocess_esxi.conf
View File

@@ -1,31 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
#
# This configuration file takes ESXi syslog messages and filters them. There is no input as the logs would have came in via syslog
filter {
# This is an example of using an IP address range to classify a syslog message to a specific type of log
# This is helpful as so many devices only send logs via syslog
if [host] =~ "10\.[0-1]\.9\." {
mutate {
replace => ["type", "esxi"]
}
}
if [host] =~ "\.234$" {
mutate {
replace => ["type", "esxi"]
}
}
if [type] == "esxi" {
grok {
match => { "message" => "(?:%{SYSLOGTIMESTAMP:timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) (?:%{SYSLOGHOST:logsource}) (?:%{SYSLOGPROG}): (?<messagebody>(?:\[(?<esxi_thread_id>[0-9A-Z]{8,8}) %{DATA:esxi_loglevel} \'%{DATA:esxi_service}\'\] %{GREEDYDATA:esxi_message}|%{GREEDYDATA}))"}
# pattern => ['(?:%{SYSLOGTIMESTAMP:timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) (?:%{SYSLOGHOST:logsource}) (?:%{SYSLOGPROG}): (?<messagebody>(?:\[(?<esxi_thread_id>[0-9A-Z]{8,8}) %{DATA:esxi_loglevel} \'%{DATA:esxi_service}\'\] %{GREEDYDATA:esxi_message}|%{GREEDYDATA}))']
}
mutate {
#add_tag => [ "conf_file_1029"]
}
}
}

21
salt/logstash/pipelines/config/so/1030_preprocess_greensql.conf
View File

@@ -1,21 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
filter {
if [type] == "greensql" {
# This section is parsing out the fields for GreenSQL syslog data
grok {
match => { "message" => "<%{INT:Code}>%{DATA:Category}\[%{INT:Transcation}\]:\s*Database=%{DATA:Database}\sUser=%{DATA:UserName}\sApplication Name=%{DATA:Application}\sSource IP=%{IPV4:SrcIp}\sSource Port=%{INT:SrcPort}\sTarget IP=?%{IPV4:DstIp}\sTarget Port=%{DATA:DstPort}\sQuery=%{GREEDYDATA:Query}"}
match => { "message" => "<%{INT:Code}>%{DATA:Category}\[%{INT:Transcation}\]:\sAdmin_Name=%{DATA:UserName}\sIP_Address=%{IPV4:SrcIp}\sUser_Agent=%{DATA:UserAgent}\sMessage=%{DATA:StatusMessage}\sDescription=%{DATA:Description}\sSeverity=%{GREEDYDATA:Severity}"}
}
# Remove the message field as it is unnecessary
#mutate {
# remove_field => [ "message"]
#}
mutate {
#add_tag => [ "conf_file_1030"]
}
}
}

21
salt/logstash/pipelines/config/so/1031_preprocess_iis.conf
View File

@@ -1,21 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
filter {
if [type] == "iis" {
# The log is expected to have come from NXLog and in JSON format. This allows for automatic parsing of fields
json {
source => "message"
}
# This removes the message field as it is unneccesary and tags the packet as web
mutate {
# remove_field => [ "message"]
add_tag => [ "web" ]
}
mutate {
#add_tag => [ "conf_file_1031"]
}
}
}

26
salt/logstash/pipelines/config/so/1032_preprocess_mcafee.conf
View File

@@ -1,26 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
#
# This file looks for McAfee EPO logs
filter {
if [type] == "mcafee" {
# NXLog should be sending the logs in JSON format so they auto parse
json {
source => "message"
}
# This section converts the UTC fields to the proper time format
date {
match => [ "ReceivedUTC", "YYYY-MM-dd HH:mm:ss" ]
target => [ "ReceivedUTC" ]
}
date {
match => [ "DetectedUTC", "YYYY-MM-dd HH:mm:ss" ]
target => [ "DetectedUTC" ]
}
mutate {
#add_tag => [ "conf_file_1032"]
}
}
}

125
salt/logstash/pipelines/config/so/1033_preprocess_snort.conf
View File

@@ -1,125 +0,0 @@
filter {
if [engine] == "suricata" {
json {
source => "message"
}
mutate {
# Make this compatible with event.id as a string
convert => { "[flow_id]" => "string" }
rename => {
"proto" => "[network][transport]"
"event_type" => "[event][dataset]"
"flow_id" => "[event][id]"
"community_id" => "[network][community_id]"
}
lowercase => [ "[network][transport]" ]
merge => {"[event][id]" => "[related][id]" }
add_field => {
"[related][domain]" => []
"[related][ip]" => []
"[related][id]" => []
"[event][module]" => "suricata"
"[event][created]" => "%{[@timestamp]}"
"[event][version]" => "1.0.0"
"[event][category]" => "network"
}
}
# Set the timestamp from the event
date {
match => [ "timestamp", "ISO8601" ]
tag_on_failure => [ "_dateparsefailure", "_parsefailure", "_suricata_dateparsefailure" ]
remove_field => [ "timestamp" ]
}
# Suricata uses top-level src/dest to track flow
if [src_ip] {
mutate {
rename => {
"[src_ip]" => "[source][ip]"
"[src_port]" => "[source][port]"
}
merge => { "[related][ip]" => "[source][ip]" }
}
}
if [dest_ip] {
mutate {
rename => {
"[dest_ip]" => "[destination][ip]"
"[dest_port]" => "[destination][port]"
}
merge => { "[related][ip]" => "[destination][ip]" }
}
}
if [vlan] {
mutate {
rename => { "[vlan]" => "[vlan][id]" }
}
}
if [app_proto] {
if [app_proto] == "failed" {
# delete failed detections to be consistent with zeek
mutate { rename => { "app_proto" => "[error][message]" } }
}
else {
mutate { rename => {"app_proto" => "[network][protocol]"}}
}
}
if [event_type] == "alert" {
if [alert][severity] == 1 {
mutate {
add_field => { "severity" => "High" }
}
}
if [alert][severity] == 2 {
mutate {
add_field => { "severity" => "Medium" }
}
}
if [alert][severity] == 3 {
mutate {
add_field => { "severity" => "Low" }
}
}
# If the alert is a Snort GPL alert break it apart for easier reading and categorization
if [alert][signature] =~ "GPL " {
# This will parse out the category type from the alert
grok {
match => { "[alert][signature]" => "GPL\s+%{DATA:category}\s" }
}
# This will store the category
mutate {
add_field => { "rule_type" => "Snort GPL" }
lowercase => [ "category" ]
}
}
# If the alert is an Emerging Threat alert break it apart for easier reading and categorization
if [alert][signature] =~ "ET " {
# This will parse out the category type from the alert
grok {
match => { "[alert][signature]" => "ET\s+%{DATA:category}\s" }
}
# This will store the category
mutate {
add_field => { "rule_type" => "Emerging Threats" }
lowercase => [ "category" ]
}
}
# This section adds URLs to lookup information about a rule online
if [rule_type] == "Snort GPL" {
mutate {
add_field => [ "signature_info", "https://www.snort.org/search?query=%{[alert][gid]}-%{[alert][signature_id]}" ]
}
}
if [rule_type] == "Emerging Threats" {
mutate {
add_field => [ "signature_info", "http://doc.emergingthreats.net/%{[alert][signature_id]}" ]
}
}
}
mutate {
remove_field => [ "alert" ]
}
}
}

16
salt/logstash/pipelines/config/so/1034_preprocess_syslog.conf
View File

@@ -1,16 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 5/22/2017
filter {
if [type] == "syslog" {
# This drops syslog messages regarding license messages. You may want to comment it out.
#if [message] =~ "license" {
# drop { }
#}
mutate {
#convert => [ "status_code", "integer" ]
}
}
}

77
salt/logstash/pipelines/config/so/1100_preprocess_bro_conn.conf
View File

@@ -1,77 +0,0 @@
# Original Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
#
# This conf file is based on accepting logs for conn.log from Bro systems
filter {
if [type] == "bro_conn" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
rename => { "proto" => "protocol" }
#service
#duration
rename => { "orig_bytes" => "original_bytes" }
rename => { "resp_bytes" => "respond_bytes" }
rename => { "conn_state" => "connection_state" }
#local_orig
rename => { "local_resp" => "local_respond" }
#missed_bytes
#history
rename => { "orig_pkts" => "original_packets" }
rename => { "orig_ip_bytes" => "original_ip_bytes" }
rename => { "resp_pkts" => "respond_packets" }
rename => { "resp_ip_bytes" => "respond_ip_bytes" }
#tunnel_parents
rename => { "orig_cc" => "original_country_code" }
rename => { "resp_cc" => "respond_country_code" }
rename => { "sensorname" => "sensor_name" }
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","protocol","service","duration","original_bytes","respond_bytes","connection_state","local_orig","local_respond","missed_bytes","history","original_packets","original_ip_bytes","respond_packets","respond_ip_bytes","tunnel_parents","sensor_name"]
# If you use a custom delimiter, change the following value in between the quotes to your delimiter. Otherwise, insert a literal <tab> in between the two quotes on your logstash system, use a text editor like nano that doesn't convert tabs to spaces.
separator => " "
}
}
translate {
field => "connection_state"
destination => "connection_state_description"
dictionary => [
"S0", "Connection attempt seen, no reply",
"S1", "Connection established, not terminated",
"S2", "Connection established and close attempt by originator seen (but no reply from responder)",
"S3", "Connection established and close attempt by responder seen (but no reply from originator)",
"SF", "Normal SYN/FIN completion",
"REJ", "Connection attempt rejected",
"RSTO", "Connection established, originator aborted (sent a RST)",
"RSTR", "Established, responder aborted",
"RSTOS0", "Originator sent a SYN followed by a RST, we never saw a SYN-ACK from the responder",
"RSTRH", "Responder sent a SYN ACK followed by a RST, we never saw a SYN from the (purported) originator",
"SH", "Originator sent a SYN followed by a FIN, we never saw a SYN ACK from the responder (hence the connection was 'half' open)",
"SHR", "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator",
"OTH", "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed)"
]
}
mutate {
#add_tag => [ "conf_file_1100"]
}
}
}

56
salt/logstash/pipelines/config/so/1101_preprocess_bro_dhcp.conf
View File

@@ -1,56 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks and Wes Lambert
# Last Update: 1/3/2019
#
# This conf file is based on accepting logs for dhcp.log from Bro systems
filter {
if [type] == "bro_dhcp" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#mac
#assigned_ip
#lease_time
rename => { "trans_id" => "transaction_id" }
# new dhcp log format
rename => { "assigned_addr" => "assigned_ip" }
rename => { "client_addr" => "source_ip" }
rename => { "server_addr" => "destination_ip" }
rename => { "requested_addr" => "requested_ip" }
rename => { "domain" => "domain_name" }
rename => { "host_name" => "hostname" }
rename => { "msg_types" => "message_types" }
rename => { "uids" => "uid" }
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
# Bro logs in TSV format
csv {
columns => [ "timestamp", "uid", "source_ip", "destination_ip", "mac", "hostname", "client_fqdn", "domain_name", "requested_ip", "assigned_ip", "lease_time","client_message", "server_message", "message_types", "duration" ]
separator => " "
}
# Remove fields with empty values (-) to prevent field data type conflict
ruby {
code =>"
hash = event.to_hash.each do |key,value|
if value == '-'
event.remove(key)
end
end"
}
}
}
}

74
salt/logstash/pipelines/config/so/1102_preprocess_bro_dns.conf
View File

@@ -1,74 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for dns.log from Bro systems
filter {
if [type] == "bro_dns" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
rename => { "proto" => "protocol" }
rename => { "trans_id" => "transaction_id" }
#rtt field
#query field
rename => { "qclass" => "query_class" }
rename => { "qclass_name" => "query_class_name" }
rename => { "qtype" => "query_type" }
rename => { "qtype_name" => "query_type_name" }
#rcode
#rcode_name
rename => { "AA" => "aa" }
rename => { "TC" => "tc" }
rename => { "RD" => "rd" }
rename => { "RA" => "ra" }
rename => { "Z" => "z" }
#answers
rename => { "TTLs" => "ttls" }
#rejected
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","protocol","transaction_id","rtt","query","query_class","query_class_name","query_type","query_type_name","rcode","rcode_name","aa","tc","rd","ra","z","answers","ttls","rejected"]
#If you use a custom delimiter, change the following value in between the quotes to your delimiter. Otherwise, insert a literal <tab> in between the two quotes on your logstash system, use a text editor like nano that doesn't convert tabs to spaces.
separator => " "
}
}
mutate {
add_tag => [ "dns" ]
}
if [ttls] == "-" {
mutate {
remove_field => [ "ttls" ]
}
}
if [rtt] == "-" {
mutate {
remove_field => [ "rtt" ]
}
}
#mutate {
#convert => [ "rtt", "float" ]
#}
mutate {
#add_tag => [ "conf_file_1102"]
}
}
}

42
salt/logstash/pipelines/config/so/1103_preprocess_bro_dpd.conf
View File

@@ -1,42 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for dpd.log from Bro systems
filter {
if [type] == "bro_dpd" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
rename => { "proto" => "protocol" }
#analyzer
#failure_reason
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","protocol","analyzer","failure_reason"]
separator => " "
}
}
mutate {
#add_tag => [ "conf_file_1103"]
}
}
}

64
salt/logstash/pipelines/config/so/1104_preprocess_bro_files.conf
View File

@@ -1,64 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for files.log from Bro systems
filter {
if [type] == "bro_files" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#fuid
rename => { "tx_hosts" => "file_ip" }
rename => { "rx_hosts" => "destination_ip" }
rename => { "conn_uids" => "connection_uids" }
#source field
#depth field
rename => { "analyzers" => "analyzer" }
rename => { "mime_type" => "mimetype" }
rename => { "filename" => "file_name" }
#duration
#local_orig
#is_orig
#seen_bytes
#total_bytes
#missing_bytes
#overflow_bytes
rename => { "timedout" => "timed_out" }
#parent_fuid
#md5
#sha1
#sha256
#extracted
#extracted_cutoff
#extracted_size
}
} else {
csv {
columns => ["timestamp","fuid","file_ip","destination_ip","connection_uids","source","depth","analyzer","mimetype","file_name","duration","local_orig","is_orig","seen_bytes","total_bytes","missing_bytes","overflow_bytes","timed_out","parent_fuid","md5","sha1","sha256","extracted","extracted_cutoff","extracted_size"]
separator => " "
}
if [destination_ip] =~ /,/ {
mutate {
split => { "destination_ip" => "," }
}
}
if [file_ip] =~ /,/ {
mutate {
split => { "file_ip" => "," }
}
}
}
mutate {
#add_tag => [ "conf_file_1104"]
}
}
}

56
salt/logstash/pipelines/config/so/1105_preprocess_bro_ftp.conf
View File

@@ -1,56 +0,0 @@
# Original Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for ftp.log from Bro systems
filter {
if [type] == "bro_ftp" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
rename => { "user" => "username" }
#password
rename => { "command" => "ftp_command" }
rename => { "arg" => "ftp_argument" }
rename => { "mime_type" => "mimetype" }
#file_size
#reply_code
rename => { "reply_msg" => "reply_message" }
rename => { "data_channel.passive" => "data_channel_passive" }
rename => { "data_channel.orig_h" => "data_channel_source_ip" }
rename => { "data_channel.resp_h" => "data_channel_destination_ip" }
rename => { "data_channel.resp_p" => "data_channel_destination_port" }
#fuid
}
mutate {
convert => { "reply" => "string" }
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","username","password","ftp_command","ftp_argument","mimetype","file_size","reply_code","reply_message","data_channel_passive","data_channel_source_ip","data_channel_destination_ip","data_channel_destination_port","fuid"]
separator => " "
}
}
mutate {
#add_tag => [ "conf_file_1105"]
}
}
}

77
salt/logstash/pipelines/config/so/1106_preprocess_bro_http.conf
View File

@@ -1,77 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
#
# This conf file is based on accepting logs for http.log from Bro systems
filter {
if [type] == "bro_http" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
# Rename logstash tags field to avoid being overwritten by Bro's http tags field
mutate {
rename => { "tags" => "tags-orig" }
}
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#trans_depth
#method
rename => { "host" => "virtual_host" }
#uri
#referrer
#version
#convert => { "version" => "string" }
rename => { "user_agent" => "useragent" }
#origin
rename => { "request_body_len" => "request_body_length" }
rename => { "response_body_len" => "response_body_length" }
#status_code
#status_message
rename => { "status_msg" => "status_message" }
#info_code
rename => { "info_msg" => "info_message" }
#tags
# Rename http tags field to http-tags
rename => { "tags" => "http-tags" }
# Rename logstash tags field to tags
rename => { "tags-orig" => "tags" }
#username
#password
#proxied
#orig_fuids
#orig_filenames
#orig_mime_types
#resp_fuids
#resp_filenames
#resp_mime_types
}
if [http-tags] {
mutate {
remove_field => [ "http-tags" ]
}
}
} else {
grok {
match => [ "message", "(?<timestamp>(.*?))\t(?<uid>(.*?))\t(?<source_ip>(.*?))\t(?<source_port>(.*?))\t(?<destination_ip>(.*?))\t(?<destination_port>(.*?))\t(?<trans_depth>(.*?))\t(?<method>(.*?))\t(?<virtual_host>(.*?))\t(?<uri>(.*?))\t(?<referrer>(.*?))\t(?<version>(.*?))\t(?<useragent>(.*?))\t(?<origin>(.*?))\t(?<request_body_length>(.*?))\t(?<response_body_length>(.*?))\t(?<status_code>(.*?))\t(?<status_message>(.*?))\t(?<info_code>(.*?))\t(?<info_message>(.*?))\t(?<tags>(.*))\t(?<username>(.*))\t(?<password>(.*))\t(?<proxied>(.*))\t(?<orig_fuids>(.*))\t(?<orig_filenames>(.*?))\t(?<orig_mime_types>(.*))\t(?<resp_fuids>(.*))\t(?<resp_filenames>(.*?))\t(?<resp_mime_types>(.*))" ]
}
}
if [useragent] == "-" {
mutate {
remove_field => [ "useragent" ]
}
}
mutate {
#add_tag => [ "conf_file_1106"]
}
}
}

46
salt/logstash/pipelines/config/so/1107_preprocess_bro_irc.conf
View File

@@ -1,46 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for irc.log from Bro systems
filter {
if [type] == "bro_irc" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#nick
rename => { "user" => "irc_username" }
rename => { "command" => "irc_command" }
#value
rename => { "addl" => "additional_info" }
#dcc_file_name
#dcc_file_size
#dcc_mime_type
#fuid
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","nick","irc_username","irc_command","value","additional_info","dcc_file_name","dcc_file_size","dcc_mime_type","fuid"]
separator => " "
}
}
mutate {
#add_tag => [ "conf_file_1107"]
}
}
}

56
salt/logstash/pipelines/config/so/1108_preprocess_bro_kerberos.conf
View File

@@ -1,56 +0,0 @@
# Original Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for kerberos.log from Bro systems
filter {
if [type] == "bro_kerberos" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#request_type
#client
#service
rename => { "success" => "kerberos_success" }
rename => { "error_msg" => "error_message" }
rename => { "from" => "valid_from" }
rename => { "till" => "valid_till" }
#cipher
#forwardable
#renewable
rename => { "client_cert_subject" => "client_certificate_subject" }
rename => { "client_cert_fuid" => "client_certificate_fuid" }
rename => { "server_cert_subject" => "server_certificate_subject" }
rename => { "server_cert_fuid" => "server_certificate_fuid" }
}
mutate {
convert => { "kerberos_success" => "string" }
convert => { "renewable" => "string" }
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","request_type","client","service","kerberos_success","error_message","valid_from","valid_till","cipher","forwardable","renewable","client_certificate_subject","client_certificate_fuid","server_certificate_subject","server_certificate_fuid"]
separator => " "
}
}
mutate {
#add_tag => [ "conf_file_1108"]
}
}
}

56
salt/logstash/pipelines/config/so/1109_preprocess_bro_notice.conf
View File

@@ -1,56 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for notice.log from Bro systems
filter {
if [type] == "bro_notice" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#fuid
rename => { "mime" => "file_mime_type" }
rename => { "desc" => "file_description" }
rename => { "proto" => "protocol" }
rename => { "note" => "note" }
rename => { "msg" => "msg" }
rename => { "sub" => "sub_msg" }
rename => { "src" => "source_ip" }
rename => { "dst" => "destination_ip" }
#p
#n
rename => { "peer_descr" => "peer_description" }
rename => { "actions" => "action" }
#suppress_for
#destination_country_code
#destination_region
#destination_city
#destination_latitude
#destination_longitude
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","fuid","file_mime_type","file_description","protocol","note","msg","sub_msg","source_ip","destination_ip","p","n","peer_description","action","suppress_for","destination_country_code","destination_region","destination_city","destination_latitude","destination_longitude"]
separator => " "
}
}
mutate {
#add_tag => [ "conf_file_1109"]
}
}
}

52
salt/logstash/pipelines/config/so/1110_preprocess_bro_rdp.conf
View File

@@ -1,52 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
#
# This conf file is based on accepting logs for rdp.log from Bro systems
filter {
if [type] == "bro_rdp" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#cookie
#result
#security_protocol
#client_channels
#keyboard_layout
#client_build
#client_name
rename => { "client_dig_product_id" => "client_digital_product_id" }
#desktop_width
#desktop_height
#requested_color_depth
rename => { "cert_type" => "certificate_type" }
rename => { "cert_count" => "certificate_count" }
rename => { "cert_permanent" => "certificate_permanent" }
#encryption_level
#encryption_method
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","cookie","result","security_protocol","client_channels","keyboard_layout","client_build","client_name","client_digital_product_id","desktop_width","desktop_height","requested_color_depth","certificate_type","certificate_count","certificate_permanent","encryption_level","encryption_method"]
separator => " "
}
}
mutate {
#add_tag => [ "conf_file_1110"]
}
}
}

43
salt/logstash/pipelines/config/so/1111_preprocess_bro_signatures.conf
View File

@@ -1,43 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for signatures.log from Bro systems
filter {
if [type] == "bro_signatures" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#note
rename => { "sig_id" => "signature_id" }
rename => { "event_msg" => "event_message" }
rename => { "sub_msg" => "sub_message" }
rename => { "sig_count" => "signature_count" }
#host_count
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","note","signature_id","event_message","sub_message","signature_count","host_count"]
separator => " "
}
}
mutate {
#add_tag => [ "conf_file_1111"]
}
}
}

65
salt/logstash/pipelines/config/so/1112_preprocess_bro_smtp.conf
View File

@@ -1,65 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for smtp.log from Bro systems
filter {
if [type] == "bro_smtp" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#trans_depth
#helo
rename => { "mailfrom" => "mail_from" }
rename => { "rcptto" => "recipient_to" }
rename => { "date" => "mail_date" }
#from
#to
#cc
#reply_to
rename => { "msg_id" => "message_id" }
#in_reply_to
#subject
#x_originating_ip
#first_received
#second_received
#last_reply
#path
rename => { "user_agent" => "useragent" }
#tls
#fuids
#is_webmail
}
mutate {
convert => { "tls" => "string" }
convert => { "is_webmail" => "string" }
}
} else {
grok {
match => [ "message", "(?<timestamp>(.*?))\t(?<uid>(.*?))\t(?<source_ip>(.*?))\t(?<source_port>(.*?))\t(?<destination_ip>(.*?))\t(?<destination_port>(.*?))\t(?<trans_depth>(.*?))\t(?<helo>(.*?))\t(?<mail_from>(.*?))\t(?<recipient_to>(.*?))\t(?<mail_date>(.*?))\t(?<from>(.*?))\t(?<to>(.*?))\t(?<cc>(.*?))\t(?<reply_to>(.*?))\t(?<message_id>(.*?))\t(?<in_reply_to>(.*?))\t(?<subject>(.*?))\t(?<x_originating_ip>(.*?))\t(?<first_received>(.*))\t(?<second_received>(.*))\t(?<last_reply>(.*))\t(?<path>(.*))\t(?<useragent>(.*))\t(?<tls>(.*))\t(?<fuids>(.*))\t(?<is_webmail>(.*))" ]
}
}
if [useragent] == "-" {
mutate {
remove_field => [ "useragent" ]
}
}
mutate {
#add_tag => [ "conf_file_1112"]
}
}
}

47
salt/logstash/pipelines/config/so/1113_preprocess_bro_snmp.conf
View File

@@ -1,47 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for snmp.log from Bro systems
filter {
if [type] == "bro_snmp" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#duration
#version
#convert => { "version" => "string" }
#community
#get_requests
#get_bulk_requests
#get_responses
#set_requests
#display_string
#up_since
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","duration","version","community","get_requests","get_bulk_requests","get_responses","set_requests","display_string","up_since"]
separator => " "
}
}
mutate {
#add_tag => [ "conf_file_1113"]
}
}
}

49
salt/logstash/pipelines/config/so/1114_preprocess_bro_software.conf
View File

@@ -1,49 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for software.log from Bro systems
filter {
if [type] == "bro_software" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "host" => "source_ip" }
rename => { "host_p" => "source_port" }
#software_type
#name
rename => { "version.major" => "version_major" }
rename => { "version.minor" => "version_minor" }
rename => { "version.minor2" => "version_minor2" }
rename => { "version.minor3" => "version_minor3" }
rename => { "version.addl" => "version_additional_info" }
#unparsed_version
}
mutate {
convert => { "version_major" => "string" }
convert => { "version_minor" => "string" }
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","source_ip","source_port","software_type","name","version_major","version_minor","version_minor2","version_minor3","version_additional_info","unparsed_version"]
separator => " "
}
}
mutate {
#add_tag => [ "conf_file_1114"]
}
}
}

66
salt/logstash/pipelines/config/so/1115_preprocess_bro_ssh.conf
View File

@@ -1,66 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks and Wes Lambert
# Last Update: 10/30/2018
#
# This conf file is based on accepting logs for ssh.log from Bro systems
filter {
if [type] == "bro_ssh" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#version
#convert => { "version" => "string" }
rename => { "auth_success" => "authentication_success" }
rename => { "auth_attempts" => "authentication_attempts" }
#direction
#client
#server
rename => { "cipher_alg" => "cipher_algorithm" }
rename => { "compression_alg" => "compression_algorithm" }
rename => { "cshka" => "client_host_key_algorithms" }
rename => { "host_key_alg" => "host_key_algorithm" }
rename => { "hasshAlgorithms" => "hassh_algorithms" }
rename => { "hasshServer" => "hassh_server" }
rename => { "hasshServerAlgorithms" => "hassh_server_algorithms" }
rename => { "hasshVersion" => "hassh_version" }
rename => { "kex_alg" => "kex_algorithm" }
rename => { "mac_alg" => "mac_algorithm" }
rename => { "sshka" => "server_host_key_algorithms" }
#host_key
#destination_country_code
#destination_region
#destination_city
#destination_latitude
#destination_longitude
}
mutate {
convert => { "authentication_success" => "string" }
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","version","authentication_success","authentication_attempts","direction","client","server","cipher_algorithm","mac_algorithm","compression_algorithm","kex_algorithm","host_key_algorithm","host_key","destination_country_code","destination_region","destination_city","destination_latitude","destination_longitude","hassh_version","hassh","hassh_server","client_host_key_algorithms","hassh_algorithms","server_host_key_algorithms","hassh_server_algorithms"]
separator => " "
}
}
mutate {
#add_tag => [ "conf_file_1115"]
}
}
}

186
salt/logstash/pipelines/config/so/1116_preprocess_bro_ssl.conf
View File

@@ -1,186 +0,0 @@
# Original Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 10/30/2018
#
# This conf file is based on accepting logs for ssl.log from Bro systems
filter {
if [type] == "bro_ssl" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#version
#convert => { "version" => "string" }
#cipher
#curve
#server_name
#resumed
#last_alert
#next_protocol
#established
rename => { "cert_chain_fuids" => "certificate_chain_fuids" }
rename => { "client_cert_chain_fuids" => "client_certificate_chain_fuids" }
rename => { "subject" => "certificate_subject" }
rename => { "issuer" => "certificate_issuer" }
#client_subject
#client_issuer
#validation_status
#ja3
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","version","cipher","curve","server_name","resumed","last_alert","next_protocol","established","certificate_chain_fuids","client_certificate_chain_fuids","certificate_subject","certificate_issuer","client_subject","client_issuer","validation_status","ja3","ja3s"]
separator => " "
}
}
mutate {
gsub => [ "subject", "\\\\,", "|" ]
}
kv {
include_keys => [ "CN", "C", "O", "OU", "ST", "SN", "L", "DC", "GN", "pseudonym", "serialNumber", "title", "initials" ]
field_split => ","
source => "certificate_issuer"
}
mutate {
rename => { "CN" => "issuer_common_name"}
rename => { "C" => "issuer_country_code"}
rename => { "O" => "issuer_organization"}
rename => { "OU" => "issuer_organization_unit"}
rename => { "ST" => "issuer_state"}
rename => { "SN" => "issuer_surname"}
rename => { "L" => "issuer_locality"}
rename => { "DC" => "issuer_distinguished_name"}
rename => { "GN" => "issuer_given_name"}
rename => { "pseudonym" => "issuer_pseudonym"}
rename => { "serialNumber" => "issuer_serial_number"}
rename => { "title" => "issuer_title"}
rename => { "initials" => "issuer_initials"}
}
kv {
include_keys => [ "CN", "C", "O", "OU", "ST", "SN", "L", "GN", "pseudonym", "serialNumber", "title", "initials" ]
field_split => ","
source => "certificate_subject"
}
mutate {
rename => { "CN" => "certificate_common_name"}
rename => { "C" => "certificate_country_code"}
rename => { "O" => "certificate_organization"}
rename => { "OU" => "certificate_organization_unit"}
rename => { "ST" => "certificate_state"}
rename => { "SN" => "certificate_surname"}
rename => { "L" => "certificate_locality"}
rename => { "GN" => "certificate_given_name"}
rename => { "pseudonym" => "certificate_pseudonym"}
rename => { "serialNumber" => "certificate_serial_number"}
rename => { "title" => "certificate_title"}
rename => { "initials" => "certificate_initials"}
}
if [certificate_subject] == "-" {
mutate {
remove_field => [ "certificate_subject" ]
}
}
if [certificate_issuer] == "-" {
mutate {
remove_field => [ "certificate_issuer" ]
}
}
if [certificate_common_name] {
ruby {
code => "event.set('certificate_common_name_length', event.get('certificate_common_name').length)"
}
}
if [issuer_common_name] {
ruby {
code => "event.set('issuer_common_name_length', event.get('issuer_common_name').length)"
}
}
if [server_name] {
if [server_name] == "-" {
mutate {
remove_field => [ "server_name" ]
}
} else {
ruby {
code => "event.set('server_name_length', event.get('server_name').length)"
}
}
}
if [certificate_chain_fuids] {
if [certificate_chain_fuids] == "-" {
mutate {
remove_field => [ "certificate_chain_fuids" ]
}
} else {
ruby {
code => "event.set('certificate_chain_count', event.get('certificate_chain_fuids').count(',') + 1)"
}
mutate {
convert => [ "certificate_chain_length", "integer" ]
}
}
}
if [client_certificate_chain_fuids] == "-" {
mutate {
remove_field => [ "client_certificate_chain_fuids" ]
}
}
if [client_issuer] == "-" {
mutate {
remove_field => [ "client_issuer" ]
}
}
if [client_subject] == "-" {
mutate {
remove_field => [ "client_subject" ]
}
}
if [curve] == "-" {
mutate {
remove_field => [ "curve" ]
}
}
if [issuer] == "-" {
mutate {
remove_field => [ "issuer" ]
}
}
if [query] == "-" {
mutate {
remove_field => [ "query" ]
}
}
if [subject] == "-" {
mutate {
remove_field => [ "subject" ]
}
}
if [validation_status] == "-" {
mutate {
remove_field => [ "validation_status" ]
}
}
if [ja3] == "-" {
mutate {
remove_field => [ "ja3" ]
}
}
mutate {
#add_tag => [ "conf_file_1116"]
}
}
}

41
salt/logstash/pipelines/config/so/1117_preprocess_bro_syslog.conf
View File

@@ -1,41 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for syslog.log from Bro systems
filter {
if [type] == "bro_syslog" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
rename => { "proto" => "protocol" }
#facility
#severity
#message
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","protocol","facility","severity","message"]
separator => " "
}
}
mutate {
#add_tag => [ "conf_file_1117"]
}
}
}

40
salt/logstash/pipelines/config/so/1118_preprocess_bro_tunnel.conf
View File

@@ -1,40 +0,0 @@
# Original Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for tunnel.log from Bro systems
# Security Onion syslog-ng.conf sets type to "bro_tunnels"
filter {
if [type] == "bro_tunnels" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#tunnel_type
#action
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","tunnel_type","action"]
separator => " "
}
}
mutate {
#add_tag => [ "conf_file_1118"]
}
}
}

42
salt/logstash/pipelines/config/so/1119_preprocess_bro_weird.conf
View File

@@ -1,42 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for weird.log from Bro systems
filter {
if [type] == "bro_weird" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#name
rename => { "addl" => "additional_info" }
#notice
#peer
}
mutate {
convert => { "notice" => "string" }
}
} else {
grok {
match => [ "message", "(?<timestamp>(.*?))\t(?<uid>(.*?))\t(?<source_ip>(.*?))\t(?<source_port>(.*?))\t(?<destination_ip>(.*?))\t(?<destination_port>(.*?))\t(?<name>(.*?))\t(?<additional_info>(.*?))\t(?<notice>(.*?))\t(?<peer>(.*))" ]
}
}
mutate {
#add_tag => [ "conf_file_1119"]
}
}
}

57
salt/logstash/pipelines/config/so/1121_preprocess_bro_mysql.conf
View File

@@ -1,57 +0,0 @@
# Original Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for mysql.log from Bro systems
#
# Parse using grok
filter {
if [type] == "bro_mysql" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
rename => { "cmd" => "mysql_command" }
rename => { "arg" => "mysql_argument" }
rename => { "success" => "mysql_success" }
#rows
#response
}
mutate {
convert => { "mysql_success" => "string" }
}
} else {
grok {
match => [ "message", "(?<timestamp>(.*?))\t(?<uid>(.*?))\t(?<source_ip>(.*?))\t(?<source_port>(.*?))\t(?<destination_ip>(.*?))\t(?<destination_port>(.*?))\t(?<mysql_command>(.*?))\t(?<mysql_argument>(.*?))\t(?<mysql_success>(.*?))\t(?<rows>(.*?))\t(?<response>(.*))" ]
}
}
mutate {
#add_tag => [ "conf_file_1121"]
}
}
}
# Reverting to grok for now, due to double-quoted values in log file
# Parse using csv filter
#filter {
# if [type] == "bro_mysql" {
# csv {
# columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","mysql_command","mysql_argument","mysql_success","rows","response"]
# separator => " "
# quote_char=
# }
# }
#}

62
salt/logstash/pipelines/config/so/1122_preprocess_bro_socks.conf
View File

@@ -1,62 +0,0 @@
# Original Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for socks.log from Bro systems
# Parse using csv
filter {
if [type] == "bro_socks" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#version
#convert => { "version" => "string" }
rename => { "user" => "username" }
#password
rename => { "status" => "server_status" }
rename => { "request.host" => "request_host" }
rename => { "request.name" => "request_name" }
rename => { "request_p" => "request_port" }
rename => { "bound.host" => "bound_host" }
rename => { "bound.name" => "bound_name" }
rename => { "bound_p" => "bound_port" }
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","version","username","password","server_status","request_host","request_name","request_port","bound_host","bound_name","bound_port"]
separator => " "
}
}
mutate {
#add_tag => [ "conf_file_1122"]
}
}
}
# Parse using grok
#filter {
# if [type] == "bro_socks" {
# # This is the initial parsing of the log
# grok {
# match => [ "message", "(?<timestamp>(.*?))\t(?<uid>(.*?))\t(?<source_ip>(.*?))\t(?<source_port>(.*?))\t(?<destination_ip>(.*?))\t(?<destination_port>(.*?))\t(?<version>(.*?))\t(?<user>(.*?))\t(?<password>(.*?))\t(?<status>(.*))\t(?<request_host>(.*))\t(?<request_name>(.*))\t(?<request_port>(.*))\t(?<bound_host>(.*))\t(?<bound_name>(.*))\t(?<bound_port>(.*))" ]
# }
# mutate {
# #add_tag => [ "conf_file_1122"]
# }
# }
#}

154
salt/logstash/pipelines/config/so/1123_preprocess_bro_x509.conf
View File

@@ -1,154 +0,0 @@
# Original Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for x509.log from Bro systems
filter {
if [type] == "bro_x509" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#id
rename => { "certificate.version" => "certificate_version" }
rename => { "certificate.serial" => "certificate_serial" }
rename => { "certificate.subject" => "certificate_subject" }
rename => { "certificate.issuer" => "certificate_issuer" }
rename => { "certificate.not_valid_before" => "certificate_not_valid_before" }
rename => { "certificate.not_valid_after" => "certificate_not_valid_after" }
rename => { "certificate.key_alg" => "certificate_key_algorithm" }
rename => { "certificate.sig_alg" => "certificate_signing_algorithm" }
rename => { "certificate.key_type" => "certificate_key_type" }
rename => { "certificate.key_length" => "certificate_key_length" }
rename => { "certificate.exponent" => "certificate_exponent" }
rename => { "certificate.curve" => "certificate_curve" }
rename => { "id" => "fuid" }
rename => { "san.dns" => "san_dns" }
rename => { "san.uri" => "san_uri" }
rename => { "san.email" => "san_email" }
rename => { "san.ip" => "san_ip" }
rename => { "basic_constraints.ca" => "basic_constraints_ca" }
rename => { "basic_constraints.path_length" => "basic_constraints_path_length" }
}
} else {
grok {
match => [ "message", "(?<timestamp>(.*?))\t(?<fuid>(.*?))\t(?<certificate_version>(.*?))\t(?<certificate_serial>(.*?))\t(?<certificate_subject>(.*?))\t(?<certificate_issuer>(.*?))\t(?<certificate_not_valid_before>(.*?))\t(?<certificate_not_valid_after>(.*?))\t(?<certificate_key_algorithm>(.*?))\t(?<certificate_signing_algorithm>(.*))\t(?<certificate_key_type>(.*))\t(?<certificate_key_length>(.*))\t(?<certificate_exponent>(.*))\t(?<certificate_curve>(.*))\t(?<san_dns>(.*))\t(?<san_uri>(.*))\t(?<san_email>(.*))\t(?<san_ip>(.*))\t(?<basic_constraints_ca>(.*))\t(?<basic_constraints_path_length>(.*))" ]
}
}
mutate {
gsub => [ "certificate_issuer", "\\\\,", "|" ]
gsub => [ "certificate_subject", "\\\\,", "|" ]
}
kv {
include_keys => [ "CN", "C", "O", "OU", "ST", "SN", "L", "DC", "GN", "pseudonym", "serialNumber", "title", "initials" ]
field_split => ","
source => "certificate_issuer"
}
mutate {
rename => { "CN" => "issuer_common_name"}
rename => { "C" => "issuer_country_code"}
rename => { "O" => "issuer_organization"}
rename => { "OU" => "issuer_organization_unit"}
rename => { "ST" => "issuer_state"}
rename => { "SN" => "issuer_surname"}
rename => { "L" => "issuer_locality"}
rename => { "DC" => "issuer_distinguished_name"}
rename => { "GN" => "issuer_given_name"}
rename => { "pseudonym" => "issuer_pseudonym"}
rename => { "serialNumber" => "issuer_serial_number"}
rename => { "title" => "issuer_title"}
rename => { "initials" => "issuer_initials"}
}
kv {
include_keys => [ "CN", "C", "O", "OU", "ST", "SN", "L", "GN", "pseudonym", "serialNumber", "title", "initials" ]
field_split => ","
source => "certificate_subject"
}
mutate {
rename => { "CN" => "certificate_common_name"}
rename => { "C" => "certificate_country_code"}
rename => { "O" => "certificate_organization"}
rename => { "OU" => "certificate_organization_unit"}
rename => { "ST" => "certificate_state"}
rename => { "SN" => "certificate_surname"}
rename => { "L" => "certificate_locality"}
rename => { "GN" => "certificate_given_name"}
rename => { "pseudonym" => "certificate_pseudonym"}
rename => { "serialNumber" => "certificate_serial_number"}
rename => { "title" => "certificate_title"}
rename => { "initials" => "certificate_initials"}
convert => [ "certificate_key_length", "integer" ]
convert => [ "certificate_not_valid_after", "integer" ]
convert => [ "certificate_not_valid_before", "integer" ]
}
if [query] == "-" {
mutate {
remove_field => [ "query" ]
}
}
if [san_dns] == "-" {
mutate {
remove_field => [ "san_dns" ]
}
}
if [san_email] == "-" {
mutate {
remove_field => [ "san_email" ]
}
}
if [san_uri] == "-" {
mutate {
remove_field => [ "san_uri" ]
}
}
if [san_ip] == "-" {
mutate {
remove_field => [ "san_ip" ]
}
}
if [certificate_common_name] {
ruby {
code => "event.set('certificate_common_name_length', event.get('certificate_common_name').length)"
}
}
if [issuer_common_name] {
ruby {
code => "event.set('issuer_common_name_length', event.get('issuer_common_name').length)"
}
}
if [certificate_not_valid_after] == "-" {
mutate {
remove_field => [ "certificate_not_valid_after" ]
}
}
if [certificate_not_valid_before] == "-" {
mutate {
remove_field => [ "certificate_not_valid_before" ]
}
}
if [certificate_not_valid_after] and [certificate_not_valid_before] {
ruby {
code => "event.set('certificate_number_days_valid', ((event.get('certificate_not_valid_after') - event.get('certificate_not_valid_before')) / 86400).ceil)"
}
date {
match => [ "certificate_not_valid_after", "UNIX" ]
target => "certificate_not_valid_after"
}
date {
match => [ "certificate_not_valid_before", "UNIX" ]
target => "certificate_not_valid_before"
}
}
mutate {
#add_tag => [ "conf_file_1123"]
}
}
}

46
salt/logstash/pipelines/config/so/1124_preprocess_bro_intel.conf
View File

@@ -1,46 +0,0 @@
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for intel.log from Bro systems
filter {
if [type] == "bro_intel" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
rename => { "seen.indicator" => "indicator" }
rename => { "seen.indicator_type" => "indicator_type" }
rename => { "seen.where" => "seen_where" }
rename => { "seen.node" => "seen_node" }
#matched
#sources
#fuid
rename => { "file_mime_type" => "mimetype" }
rename => { "file_desc" => "file_description" }
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","indicator","indicator_type","seen_where","seen_node","matched","sources","fuid","mimetype","file_description"]
separator => " "
}
}
mutate {
#add_tag => [ "conf_file_1124"]
}
}
}

49
salt/logstash/pipelines/config/so/1125_preprocess_bro_modbus.conf
View File

@@ -1,49 +0,0 @@
# Author: Wes Lambert
# Adapted from existing filters provided by Justin Henderson
#
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for modbus.log from Bro systems
#
filter {
if [type] == "bro_modbus" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
rename => { "func" => "function" }
#exception
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","function","exception"]
separator => " "
}
}
}
}
# Parse using grok
#filter {
# if [type] == "bro_modbus" {
# grok {
# match => [ "message", "(?<timestamp>(.*?))\t(?<uid>(.*?))\t(?<source_ip>(.*?))\t(?<source_port>(.*?))\t(?<destination_ip>(.*?))\t(?<destination_port>(.*?))\t(?<func>(.*?))\t(?<exception>(.*?))$" ]
# }
#mutate {
#add_tag => [ "conf_file_1125"]
#}
# }
#}

66
salt/logstash/pipelines/config/so/1126_preprocess_bro_sip.conf
View File

@@ -1,66 +0,0 @@
# Author: Wes Lambert
#
# Adapted from existing filters provided by Justin Henderson
#
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for sip.log from Bro systems
#
filter {
if [type] == "bro_sip" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#trans_depth
#method
#uri
#date
#request_from
#request_to
#response_from
#response_to
#reply_to
#call_id
#seq
#subject
#request_path
#response_path
#user_agent
#status_code
#status_msg
#warning
rename => { "request_body_len" => "request_body_length" }
rename => { "response_body_len" => "response_body_length" }
#content_type
}
} else {
grok {
match => [ "message", "(?<timestamp>(.*?))\t(?<uid>(.*?))\t(?<source_ip>(.*?))\t(?<source_port>(.*?))\t(?<destination_ip>(.*?))\t(?<destination_port>(.*?))\t(?<trans_depth>(.*?))\t(?<method>(.*?))\t(?<uri>(.*?))\t(?<date>(.*?))\t(?<request_from>(.*?))\t(?<request_to>(.*?))\t(?<response_from>(.*?))\t(?<response_to>(.*?))\t(?<reply_to>(.*?))\t(?<call_id>(.*?))\t(?<seq>(.*?))\t(?<subject>(.*?))\t(?<request_path>(.*?))\t(?<response_path>(.*?))\t(?<user_agent>(.*?))\t(?<status_code>(.*?))\t(?<status_msg>(.*?))\t(?<warning>(.*?))\t(?<request_body_length>(.*?))\t(?<response_body_length>(.*?))\t(?<content_type>(.*?))$" ]
}
}
mutate {
add_tag => [ "conf_file_1126"]
}
}
}
# Parse using csv filter
#filter {
# if [type] == "bro_sip" {
# csv {
# columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","trans_depth","method","uri","date","request_from","request_to","response_from","response_to","reply_to","call_id","seq","subject","request_path","response_path","user_agent","status_code","status_msg","warning","request_body_len","response_body_len","content_type"]
# separator => " "
# }
# }
#}

73
salt/logstash/pipelines/config/so/1127_preprocess_bro_radius.conf
View File

@@ -1,73 +0,0 @@
# Author: Wes Lambert
#
# Adapted from existing filters provided by Justin Henderson
#
# Updated by: Doug Burks
#
# This conf file is based on accepting logs for radius.log from Bro systems
#
filter {
if [type] == "bro_radius" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#username
#mac
#framed_addr
#tunnel_client
#connect_info
rename => { "reply_msg" => "reply_message" }
#result
#ttl
#logged
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","username","mac","framed_addr","tunnel_client","connect_info","reply_message","result","ttl","logged"]
separator => " "
}
if [tunnel_client] == "-" {
mutate {
remove_field => [ "tunnel_client" ]
}
}
}
# Remove the ttl and framed_addr fields
if [ttl] {
mutate {
remove_field => [ "ttl" ]
}
}
if [framed_addr] {
mutate {
remove_field => [ "framed_addr" ]
}
}
}
}
# Parse using grok
#filter {
# if [type] == "bro_radius" {
# grok {
# match => [ "message", "(?<timestamp>(.*?))\t(?<uid>(.*?))\t(?<source_ip>(.*?))\t(?<source_port>(.*?))\t(?<destination_ip>(.*?))\t(?<destination_port>(.*?))\t(?<username>(.*?))\t(?<mac>(.*?))\t(?<tunnel_client>(.*?))\t(?<logged>(.*?))\t(?<connect_info>(.*?))$" ]
# }
# mutate {
# #add_tag => [ "conf_file_1127"]
# }
# }
#}

46
salt/logstash/pipelines/config/so/1128_preprocess_bro_pe.conf
View File

@@ -1,46 +0,0 @@
# Author: Wes Lambert
#
# Adapted from existing filters provided by Justin Henderson
#
# Updated by: Doug Burks
#
# This conf file is based on accepting logs for pe.log from Bro systems
#
filter {
if [type] == "bro_pe" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
rename => { "id" => "fuid" }
#machine
#compile_ts
#os
#subsystem
#is_exe
#is_64bit
#uses_aslr
#uses_dep
#uses_code_integrity
#uses_seh
#has_import_table
#has_export_table
#has_cert_table
#has_debug_data
#section_names
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","fuid","machine","compile_ts","os","subsystem","is_exe","is_64bit","uses_aslr","uses_dep","uses_code_integrity","uses_seh","has_import_table","has_export_table","has_cert_table","has_debug_data","section_names"]
separator => " "
}
}
}
}

65
salt/logstash/pipelines/config/so/1129_preprocess_bro_rfb.conf
View File

@@ -1,65 +0,0 @@
# Author: Wes Lambert
#
# Adapted from existing filters provided by Justin Henderson
#
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for rfb.log from Bro systems
#
# Parse using csv filter
filter {
if [type] == "bro_rfb" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#client_major_version
#client_minor_version
#server_major_version
#server_minor_version
#authentication_method
#auth
#share_flag
#desktop_name
#width
#height
}
mutate {
convert => { "auth" => "string" }
convert => { "share_flag" => "string" }
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","client_major_version","client_minor_version","server_major_version","server_minor_version","authentication_method","auth","share_flag","desktop_name","width","height"]
separator => " "
}
}
}
}
# Parse using grok
#filter {
# if [type] == "bro_rfb" {
# grok {
# match => [ "message", "(?<timestamp>(.*?))\t(?<uid>(.*?))\t(?<source_ip>(.*?))\t(?<source_port>(.*?))\t(?<destination_ip>(.*?))\t(?<destination_port>(.*?))\t(?<client_major_version>(.*?))\t(?<client_minor_version>(.*?))\t(?<server_major_version>(.*?))\t(?<server_minor_version>(.*?))\t(?<authentication_method>(.*?))\t(?<auth>(.*?))\t(?<share_flag>(.*?))\t(?<desktop_name>(.*?))\t(?<width>(.*?))\t(?<height>(.*?))$" ]
# }
# mutate {
# #add_tag => [ "conf_file_1129"]
# }
# }
#}

51
salt/logstash/pipelines/config/so/1130_preprocess_bro_dnp3.conf
View File

@@ -1,51 +0,0 @@
# Author: Wes Lambert
#
# Adapted from existing filters provided by Justin Henderson
#
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for dnp3.log from Bro systems
#
filter {
if [type] == "bro_dnp3" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#fc_request
#fc_reply
#iin
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","fc_request","fc_reply","iin"]
separator => " "
}
}
}
}
# Parse using grok
#filter {
# if [type] == "bro_dnp3" {
# grok {
# match => [ "message", "(?<timestamp>(.*?))\t(?<uid>(.*?))\t(?<source_ip>(.*?))\t(?<source_port>(.*?))\t(?<destination_ip>(.*?))\t(?<destination_port>(.*?))\t(?<fc_request>(.*?))\t(?<fc_reply>(.*?))\t(?<iin>(.*?))$" ]
# }
# mutate {
# #add_tag => [ "conf_file_1130"]
# }
# }
#}

46
salt/logstash/pipelines/config/so/1131_preprocess_bro_smb_files.conf
View File

@@ -1,46 +0,0 @@
# Author: Wes Lambert
#
# Adapted from existing filters provided by Justin Henderson
#
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for smb_files.log from Bro systems
#
filter {
if [type] == "bro_smb_files" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#fuid
#action
#path
#name
#size
#prev_name
rename => { "times.modified" => "times_modified" }
rename => { "times.accessed" => "times_accessed" }
rename => { "times.created" => "times_created" }
rename => { "times.changed" => "times_changed" }
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","fuid","action","path","name","size","prev_name","times_modified","times_accessed","times_created","times_changed"]
separator => " "
}
}
}
}

40
salt/logstash/pipelines/config/so/1132_preprocess_bro_smb_mapping.conf
View File

@@ -1,40 +0,0 @@
# Author: Wes Lambert
#
# Adapted from existing filters provided by Justin Henderson
#
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for smb_mapping.log from Bro systems
#
filter {
if [type] == "bro_smb_mapping" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#path
#service
#native_file_system
#share_type
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","path","service","native_file_system","share_type"]
separator => " "
}
}
}
}

50
salt/logstash/pipelines/config/so/1133_preprocess_bro_ntlm.conf
View File

@@ -1,50 +0,0 @@
# Author: Wes Lambert
#
# Adapted from existing filters provided by Justin Henderson
#
# Updated by: Doug Burks and Wes Lambert
# Last Update: 1/2/2019
#
# This conf file is based on accepting logs for ntlm.log from Bro systems
#
filter {
if [type] == "bro_ntlm" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#hostname
rename => { "domainname" => "domain_name" }
rename => { "success" => "ntlm_success" }
#status
}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => [ "timestamp", "uid", "source_ip", "source_port", "destination_ip", "destination_port", "username", "hostname", "domain_name", "server_nb_computer_name", "server_dns_computer_name", "server_tree_name", "ntlm_success"]
separator => " "
}
ruby {
code =>"
hash = event.to_hash.each do |key,value|
if value == '-'
event.remove(key)
end
end"
}
}
}
}

54
salt/logstash/pipelines/config/so/1134_preprocess_bro_dce_rpc.conf
View File

@@ -1,54 +0,0 @@
# Author: Wes Lambert
#
# Adapted from existing filters provided by Justin Henderson
#
# Updated by: Doug Burks
# Last Update: 2/7/2018
#
# This conf file is based on accepting logs for dce_rpc.log from Bro systems
#
filter {
if [type] == "bro_dce_rpc" {
# If message looks like json, try to parse it as such. Otherwise, fall back to csv or grok.
if [message] =~ /^{.*}$/ {
json {
source => "message"
}
mutate {
rename => { "ts" => "timestamp" }
#uid
rename => { "id.orig_h" => "source_ip" }
rename => { "id.orig_p" => "source_port" }
rename => { "id.resp_h" => "destination_ip" }
rename => { "id.resp_p" => "destination_port" }
#rtt
#named_pipe
#endpoint
#operation
}
#mutate {
#convert => { "rtt" => "float" }
#}
} else {
mutate {
gsub => [ "message", "[\"']", "" ]
}
csv {
columns => ["timestamp","uid","source_ip","source_port","destination_ip","destination_port","rtt","named_pipe","endpoint","operation"]
separator => " "
}
if [rtt] == "-" {
mutate {
remove_field => [ "rtt" ]
}
}
#mutate {
#convert => [ "rtt", "float" ]
#}
}
}
}