diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 000000000..6dac78507 Binary files /dev/null and b/.DS_Store differ diff --git a/pillar/sensors/example.sls b/pillar/sensors/example.sls new file mode 100644 index 000000000..77068ef91 --- /dev/null +++ b/pillar/sensors/example.sls @@ -0,0 +1,6 @@ +# Example Pillar file for a sensor +sensor: + interface: CHANGEME + lbprocs: CHANGEME + + diff --git a/pillar/top.sls b/pillar/top.sls new file mode 100644 index 000000000..868da6033 --- /dev/null +++ b/pillar/top.sls @@ -0,0 +1,4 @@ +base: + 'G@role:so-sensor' + - sensor.schedule + - sensors.{{ hostname }} diff --git a/salt/bro/files/node.cfg b/salt/bro/files/node.cfg new file mode 100644 index 000000000..c9a0a5665 --- /dev/null +++ b/salt/bro/files/node.cfg @@ -0,0 +1,19 @@ +{%- set interface = salt['pillar.get'](sensor:interface) %} +{%- set lbprocs = salt['pillar.get'](sensor:lbprocs) %} +[manager] +type=manager +host=localhost + +[proxy] +type=proxy +host=localhost + +[sotest-eth1] +type=worker +host=localhost +interface=af_packet::{{ interface }} +lb_method=custom +lb_procs={{ lbprocs }} +af_packet_fanout_id=23 +af_packet_fanout_mode=AF_Packet::FANOUT_HASH +af_packet_buffer_size=128*1024*1024 diff --git a/salt/bro/init.sls b/salt/bro/init.sls new file mode 100644 index 000000000..cca953e7b --- /dev/null +++ b/salt/bro/init.sls @@ -0,0 +1,35 @@ +# Bro Salt State +# Add Bro User +bro: + user.present: + - uid: 937 + - gid: 937 + - home: /home/bro + +file.directory: + - name: /opt/so/conf/bro + - user: 937 + - group: 939 + +file.directory: + - name: /opt/so/conf/bro/policy + - user: 937 + - group: 939 + +# Add the container + +so-bro: + dockerng.running: + - image: {{ dockerrepo }}/so-bro:{{ broversion }} + - hostname: bro + - user: bro + - priviledged: true + - binds: + - /nsm/bro/logs:/nsm/bro/logs:rw + - /nsm/bro/spool:/nsm/bro/spool:rw + - /opt/so/conf/bro/etc:/opt/bro/etc:ro + - /opt/so/conf/bro/etc/node.cfg:/opt/bro/etc/node.cfg:ro + - /opt/so/conf/share/bro:/opt/bro/share/bro:ro + - network_mode: host + +# Add Bro cron diff --git a/salt/common/init.sls b/salt/common/init.sls new file mode 100644 index 000000000..a9d18118c --- /dev/null +++ b/salt/common/init.sls @@ -0,0 +1,37 @@ +# Create a state directory + +statedir: + file.directory: + - name: /opt/so/state + +salttmp: + file.directory: + - name: /opt/so/tmp + +# Install packages needed for the sensor + +sensorpkgs: + pkg.installed: + - skip_suggestions: True + - pkgs: + - docker + +# Always keep these packages up to date + +alwaysupdated: + pkg.latest: + - openssl + - openssh-server + - bash + - skip_suggestions: True + +# Set time to UTC + +Etc/UTC: + timezone.system + +# Set up docker network +dockernet: + dockerng.network_present + - name: so-docker-net + - driver: bridge diff --git a/salt/logstash/files/beats-template.json b/salt/logstash/files/beats-template.json new file mode 100644 index 000000000..b0f9edc20 --- /dev/null +++ b/salt/logstash/files/beats-template.json @@ -0,0 +1,182 @@ +{ + "index_patterns" : "beats-*", + "version" : 50001, + "settings" : { + "number_of_replicas": 0, + "number_of_shards": 1, + "index.refresh_interval" : "5s" + }, + "mappings": { + "doc": { + "_meta": { + "version": "5.6.4" + }, + "date_detection": false, + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "activity_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "beat": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "computer_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "event_id": { + "type": "long" + }, + "keywords": { + "ignore_above": 1024, + "type": "keyword" + }, + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "log_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "message_error": { + "ignore_above": 1024, + "type": "keyword" + }, + "meta": { + "properties": { + "cloud": { + "properties": { + "availability_zone": { + "ignore_above": 1024, + "type": "keyword" + }, + "instance_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "machine_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "project_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "region": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "opcode": { + "ignore_above": 1024, + "type": "keyword" + }, + "process_id": { + "type": "long" + }, + "provider_guid": { + "ignore_above": 1024, + "type": "keyword" + }, + "record_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "related_activity_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "source_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, + "task": { + "ignore_above": 1024, + "type": "keyword" + }, + "thread_id": { + "type": "long" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "identifier": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "version": { + "type": "long" + }, + "xml": { + "norms": false, + "type": "text" + } + } + } + }, + "order": 0, + "settings": { + "index.mapping.total_fields.limit": 10000, + "index.refresh_interval": "5s" + } +} diff --git a/salt/logstash/files/conf.d/0000_input_syslogng.conf b/salt/logstash/files/conf.d/0000_input_syslogng.conf new file mode 100644 index 000000000..791045f46 --- /dev/null +++ b/salt/logstash/files/conf.d/0000_input_syslogng.conf @@ -0,0 +1,19 @@ +# Original Author: Justin Henderson +# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics +# Updated by: Doug Burks +# Last Update: 5/15/2017 + +input { + tcp { + port => 6050 + codec => json + tags => "syslogng" + } +} +filter { + if "syslogng" in [tags] { + mutate { + #add_tag => [ "conf_file_0000"] + } + } +} diff --git a/salt/logstash/files/conf.d/0001_input_json.conf b/salt/logstash/files/conf.d/0001_input_json.conf new file mode 100644 index 000000000..4df89d293 --- /dev/null +++ b/salt/logstash/files/conf.d/0001_input_json.conf @@ -0,0 +1,19 @@ +# Author: Justin Henderson +# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics +# Email: justin@hasecuritysolution.com +# Last Update: 12/9/2016 + +input { + tcp { + port => 6051 + codec => json + tags => [ "json" ] + } +} +filter { + if "json" in [tags] { + mutate { + #add_tag => [ "conf_file_0001"] + } + } +} \ No newline at end of file diff --git a/salt/logstash/files/conf.d/0002_input_windows_json.conf b/salt/logstash/files/conf.d/0002_input_windows_json.conf new file mode 100644 index 000000000..54b700bd5 --- /dev/null +++ b/salt/logstash/files/conf.d/0002_input_windows_json.conf @@ -0,0 +1,22 @@ +# Author: Justin Henderson +# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics +# Email: justin@hasecuritysolution.com +# Last Update: 12/9/2016 + +input { + tcp { + port => 6052 + type => "windows" + tags => [ "json" ] + codec => json { + charset => "CP1252" + } + } +} +filter { + if [type] == "windows" { + mutate { + #add_tag => [ "conf_file_0002"] + } + } +} \ No newline at end of file diff --git a/salt/logstash/files/conf.d/0003_input_syslog.conf b/salt/logstash/files/conf.d/0003_input_syslog.conf new file mode 100644 index 000000000..dbd1c29bb --- /dev/null +++ b/salt/logstash/files/conf.d/0003_input_syslog.conf @@ -0,0 +1,18 @@ +# Original Author: Justin Henderson +# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics +# Updated by: Doug Burks +# Last Update: 5/15/2017 + +#input { +# udp { +# port => 1514 +# tags => "syslog" +# } +#} +#filter { +# if "syslog" in [tags] { +# mutate { +# #add_tag => [ "conf_file_0003"] +# } +# } +#} diff --git a/salt/logstash/files/conf.d/0005_input_suricata.conf b/salt/logstash/files/conf.d/0005_input_suricata.conf new file mode 100644 index 000000000..d3d23063a --- /dev/null +++ b/salt/logstash/files/conf.d/0005_input_suricata.conf @@ -0,0 +1,19 @@ +# Author: Justin Henderson +# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics +# Email: justin@hasecuritysolution.com +# Last Update: 12/9/2016 + +input { + tcp { + port => 6053 + codec => json + type => "suricata" + } +} +filter { + if [type] == "suricata" { + mutate { + #add_tag => [ "conf_file_0005"] + } + } +} \ No newline at end of file diff --git a/salt/logstash/files/conf.d/0006_input_beats.conf b/salt/logstash/files/conf.d/0006_input_beats.conf new file mode 100644 index 000000000..d4a57c998 --- /dev/null +++ b/salt/logstash/files/conf.d/0006_input_beats.conf @@ -0,0 +1,11 @@ +# Author: Justin Henderson +# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics +# Email: justin@hasecuritysolutions.com +# Last Update: 12/11/2017 + +input { + beats { + port => "5044" + tags => [ "beat" ] + } +} diff --git a/salt/logstash/files/conf.d/0007_input_import.conf b/salt/logstash/files/conf.d/0007_input_import.conf new file mode 100644 index 000000000..c502b57ca --- /dev/null +++ b/salt/logstash/files/conf.d/0007_input_import.conf @@ -0,0 +1,182 @@ +# Updated by: Doug Burks +# Last Update: 5/16/2017 + +input { + file { + path => "/nsm/import/bro/conn*" + type => "bro_conn" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/dce_rpc*" + type => "bro_dce_rpc" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/dhcp*" + type => "bro_dhcp" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/dnp3*" + type => "bro_dnp3" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/dns*" + type => "bro_dns" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/dpd*" + type => "bro_dpd" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/files*" + type => "bro_files" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/ftp*" + type => "bro_ftp" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/http*" + type => "bro_http" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/intel*" + type => "bro_intel" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/irc*" + type => "bro_irc" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/kerberos*" + type => "bro_kerberos" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/modbus*" + type => "bro_modbus" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/mysql*" + type => "bro_mysql" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/notice*" + type => "bro_notice" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/ntlm*" + type => "bro_ntlm" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/pe*" + type => "bro_pe" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/radius*" + type => "bro_radius" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/rdp*" + type => "bro_rdp" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/rfb*" + type => "bro_rfb" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/signatures*" + type => "bro_signatures" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/sip*" + type => "bro_sip" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/smb_files*" + type => "bro_smb_files" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/smb_mapping*" + type => "bro_smb_mapping" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/smtp*" + type => "bro_smtp" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/snmp*" + type => "bro_snmp" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/socks*" + type => "bro_socks" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/software*" + type => "bro_software" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/ssh*" + type => "bro_ssh" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/ssl*" + type => "bro_ssl" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/syslog*" + type => "bro_syslog" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/tunnel*" + type => "bro_tunnels" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/weird*" + type => "bro_weird" + tags => ["bro", "import"] + } + file { + path => "/nsm/import/bro/x509*" + type => "bro_x509" + tags => ["bro", "import"] + } +} +filter { + if "import" in [tags] { + mutate { + #add_tag => [ "conf_file_0006"] + } + } +} diff --git a/salt/logstash/files/conf.d/1000_preprocess_log_elapsed.conf b/salt/logstash/files/conf.d/1000_preprocess_log_elapsed.conf new file mode 100644 index 000000000..d098eb11a --- /dev/null +++ b/salt/logstash/files/conf.d/1000_preprocess_log_elapsed.conf @@ -0,0 +1,13 @@ +# Author: Justin Henderson +# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics +# Email: justin@hasecuritysolution.com +# Last Update: 12/9/2016 + +filter { + ruby { + code => "event.set('task_start', Time.now.to_f)" + } + mutate { + #add_tag => [ "conf_file_1000"] + } +} diff --git a/salt/logstash/files/conf.d/1001_preprocess_syslogng.conf b/salt/logstash/files/conf.d/1001_preprocess_syslogng.conf new file mode 100644 index 000000000..d2467a3f8 --- /dev/null +++ b/salt/logstash/files/conf.d/1001_preprocess_syslogng.conf @@ -0,0 +1,30 @@ +# Updated by: Doug Burks +# Last Update: 5/15/2017 + +filter { + if "syslogng" in [tags] { + mutate { + rename => { "MESSAGE" => "message" } + rename => { "PROGRAM" => "type" } + rename => { "FACILITY" => "syslog-facility" } + rename => { "FILE_NAME" => "syslog-file_name" } + rename => { "HOST" => "syslog-host" } + rename => { "HOST_FROM" => "syslog-host_from" } + rename => { "LEGACY_MSGHDR" => "syslog-legacy_msghdr" } + rename => { "PID" => "syslog-pid" } + rename => { "PRIORITY" => "syslog-priority" } + rename => { "SOURCEIP" => "syslog-sourceip" } + rename => { "TAGS" => "syslog-tags" } + #add_tag => [ "conf_file_1000"] + } + if "bro_" in [type] { + mutate { + add_tag => [ "bro"] + } + } else { + mutate { + add_tag => [ "syslog"] + } + } + } +} diff --git a/salt/logstash/files/conf.d/1002_preprocess_json.conf b/salt/logstash/files/conf.d/1002_preprocess_json.conf new file mode 100644 index 000000000..8aff64715 --- /dev/null +++ b/salt/logstash/files/conf.d/1002_preprocess_json.conf @@ -0,0 +1,18 @@ +# Author: Justin Henderson +# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics +# Email: justin@hasecuritysolution.com +# Last Update: 12/9/2016 + +filter { + if "json" in [tags]{ + json { + source => "message" + } + mutate { + remove_tag => [ "json" ] + } + mutate { + #add_tag => [ "conf_file_1001"] + } + } +} diff --git a/salt/logstash/files/conf.d/1003_preprocess_bro.conf b/salt/logstash/files/conf.d/1003_preprocess_bro.conf new file mode 100644 index 000000000..e24da1329 --- /dev/null +++ b/salt/logstash/files/conf.d/1003_preprocess_bro.conf @@ -0,0 +1,24 @@ +# Author: Justin Henderson +# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics +# Updated by: Doug Burks +# Last Update: 5/15/2017 + +filter { + if "bro" in [tags] { + # If a log comes in with a message starting with # then drop it as it doesn'then + # contain anything and is the header of a rotated bro log + if [message] =~ /^#/ { + drop { } + } else { + # Replace the host field with the host found in the bro log + if [bro_host] { +# mutate { +# replace => [ "host", "%{bro_host}" ] +# } + } + } + mutate { + #add_tag => [ "conf_file_1002"] + } + } +} diff --git a/salt/logstash/files/conf.d/1004_preprocess_syslog_types.conf b/salt/logstash/files/conf.d/1004_preprocess_syslog_types.conf new file mode 100644 index 000000000..5b47968b2 --- /dev/null +++ b/salt/logstash/files/conf.d/1004_preprocess_syslog_types.conf @@ -0,0 +1,19 @@ +filter { + if "syslog" in [tags] { + if [host] == "172.16.1.1" { + mutate { + add_field => { "type" => "fortinet" } + add_tag => [ "firewall" ] + } + } + if [host] == "10.0.0.101" { + mutate { + add_field => { "type" => "brocade" } + add_tag => [ "switch" ] + } + } + mutate { + #add_tag => [ "conf_file_1003"] + } + } +} diff --git a/salt/logstash/files/conf.d/1026_preprocess_dhcp.conf b/salt/logstash/files/conf.d/1026_preprocess_dhcp.conf new file mode 100644 index 000000000..6ba00012f --- /dev/null +++ b/salt/logstash/files/conf.d/1026_preprocess_dhcp.conf @@ -0,0 +1,156 @@ +# Author: Justin Henderson +# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics +# Email: justin@hasecuritysolution.com +# Last Update: 12/9/2016 +# +# This conf file is based on accepting logs for DHCP. It is currently based on Windows DHCP only. +filter { + if [type] == "dhcp" { + mutate { + add_field => { "Hostname" => "%{host}" } + } + mutate { + strip => "message" + } + } + # If the message contains nothing then drop it + if [message] =~ /^$/ { + drop { } + } + # If the message starts with # then drop it as it is the header of the DHCP log. + # This behavior is normal when the log is rotated. + if [message] =~ /^#/ { + drop { } + } else { + if [type] == "dhcp" { + # This is the initial parsing of the log + grok { + # Server 2008+ + match => { "message" => "%{DATA:id},%{DATE_US:date},(?