Merge branch 'dev' into feature/setup-changes

# Conflicts:
#	setup/so-setup

files/firewall/hostgroups.local.yaml

@@ -13,6 +13,7 @@ firewall:
         delete:
         insert:
     elasticsearch_rest:
+      ips:
         delete:
         insert:
     fleet:

salt/common/tools/sbin/so-allow

@@ -15,7 +15,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-source ./so-common
+. /usr/sbin/so-common
 
 local_salt_dir=/opt/so/saltstack/local
 

salt/common/tools/sbin/so-import-pcap

@@ -21,18 +21,9 @@
 
 function usage {
   cat << EOF
-Usage:
+Usage: $0 <pcap-file-1> [pcap-file-2] [pcap-file-N]
-Please supply at least one pcap file.
-
-For example, to import a single pcap named import.pcap:
-so-import-pcap import.pcap
-
-To import multiple pcaps:
-so-import-pcap import1.pcap import2.pcap
-
-** IMPORTANT **
-Security Onion installations contain processes that automatically discard old data. Therefore, imports of old network traffic might immediately be erased, unless those processes are first disabled. 
 
+Imports one or more PCAP files for analysis. If available, curator will be automatically stopped.
 EOF
 }
 
@@ -76,11 +67,13 @@ function zeek() {
 
   NSM_PATH=/nsm/import/${HASH}/zeek
   mkdir -p $NSM_PATH/logs
-  mkdir -p $NSM_PATH/spool
   mkdir -p $NSM_PATH/extracted
+  mkdir -p $NSM_PATH/spool
   chown -R zeek:socore $NSM_PATH
   docker run --rm \
-    -v $NSM_PATH:/nsm:rw \
+    -v $NSM_PATH/logs:/nsm/zeek/logs:rw \
+    -v $NSM_PATH/spool:/nsm/zeek/spool:rw \
+    -v $NSM_PATH/extracted:/nsm/zeek/extracted:rw \
     -v $PCAP:/input.pcap:ro \
     -v /opt/so/conf/zeek/local.zeek:/opt/zeek/share/zeek/site/local.zeek:ro \
     -v /opt/so/conf/zeek/node.cfg:/opt/zeek/etc/node.cfg:ro \
@@ -99,13 +92,13 @@ function zeek() {
 # if no parameters supplied, display usage
 if [ $# -eq 0 ]; then
   usage
-  exit
+  exit 1
 fi
 
-# ensure this is a manager node
+# ensure this is a sensor node
-if [ ! -d /opt/so/conf/soc ]; then
+if [ ! -d /opt/so/conf/suricata ]; then
-  echo "This procedure must be run on a manager node."
+  echo "This command must be run on a sensor node."
-  exit
+  exit 3
 fi
 
 # verify that all parameters are files
@@ -113,10 +106,18 @@ for i in "$@"; do
   if ! [ -f "$i" ]; then
     usage
     echo "\"$i\" is not a valid file!"
-    exit
+    exit 2
   fi
 done
 
+if ! [ -d /opt/so/conf/curator ]; then
+  echo "Curator is not installed on this node and cannot be stopped automatically."
+else
+  echo -n "Stopping curator..."
+  so-curator-stop > /dev/null 2>&1
+  echo "Done"
+fi
+
 # track if we have any valid or invalid pcaps
 INVALID_PCAPS="no"
 VALID_PCAPS="no"

salt/elastalert/files/rules/so/nids2hive.yaml

@@ -1,52 +0,0 @@
-{% set es = salt['pillar.get']('static:masterip', '') %}
-{% set hivehost = salt['pillar.get']('static:masterip', '') %}
-{% set hivekey = salt['pillar.get']('static:hivekey', '') %}
-{% set MASTER = salt['pillar.get']('master:url_base', '') %}
-
-# hive.yaml
-# Elastalert rule to forward IDS alerts from Security Onion to a specified TheHive instance.
-#
-es_host: {{es}}
-es_port: 9200
-name: NIDS-Alert
-type: frequency
-index: "so-ids-*"
-num_events: 1
-timeframe:
-    minutes: 10
-buffer_time:
-    minutes: 10
-allow_buffer_time_overlap: true
-query_key: ["rule.uuid"]
-realert:
-    days: 1
-filter:
-- query:
-   query_string:
-      query: "event.module: suricata"
-
-alert: hivealerter
-
-hive_connection:
-  hive_host: http://{{hivehost}}
-  hive_port: 9000/thehive
-  hive_apikey: {{hivekey}}
-  
-hive_proxies:
-  http: ''
-  https: ''
-
-hive_alert_config:
-  title: '{match[rule][name]}'
-  type: 'NIDS'
-  source: 'SecurityOnion'
-  description: "`Hunting Pivot:` \n\n <https://{{MASTER}}/#/hunt?q=event.module%3A%20suricata%20AND%20rule.uuid%3A{match[rule][uuid]}%20%7C%20groupby%20source.ip%20destination.ip%20rule.name>  \n\n `Kibana Dashboard - Signature Drilldown:` \n\n <https://{{MASTER}}/kibana/app/kibana#/dashboard/ed6f7e20-e060-11e9-8f0c-2ddbf5ed9290?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(_source),index:'*:logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:'sid:')),sort:!('@timestamp',desc))> \n\n `Kibana Dashboard - Community_ID:` \n\n <https://{{MASTER}}/kibana/app/kibana#/dashboard/30d0ac90-729f-11ea-8dd2-9d8795a1200b?_g=(filters:!(('$state':(store:globalState),meta:(alias:!n,disabled:!f,index:'*:so-*',key:network.community_id,negate:!f,params:(query:'{match[network][community_id]}'),type:phrase),query:(match_phrase:(network.community_id:'{match[network][community_id]}')))),refreshInterval:(pause:!t,value:0),time:(from:now-7d,to:now))>  \n\n `IPs: `{match[source][ip]}:{match[source][port]} --> {match[destination][ip]}:{match[destination][port]} \n\n `Signature:`{match[rule][rule]}"
-  severity: 2
-  tags: ['{match[rule][uuid]}','{match[source][ip]}','{match[destination][ip]}']
-  tlp: 3
-  status: 'New'
-  follow: True
-
-hive_observable_data_mapping:
-  - ip: '{match[source][ip]}'
-  - ip: '{match[destination][ip]}'

salt/elastalert/files/rules/so/suricata_thehive.yaml

@@ -0,0 +1,51 @@
+{% set es = salt['pillar.get']('static:masterip', '') %}
+{% set hivehost = salt['pillar.get']('static:masterip', '') %}
+{% set hivekey = salt['pillar.get']('static:hivekey', '') %}
+{% set MASTER = salt['pillar.get']('master:url_base', '') %}
+
+# Elastalert rule to forward Suricata alerts from Security Onion to a specified TheHive instance.
+#
+es_host: {{es}}
+es_port: 9200
+name: Suricata-Alert
+type: frequency
+index: "so-ids-*"
+num_events: 1
+timeframe:
+    minutes: 10
+buffer_time:
+    minutes: 10
+allow_buffer_time_overlap: true
+query_key: ["rule.uuid","source.ip","destination.ip"]
+realert:
+    days: 1
+filter:
+- query:
+   query_string:
+      query: "event.module: suricata AND rule.severity:(1 OR 2)"
+
+alert: hivealerter
+
+hive_connection:
+  hive_host: http://{{hivehost}}
+  hive_port: 9000/thehive
+  hive_apikey: {{hivekey}}
+  
+hive_proxies:
+  http: ''
+  https: ''
+
+hive_alert_config:
+  title: '{match[rule][name]}'
+  type: 'NIDS'
+  source: 'SecurityOnion'
+  description: "`SOC Hunt Pivot:` \n\n <https://{{MASTER}}/#/hunt?q=network.community_id%3A%20%20%22{match[network][community_id]}%22%20%7C%20groupby%20source.ip%20destination.ip,event.module,%20event.dataset>  \n\n `Kibana Dashboard Pivot:` \n\n <https://{{MASTER}}/kibana/app/kibana#/dashboard/30d0ac90-729f-11ea-8dd2-9d8795a1200b?_g=(filters:!(('$state':(store:globalState),meta:(alias:!n,disabled:!f,index:'*:so-*',key:network.community_id,negate:!f,params:(query:'{match[network][community_id]}'),type:phrase),query:(match_phrase:(network.community_id:'{match[network][community_id]}')))),refreshInterval:(pause:!t,value:0),time:(from:now-7d,to:now))>  \n\n `IPs: `{match[source][ip]}:{match[source][port]} --> {match[destination][ip]}:{match[destination][port]} \n\n `Signature:`{match[rule][rule]}"
+  severity: 2
+  tags: ['{match[rule][uuid]}','{match[source][ip]}','{match[destination][ip]}']
+  tlp: 3
+  status: 'New'
+  follow: True
+
+hive_observable_data_mapping:
+  - ip: '{match[source][ip]}'
+  - ip: '{match[destination][ip]}'

salt/elastalert/files/rules/so/wazuh_thehive.yaml

@@ -0,0 +1,49 @@
+{% set es = salt['pillar.get']('static:masterip', '') %}
+{% set hivehost = salt['pillar.get']('static:masterip', '') %}
+{% set hivekey = salt['pillar.get']('static:hivekey', '') %}
+{% set MASTER = salt['pillar.get']('master:url_base', '') %}
+
+# Elastalert rule to forward high level Wazuh alerts from Security Onion to a specified TheHive instance.
+#
+es_host: {{es}}
+es_port: 9200
+name: Wazuh-Alert
+type: frequency
+index: "so-ossec-*"
+num_events: 1
+timeframe:
+    minutes: 10
+buffer_time:
+    minutes: 10
+allow_buffer_time_overlap: true
+realert:
+    days: 1
+filter:
+- query:
+   query_string:
+      query: "event.module: ossec AND rule.level>=8"
+
+alert: hivealerter
+
+hive_connection:
+  hive_host: http://{{hivehost}}
+  hive_port: 9000/thehive
+  hive_apikey: {{hivekey}}
+  
+hive_proxies:
+  http: ''
+  https: ''
+
+hive_alert_config:
+  title: '{match[rule][name]}'
+  type: 'wazuh'
+  source: 'SecurityOnion'
+  description: "`SOC Hunt Pivot:` \n\n <https://{{MASTER}}/#/hunt?q=event.module%3A%20ossec%20AND%20rule.id%3A{match[rule][id]}%20%7C%20groupby%20host.name%20rule.name>  \n\n `Kibana Dashboard Pivot:` \n\n <https://{{MASTER}}/kibana/app/kibana#/dashboard/ed6f7e20-e060-11e9-8f0c-2ddbf5ed9290?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(_source),index:'*:logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:'sid:')),sort:!('@timestamp',desc))>"
+  severity: 2
+  tags: ['{match[rule][id]}','{match[host][name]}']
+  tlp: 3
+  status: 'New'
+  follow: True
+
+hive_observable_data_mapping:
+  - other: '{match[host][name]}'

salt/elasticsearch/files/ingest/zeek.ftp

@@ -1,5 +1,5 @@
 {
-  "description" : "zeek.http",
+  "description" : "zeek.ftp",
   "processors" : [
     { "remove":         { "field": ["host"],     "ignore_failure": true                                                                  } },
     { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },

salt/elasticsearch/files/ingest/zeek.smb_mapping

@@ -1,5 +1,5 @@
 {
-  "description" : "zeek.smb_files",
+  "description" : "zeek.smb_mapping",
   "processors" : [
     { "remove":         { "field": ["host"],     "ignore_failure": true                                                                  } },
     { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },

salt/elasticsearch/files/ingest/zeek.ssh

@@ -1,5 +1,5 @@
 {
-  "description" : "zeek.conn",
+  "description" : "zeek.ssh",
   "processors" : [
     { "remove":         { "field": ["host"],     "ignore_failure": true                                                                  } },
     { "json":		{ "field": "message",				"target_field": "message2",			"ignore_failure": true	} },

salt/filebeat/etc/filebeat.yml

@@ -128,8 +128,8 @@ filebeat.inputs:
         imported: true
     processors:
     - dissect:
-        tokenizer: "/nsm/import/%{import_id}/zeek/logs/%{import_source}"
+        tokenizer: "/nsm/import/%{import.id}/zeek/logs/%{import.file}"
-        field: "source"
+        field: "log.file.path"
         target_prefix: ""
     - drop_fields:
         fields: ["source", "prospector", "input", "offset", "beat"]
@@ -166,8 +166,8 @@ filebeat.inputs:
       imported: true 
     processors:
     - dissect:
-        tokenizer: "/nsm/import/%{import_id}/suricata/%{import_source}"
+        tokenizer: "/nsm/import/%{import.id}/suricata/%{import.file}"
-        field: "source"
+        field: "log.file.path"
         target_prefix: ""
     - drop_fields:
         fields: ["source", "prospector", "input", "offset", "beat"]

salt/filebeat/init.sls

@@ -53,12 +53,11 @@ so-filebeat:
     - user: root
     - extra_hosts: {{ MASTER }}:{{ MASTERIP }}
     - binds:
+      - /nsm:/nsm:ro
       - /opt/so/log/filebeat:/usr/share/filebeat/logs:rw
       - /opt/so/conf/filebeat/etc/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
-      - /nsm:/nsm:ro
       - /opt/so/wazuh/logs/alerts:/wazuh/alerts:ro
       - /opt/so/wazuh/logs/archives:/wazuh/archives:ro
-      - /nsm/osquery/fleet/:/nsm/osquery/fleet:ro
       - /opt/so/conf/filebeat/etc/pki/filebeat.crt:/usr/share/filebeat/filebeat.crt:ro
       - /opt/so/conf/filebeat/etc/pki/filebeat.key:/usr/share/filebeat/filebeat.key:ro
       - /etc/ssl/certs/intca.crt:/usr/share/filebeat/intraca.crt:ro

salt/soctopus/files/SOCtopus.conf

@@ -60,7 +60,7 @@ slack_url = YOURSLACKWORKSPACE
 slack_webhook = YOURSLACKWEBHOOK
 
 [playbook]
-playbook_url = https://{{MASTER}}/playbook
+playbook_url = http://{{MASTER}}:3200/playbook
 playbook_key = de6639318502476f2fa5aa06f43f51fb389a3d7f
 playbook_verifycert = no
 playbook_unit_test_index = playbook-testing

salt/soctopus/init.sls

@@ -10,7 +10,7 @@ soctopusdir:
     - group: 939
     - makedirs: True
 
-soctopussync:
+soctopus-sync:
   file.recurse:
     - name: /opt/so/conf/soctopus/templates
     - source: salt://soctopus/files/templates
@@ -24,7 +24,6 @@ soctopusconf:
     - source: salt://soctopus/files/SOCtopus.conf
     - user: 939
     - group: 939
-    - replace: False
     - mode: 600
     - template: jinja
 

setup/so-setup

@@ -532,18 +532,29 @@ fi
 		salt-call state.apply -l info soctopus >> $setup_log 2>&1
 	fi
 
-	if [[ "$OSQUERY" = 1 ]]; then
+	if [[ "$PLAYBOOK" = 1 ]]; then
 		set_progress_str 73 "$(print_salt_state_apply 'mysql')"
 		salt-call state.apply -l info mysql >> $setup_log 2>&1
 		
-		set_progress_str 73 "$(print_salt_state_apply 'fleet')"
+		set_progress_str 73 "$(print_salt_state_apply 'playbook')"
+		salt-call state.apply -l info playbook >> $setup_log 2>&1
+		so-playbook-ruleupdate >> /root/setup_playbook_rule_update.log 2>&1 &
+	fi
+	
+	if [[ "$OSQUERY" = 1 ]]; then
+		if [[ "$PLAYBOOK" != 1 ]]; then
+		set_progress_str 74 "$(print_salt_state_apply 'mysql')"
+		salt-call state.apply -l info mysql >> $setup_log 2>&1
+		fi
+
+		set_progress_str 75 "$(print_salt_state_apply 'fleet')"
 		salt-call state.apply -l info fleet >> $setup_log 2>&1
 
-		set_progress_str 73 "$(print_salt_state_apply 'redis')"
+		set_progress_str 76 "$(print_salt_state_apply 'redis')"
 		salt-call state.apply -l info redis >> $setup_log 2>&1
 
 		if [[ $is_fleet_standalone && $FLEETCUSTOMHOSTNAME != '' ]]; then
-			set_progress_str 73 "$(print_salt_state_apply 'fleet.event_update-custom-hostname')"
+			set_progress_str 77 "$(print_salt_state_apply 'fleet.event_update-custom-hostname')"
 			pillar_override="{\"static\":{\"fleet_custom_hostname\": \"$FLEETCUSTOMHOSTNAME\"}}"
 			salt-call state.apply -l info fleet.event_update-custom-hostname pillar="$pillar_override" >> $setup_log 2>&1
 		fi
@@ -554,28 +565,23 @@ fi
 	fi
 
 	if [[ "$WAZUH" = 1 ]]; then
-		set_progress_str 75 "$(print_salt_state_apply 'wazuh')"
+		set_progress_str 78 "$(print_salt_state_apply 'wazuh')"
 		salt-call state.apply -l info wazuh >> $setup_log 2>&1
 	fi
 
 	if [[ "$THEHIVE" = 1 ]]; then
-		set_progress_str 76 "$(print_salt_state_apply 'thehive')"
+		set_progress_str 79 "$(print_salt_state_apply 'thehive')"
 		salt-call state.apply -l info thehive >> $setup_log 2>&1
 	fi
 
 	if [[ "$STRELKA" = 1 ]]; then
-		set_progress_str 77 "$(print_salt_state_apply 'strelka')"
+		set_progress_str 80 "$(print_salt_state_apply 'strelka')"
 		salt-call state.apply -l info strelka >> $setup_log 2>&1
 		if [[ $STRELKARULES == 1 ]]; then
 			/usr/sbin/so-yara-update >> $setup_log 2>&1	
 		fi
 	fi
 
-	if [[ "$PLAYBOOK" = 1 ]]; then
-		set_progress_str 78 "$(print_salt_state_apply 'playbook')"
-		salt-call state.apply -l info playbook >> $setup_log 2>&1
-	fi
-
 	if [[ $is_master || $is_helix ]]; then		
 		set_progress_str 81 "$(print_salt_state_apply 'utility')"
 		salt-call state.apply -l info utility >> $setup_log 2>&1