Merge branch 'dev' of https://github.com/Security-Onion-Solutions/securityonion-saltstack into dev

2026-05-22 03:01:52 +02:00 · 2020-03-17 08:59:19 -04:00
parent 417e3ee19f ad0aa85e79
commit 632aad2787
31 changed files with 631 additions and 22784 deletions
@@ -17,7 +17,5 @@ logstash:
        - so/9700_output_strelka.conf.jinja
  templates:
    - so/so-beats-template.json
-    - so/so-ossec-template.json
-    - so/so-strelka-template.json
-    - so/so-template.json
+    - so/so-common-template.json
    - so/so-zeek-template.json
@@ -0,0 +1,54 @@
+{% set MASTERIP = salt['pillar.get']('master:mainip', '') %}
+#!/bin/bash
+# Copyright 2014,2015,2016,2017,2018,2019 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+ELASTICSEARCH_HOST="{{ MASTERIP}}"
+ELASTICSEARCH_PORT=9200
+#ELASTICSEARCH_AUTH=""
+
+# Define a default directory to load pipelines from
+ELASTICSEARCH_TEMPLATES="/opt/so/saltstack/salt/logstash/pipelines/templates/so/"
+
+# Wait for ElasticSearch to initialize
+echo -n "Waiting for ElasticSearch..."
+COUNT=0
+ELASTICSEARCH_CONNECTED="no"
+while [[ "$COUNT" -le 240 ]]; do
+	curl --output /dev/null --silent --head --fail http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
+	if [ $? -eq 0 ]; then
+		ELASTICSEARCH_CONNECTED="yes"
+		echo "connected!"
+		break
+	else
+		((COUNT+=1))
+		sleep 1
+		echo -n "."
+	fi
+done
+if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
+	echo
+	echo -e "Connection attempt timed out.  Unable to connect to ElasticSearch.  \nPlease try: \n  -checking log(s) in /var/log/elasticsearch/\n  -running 'sudo docker ps' \n  -running 'sudo so-elastic-restart'"
+	echo
+fi
+
+cd ${ELASTICSEARCH_TEMPLATES}
+
+
+echo "Loading templates..."
+for i in *; do TEMPLATE=$(echo $i | cut -d '-' -f2); echo "so-$TEMPLATE"; curl ${ELASTICSEARCH_AUTH} -s -XPUT http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/so-$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done
+echo
+
+cd - >/dev/null
@@ -1,17 +1,10 @@
 {
  "description" : "common",
  "processors" : [
-    { 
-	"rename": {
-		"field": "type",
-		"target_field": "event_type",
-		"ignore_missing": true
-	}
-    },
    { 	
 	"geoip":  { 
 		"field": "destination.ip",
-		"target_field": "destination_geo",
+		"target_field": "geo",
 		"database_file": "GeoLite2-City.mmdb",
 		"ignore_missing": true, 
 		"properties": ["ip", "country_iso_code", "country_name", "continent_name", "region_iso_code", "region_name", "city_name", "timezone", "location"]
@@ -20,7 +13,7 @@
    { 
 	"geoip":  {
 		"field": "source.ip",
-		"target_field": "source_geo",
+		"target_field": "geo",
 		"database_file": "GeoLite2-City.mmdb",
 		"ignore_missing": true,
 		"properties": ["ip", "country_iso_code", "country_name", "continent_name", "region_iso_code", "region_name", "city_name", "timezone", "location"]
@@ -41,10 +34,12 @@
 	        "ignore_failure": true,
        	"index_name_format": "yyyy.MM.dd"
        }
-    },
+    },    
+    { "rename":         { "field": "module",              "target_field": "event.module",                  "ignore_missing": true  } },
+    { "rename":         { "field": "dataset",              "target_field": "event.dataset",                  "ignore_missing": true  } },
    { 
 	"remove": {
-		"field": [ "index_name_prefix"],
+		"field": [ "index_name_prefix", "beat"],
 		"ignore_failure": false
 	}
    }
@@ -0,0 +1,53 @@
+{
+  "description" : "ossec",
+  "processors" : [
+    { "json":		{ "field": "message",					"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.agent", 				"target_field": "agent",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.data", 				"target_field": "data",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.decoder", 				"target_field": "decoder",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.full_log", 			"target_field": "log.full",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id", 				"target_field": "log.id.id",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.location", 			"target_field": "location",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.manager", 				"target_field": "manager",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.predecoder", 			"target_field": "predecoder",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.timestamp", 			"target_field": "timestamp",		"ignore_missing": true 	} },
+    { "rename":         { "field": "message2.previous_log",                        "target_field": "log.previous_log",            "ignore_missing": true  } },
+    { "rename":         { "field": "message2.previous_output",                        "target_field": "log.previous_output",            "ignore_missing": true  } },
+    { "rename": 	{ "field": "message2.rule", 				"target_field": "rule",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.command", 				"target_field": "command",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.dstip", 				"target_field": "destination.ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.dstport", 				"target_field": "destination.port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.dstuser", 				"target_field": "user.escalated",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.srcip", 				"target_field": "source.ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.win.eventdata.destinationHostname", 	"target_field": "destination.hostname",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.win.eventdata.destinationIp", 		"target_field": "destination.ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.win.eventdata.destinationPort", 	"target_field": "destination.port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.win.eventdata.image", 			"target_field": "image_path",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.win.eventdata.parentImage", 		"target_field": "parent_image_path",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.win.eventdata.sourceHostname", 	"target_field": "source.hostname",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.win.eventdata.sourceIp", 		"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.win.eventdata.sourcePort", 		"target_field": "source.port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.win.eventdata.targetFilename",		"target_field": "file.target",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.win.eventdata.user", 			"target_field": "user.name",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "data.win.system.eventID", 			"target_field": "event.code",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "predecoder.program_name", 			"target_field": "process.name",		"ignore_missing": true 	} },
+    { "set": 		{ "if": "ctx.rule.level == 1", 		"field": "rule.category", "value": "None" 				} },
+    { "set": 		{ "if": "ctx.rule.level == 2", 		"field": "rule.category", "value": "System low priority notification" 	} },
+    { "set": 		{ "if": "ctx.rule.level == 3", 		"field": "rule.category", "value": "Successful/authorized event" 	} },
+    { "set": 		{ "if": "ctx.rule.level == 4", 		"field": "rule.category", "value": "System low priority error" 	} },
+    { "set": 		{ "if": "ctx.rule.level == 5", 		"field": "rule.category", "value": "User generated error" 		} },
+    { "set": 		{ "if": "ctx.rule.level == 6", 		"field": "rule.category", "value": "Low relevance attack" 		} },
+    { "set": 		{ "if": "ctx.rule.level == 7", 		"field": "rule.category", "value": "\"Bad word\" matching" 		} },
+    { "set": 		{ "if": "ctx.rule.level == 8", 		"field": "rule.category", "value": "First time seen" 			} },
+    { "set": 		{ "if": "ctx.rule.level == 9", 		"field": "rule.category", "value": "Error from invalid source" 	} },
+    { "set": 		{ "if": "ctx.rule.level == 10", 		"field": "rule.category", "value": "Multiple user generated errors" 	} },
+    { "set": 		{ "if": "ctx.rule.level == 11", 		"field": "rule.category", "value": "Integrity checking warning" 	} },
+    { "set": 		{ "if": "ctx.rule.level == 12", 		"field": "rule.category", "value": "High importance event" 		} },
+    { "set": 		{ "if": "ctx.rule.level == 13", 		"field": "rule.category", "value": "Unusal error (high importance)" 	} },
+    { "set": 		{ "if": "ctx.rule.level == 14", 		"field": "rule.category", "value": "High importance security event" 	} },
+    { "set": 		{ "if": "ctx.rule.level == 15", 		"field": "rule.category", "value": "Severe attack" 			} },
+    { "append": 	{ "if": "ctx.rule.level != null",		"field": "tags", "value": ["alert"] 					} },
+    { "remove": { "field": [ "predecoder", "decoder" ], "ignore_missing": true, "ignore_failure": false } },
+    { "pipeline":	{ "name": "common" } }
+  ]
+}
@@ -0,0 +1,12 @@
+{
+  "description" : "strelka",
+  "processors" : [
+    { "json":           { "field": "message",                        "target_field": "message2",                   "ignore_failure": true  } },
+    { "rename":         { "field": "message2.file",        "target_field": "file",          "ignore_missing": true  } },
+    { "rename":         { "field": "message2.scan",        "target_field": "scan",          "ignore_missing": true  } },
+    { "rename":         { "field": "message2.request",        "target_field": "request",          "ignore_missing": true  } },
+    { "rename":         { "field": "scan.hash",        "target_field": "file.hash",          "ignore_missing": true  } },
+    { "remove":         { "field": ["host", "path"],                                         "ignore_missing": true  } },
+    { "pipeline":       { "name": "common"                                                                                   } }
+  ]
+}
@@ -0,0 +1,11 @@
+{
+  "description" : "suricata.alert",
+  "processors" : [
+    { "json":           { "field": "message",                   "target_field": "message2",             "ignore_failure": true  } },
+    { "rename":{ "field": "message2.comunity_id",       "target_field": "network.comunity_id",  "ignore_failure": true  } },
+    { "rename":{ "field": "message2.alert",	"target_field": "rule",	"ignore_failure": true 	} },
+    { "rename":{ "field": "rule.signature",     "target_field": "rule.name", "ignore_failure": true  } },
+    { "rename":{ "field": "rule.ref",     "target_field": "rule.version", "ignore_failure": true  } },
+    { "pipeline": { "name": "suricata.common" } }
+  ]
+}
@@ -0,0 +1,14 @@
+{
+  "description" : "suricata.common",
+  "processors" : [
+    { "rename":{ "field": "message2.proto",		"target_field": "network.transport",		"ignore_failure": true 	} },
+    { "rename":{ "field": "message2.flow_id",		"target_field": "event.id",		"ignore_failure": true 	} },
+    { "rename":{ "field": "message2.comunity_id",	"target_field": "network.comunity_id",	"ignore_failure": true 	} },
+    { "rename":{ "field": "message2.src_ip",       "target_field": "source.ip",  "ignore_failure": true  } },
+    { "rename":{ "field": "message2.src_port",       "target_field": "source.port",  "ignore_failure": true  } },
+    { "rename":{ "field": "message2.dest_ip",       "target_field": "destination.ip",  "ignore_failure": true  } },
+    { "rename":{ "field": "message2.dest_port",       "target_field": "destination.port",  "ignore_failure": true  } },
+     { "remove":         { "field": ["message2"],     "ignore_failure": true                                                                  } },
+    { "pipeline": { "name": "common" } }
+  ]
+}
@@ -1,7 +1,7 @@
 {
  "description" : "zeek.common",
  "processors" : [
-    { "rename":		{ "field": "@timestamp",	"target_field": "es.timestamp",	"ignore_missing": true					} },
+    { "rename":		{ "field": "@timestamp",	"target_field": "ingest.timestamp",	"ignore_missing": true					} },
    { "rename":         { "field": "message2.uid",              "target_field": "log.id.uid",                  "ignore_missing": true  } },
    { "dot_expander":   { "field": "id.orig_h",                 "path": "message2",                     "ignore_failure": true  } },
    { "rename":         { "field": "message2.id.orig_h",        "target_field": "source.ip",            "ignore_missing": true  } },
@@ -15,8 +15,6 @@
    { "dot_expander":   { "field": "id.resp_p",                 "path": "message2",                     "ignore_failure": true  } },
    { "rename":         { "field": "message2.id.resp_p",        "target_field": "destination.port",     "ignore_missing": true  } },
    { "set":         { "field": "server.port",        "value": "{{destination.port}}"  } },
-    { "set":         { "field": "event.module",        "value": "zeek"  } },
-    { "grok": 	     { "field": "event_type",		"patterns": ["zeek.%{WORD:event.dataset}"]					} },
    { "date":		{ "field": "message2.ts",	"target_field": "@timestamp",	"formats": ["ISO8601", "UNIX"], "ignore_failure": true	} },
    { "remove":		{ "field": ["message2.ts", "path"],	"ignore_failure": true									} },
    { "pipeline":	{ "name": "common" 													} }
@@ -1,7 +1,7 @@
 {
  "description" : "zeek.conn",
  "processors" : [
-    { "remove":         { "field": ["host"],     "ignore_failure": false                                                                  } },
+    { "remove":         { "field": ["host"],     "ignore_failure": true                                                              } },
    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
    { "rename": 	{ "field": "message2.proto", 		"target_field": "network.transport",		"ignore_missing": true 	} },
    { "rename": 	{ "field": "message2.service", 		"target_field": "network.protocol",		"ignore_missing": true 	} },
@@ -21,20 +21,6 @@
    { "rename": 	{ "field": "message2.orig_cc", 		"target_field": "client.country_code","ignore_missing": true 	} },
    { "rename": 	{ "field": "message2.resp_cc", 		"target_field": "server.country_code",	"ignore_missing": true 	} },
    { "rename": 	{ "field": "message2.sensorname", 	"target_field": "observer.name",		"ignore_missing": true 	} },
-    { "script":		{ "lang": "painless", "source":	"ctx.connection.bytes.total = (ctx.client.bytes + ctx.server.bytes)", "ignore_failure": true } },
-    { "set": { "if": "ctx.connection.state == 'S0'", 	"field": "connection.state_description", "value": "Connection attempt seen, no reply" } },
-    { "set": { "if": "ctx.connection.state == 'S1'", 	"field": "connection.state_description", "value": "Connection established, not terminated" } },
-    { "set": { "if": "ctx.connection.state == 'S2'", 	"field": "connection.state_description", "value": "Connection established and close attempt by originator seen (but no reply from responder)" } },
-    { "set": { "if": "ctx.connection.state == 'S3'", 	"field": "connection.state_description", "value": "Connection established and close attempt by responder seen (but no reply from originator)" } },
-    { "set": { "if": "ctx.connection.state == 'SF'", 	"field": "connection.state_description", "value": "Normal SYN/FIN completion" } },
-    { "set": { "if": "ctx.connection.state == 'REJ'", 	"field": "connection.state_description", "value": "Connection attempt rejected" } },
-    { "set": { "if": "ctx.connection.state == 'RSTO'", 	"field": "connection.state_description", "value": "Connection established, originator aborted (sent a RST)" } },
-    { "set": { "if": "ctx.connection.state == 'RSTR'", 	"field": "connection.state_description", "value": "Established, responder aborted" } },
-    { "set": { "if": "ctx.connection.state == 'RSTOS0'","field": "connection.state_description", "value": "Originator sent a SYN followed by a RST, we never saw a SYN-ACK from the responder" } },
-    { "set": { "if": "ctx.connection.state == 'RSTRH'", "field": "connection.state_description", "value": "Responder sent a SYN ACK followed by a RST, we never saw a SYN from the (purported) originator" } },
-    { "set": { "if": "ctx.connection.state == 'SH'", 	"field": "connection.state_description", "value": "Originator sent a SYN followed by a FIN, we never saw a SYN ACK from the responder (hence the connection was 'half' open)" } },
-    { "set": { "if": "ctx.connection.state == 'SHR'", 	"field": "connection.state_description", "value": "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator" } },
-    { "set": { "if": "ctx.connection.state == 'OTH'", 	"field": "connection.state_description", "value": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed)" } },
    { "pipeline": 	{ "name": "zeek.common" } }
  ]
 }
@@ -3,7 +3,6 @@
  "processors" : [
    { "remove":         { "field": ["host"],     "ignore_failure": true                                                                  } },
    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
-    { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source.ip",		"ignore_missing": true 	} },
    { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
@@ -13,8 +12,8 @@
    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
    { "rename": 	{ "field": "message2.proto", 		"target_field": "network.protocol",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.analyzer", 	"target_field": "analyzer",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.failure_reason", 	"target_field": "failure_reason",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.analyzer", 	"target_field": "observer.analyzer",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.failure_reason", 	"target_field": "error.reason",	"ignore_missing": true 	} },
    { "pipeline":       { "name": "zeek.common"                                                                                   } }
  ]
 }
@@ -2,7 +2,7 @@
  "description" : "zeek.http",
  "processors" : [
    { "remove":         { "field": ["host"],     "ignore_failure": true                                                                  } },
-    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "json":           { "field": "message",                   "target_field": "message2",             "ignore_failure": true  } },
    { "rename": 	{ "field": "message2.trans_depth",	"target_field": "http.trans_depth",		"ignore_missing": true 	} },
    { "rename": 	{ "field": "message2.method", 		"target_field": "http.method",		"ignore_missing": true 	} },
    { "rename": 	{ "field": "message2.host", 		"target_field": "http.virtual_host",		"ignore_missing": true 	} },
@@ -2,7 +2,6 @@
  "description" : "zeek.tunnel",
  "processors" : [
    { "remove":         { "field": ["host"],     "ignore_failure": true                                                                  } },
-    { "set":		{ "field": "event_type", "value": "zeek.tunnels"								} },
    { "pipeline":       { "name": "zeek.tunnels"                                                                                 } }
  ]
 }
@@ -12,8 +12,8 @@
    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.tunnel_type", 	"target_field": "tunnel_type",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.action", 		"target_field": "action",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.tunnel_type", 	"target_field": "tunnel.type",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.action", 		"target_field": "event.action",		"ignore_missing": true 	} },
    { "pipeline":       { "name": "zeek.common"                                                                                   } }
  ]
 }
@@ -2,7 +2,6 @@
  "description" : "zeek.weird",
  "processors" : [
    { "remove":         { "field": ["host"],     "ignore_failure": true                                                                  } },
-    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
    { "rename": 	{ "field": "message2.name", 		"target_field": "weird.name",			"ignore_missing": true 	} },
    { "rename": 	{ "field": "message2.addl", 		"target_field": "weird.additional_info",	"ignore_missing": true 	} },
    { "rename": 	{ "field": "message2.notice", 		"target_field": "weird.notice",		"ignore_missing": true 	} },
@@ -143,3 +143,9 @@ so-elasticsearch-pipelines-file:
 so-elasticsearch-pipelines:
 cmd.run:
   - name: /opt/so/conf/elasticsearch/so-elasticsearch-pipelines {{ esclustername }}
+
+so-elasticsearch-templates:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-templates
+    - cwd: /
+
@@ -80,9 +80,13 @@ filebeat.prospectors:
    paths:
      - /nsm/zeek/logs/current/{{ LOGNAME }}.log
    fields:
-      type: bro_{{ LOGNAME }}
+        module: zeek 
+        dataset: {{ LOGNAME }}  
+    processors:
+    - drop_fields:
+        fields: ["source", "prospector", "input", "offset", "beat"]
+ 
    fields_under_root: true
-    tags: ["bro"]
    clean_removed: false
    close_removed: false

@@ -93,8 +97,13 @@ filebeat.prospectors:
    paths:
      - /suricata/eve.json
    fields:
-      type: ids
-      engine: suricata
+      module: suricata
+      dataset: alert
+
+    processors:
+    - drop_fields:
+        fields: ["source", "prospector", "input", "offset", "beat"]
+
    fields_under_root: true
    clean_removed: false
    close_removed: false
@@ -106,19 +115,25 @@ filebeat.prospectors:
    paths:
      - /wazuh/alerts/alerts.json
    fields:
-      type: ossec
+      module: ossec
+      dataset: alert
+
+    processors:
+    - drop_fields:
+        fields: ["source", "prospector", "input", "offset", "beat"]
+
    fields_under_root: true
    clean_removed: false
    close_removed: false

-  - type: log
-    paths:
-      - /wazuh/archives/archives.json
-    fields:
-      type: ossec_archive
-    fields_under_root: true
-    clean_removed: false
-    close_removed: false
+# - type: log
+#    paths:
+#      - /wazuh/archives/archives.json
+#    fields:
+#      type: ossec_archive
+#    fields_under_root: true
+#    clean_removed: false
+#    close_removed: false

 {%- endif %}

@@ -129,6 +144,11 @@ filebeat.prospectors:
      - /osquery/logs/result.log
    fields:
      type: osquery
+
+    processors:
+    - drop_fields:
+        fields: ["source", "prospector", "input", "offset", "beat"]
+
    fields_under_root: true
    clean_removed: false
    close_removed: false
@@ -141,34 +161,56 @@ filebeat.prospectors:
    paths:
      - /opt/so/log/strelka/strelka.log
    fields:
-      type: strelka
+      module: strelka
+      dataset: file
+
+    processors:
+    - drop_fields:
+        fields: ["source", "prospector", "input", "offset", "beat"]    
+
    fields_under_root: true
    clean_removed: false
    close_removed: false

 {%- endif %}
 #----------------------------- Logstash output ---------------------------------
-output.logstash:
-  # Boolean flag to enable or disable the output module.
+output.elasticsearch:
  enabled: true
+  hosts: ["{{ MASTER }}:9200"]
+  pipelines:
+    - pipeline: "%{[module]}.%{[dataset]}"
+  indices:
+    - index: "so-zeek-%{+yyyy.MM.dd}"
+      when.contains:
+        module: "zeek"
+    - index: "so-ids-%{+yyyy.MM.dd}"
+      when.contains:
+        module: "suricata"
+    - index: "so-ossec-%{+yyyy.MM.dd}"
+      when.contains:
+        module: "ossec"
+
+#output.logstash:
+  # Boolean flag to enable or disable the output module.
+  #enabled: true

  # The Logstash hosts
-  hosts: ["{{ MASTER }}:5644"]
+  #hosts: ["{{ MASTER }}:5644"]

  # Number of workers per Logstash host.
-  worker: 1
+  #worker: 1

  # Set gzip compression level.
-  compression_level: 3
+  #compression_level: 3

  # Enable SSL support. SSL is automatically enabled, if any SSL setting is set.
-  ssl.enabled: true
+  #ssl.enabled: true

  # Configure SSL verification mode. If `none` is configured, all server hosts
  # and certificates will be accepted. In this mode, SSL based connections are
  # susceptible to man-in-the-middle attacks. Use only for testing. Default is
  # `full`.
-  ssl.verification_mode: full
+  #ssl.verification_mode: full

  # List of supported/valid TLS versions. By default all TLS versions 1.0 up to
  # 1.2 are enabled.
@@ -176,16 +218,14 @@ output.logstash:

  # Optional SSL configuration options. SSL is off by default.
  # List of root certificates for HTTPS server verifications
-  ssl.certificate_authorities: ["/usr/share/filebeat/intraca.crt"]
+  #ssl.certificate_authorities: ["/usr/share/filebeat/intraca.crt"]

  # Certificate for SSL client authentication
-  ssl.certificate: "/usr/share/filebeat/filebeat.crt"
+  #ssl.certificate: "/usr/share/filebeat/filebeat.crt"

  # Client Certificate Key
-  ssl.key: "/usr/share/filebeat/filebeat.key"
+  #ssl.key: "/usr/share/filebeat/filebeat.key"

-# Elasticsearch template settings
-#setup.template.settings:

  # A dictionary of settings to place into the settings.index dictionary
  # of the Elasticsearch template. For more details, please check
@@ -1,24 +0,0 @@
-{
-  "index_patterns": ["logstash-strelka-*"],
-  "version":50001,
-  "order" : 0,
-  "settings":{
-    "number_of_replicas":0,
-    "number_of_shards":1,
-    "index.refresh_interval":"30s"
-  },
-  "mappings":{
-    "doc":{
-      "dynamic": false,
-      "date_detection": false,
-      "properties":{
-        "@timestamp":{
-         "type":"date"
-        },
-        "@version":{
-          "type":"keyword"
-        }
-      }
-    }
-  }
-}
@@ -0,0 +1,355 @@
+{
+  "index_patterns": ["so-ids-*", "so-firewall-*", "so-syslog-*", "so-zeek-*", "so-import-*", "so-ossec-*", "so-strelka", "so-beats-*"],
+  "version":50001,
+  "order" : 10,
+  "settings":{
+    "number_of_replicas":0,
+    "number_of_shards":1,
+    "index.refresh_interval":"30s"
+  },
+  "mappings":{
+    "doc":{
+       "dynamic": false,
+       "date_detection": false,
+       "properties":{
+        "@timestamp":{
+          "type":"date"
+        },
+        "@version":{
+          "type":"keyword"
+        },
+        "geoip":{
+          "dynamic":true,
+          "properties":{
+            "ip":{
+              "type":"ip"
+            },
+            "location":{
+              "type":"geo_point"
+            },
+            "latitude":{
+              "type":"half_float"
+            },
+            "longitude":{
+              "type":"half_float"
+            }
+          }
+        },
+        "destination_geo":{
+          "dynamic":true,
+          "properties":{
+            "ip":{
+              "type":"ip"
+            },
+            "location":{
+              "type":"geo_point"
+            },
+            "latitude":{
+              "type":"half_float"
+            },
+            "longitude":{
+              "type":"half_float"
+            }
+          }
+        },
+        "source_geo":{
+          "dynamic":true,
+          "properties":{
+            "ip":{
+              "type":"ip"
+            },
+            "location":{
+              "type":"geo_point"
+            },
+            "latitude":{
+              "type":"half_float"
+            },
+            "longitude":{
+              "type":"half_float"
+            }
+          }
+        },
+        "agent":{
+          "type":"object",
+          "dynamic": true
+        },
+        "as":{
+          "type":"object",
+          "dynamic": true
+        },
+        "alert":{
+          "type":"object",
+          "dynamic": true
+        },
+	"client":{
+          "type":"object",
+          "dynamic": true
+        },
+        "cloud":{
+          "type":"object",
+          "dynamic": true
+        },
+        "code_signature":{
+          "type":"object",
+          "dynamic": true
+        },
+        "connection":{
+          "type":"object",
+          "dynamic": true
+        },
+        "container":{
+          "type":"object",
+          "dynamic": true
+        },
+        "data":{
+          "type":"object",
+          "dynamic": true
+        },
+        "dce_rpc":{
+          "type":"object",
+          "dynamic": true
+        },
+        "destination":{
+          "type":"object",
+          "dynamic": true
+        },
+        "dhcp":{
+          "type":"object",
+          "dynamic": true
+        },
+        "dnp3":{
+          "type":"object",
+          "dynamic": true
+        },
+        "dns":{
+          "type":"object",
+          "dynamic": true
+        },
+        "dll":{
+          "type":"object",
+          "dynamic": true
+        },
+        "ecs":{
+          "type":"object",
+          "dynamic": true
+        },
+        "error":{
+          "type":"object",
+          "dynamic": true
+        },
+        "event":{
+          "type":"object",
+          "dynamic": true
+        }, 
+        "file":{
+          "type":"object",
+          "dynamic": true
+        },
+        "flow":{
+          "type":"object",
+          "dynamic": true
+        },
+        "ftp":{
+          "type":"object",
+          "dynamic": true
+        },
+        "geo":{
+          "type":"object",
+          "dynamic": true
+        },
+        "group":{
+          "type":"object",
+          "dynamic": true
+        },
+        "hash":{
+          "type":"object",
+          "dynamic": true
+        },
+        "host":{
+          "type":"object",
+          "dynamic": true
+        },
+        "http":{
+          "type":"object",
+          "dynamic": true
+        },
+        "ingest":{
+          "type":"object",
+          "dynamic": true
+        },
+        "interface":{
+          "type":"object",
+          "dynamic": true
+        },
+        "kerberos":{
+          "type":"object",
+          "dynamic": true
+        },
+        "log":{
+          "type":"object",
+          "dynamic": true
+        },
+        "manager":{
+          "type":"object",
+          "dynamic": true
+        },
+        "modbus":{
+          "type":"object",
+          "dynamic": true
+        },
+        "network":{
+          "type":"object",
+          "dynamic": true
+        },
+        "ntlm":{
+          "type":"object",
+          "dynamic": true
+        },
+        "observer":{
+          "type":"object",
+          "dynamic": true
+        },
+        "organization":{
+          "type":"object",
+          "dynamic": true
+        },
+        "os":{
+          "type":"object",
+          "dynamic": true
+        },
+        "package":{
+          "type":"object",
+          "dynamic": true
+        },
+        "pe":{
+          "type":"object",
+          "dynamic": true
+        },
+        "process":{
+          "type":"object",
+          "dynamic": true
+        },
+        "radius":{
+          "type":"object",
+          "dynamic": true
+        },
+        "rdp":{
+          "type":"object",
+          "dynamic": true
+        },
+        "registry":{
+          "type":"object",
+          "dynamic": true
+        },
+        "related":{
+          "type":"object",
+          "dynamic": true
+        },
+        "rfb":{
+          "type":"object",
+          "dynamic": true
+        },
+        "rule":{
+          "type":"object",
+          "dynamic": true
+        },
+        "server":{
+          "type":"object",
+          "dynamic": true
+        },
+        "service":{
+          "type":"object",
+          "dynamic": true
+        },
+        "sip":{
+          "type":"object",
+          "dynamic": true
+        },
+        "smb":{
+          "type":"object",
+          "dynamic": true
+        },
+        "smtp":{
+          "type":"object",
+          "dynamic": true
+        },
+        "snmp":{
+          "type":"object",
+          "dynamic": true
+        },
+        "socks":{
+          "type":"object",
+          "dynamic": true
+        },
+        "software":{
+          "type":"object",
+          "dynamic": true
+        },
+	"source":{
+          "type":"object",
+          "dynamic": true
+        },
+        "ssh":{
+          "type":"object",
+          "dynamic": true
+        },
+        "ssl":{
+          "type":"object",
+          "dynamic": true
+        },
+        "tags":{
+          "type":"text",
+          "fields":{
+            "keyword":{
+              "type":"keyword"
+            }
+          }
+        },
+        "threat":{
+          "type":"object",
+          "dynamic": true
+        },
+        "tls":{
+          "type":"object",
+          "dynamic": true
+        },
+        "trace":{
+          "type":"object",
+          "dynamic": true
+        },
+        "tunnel":{
+          "type":"object",
+          "dynamic": true
+        },
+        "user":{
+          "type":"object",
+          "dynamic": true
+        },
+        "user_agent":{
+          "type":"object",
+          "dynamic": true
+        },
+        "version":{
+          "type":"object",
+          "dynamic": true
+        },
+        "vlan":{
+          "type":"object",
+          "dynamic": true
+        },
+        "vulnerability":{
+          "type":"object",
+          "dynamic": true
+        },
+        "weird":{
+          "type":"object",
+          "dynamic": true
+        },
+        "x509":{
+          "type":"object",
+          "dynamic": true
+        }
+      }
+    }
+  }
+}
@@ -1,24 +0,0 @@
-{
-  "index_patterns": ["so-strelka-*"],
-  "version":50001,
-  "order" : 0,
-  "settings":{
-    "number_of_replicas":0,
-    "number_of_shards":1,
-    "index.refresh_interval":"30s"
-  },
-  "mappings":{
-    "doc":{
-      "dynamic": false,
-      "date_detection": false,
-      "properties":{
-        "@timestamp":{
-         "type":"date"
-        },
-        "@version":{
-          "type":"keyword"
-        }
-      }
-    }
-  }
-}
@@ -55,7 +55,7 @@ so-playbooksynccron:

 so-playbookruleupdatecron:
  cron.present:
-    - name /usr/sbin/so-playbook-ruleupdate
+    - name: /usr/sbin/so-playbook-ruleupdate
    - user: root
    - minute: '1'
    - hour: '6'
@@ -60,7 +60,7 @@ base:
    - wazuh
    {%- endif %}
    - elasticsearch
-    - logstash
+    - filebeat
    - kibana
    - pcap
    - suricata
@@ -0,0 +1,17 @@
+zeek:
+  zeekctl:
+    MailTo: root@localhost
+    MailConnectionSummary: 1
+    MinDiskSpace: 5
+    MailHostUpDown: 1
+    LogRotationInterval: 3600
+    LogExpireInterval: 0
+    StatsLogEnable: 1
+    StatsLogExpireInterval: 0
+    StatusCmdShowAll: 0
+    CrashExpireInterval: 0
+    SitePolicyScripts: local.zeek
+    LogDir: /nsm/zeek/logs
+    SpoolDir: /nsm/zeek/spool
+    CfgDir: /opt/zeek/etc
+    CompressLogs: 1
@@ -0,0 +1,9 @@
+## Global ZeekControl configuration file.
+
+{%- set ALLOWEDOPTIONS = ['commtimeout','commandtimeout','compresscmd','compressextension','compresslogs','compresslogsinflight','controltopic','crashexpireinterval','croncmd','debug','env_vars','havenfs','keeplogs','logdir','logexpireinterval','logrotationinterval','mailalarmsinterval','mailalarmsto','mailarchivelogfail','mailconnectionsummary','mailfrom','mailhostupdown','mailreceivingpackets','mailreplyto','mailsubjectprefix','mailto','makearchivename','memlimit','mindiskspace','pfringclusterid','pfringclustertype','pfringfirstappinstance','prefixes','savetraces','sendmail','sitepluginpath','sitepolicypath','sitepolicyscripts','statslogenable','statslogexpireinterval','statuscmdshowall','stoptimeout','stopwait','timefmt','timemachinehost','timemachineport','zeekargs','zeekport','bindir','capstatspath','cfgdir','debuglog','defaultstoredir','helperdir','libdir','libdir64','libdirinternal','localnetscfg','lockfile','logexpireminutes','nodecfg','os','pcapbufsize','pcapsnaplen','plugindir','pluginzeekdir','policydir','policydirsiteinstall','policydirsiteinstallauto','postprocdir','scriptsdir','spooldir','standalone','statefile','staticdir','statsdir','statslog','time','tmpdir','tmpexecdir','tracesummary','version','zeek','zeekbase'] %}
+
+{%- for option in ZEEKCTL %}
+  {%- if option|lower in ALLOWEDOPTIONS %}
+{{ option }} = {{ ZEEKCTL[option] }}
+  {%- endif %}
+{%- endfor %}
@@ -3,7 +3,12 @@
 {% set BPF_ZEEK = salt['pillar.get']('zeek:bpf', {}) %}
 {% set BPF_STATUS = 0  %}
 {% set INTERFACE = salt['pillar.get']('sensor:interface', 'bond0') %}
+
+{% import_yaml 'zeek/defaults.yml' as ZEEKDEFAULTS %}
+{% set ZEEK = salt['pillar.get']('zeek', default=ZEEKDEFAULTS.zeek, merge=True) %}
+
 # Zeek Salt State
+
 # Add Zeek group
 zeekgroup:
  group.present:
@@ -63,6 +68,16 @@ zeekpolicysync:
    - group: 939
    - template: jinja

+zeekctlcfg:
+  file.managed:
+    - name: /opt/so/conf/zeek/zeekctl.cfg
+    - source: salt://zeek/files/zeekctl.cfg.jinja
+    - user: 937
+    - group: 939
+    - template: jinja
+    - defaults:
+        ZEEKCTL: {{ ZEEK.zeekctl | tojson }}
+
 # Sync node.cfg
 nodecfgsync:
  file.managed:
@@ -132,6 +147,7 @@ so-zeek:
      - /nsm/zeek/extracted:/nsm/zeek/extracted:rw
      - /opt/so/conf/zeek/local.zeek:/opt/zeek/share/zeek/site/local.zeek:ro
      - /opt/so/conf/zeek/node.cfg:/opt/zeek/etc/node.cfg:ro
+      - /opt/so/conf/zeek/zeekctl.cfg:/opt/zeek/etc/zeekctl.cfg:ro
      - /opt/so/conf/zeek/policy/securityonion:/opt/zeek/share/zeek/policy/securityonion:ro
      - /opt/so/conf/zeek/policy/custom:/opt/zeek/share/zeek/policy/custom:ro
      - /opt/so/conf/zeek/policy/cve-2020-0601:/opt/zeek/share/zeek/policy/cve-2020-0601:ro
@@ -141,5 +157,6 @@ so-zeek:
    - watch:
      - file: /opt/so/conf/zeek/local.zeek
      - file: /opt/so/conf/zeek/node.cfg
+      - file: /opt/so/conf/zeek/zeekctl.cfg
      - file: /opt/so/conf/zeek/policy
      - file: /opt/so/conf/zeek/bpf