Merge remote-tracking branch 'remotes/origin/dev' into issue/749

salt/common/tools/sbin/so-start

@@ -32,5 +32,5 @@ fi
 case $1 in
    "all") salt-call state.highstate queue=True;;
    "steno") if docker ps | grep -q so-$1; then printf "\n$1 is already running!\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply pcap queue=True; fi ;; 
-   *) if docker ps | grep -q so-$1; then printf "\n$1 is already running\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply $1 queue=True; fi ;; 
+   *) if docker ps | grep -E -q '^so-$1$'; then printf "\n$1 is already running\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply $1 queue=True; fi ;; 
 esac

salt/common/tools/sbin/so-zeek-stats

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Show Zeek stats (capstats, netstats)
+
+show_stats() {
+  echo '##############'
+  echo '# Zeek Stats #'
+  echo '##############'
+  echo
+  echo "Average throughput:"
+  echo
+  docker exec -it so-zeek /opt/zeek/bin/zeekctl capstats
+  echo
+  echo "Average packet loss:"
+  echo
+  docker exec -it so-zeek /opt/zeek/bin/zeekctl netstats
+  echo
+}
+
+if docker ps | grep -q zeek; then
+  show_stats
+else
+  echo "Zeek is not running! Try starting it with 'so-zeek-start'." && exit 1;
+fi

salt/elasticsearch/files/ingest/syslog

@@ -0,0 +1,13 @@
+{
+  "description" : "syslog",
+  "processors" : [
+    {
+        "dissect": {
+                "field": "message",
+                "pattern" : "%{message}",
+                "on_failure": [ { "drop" : { } } ]
+        }
+    },
+    { "pipeline":       { "name": "common"                                             } }
+  ]
+}

salt/elasticsearch/files/ingest/zeek.radius

@@ -5,7 +5,7 @@
     { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
     { "rename": 	{ "field": "message2.username", 	"target_field": "user.name",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.mac", 		"target_field": "host.mac",			"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.framed_addr", 	"target_field": "framed_addr",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.framed_addr", 	"target_field": "radius.framed_address",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.remote_ip", 	"target_field": "destination.ip",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.connect_info", 	"target_field": "radius.connect_info",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.reply_msg", 	"target_field": "radius.reply_message",	"ignore_missing": true 	} },

salt/filebeat/etc/filebeat.yml

@@ -75,6 +75,19 @@ filebeat.modules:
 filebeat.inputs:
 #------------------------------ Log prospector --------------------------------
 {%- if grains['role'] == 'so-sensor' or grains['role'] == "so-eval" or grains['role'] == "so-helix" or grains['role'] == "so-heavynode" or grains['role'] == "so-standalone" %}
+  - type: syslog
+    enabled: true
+    protocol.udp:
+      host: "0.0.0.0:514"
+    fields:
+      module: syslog
+      dataset: syslog
+    pipeline: "syslog"
+    index: "so-syslog-%{+yyyy.MM.dd}"
+    processors:
+    - drop_fields:
+        fields: ["source", "prospector", "input", "offset", "beat"]
+
 {%- if BROVER != 'SURICATA' %}
 {%- for LOGNAME in salt['pillar.get']('brologs:enabled', '')  %}
   - type: log

salt/filebeat/init.sls

@@ -64,5 +64,7 @@ so-filebeat:
       - /opt/so/conf/filebeat/etc/pki/filebeat.crt:/usr/share/filebeat/filebeat.crt:ro
       - /opt/so/conf/filebeat/etc/pki/filebeat.key:/usr/share/filebeat/filebeat.key:ro
       - /etc/ssl/certs/intca.crt:/usr/share/filebeat/intraca.crt:ro
+    - port_bindings:
+        - 0.0.0.0:514:514/udp
     - watch:
       - file: /opt/so/conf/filebeat/etc/filebeat.yml

salt/firewall/init.sls

@@ -136,6 +136,18 @@ enable_wazuh_manager_1514_udp_{{ip}}:
     - position: 1
     - save: True
 
+# Allow syslog
+enable_syslog_514_{{ip}}:
+  iptables.insert:
+    - table: filter
+    - chain: DOCKER-USER
+    - jump: ACCEPT
+    - proto: tcp
+    - source: {{ ip }}
+    - dport: 514
+    - position: 1
+    - save: True
+
 # Rules if you are a Master
 {% if grains['role'] in ['so-master', 'so-eval', 'so-helix', 'so-mastersearch', 'so-standalone'] %}
 #This should be more granular

salt/soc/files/soc/soc.json

@@ -99,7 +99,7 @@
           { "name": "Connections", "description": "Connections grouped by destination country", "query": "event.module:zeek AND event.dataset:conn | groupby destination.geo.country_name"},
           { "name": "Connections", "description": "Connections grouped by source country", "query": "event.module:zeek AND event.dataset:conn | groupby source.geo.country_name"},
           { "name": "DCE_RPC", "description": "DCE_RPC grouped by operation", "query": "event.module:zeek AND event.dataset:dce_rpc | groupby dce_rpc.operation"},
-          { "name": "DHCP", "description": "DHCP leases", "query": "event.module:zeek AND event.dataset:dhcp | groupby host.hostname host.domain dhcp.requested_address"},
+          { "name": "DHCP", "description": "DHCP leases", "query": "event.module:zeek AND event.dataset:dhcp | groupby host.hostname host.domain"},
           { "name": "DHCP", "description": "DHCP grouped by message type", "query": "event.module:zeek AND event.dataset:dhcp | groupby dhcp.message_types"},
           { "name": "DNP3", "description": "DNP3 grouped by reply", "query": "event.module:zeek AND event.dataset:dnp3 | groupby dnp3.fc_reply"},
           { "name": "DNS", "description": "DNS queries grouped by port ", "query": "event.module:zeek AND event.dataset:dns | groupby dns.query.name destination.port"},
@@ -122,8 +122,7 @@
           { "name": "KERBEROS", "description": "KERBEROS grouped by service", "query": "event.module:zeek AND event.dataset:kerberos | groupby kerberos.service"},
           { "name": "MODBUS", "description": "MODBUS grouped by function", "query": "event.module:zeek AND event.dataset:modbus | groupby modbus.function"},
           { "name": "MYSQL", "description": "MYSQL grouped by command", "query": "event.module:zeek AND event.dataset:mysql | groupby mysql.command"},
-          { "name": "NOTICE", "description": "Zeek notice logs grouped by note", "query": "event.module:zeek AND event.dataset:notice | groupby notice.note"},
-          { "name": "NOTICE", "description": "Zeek notice logs grouped by message", "query": "event.module:zeek AND event.dataset:notice | groupby notice.message"},
+          { "name": "NOTICE", "description": "Zeek notice logs grouped by note and message", "query": "event.module:zeek AND event.dataset:notice | groupby notice.note notice.message"},
           { "name": "NTLM", "description": "NTLM grouped by computer name", "query": "event.module:zeek AND event.dataset:ntlm | groupby ntlm.server.dns.name"},
           { "name": "PE", "description": "PE files list", "query": "event.module:zeek AND event.dataset:pe | groupby file.machine file.os file.subsystem"},
           { "name": "RADIUS", "description": "RADIUS grouped by username", "query": "event.module:zeek AND event.dataset:radius | groupby user.name.keyword"},

salt/soctopus/init.sls

@@ -1,5 +1,7 @@
 {% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %}
 {% set MASTER = salt['grains.get']('master') %}
+{%- set MASTER_URL = salt['pillar.get']('master:url_base', '') %}
+{%- set MASTER_IP = salt['pillar.get']('static:masterip', '') %}
 
 soctopusdir:
   file.directory:
@@ -69,3 +71,5 @@ so-soctopus:
       - /opt/so/conf/navigator/nav_layer_playbook.json:/etc/playbook/nav_layer_playbook.json:rw
     - port_bindings:
       - 0.0.0.0:7000:7000
+    - extra_hosts:
+      - {{MASTER_URL}}:{{MASTER_IP}}

salt/top.sls

@@ -157,6 +157,9 @@ base:
     {%- if PLAYBOOK != 0 %}
     - playbook
     {%- endif %}
+    {%- if NAVIGATOR != 0 %}
+    - navigator
+    {%- endif %}
     {%- if FREQSERVER != 0 %}
     - freqserver
     {%- endif %}

setup/automation/pm_standalone_defaults

@@ -21,6 +21,8 @@ address_type=DHCP
 ADMINUSER=onionuser
 ADMINPASS1=onionuser
 ADMINPASS2=onionuser
+ALLOW_CIDR=0.0.0.0/0
+ALLOW_ROLE=a
 BASICBRO=7
 BASICSURI=7
 # BLOGS=
@@ -34,7 +36,6 @@ HNMASTER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12
 HNSENSOR=inherit
 HOSTNAME=standalone
 install_type=STANDALONE
-IP=192.168.0.0/16
 # LSINPUTBATCHCOUNT=
 # LSINPUTTHREADS=
 # LSPIPELINEBATCH=
@@ -66,6 +67,7 @@ PLAYBOOK=1
 REDIRECTINFO=IP
 RULESETUP=ETOPEN
 # SHARDCOUNT=
+SKIP_REBOOT=1
 SOREMOTEPASS1=onionuser
 SOREMOTEPASS2=onionuser
 STRELKA=1

setup/so-functions

@@ -206,7 +206,7 @@ check_admin_pass() {
 	check_pass_match "$ADMINPASS1" "$ADMINPASS2" "APMATCH"
 }
 
-check_hive_init_then_reboot() {
+check_hive_init() {
 
 	wait_for_file /opt/so/state/thehive.txt 20 5
 	local return_val=$?
@@ -216,7 +216,6 @@ check_hive_init_then_reboot() {
 
 	docker stop so-thehive
 	docker rm so-thehive
-	shutdown -r now
 }
 
 check_network_manager_conf() {
@@ -981,59 +980,6 @@ node_pillar() {
 	cat "$pillar_file" >> "$setup_log" 2>&1
 }
 
-parse_options() {
-	case "$1" in
-		--turbo=*)
-			local proxy
-			proxy=$(echo "$1" | tr -d '"' | awk -F'--turbo=' '{print $2}')
-			proxy_url="http://$proxy"
-			TURBO="$proxy_url"
-			;;
-		--proxy=*)
-			local proxy
-			proxy=$(echo "$1" | tr -d '"' | awk -F'--proxy=' '{print $2}')
-
-			local proxy_protocol
-			proxy_protocol=$(echo "$proxy" | awk 'match($0, /http|https/) { print substr($0, RSTART, RLENGTH) }')
-
-			if [[ ! $proxy_protocol =~ ^(http|https)$ ]]; then
-				echo "Invalid proxy protocol"
-				echo "Ignoring proxy"
-				return
-			fi
-
-			if [[ $2 == --proxy-user=* && $3 == --proxy-pass=* ]]; then
-				local proxy_user
-				local proxy_password
-				proxy_user=$(echo "$2" | tr -d '"' | awk -F'--proxy-user=' '{print $2}')
-				proxy_password=$(echo "$3" | tr -d '"' | awk -F'--proxy-pass=' '{print $2}')
-
-				local proxy_addr
-				proxy_addr=$(echo "$proxy" | awk -F'http\:\/\/|https\:\/\/' '{print $2}')
-
-				export http_proxy="${proxy_protocol}://${proxy_user}:${proxy_password}@${proxy_addr}"
-
-			elif [[ (-z $2 || -z $3) && (-n $2 || -n $3) || ( -n $2 && -n $3 && ($2 != --proxy-user=* || $3 != --proxy-pass=*) ) ]]; then
-				echo "Invalid options passed for proxy. Order is --proxy-user=<user> --proxy-pass=<password>"
-				echo "Ignoring proxy"
-				return
-
-			else
-				export http_proxy="$proxy"
-			fi
-
-			export {https,ftp,rsync,all}_proxy="$http_proxy"
-			;;
-		"--allow-analyst"|"--allow=a")
-			export allow='a'
-			;;
-		*)
-			if  [[ $1 = --* ]]; then
-				echo "Invalid option"
-			fi
-	esac
-}
-
 patch_pillar() {
 
 	local pillar_file=$temp_install_dir/pillar/minions/$MINION_ID.sls

setup/so-setup

@@ -21,15 +21,35 @@ source ./so-common-functions
 source ./so-whiptail
 source ./so-variables
 
+# Parse command line arguments
 setup_type=$1
-export setup_type
-
 automation=$2
 
-automated=no
+while [[ $# -gt 0 ]]; do
+	arg="$1"
+	shift
+	case "$arg" in
+		"--turbo="* ) 
+			export TURBO="http://${arg#*=}";;
+		"--proxy="* ) 
+			export {http,https,ftp,rsync,all}_proxy="${arg#*=}";;
+		"--allow-role="* )
+			export ALLOW_ROLE="${arg#*=}";;
+		"--allow-cidr="* )
+			export ALLOW_CIDR="${arg#*=}";;
+		"--skip-reboot" )
+			export SKIP_REBOOT=1;;
+		* )
+			if [[ "$arg" == "--"* ]]; then
+				echo "Invalid option"
+			fi
+	esac
+done
 
+# Begin Installation pre-processing
 echo "---- Starting setup at $(date -u) ----" >> $setup_log 2>&1
 
+automated=no
 function progress() {
 	if [ $automated == no ]; then
 		whiptail --title "Security Onion Install" --gauge 'Please wait while installing' 6 60 0
@@ -43,7 +63,7 @@ if [[ -f automation/$automation && $(basename $automation) == $automation ]]; th
 	source automation/$automation
 	automated=yes
 
-	echo "Checking network configuration" >> $setup_log 2>&1g
+	echo "Checking network configuration" >> $setup_log 2>&1
 	ip a >> $setup_log 2>&1
 
 	attempt=1
@@ -78,11 +98,6 @@ export PATH=$PATH:../salt/common/tools/sbin
 
 got_root
 
-if [[ $# -gt 1 ]]; then
-	set -- "${@:2}"
-	parse_options "$@" >> $setup_log 2>&1
-fi
-
 detect_os
 
 if [ "$OS" == ubuntu ]; then
@@ -550,15 +565,17 @@ fi
 success=$(tail -10 $setup_log | grep Failed | awk '{ print $2}')
 if [[ "$success" = 0 ]]; then
 	whiptail_setup_complete
-	if [[ -n $allow ]]; then 
-		so-allow -$allow >> $setup_log 2>&1
+	if [[ -n $ALLOW_ROLE && -n $ALLOW_CIDR ]]; then 
+		export IP=$ALLOW_CIDR
+		so-allow -$ALLOW_ROLE >> $setup_log 2>&1
 	fi
 	if [[ $THEHIVE == 1 ]]; then
-		check_hive_init_then_reboot
-	else
-		shutdown -r now
+		check_hive_init
 	fi
 else
 	whiptail_setup_failed
+fi
+
+if [[ -z $SKIP_REBOOT ]]; then
 	shutdown -r now
 fi