Merge branch 'dev' of https://github.com/Security-Onion-Solutions/securityonion into dev

salt/common/tools/sbin/so-playbook-reset

@@ -22,5 +22,5 @@ salt-call state.apply playbook.db_init,playbook,playbook.automation_user_create
 /usr/sbin/so-soctopus-restart
 
 echo "Importing Plays - this will take some time...."
-sleep 5
+wait 5
 /usr/sbin/so-playbook-ruleupdate

salt/elasticsearch/files/ingest/ossec

@@ -63,7 +63,7 @@
     { "rename":         { "field": "fields.module",              "target_field": "event.module",                  "ignore_failure": true, "ignore_missing": true  } },
     { "pipeline":      { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'",   "name": "sysmon"  }  },
     { "pipeline":      { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational'",  "name":"win.eventlogs" }  },
-    { "set":           { "if": "ctx.containsKey('rule') && ctx.rule != null",   "field": "event.dataset", "value": "alert", "override": true }  },
+    { "set":           { "if": "ctx.rule != null && ctx.rule.name != null",   "field": "event.dataset", "value": "alert", "override": true }  },
     { "pipeline":       { "name": "common" } }    
   ]
 }

salt/elasticsearch/files/ingest/win.eventlogs

@@ -6,7 +6,7 @@
     { "set":           { "if": "ctx.winlog?.computer_name != null",   "field": "observer.name", "value": "{{winlog.computer_name}}", "override": true }  },
     { "set":           { "field": "event.code", "value": "{{winlog.event_id}}", "override": true }  },  
     { "set":           { "field": "event.category", "value": "host", "override": true }  },   
-    { "rename": 	   { "field": "winlog.event_data.SubjectUserName", 			  "target_field": "user.name",		"ignore_missing": true 	} },
+    { "rename": 	   { "field": "winlog.event_data.SubjectUserName", 			  "target_field": "user.name",		"ignore_failure": true,   "ignore_missing": true 	} },
     { "rename": 	   { "field": "winlog.event_data.User", 			            "target_field": "user.name",		"ignore_missing": true 	} }
   ]
 }

salt/playbook/init.sls

@@ -38,7 +38,7 @@ query_playbookdbuser_grants:
 query_updatwebhooks:
   mysql_query.run:
     - database: playbook
-    - query:    "update webhooks set url = 'http://{{MANAGERIP}}:7000/playbook/webhook' where project_id in (1,2)"
+    - query:    "update webhooks set url = 'http://{{MANAGERIP}}:7000/playbook/webhook' where project_id = 1"
     - connection_host: {{ MAINIP }}
     - connection_port: 3306
     - connection_user: root

salt/zeek/policy/securityonion/file-extraction/extract.zeek

@@ -1,4 +1,5 @@
-{%- import_yaml "zeek/fileextraction_defaults.yaml" as zeek with context %}
+{% import_yaml "zeek/fileextraction_defaults.yaml" as zeek_default -%}
+{% set zeek = salt['grains.filter_by'](zeek_default, default='zeek', merge=salt['pillar.get']('zeek', {})) -%}
 # Directory to stage Zeek extracted files before processing
 redef FileExtract::prefix = "/nsm/zeek/extracted/";
 # Set a limit to the file size
@@ -6,7 +7,7 @@ redef FileExtract::default_limit = 9000000;
 # These are the mimetypes we want to rip off the networks
 export {
     global _mime_whitelist: table[string] of string = {
-        {%- for li in zeek.zeek.policy.file_extraction %}
+        {%- for li in zeek.policy.file_extraction %}
           {%- if not loop.last %}
           {%- for k,v in li.items() %}
         ["{{ k }}"] = "{{ v }}",

setup/automation/aws_standalone_defaults

@@ -26,7 +26,7 @@ ALLOW_ROLE=a
 BASICZEEK=7
 BASICSURI=7
 # BLOGS=
-BNICS=ens6
+BNICS=eth1
 ZEEKVERSION=ZEEK
 # CURCLOSEDAYS=
 # EVALADVANCED=BASIC
@@ -46,7 +46,7 @@ MANAGERUPDATES=1
 # MGATEWAY=
 # MIP=
 # MMASK=
-MNIC=ens5
+MNIC=eth0
 # MSEARCH=
 # MSRV=
 # MTU=

setup/so-common-functions

@@ -1,52 +0,0 @@
-#!/bin/bash
-
-source ./so-variables
-source ../salt/common/tools/sbin/so-common
-source ../salt/common/tools/sbin/so-image-common
-
-# Helper functions
-
-filter_unused_nics() {
-
-	if [[ $MNIC ]]; then local grep_string="$MNIC\|bond0"; else local grep_string="bond0"; fi
-
-	# If we call this function and NICs have already been assigned to the bond interface then add them to the grep search string
-	if [[ $BNICS ]]; then
-		grep_string="$grep_string"
-		for BONDNIC in "${BNICS[@]}"; do
-			grep_string="$grep_string\|$BONDNIC"
-		done
-	fi
-
-	# Finally, set filtered_nics to any NICs we aren't using (and ignore interfaces that aren't of use)
-	filtered_nics=$(ip link | awk -F: '$0 !~ "lo|vir|veth|br|docker|wl|^[^0-9]"{print $2}' | grep -vwe "$grep_string"  | sed 's/ //g')
-	readarray -t filtered_nics <<< "$filtered_nics"
-	
-	nic_list=()
-	for nic in "${filtered_nics[@]}"; do
-		case $(cat "/sys/class/net/${nic}/carrier" 2>/dev/null) in
-			1)
-				nic_list+=("$nic" "Link UP " "OFF")
-				;;
-			0)
-				nic_list+=("$nic" "Link DOWN " "OFF")
-				;;
-			*)
-				nic_list+=("$nic" "Link UNKNOWN " "OFF")
-				;;
-		esac
-	done
-
-	export nic_list
-}
-
-calculate_useable_cores() {
-
-	# Calculate reasonable core usage
-	local cores_for_zeek=$(( (num_cpu_cores/2) - 1 ))
-	local lb_procs_round
-	lb_procs_round=$(printf "%.0f\n" $cores_for_zeek)
-
-	if [ "$lb_procs_round" -lt 1 ]; then lb_procs=1; else lb_procs=$lb_procs_round; fi
-    export lb_procs
-}

setup/so-functions

@@ -15,13 +15,7 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-source ./so-whiptail
+# README - DO NOT DEFINE GLOBAL VARIABLES IN THIS FILE. Instead use so-variables.
-source ./so-variables
-source ./so-common-functions
-
-CONTAINER_REGISTRY=quay.io
-
-SOVERSION=$(cat ../VERSION)
 
 log() {
 	msg=$1
@@ -48,6 +42,51 @@ logCmd() {
 	$cmd >> "$setup_log" 2>&1
 }
 
+filter_unused_nics() {
+
+	if [[ $MNIC ]]; then local grep_string="$MNIC\|bond0"; else local grep_string="bond0"; fi
+
+	# If we call this function and NICs have already been assigned to the bond interface then add them to the grep search string
+	if [[ $BNICS ]]; then
+		grep_string="$grep_string"
+		for BONDNIC in "${BNICS[@]}"; do
+			grep_string="$grep_string\|$BONDNIC"
+		done
+	fi
+
+	# Finally, set filtered_nics to any NICs we aren't using (and ignore interfaces that aren't of use)
+	filtered_nics=$(ip link | awk -F: '$0 !~ "lo|vir|veth|br|docker|wl|^[^0-9]"{print $2}' | grep -vwe "$grep_string"  | sed 's/ //g')
+	readarray -t filtered_nics <<< "$filtered_nics"
+	
+	nic_list=()
+	for nic in "${filtered_nics[@]}"; do
+		case $(cat "/sys/class/net/${nic}/carrier" 2>/dev/null) in
+			1)
+				nic_list+=("$nic" "Link UP " "OFF")
+				;;
+			0)
+				nic_list+=("$nic" "Link DOWN " "OFF")
+				;;
+			*)
+				nic_list+=("$nic" "Link UNKNOWN " "OFF")
+				;;
+		esac
+	done
+
+	export nic_list
+}
+
+calculate_useable_cores() {
+
+	# Calculate reasonable core usage
+	local cores_for_zeek=$(( (num_cpu_cores/2) - 1 ))
+	local lb_procs_round
+	lb_procs_round=$(printf "%.0f\n" $cores_for_zeek)
+
+	if [ "$lb_procs_round" -lt 1 ]; then lb_procs=1; else lb_procs=$lb_procs_round; fi
+    export lb_procs
+}
+
 airgap_rules() {
 	# Copy the rules for suricata if using Airgap
 	mkdir -p /nsm/repo/rules
@@ -871,7 +910,7 @@ docker_registry() {
 		"  \"bip\": \"$DNETBIP\","\
 		"  \"default-address-pools\": ["\
 		"    {"\
-		"      \"base\" : \"$DOCKERNET\","\
+		"      \"base\" : \"$DOCKERNET/24\","\
 		"      \"size\" : 24"\
 		"    }"\
 		"  ]"\

setup/so-setup

@@ -24,9 +24,19 @@ fi
 
 cd "$(dirname "$0")" || exit 255
 
+# Source the generic function libraries that are also used by the product after
+# setup. These functions are intended to be reusable outside of the setup process.
+source ../salt/common/tools/sbin/so-common
+source ../salt/common/tools/sbin/so-image-common
+
+# Setup bash functionality is divided into functions and user-facing prompts. 
+# Do not attempt to re-use any of this functionality outside of setup. Instead, 
+# if needed, migrated generic functions into so-common.
 source ./so-functions
-source ./so-common-functions
 source ./so-whiptail
+
+# Finally, source the default variable definitions, which require availability of
+# functions sourced above.
 source ./so-variables
 
 # Parse command line arguments

setup/so-variables

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+SOVERSION=$(cat ../VERSION)
+
 total_mem=$(grep MemTotal /proc/meminfo | awk '{print $2}' | sed -r 's/.{3}$//')
 export total_mem
 

setup/so-whiptail

@@ -15,9 +15,6 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-source ./so-variables
-source ./so-common-functions
-
 whiptail_airgap() {
 
 	[ -n "$TESTING" ] && return