From c74ace89bae108dbccd7f4f75241fb5058ef7ef6 Mon Sep 17 00:00:00 2001
From: Josh Brower <Josh@Defensivedepth.com>
Date: Thu, 21 May 2020 14:34:00 -0400
Subject: [PATCH 01/43] Initial support  - Ingest community_id

---
 salt/elasticsearch/files/ingest/zeek.common | 9 +++++++++
 salt/elasticsearch/files/ingest/zeek.http   | 1 +
 salt/elasticsearch/files/ingest/zeek.notice | 2 +-
 salt/elasticsearch/files/ingest/zeek.smtp   | 1 +
 4 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/salt/elasticsearch/files/ingest/zeek.common b/salt/elasticsearch/files/ingest/zeek.common
index c31625db6..5859b354b 100644
--- a/salt/elasticsearch/files/ingest/zeek.common
+++ b/salt/elasticsearch/files/ingest/zeek.common
@@ -7,6 +7,15 @@
     { "dot_expander":   { "field": "id.orig_p",                 "path": "message2",                     "ignore_failure": true  } },
     { "dot_expander":   { "field": "id.resp_h",                 "path": "message2",                     "ignore_failure": true  } },
     { "dot_expander":   { "field": "id.resp_p",                 "path": "message2",                     "ignore_failure": true  } },
+    
+    {"convert":{"field":"message2.id.orig_p","type":"string"}},
+    {"convert":{"field":"message2.id.resp_p","type":"string"}},
+
+    {"community_id": {"if": "ctx.network?.transport != null", "field":["message2.id.orig_h","message2.id.orig_p","message2.id.resp_h","message2.id.resp_p","network.transport"],"target_field":"network.community_id"}},
+
+    {"convert":{"field":"message2.id.orig_p","type":"integer"}},
+    {"convert":{"field":"message2.id.resp_p","type":"integer"}},
+
     { "rename":         { "field": "message2.id.orig_h",        "target_field": "source.ip",            "ignore_missing": true  } },
     { "rename":         { "field": "message2.id.orig_p",        "target_field": "source.port",          "ignore_missing": true  } },
     { "rename":         { "field": "message2.id.resp_h",        "target_field": "destination.ip",       "ignore_missing": true  } },
diff --git a/salt/elasticsearch/files/ingest/zeek.http b/salt/elasticsearch/files/ingest/zeek.http
index a1354044c..3368e45e1 100644
--- a/salt/elasticsearch/files/ingest/zeek.http
+++ b/salt/elasticsearch/files/ingest/zeek.http
@@ -29,6 +29,7 @@
     { "script":	{ "lang": "painless", "source": "ctx.uri_length = ctx.uri.length()", 			"ignore_failure": true 	} },
     { "script":	{ "lang": "painless", "source": "ctx.useragent_length = ctx.useragent.length()",	"ignore_failure": true 	} },
     { "script":	{ "lang": "painless", "source": "ctx.virtual_host_length = ctx.virtual_host.length()", 	"ignore_failure": true	} },
+    { "set":      { "field": "network.transport",        "value": "tcp"  } },
     { "pipeline":       { "name": "zeek.common"											} }
   ]
 }
diff --git a/salt/elasticsearch/files/ingest/zeek.notice b/salt/elasticsearch/files/ingest/zeek.notice
index 4e54f325d..b662393f6 100644
--- a/salt/elasticsearch/files/ingest/zeek.notice
+++ b/salt/elasticsearch/files/ingest/zeek.notice
@@ -6,7 +6,7 @@
     { "rename": 	{ "field": "message2.fuid", 		"target_field": "log.id.fuid",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.mime", 		"target_field": "file.mimetype",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.desc", 		"target_field": "file.description",	"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.proto", 		"target_field": "network.protocol",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.proto", 		"target_field": "network.transport",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.note", 		"target_field": "notice.note",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.msg", 		"target_field": "notice.message",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.sub", 		"target_field": "notice.sub_message",		"ignore_missing": true 	} },
diff --git a/salt/elasticsearch/files/ingest/zeek.smtp b/salt/elasticsearch/files/ingest/zeek.smtp
index 473b4cce5..9bfb1e3e1 100644
--- a/salt/elasticsearch/files/ingest/zeek.smtp
+++ b/salt/elasticsearch/files/ingest/zeek.smtp
@@ -25,6 +25,7 @@
     { "rename": 	{ "field": "message2.tls", 		"target_field": "smtp.tls",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.fuids", 		"target_field": "log.id.fuids",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.is_webmail",	"target_field": "smtp.is_webmail",		"ignore_missing": true 	} },
+    { "set":      { "field": "network.transport",        "value": "tcp"  } },
     { "pipeline":       { "name": "zeek.common"                                                                                   } }
   ]
 }

From bff86ea802ad731524441452409417ae743b4cce Mon Sep 17 00:00:00 2001
From: Josh Brower <Josh@Defensivedepth.com>
Date: Thu, 21 May 2020 14:35:25 -0400
Subject: [PATCH 02/43] zeek.common ingest parser fix

---
 salt/elasticsearch/files/ingest/zeek.common | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/salt/elasticsearch/files/ingest/zeek.common b/salt/elasticsearch/files/ingest/zeek.common
index 5859b354b..b0ac0d12a 100644
--- a/salt/elasticsearch/files/ingest/zeek.common
+++ b/salt/elasticsearch/files/ingest/zeek.common
@@ -6,16 +6,8 @@
     { "dot_expander":   { "field": "id.orig_h",                 "path": "message2",                     "ignore_failure": true  } },
     { "dot_expander":   { "field": "id.orig_p",                 "path": "message2",                     "ignore_failure": true  } },
     { "dot_expander":   { "field": "id.resp_h",                 "path": "message2",                     "ignore_failure": true  } },
-    { "dot_expander":   { "field": "id.resp_p",                 "path": "message2",                     "ignore_failure": true  } },
-    
-    {"convert":{"field":"message2.id.orig_p","type":"string"}},
-    {"convert":{"field":"message2.id.resp_p","type":"string"}},
-
+    { "dot_expander":   { "field": "id.resp_p",                 "path": "message2",                     "ignore_failure": true  } },    
     {"community_id": {"if": "ctx.network?.transport != null", "field":["message2.id.orig_h","message2.id.orig_p","message2.id.resp_h","message2.id.resp_p","network.transport"],"target_field":"network.community_id"}},
-
-    {"convert":{"field":"message2.id.orig_p","type":"integer"}},
-    {"convert":{"field":"message2.id.resp_p","type":"integer"}},
-
     { "rename":         { "field": "message2.id.orig_h",        "target_field": "source.ip",            "ignore_missing": true  } },
     { "rename":         { "field": "message2.id.orig_p",        "target_field": "source.port",          "ignore_missing": true  } },
     { "rename":         { "field": "message2.id.resp_h",        "target_field": "destination.ip",       "ignore_missing": true  } },

From 967148890cc2e66f30c9fdfbfd5b48fccf9288a9 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Fri, 22 May 2020 11:39:32 -0400
Subject: [PATCH 03/43] [fix] Only prompt user about network install on network
 install

---
 setup/proxies/docker.conf | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 setup/proxies/docker.conf

diff --git a/setup/proxies/docker.conf b/setup/proxies/docker.conf
new file mode 100644
index 000000000..9ab2c4b4c
--- /dev/null
+++ b/setup/proxies/docker.conf
@@ -0,0 +1,2 @@
+[Service]
+ExecStart=/usr/bin/dockerd /usr/bin/dockerd -H fd:// --registry-mirror "$proxy_addr"

From b029d9aca62b9bafd9fb2f1915287bb195967c2b Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Fri, 22 May 2020 11:40:16 -0400
Subject: [PATCH 04/43] [feat] Initial work for running setup through proxy

---
 setup/so-functions | 49 ++++++++++++++++++++++++++++++++++++++++++++++
 setup/so-setup     |  7 ++++++-
 setup/so-whiptail  |  2 +-
 so-setup-network   |  4 ++--
 4 files changed, 58 insertions(+), 4 deletions(-)

diff --git a/setup/so-functions b/setup/so-functions
index a20953035..1e7bf7951 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -948,6 +948,35 @@ node_pillar() {
 	cat "$pillar_file" >> "$setup_log" 2>&1
 }
 
+parse_options() {
+	case "$1" in
+		--turbo=*)
+			local proxy
+			proxy=$(echo "$1" | awk -F'--turbo=' '{print $2}')
+			use_proxy "http://$proxy"
+			;;
+		--proxy=*)
+			echo "Unimplimented"
+			return
+
+			if [[ $2 != --proxy-user=* ]] || [[ $3 != --proxy-pass=* ]]; then
+				echo "Invalid options passed for proxy. Order is --proxy-user=<user> --proxy-pass=<password>"
+			else
+				local proxy
+				local user
+				local password
+				proxy=$(echo "$1" | awk -F'--proxy=' '{print $2}')
+				user=$(echo "$2" | awk -F'--proxy-user=' '{print $2}')
+				password=$(echo "$3" | awk -F'--proxy-pass=' '{print $2}')
+				
+				use_proxy "$proxy" "$user" "$password"
+			fi
+			;;
+		*)
+			echo "Invalid option"
+	esac
+}
+
 patch_pillar() {
 
 	local pillar_file=$temp_install_dir/pillar/minions/$MINION_ID.sls
@@ -1443,6 +1472,26 @@ update_packages() {
 	fi
 }
 
+use_proxy() {
+	local proxy_addr=$1
+	#TODO: add options for username + pass
+
+	if [[ $OS == 'centos' ]]; then
+		printf '%s\n'\
+			"proxy=\"$proxy_addr\"" >> /etc/yum.conf
+	else
+		printf '%s\n'\
+			"Acquire {"\
+			"HTTP::proxy \"$proxy_addr\";"\
+			"HTTPS::proxy \"$proxy_addr\";"\
+			"}" > /etc/apt/apt.conf.d/proxy.conf
+	fi
+	mkdir -p /etc/systemd/system/docker.service.d
+	printf '%s\n'\
+		"[Service]"\
+		"ExecStart=/usr/bin/dockerd /usr/bin/dockerd -H fd:// --registry-mirror \"$proxy_addr\"" > /etc/systemd/system/docker.service.d/docker.conf
+}
+
 ls_heapsize() {
 
 	if [ "$total_mem" -ge 32000 ]; then
diff --git a/setup/so-setup b/setup/so-setup
index 406d69763..5ec148208 100755
--- a/setup/so-setup
+++ b/setup/so-setup
@@ -48,7 +48,7 @@ fi
 
 setterm -blank 0
 
-if (whiptail_you_sure); then
+if [ "$setup_type" == 'iso' ] || (whiptail_you_sure); then
 	true
 else
 	echo "User cancelled setup." >> $setup_log 2>&1
@@ -241,6 +241,11 @@ fi
 
 whiptail_make_changes
 
+if [[ $# -gt 1 ]]; then
+	set -- "${@:2}"
+	parse_options "$@"
+fi
+
 if [[ "$setup_type" == 'iso' ]]; then
 	# Init networking so rest of install works
 	set_hostname_iso
diff --git a/setup/so-whiptail b/setup/so-whiptail
index cfe00b67b..559f791b7 100755
--- a/setup/so-whiptail
+++ b/setup/so-whiptail
@@ -1048,7 +1048,7 @@ whiptail_you_sure() {
 
 	[ -n "$TESTING" ] && return
 
-	whiptail --title "Security Onion Setup" --yesno "Are you sure you want to install Security Onion over the internet?" 8 75
+	whiptail --title "Security Onion Setup" --yesno "Are you sure you want to continue a network install of Security Onion?" 8 75
 
 	local exitstatus=$?
 	return $exitstatus
diff --git a/so-setup-network b/so-setup-network
index ae9af4ffa..2528ff14b 100755
--- a/so-setup-network
+++ b/so-setup-network
@@ -15,6 +15,6 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-cd setup
+cd setup || exit
 
-./so-setup network
+./so-setup network "$@"

From c790b3827db56c3916752ee4bc1cf85650feea56 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Fri, 22 May 2020 13:25:17 -0400
Subject: [PATCH 05/43] [fix] yum.conf proxy should not contain quotes

---
 setup/so-functions | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup/so-functions b/setup/so-functions
index 1e7bf7951..38b0b5b9a 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -1478,7 +1478,7 @@ use_proxy() {
 
 	if [[ $OS == 'centos' ]]; then
 		printf '%s\n'\
-			"proxy=\"$proxy_addr\"" >> /etc/yum.conf
+			"proxy=$proxy_addr" >> /etc/yum.conf
 	else
 		printf '%s\n'\
 			"Acquire {"\

From 4b505827b19c9ec23d139753da0d19ab37fb1937 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Fri, 22 May 2020 13:31:58 -0400
Subject: [PATCH 06/43] [fix] Use correct ports for acng and docker registry

---
 setup/so-functions | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/setup/so-functions b/setup/so-functions
index 38b0b5b9a..934ff6cc1 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -1478,18 +1478,18 @@ use_proxy() {
 
 	if [[ $OS == 'centos' ]]; then
 		printf '%s\n'\
-			"proxy=$proxy_addr" >> /etc/yum.conf
+			"proxy=$proxy_addr:3412" >> /etc/yum.conf
 	else
 		printf '%s\n'\
 			"Acquire {"\
-			"HTTP::proxy \"$proxy_addr\";"\
-			"HTTPS::proxy \"$proxy_addr\";"\
+			"HTTP::proxy \"$proxy_addr:3412\";"\
+			"HTTPS::proxy \"$proxy_addr:3412\";"\
 			"}" > /etc/apt/apt.conf.d/proxy.conf
 	fi
-	mkdir -p /etc/systemd/system/docker.service.d
 	printf '%s\n'\
-		"[Service]"\
-		"ExecStart=/usr/bin/dockerd /usr/bin/dockerd -H fd:// --registry-mirror \"$proxy_addr\"" > /etc/systemd/system/docker.service.d/docker.conf
+		"{"\
+		"  \"registry-mirrors\": [\"$proxy_addr:5000\"]"\
+		"}" > /etc/docker/daemon.json
 }
 
 ls_heapsize() {

From 78f6261fccfae63331e38e494296209fb018ff49 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Fri, 22 May 2020 14:00:01 -0400
Subject: [PATCH 07/43] [fix] acng should be port 3142 not 3412

---
 setup/so-functions | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/setup/so-functions b/setup/so-functions
index 934ff6cc1..303590d55 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -1478,12 +1478,12 @@ use_proxy() {
 
 	if [[ $OS == 'centos' ]]; then
 		printf '%s\n'\
-			"proxy=$proxy_addr:3412" >> /etc/yum.conf
+			"proxy=$proxy_addr:3142" >> /etc/yum.conf
 	else
 		printf '%s\n'\
 			"Acquire {"\
-			"HTTP::proxy \"$proxy_addr:3412\";"\
-			"HTTPS::proxy \"$proxy_addr:3412\";"\
+			"HTTP::proxy \"$proxy_addr:3142\";"\
+			"HTTPS::proxy \"$proxy_addr:3142\";"\
 			"}" > /etc/apt/apt.conf.d/proxy.conf
 	fi
 	printf '%s\n'\

From 579b6229358f9fb11fa7e1d8a9692feddd1879e8 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Fri, 22 May 2020 14:36:55 -0400
Subject: [PATCH 08/43] [fix] Don't overwrite daemon.json, and only accept
 turbo mode on correct install types

---
 setup/so-functions | 35 +++++++++++++++++++----------------
 1 file changed, 19 insertions(+), 16 deletions(-)

diff --git a/setup/so-functions b/setup/so-functions
index 303590d55..ccfd59629 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -616,9 +616,10 @@ docker_registry() {
 	echo "Setting up Docker Registry" >> "$setup_log" 2>&1
 	mkdir -p /etc/docker >> "$setup_log" 2>&1
 	# Make the host use the master docker registry
+	if [ -n "$TURBO" ]; then local proxy="$TURBO"; else local proxy="$MSRV"; fi
 	printf '%s\n'\
 		"{"\
-		"  \"registry-mirrors\": [\"https://$MSRV:5000\"]"\
+		"  \"registry-mirrors\": [\"https://$proxy:5000\"]"\
 		"}" > /etc/docker/daemon.json
 	echo "Docker Registry Setup - Complete" >> "$setup_log" 2>&1
 
@@ -951,9 +952,15 @@ node_pillar() {
 parse_options() {
 	case "$1" in
 		--turbo=*)
-			local proxy
-			proxy=$(echo "$1" | awk -F'--turbo=' '{print $2}')
-			use_proxy "http://$proxy"
+			if [[ $is_master || $is_helix ]]; then
+				local proxy
+				proxy=$(echo "$1" | awk -F'--turbo=' '{print $2}')
+				proxy_addr="http://$proxy"
+				use_proxy "$proxy_addr"
+				TURBO="$proxy_addr"
+			else
+				echo "turbo is not supported on this install type" >> $setup_log 2>&1
+			fi
 			;;
 		--proxy=*)
 			echo "Unimplimented"
@@ -963,13 +970,13 @@ parse_options() {
 				echo "Invalid options passed for proxy. Order is --proxy-user=<user> --proxy-pass=<password>"
 			else
 				local proxy
-				local user
-				local password
+				local proxy_user
+				local proxy_password
 				proxy=$(echo "$1" | awk -F'--proxy=' '{print $2}')
-				user=$(echo "$2" | awk -F'--proxy-user=' '{print $2}')
-				password=$(echo "$3" | awk -F'--proxy-pass=' '{print $2}')
+				proxy_user=$(echo "$2" | awk -F'--proxy-user=' '{print $2}')
+				proxy_password=$(echo "$3" | awk -F'--proxy-pass=' '{print $2}')
 				
-				use_proxy "$proxy" "$user" "$password"
+				use_proxy "$proxy" "$proxy_user" "$proxy_password"
 			fi
 			;;
 		*)
@@ -1478,18 +1485,14 @@ use_proxy() {
 
 	if [[ $OS == 'centos' ]]; then
 		printf '%s\n'\
-			"proxy=$proxy_addr:3142" >> /etc/yum.conf
+			"proxy=${proxy_addr}:3142" >> /etc/yum.conf
 	else
 		printf '%s\n'\
 			"Acquire {"\
-			"HTTP::proxy \"$proxy_addr:3142\";"\
-			"HTTPS::proxy \"$proxy_addr:3142\";"\
+			"HTTP::proxy \"${proxy_addr}:3142\";"\
+			"HTTPS::proxy \"${proxy_addr}:3142\";"\
 			"}" > /etc/apt/apt.conf.d/proxy.conf
 	fi
-	printf '%s\n'\
-		"{"\
-		"  \"registry-mirrors\": [\"$proxy_addr:5000\"]"\
-		"}" > /etc/docker/daemon.json
 }
 
 ls_heapsize() {

From 19cd57bce0f2c72fd1aeab9e13503ce40cb77a27 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Fri, 22 May 2020 14:44:57 -0400
Subject: [PATCH 09/43] Support automated setup

---
 setup/automation/pm_standalone_defaults | 75 +++++++++++++++++++++++++
 setup/so-functions                      |  3 +
 setup/so-setup                          |  6 ++
 3 files changed, 84 insertions(+)
 create mode 100644 setup/automation/pm_standalone_defaults

diff --git a/setup/automation/pm_standalone_defaults b/setup/automation/pm_standalone_defaults
new file mode 100644
index 000000000..b5a6258ff
--- /dev/null
+++ b/setup/automation/pm_standalone_defaults
@@ -0,0 +1,75 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+TESTING=true
+
+address_type=DHCP
+ADMINUSER=onionuser
+ADMINPASS1=onionuser
+ADMINPASS2=onionuser
+BASICBRO=7
+BASICSURI=7
+# BLOGS=
+BNICS=eth1
+BROVERSION=ZEEK
+# CURCLOSEDAYS=
+# EVALADVANCED=BASIC
+GRAFANA=1
+# HELIXAPIKEY=
+HNMASTER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12
+HNSENSOR=inherit
+HOSTNAME=standalone
+install_type=STANDALONE
+# LSINPUTBATCHCOUNT=
+# LSINPUTTHREADS=
+# LSPIPELINEBATCH=
+# LSPIPELINEWORKERS=
+MASTERADV=BASIC
+MASTERUPDATES=1
+# MDNS=
+# MGATEWAY=
+# MIP=
+# MMASK=
+MNIC=eth0
+# MSEARCH=
+# MSRV=
+# MTU=
+NAVIGATOR=1
+NIDS=Suricata
+# NODE_ES_HEAP_SIZE=
+# NODE_LS_HEAP_SIZE=
+NODESETUP=NODEBASIC
+NSMSETUP=BASIC
+NODEUPDATES=MASTER
+# OINKCODE=
+OSQUERY=1
+# PATCHSCHEDULEDAYS=
+# PATCHSCHEDULEHOURS=
+PATCHSCHEDULENAME=auto
+PLAYBOOK=1
+# REDIRECTHOST=
+REDIRECTINFO=IP
+RULESETUP=ETOPEN
+# SHARDCOUNT=
+SOREMOTEPASS1=onionuser
+SOREMOTEPASS2=onionuser
+STRELKA=1
+THEHIVE=1
+WAZUH=1
+WEBUSER=onionuser@somewhere.invalid
+WEBPASSWD1=onionuser
+WEBPASSWD2=onionuser
\ No newline at end of file
diff --git a/setup/so-functions b/setup/so-functions
index a20953035..185f41d89 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -550,6 +550,9 @@ detect_os() {
 disable_onion_user() {
 	# Disable the default account cause security.
 	usermod -L onion
+
+	# Remove the automated setup script from crontab, if it exists
+	crontab -u onion -r
 }
 
 disable_misc_network_features() {
diff --git a/setup/so-setup b/setup/so-setup
index 406d69763..3e54d3f66 100755
--- a/setup/so-setup
+++ b/setup/so-setup
@@ -24,6 +24,12 @@ source ./so-variables
 setup_type=$1
 export setup_type
 
+automation=$2
+if [[ -f automation/$automation && $(basename $automation) == $automation ]]; then
+	echo "Preselecting variable values based on automated setup: $automation"
+	source automation/$automation
+fi
+
 case "$setup_type" in
 	iso | network) # Accepted values
 		echo "Beginning Security Onion $setup_type install"

From 862631b93a1a8d84b333448a12be9eefeae6cc38 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Fri, 22 May 2020 14:47:00 -0400
Subject: [PATCH 10/43] [fix] Add space in daemon.json

---
 setup/so-functions | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup/so-functions b/setup/so-functions
index ccfd59629..083b9db61 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -619,7 +619,7 @@ docker_registry() {
 	if [ -n "$TURBO" ]; then local proxy="$TURBO"; else local proxy="$MSRV"; fi
 	printf '%s\n'\
 		"{"\
-		"  \"registry-mirrors\": [\"https://$proxy:5000\"]"\
+		"  \"registry-mirrors\": [ \"https://$proxy:5000\" ]"\
 		"}" > /etc/docker/daemon.json
 	echo "Docker Registry Setup - Complete" >> "$setup_log" 2>&1
 

From 08564ed6d4e6807c0a51aa429542bfe350f5bfea Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Fri, 22 May 2020 14:50:42 -0400
Subject: [PATCH 11/43] [feat] Trim quotes from arguments

---
 setup/so-functions | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/setup/so-functions b/setup/so-functions
index 083b9db61..8c911c3d7 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -972,9 +972,9 @@ parse_options() {
 				local proxy
 				local proxy_user
 				local proxy_password
-				proxy=$(echo "$1" | awk -F'--proxy=' '{print $2}')
-				proxy_user=$(echo "$2" | awk -F'--proxy-user=' '{print $2}')
-				proxy_password=$(echo "$3" | awk -F'--proxy-pass=' '{print $2}')
+				proxy=$(echo "$1" | tr -d '"' | awk -F'--proxy=' '{print $2}')
+				proxy_user=$(echo "$2" | tr -d '"' | awk -F'--proxy-user=' '{print $2}')
+				proxy_password=$(echo "$3" | tr -d '"' | awk -F'--proxy-pass=' '{print $2}')
 				
 				use_proxy "$proxy" "$proxy_user" "$proxy_password"
 			fi

From ad96baab551c3f1970ded8a990369183513fd6e6 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Fri, 22 May 2020 14:50:58 -0400
Subject: [PATCH 12/43] [feat] Trim quotes from arguments

---
 setup/so-functions | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup/so-functions b/setup/so-functions
index 8c911c3d7..b29d4adef 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -954,7 +954,7 @@ parse_options() {
 		--turbo=*)
 			if [[ $is_master || $is_helix ]]; then
 				local proxy
-				proxy=$(echo "$1" | awk -F'--turbo=' '{print $2}')
+				proxy=$(echo "$1" | tr -d '"' | awk -F'--turbo=' '{print $2}')
 				proxy_addr="http://$proxy"
 				use_proxy "$proxy_addr"
 				TURBO="$proxy_addr"

From d9feb0d95c3c463a6f15c76c6e77f40eb8c4ae70 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Fri, 22 May 2020 15:09:04 -0400
Subject: [PATCH 13/43] [fix] Don't repeat protocol in daemon.json

---
 setup/so-functions | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup/so-functions b/setup/so-functions
index b29d4adef..4e01c935c 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -616,10 +616,10 @@ docker_registry() {
 	echo "Setting up Docker Registry" >> "$setup_log" 2>&1
 	mkdir -p /etc/docker >> "$setup_log" 2>&1
 	# Make the host use the master docker registry
-	if [ -n "$TURBO" ]; then local proxy="$TURBO"; else local proxy="$MSRV"; fi
+	if [ -n "$TURBO" ]; then local proxy="$TURBO"; else local proxy="https://$MSRV"; fi
 	printf '%s\n'\
 		"{"\
-		"  \"registry-mirrors\": [ \"https://$proxy:5000\" ]"\
+		"  \"registry-mirrors\": [ \"$proxy:5000\" ]"\
 		"}" > /etc/docker/daemon.json
 	echo "Docker Registry Setup - Complete" >> "$setup_log" 2>&1
 

From 29726e957f408835eaf4b6ef016e19e0d7fd6b9d Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Fri, 22 May 2020 15:16:57 -0400
Subject: [PATCH 14/43] [fix] Redirect parse_options output to setup log

---
 setup/so-setup | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup/so-setup b/setup/so-setup
index 5ec148208..4b0a4d6f6 100755
--- a/setup/so-setup
+++ b/setup/so-setup
@@ -243,7 +243,7 @@ whiptail_make_changes
 
 if [[ $# -gt 1 ]]; then
 	set -- "${@:2}"
-	parse_options "$@"
+	parse_options "$@" >> $setup_log 2>&1
 fi
 
 if [[ "$setup_type" == 'iso' ]]; then

From 13423bed18eca71c101252a0dc7d4683051eb5fb Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Fri, 22 May 2020 16:35:16 -0400
Subject: [PATCH 15/43] [fix] Add indent in proxy.conf

---
 setup/so-functions | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup/so-functions b/setup/so-functions
index 04b2652a5..ddf265153 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -1492,8 +1492,8 @@ use_proxy() {
 	else
 		printf '%s\n'\
 			"Acquire {"\
-			"HTTP::proxy \"${proxy_addr}:3142\";"\
-			"HTTPS::proxy \"${proxy_addr}:3142\";"\
+			"  HTTP::proxy \"${proxy_addr}:3142\";"\
+			"  HTTPS::proxy \"${proxy_addr}:3142\";"\
 			"}" > /etc/apt/apt.conf.d/proxy.conf
 	fi
 }

From 56f5fbdf6b095445094de6a379b80fb2d690b085 Mon Sep 17 00:00:00 2001
From: Josh Brower <Josh@Defensivedepth.com>
Date: Fri, 22 May 2020 17:11:08 -0400
Subject: [PATCH 16/43] Ingest pipeline commid fix for conn logs

---
 salt/elasticsearch/files/ingest/common | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/elasticsearch/files/ingest/common b/salt/elasticsearch/files/ingest/common
index e70d5e2d8..a65742f99 100644
--- a/salt/elasticsearch/files/ingest/common
+++ b/salt/elasticsearch/files/ingest/common
@@ -38,7 +38,7 @@
     { "rename":         { "field": "module",              "target_field": "event.module",                  "ignore_missing": true  } },
     { "rename":         { "field": "dataset",              "target_field": "event.dataset",                  "ignore_missing": true  } },
     { "rename":         { "field": "category",              "target_field": "event.category",                  "ignore_missing": true  } },
-    { "rename":         { "field": "message2.community_id",              "target_field": "network.community_id",                  "ignore_missing": true  } },
+    { "rename":         { "field": "message2.community_id", "target_field": "network.community_id",  "ignore_failure": true,  "ignore_missing": true  } },
     { 
         "remove": {
                 "field": [ "index_name_prefix", "message2", "type" ],

From 1c207afb31bd321e25a1e7e72586b7df438c3870 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Sat, 23 May 2020 12:00:24 -0400
Subject: [PATCH 17/43] Define packages in the common state

---
 salt/common/init.sls | 85 ++++++++++++++++++++++++++++++++++++++++----
 setup/so-functions   | 12 ++++---
 2 files changed, 85 insertions(+), 12 deletions(-)

diff --git a/salt/common/init.sls b/salt/common/init.sls
index 82ac4a062..a8c791c7f 100644
--- a/salt/common/init.sls
+++ b/salt/common/init.sls
@@ -28,21 +28,92 @@ salttmp:
     - group: 939
     - makedirs: True
 
-# Install packages needed for the sensor
-sensorpkgs:
+# Install epel
+{% if grains['os'] == 'CentOS' %}
+epel:
   pkg.installed:
-    - skip_suggestions: False
+    - skip_suggestions: True
+    - pkgs:
+      - epel-release
+{% endif %}
+
+# Install common packages
+commonpkgs:
+  pkg.installed:
+    - skip_suggestions: True
     - pkgs:
-      - wget
-      - jq
       {% if grains['os'] != 'CentOS' %}
       - apache2-utils
+      - wget
+      - jq
+      - python3-docker
+      - docker-ce
+      - curl
+      - ca-certificates
+      - software-properties-common
+      - apt-transport-https
+      - openssl
+      - netcat
+      - python3-mysqldb
+      - sqlite3
+      - argon2
+      - libssl-dev
+      - python3-dateutil
+      - python3-m2crypto
+      - python3-mysqldb
+      - salt-minion: 2019.2.5+ds-1
+        - hold: True
+        - update_holds: True
+      {% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-helix' or grains['role'] == 'so-mastersearch' or grains['role'] == 'so-standalone' %}
+      - salt-master: 2019.2.5+ds-1
+        - hold: True
+        - update_holds: True
+      - containerd.io: 1.2.13-2
+        - hold: True
+        - update_holds: True
+      - docker-ce: 5:19.03.9~3-0~ubuntu-bionic
+        - hold: True
+        - update_holds: True
+      {% endif %}
+      - containerd.io
+      - docker-ce      
       {% else %}
-      - net-tools
+      - wget
+      - bind-utils
+      - jq
       - tcpdump
       - httpd-tools
+      - net-tools
+      - curl
+      - sqlite
+      - argon2
+      - maridb-devel
+      - nmap-ncat
+      - python3
+			-	python36-docker
+			-	python36-dateutil
+			-	python36-m2crypto
+			-	python36-mysql
+			-	yum-utils
+			-	device-mapper-persistent-data
+			-	lvm2
+			-	openssl
+      - salt-minion: 2019.2.5
+        - hold: True
+        - update_holds: True
+      {% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-helix' or grains['role'] == 'so-mastersearch' or grains['role'] == 'so-standalone' %}
+      - salt-master: 2019.2.5
+        - hold: True
+        - update_holds: True
       {% endif %}
-
+      - containerd.io: 1.2.6-3
+        - hold: True
+        - update_holds: True
+      - docker-ce: 19.03.9-3
+        - hold: True
+        - update_holds: True
+      {% endif %}%}
+				
 # Always keep these packages up to date
 
 alwaysupdated:
diff --git a/setup/so-functions b/setup/so-functions
index ddf265153..702c3cebd 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -19,7 +19,7 @@ source ./so-whiptail
 source ./so-variables
 source ./so-common-functions
 
-SOVERSION=1.3.0
+SOVERSION=1.4.0
 
 accept_salt_key_remote() {
 	systemctl restart salt-minion
@@ -514,7 +514,7 @@ detect_os() {
 		# Install bind-utils so the host command exists
 		if ! command -v host > /dev/null 2>&1; then
 			echo "Installing required packages to run installer"
-			yum -y install bind-utils  >> "$setup_log" 2>&1
+			yum -y install bind-utils yum-plugin-versionlock >> "$setup_log" 2>&1
 		fi
 
 
@@ -583,7 +583,9 @@ docker_install() {
 		{
 			yum clean expire-cache;
 			yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo;
-			yum -y install docker-ce;
+			yum -y install docker-ce-19.03.9-3 containerd.io-1.2.6-3;
+			yum versionlock docker-ce;
+			yum versionlock containerd.io
 		} >> "$setup_log" 2>&1
 		
 	else
@@ -687,7 +689,7 @@ docker_seed_registry() {
 				# Tag it with the new registry destination
 				docker tag soshybridhunter/"$i" "$HOSTNAME":5000/soshybridhunter/"$i"
 				docker push "$HOSTNAME":5000/soshybridhunter/"$i"
-				docker rmi soshybridhunter/"$i"
+				#docker rmi soshybridhunter/"$i"
 			} >> "$setup_log" 2>&1
 		done
 	else
@@ -1107,7 +1109,7 @@ saltify() {
 			yum -y update exclude=salt*;
 			systemctl enable salt-minion;
 		} >> "$setup_log" 2>&1
-		echo "exclude=salt*" >> /etc/yum.conf
+		yum versionlock salt*
 	else
 		DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" upgrade >> "$setup_log" 2>&1
 

From 31b3563fb3608bc9b104bd6456bca84bbe8e5f81 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Sun, 24 May 2020 10:56:30 -0400
Subject: [PATCH 18/43] Fix package versions

---
 salt/common/init.sls | 8 ++++----
 setup/so-functions   | 6 +++---
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/salt/common/init.sls b/salt/common/init.sls
index a8c791c7f..17cbb1608 100644
--- a/salt/common/init.sls
+++ b/salt/common/init.sls
@@ -98,18 +98,18 @@ commonpkgs:
 			-	device-mapper-persistent-data
 			-	lvm2
 			-	openssl
-      - salt-minion: 2019.2.5
+      - salt-minion: 2019.2.5.el7
         - hold: True
         - update_holds: True
       {% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-helix' or grains['role'] == 'so-mastersearch' or grains['role'] == 'so-standalone' %}
-      - salt-master: 2019.2.5
+      - salt-master: 2019.2.5.el7
         - hold: True
         - update_holds: True
       {% endif %}
-      - containerd.io: 1.2.6-3
+      - containerd.io: 1.2.6-3.el7
         - hold: True
         - update_holds: True
-      - docker-ce: 19.03.9-3
+      - docker-ce: 19.03.9-3.el7
         - hold: True
         - update_holds: True
       {% endif %}%}
diff --git a/setup/so-functions b/setup/so-functions
index 702c3cebd..bf6db26be 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -583,9 +583,9 @@ docker_install() {
 		{
 			yum clean expire-cache;
 			yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo;
-			yum -y install docker-ce-19.03.9-3 containerd.io-1.2.6-3;
-			yum versionlock docker-ce;
-			yum versionlock containerd.io
+			yum -y install docker-ce-19.03.9-3.el7 containerd.io-1.2.6-3.el7;
+			yum versionlock docker-ce-19.03.9-3.el7;
+			yum versionlock containerd.io-1.2.6-3.el7
 		} >> "$setup_log" 2>&1
 		
 	else

From ad97092589f8d46600a4ce695d83155895cac84d Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Sun, 24 May 2020 21:38:37 -0400
Subject: [PATCH 19/43] When automating setup installation wait for network
 availability

---
 setup/so-setup | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup/so-setup b/setup/so-setup
index b3c141fae..9384c5e96 100755
--- a/setup/so-setup
+++ b/setup/so-setup
@@ -28,6 +28,7 @@ automation=$2
 if [[ -f automation/$automation && $(basename $automation) == $automation ]]; then
 	echo "Preselecting variable values based on automated setup: $automation"
 	source automation/$automation
+	sleep 30 # Re-implement with network availability probe
 fi
 
 case "$setup_type" in

From e09027e7954016e43a3f0b01ac7109e43fccaeab Mon Sep 17 00:00:00 2001
From: Wes Lambert <wlambertts@gmail.com>
Date: Tue, 26 May 2020 13:00:03 +0000
Subject: [PATCH 20/43] update Strelka cron

---
 salt/strelka/init.sls | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls
index 8bdbd8274..a9842924d 100644
--- a/salt/strelka/init.sls
+++ b/salt/strelka/init.sls
@@ -112,5 +112,5 @@ strelka_filestream:
 strelka_zeek_extracted_sync:
   cron.present:
     - user: root
-    - name: mv /nsm/zeek/extracted/complete/* /nsm/strelka
+    - name: [ -d /nsm/zeek/extracted/complete/ ] && mv /nsm/zeek/extracted/complete/* /nsm/strelka/ > /dev/null 2>&1
     - minute: '*'

From 330de4624955d2fbe0bce04869fb55f3e6376dbe Mon Sep 17 00:00:00 2001
From: Wes Lambert <wlambertts@gmail.com>
Date: Tue, 26 May 2020 13:19:15 +0000
Subject: [PATCH 21/43] update SOCtopus config for hostname/ip

---
 salt/soctopus/files/SOCtopus.conf | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/salt/soctopus/files/SOCtopus.conf b/salt/soctopus/files/SOCtopus.conf
index f2415d010..e5878cb70 100644
--- a/salt/soctopus/files/SOCtopus.conf
+++ b/salt/soctopus/files/SOCtopus.conf
@@ -1,9 +1,9 @@
-{%- set ip = salt['pillar.get']('static:masterip', '') %}
+{%- set MASTER = salt['pillar.get']('master:url_base', '') %}
 {%- set HIVEKEY = salt['pillar.get']('static:hivekey', '') %}
 {%- set CORTEXKEY = salt['pillar.get']('static:cortexorguserkey', '') %}
 
 [es]
-es_url = http://{{ip}}:9200
+es_url = http://{{MASTER}}:9200
 es_user = YOURESUSER
 es_pass = YOURESPASS
 es_index_pattern = so-*
@@ -11,7 +11,7 @@ es_verifycert = no
 
 [cortex]
 auto_analyze_alerts = no
-cortex_url = https://{{ip}}/cortex/
+cortex_url = https://{{MASTER}}/cortex/
 cortex_key = {{ CORTEXKEY }}
 supported_analyzers = Urlscan_io_Search,CERTatPassiveDNS
 
@@ -32,7 +32,7 @@ grr_user = YOURGRRUSER
 grr_pass = YOURGRRPASS
 
 [hive]
-hive_url = https://{{ip}}/thehive/
+hive_url = https://{{MASTER}}/thehive/
 hive_key = {{ HIVEKEY }}
 hive_tlp = 3
 hive_verifycert = no
@@ -59,7 +59,7 @@ slack_url = YOURSLACKWORKSPACE
 slack_webhook = YOURSLACKWEBHOOK
 
 [playbook]
-playbook_url = https://{{ip}}/playbook
+playbook_url = https://{{MASTER}}/playbook
 playbook_key = de6639318502476f2fa5aa06f43f51fb389a3d7f
 playbook_verifycert = no
 

From c28936d9e42ea3a23f5f17c49b0379981cbb6705 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Tue, 26 May 2020 09:40:29 -0400
Subject: [PATCH 22/43] temporarily abort automated installation

---
 setup/so-setup | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup/so-setup b/setup/so-setup
index 9384c5e96..69c5763f9 100755
--- a/setup/so-setup
+++ b/setup/so-setup
@@ -27,6 +27,7 @@ export setup_type
 automation=$2
 if [[ -f automation/$automation && $(basename $automation) == $automation ]]; then
 	echo "Preselecting variable values based on automated setup: $automation"
+	exit 1
 	source automation/$automation
 	sleep 30 # Re-implement with network availability probe
 fi

From 3eb62287ac6c2195cf2536bc4909a150ca232b6e Mon Sep 17 00:00:00 2001
From: Wes Lambert <wlambertts@gmail.com>
Date: Tue, 26 May 2020 14:00:16 +0000
Subject: [PATCH 23/43] update Curator config for index transition

---
 salt/curator/files/action/close.yml                    | 5 ++---
 salt/curator/files/action/delete.yml                   | 4 ++--
 salt/curator/files/bin/so-curator-closed-delete-delete | 8 ++++----
 3 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/salt/curator/files/action/close.yml b/salt/curator/files/action/close.yml
index dfe5519e8..a65e9af3d 100644
--- a/salt/curator/files/action/close.yml
+++ b/salt/curator/files/action/close.yml
@@ -24,9 +24,8 @@ actions:
       disable_action: False
     filters:
     - filtertype: pattern
-      kind: prefix
-      value: logstash-
-      exclude:
+      kind: regex 
+      value: '^(logstash-.*|so-.*)$'
     - filtertype: age
       source: name
       direction: older
diff --git a/salt/curator/files/action/delete.yml b/salt/curator/files/action/delete.yml
index e6f2f3833..030bbbfac 100644
--- a/salt/curator/files/action/delete.yml
+++ b/salt/curator/files/action/delete.yml
@@ -20,8 +20,8 @@ actions:
       disable_action: False
     filters:
     - filtertype: pattern
-      kind: prefix
-      value: logstash-
+      kind: regex
+      value: '^(logstash-.*|so-.*)$'
     - filtertype: space
       source: creation_date
       use_age: True
diff --git a/salt/curator/files/bin/so-curator-closed-delete-delete b/salt/curator/files/bin/so-curator-closed-delete-delete
index b0ec62424..3d397defc 100755
--- a/salt/curator/files/bin/so-curator-closed-delete-delete
+++ b/salt/curator/files/bin/so-curator-closed-delete-delete
@@ -33,17 +33,17 @@ LOG="/opt/so/log/curator/so-curator-closed-delete.log"
 
 # Check for 2 conditions:
 # 1. Are Elasticsearch indices using more disk space than LOG_SIZE_LIMIT?
-# 2. Are there any closed logstash- indices that we can delete?
+# 2. Are there any closed logstash-, or so- indices that we can delete?
 # If both conditions are true, keep on looping until one of the conditions is false.
 while [[ $(du -hs --block-size=1GB /nsm/elasticsearch/nodes | awk '{print $1}' ) -gt "{{LOG_SIZE_LIMIT}}" ]] &&
-curl -s {{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices | grep "^      close logstash-" > /dev/null; do
+curl -s {{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices | grep -E "^      close (logstash-|so-)" > /dev/null; do
 
 	# We need to determine OLDEST_INDEX.
-	# First, get the list of closed indices that are prefixed with "logstash-".
+	# First, get the list of closed indices that are prefixed with "logstash-" or "so-".
 	# For example: logstash-ids-YYYY.MM.DD
 	# Then, sort by date by telling sort to use hyphen as delimiter and then sort on the third field.
 	# Finally, select the first entry in that sorted list.
-	OLDEST_INDEX=$(curl -s {{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices | grep "^      close logstash-" | awk '{print $2}' | sort -t- -k3 | head -1)
+	OLDEST_INDEX=$(curl -s {{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices | grep -E "^      close (logstash-|so-)" | awk '{print $2}' | sort -t- -k3 | head -1)
 
 	# Now that we've determined OLDEST_INDEX, ask Elasticsearch to delete it.
 	curl -XDELETE {{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/${OLDEST_INDEX}

From 97306d3acdbf41e311c52f9afb5fdf1e9d39c4a6 Mon Sep 17 00:00:00 2001
From: Wes Lambert <wlambertts@gmail.com>
Date: Tue, 26 May 2020 14:05:33 +0000
Subject: [PATCH 24/43] rename indices

---
 salt/common/tools/sbin/so-elastalert-create | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/salt/common/tools/sbin/so-elastalert-create b/salt/common/tools/sbin/so-elastalert-create
index fbe9527a7..0270503bf 100755
--- a/salt/common/tools/sbin/so-elastalert-create
+++ b/salt/common/tools/sbin/so-elastalert-create
@@ -166,8 +166,7 @@ cat << EOF
 What elasticsearch index do you want to use?
 Below are the default Index Patterns used in Security Onion:
 
-*:logstash-*
-*:logstash-beats-*
+*:so-ids-*
 *:elastalert_status*
 
 EOF

From 0e51ab41cf83df46eeeac5ee3a1cebd6f4022251 Mon Sep 17 00:00:00 2001
From: Wes Lambert <wlambertts@gmail.com>
Date: Tue, 26 May 2020 14:18:58 +0000
Subject: [PATCH 25/43] Update ES watermark settings

---
 salt/elasticsearch/files/elasticsearch.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml
index 73f3c9239..271ef40cf 100644
--- a/salt/elasticsearch/files/elasticsearch.yml
+++ b/salt/elasticsearch/files/elasticsearch.yml
@@ -22,3 +22,7 @@ transport.bind_host: 0.0.0.0
 transport.publish_host: {{ nodeip }}
 transport.publish_port: 9300
 {%- endif %}
+cluster.routing.allocation.disk.threshold_enabled: true
+cluster.routing.allocation.disk.watermark.low: 95%
+cluster.routing.allocation.disk.watermark.high: 98%
+cluster.routing.allocation.disk.watermark.flood_stage: 98%

From 87407c9acefc7766f1f12ec1772822d49f965816 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Tue, 26 May 2020 11:21:59 -0400
Subject: [PATCH 26/43] rework common init.sls

---
 salt/common/init.sls | 72 +++++++++++++++++++-------------------------
 1 file changed, 31 insertions(+), 41 deletions(-)

diff --git a/salt/common/init.sls b/salt/common/init.sls
index 17cbb1608..77801dd1a 100644
--- a/salt/common/init.sls
+++ b/salt/common/init.sls
@@ -38,11 +38,11 @@ epel:
 {% endif %}
 
 # Install common packages
+{% if grains['os'] != 'CentOS' %}     
 commonpkgs:
   pkg.installed:
     - skip_suggestions: True
     - pkgs:
-      {% if grains['os'] != 'CentOS' %}
       - apache2-utils
       - wget
       - jq
@@ -61,23 +61,19 @@ commonpkgs:
       - python3-dateutil
       - python3-m2crypto
       - python3-mysqldb
-      - salt-minion: 2019.2.5+ds-1
-        - hold: True
-        - update_holds: True
-      {% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-helix' or grains['role'] == 'so-mastersearch' or grains['role'] == 'so-standalone' %}
-      - salt-master: 2019.2.5+ds-1
-        - hold: True
-        - update_holds: True
+heldpackages:
+  pkg.installed:
+    - pkgs:
       - containerd.io: 1.2.13-2
-        - hold: True
-        - update_holds: True
       - docker-ce: 5:19.03.9~3-0~ubuntu-bionic
-        - hold: True
-        - update_holds: True
-      {% endif %}
-      - containerd.io
-      - docker-ce      
-      {% else %}
+    - hold: True
+    - update_holds: True
+
+{% else %}
+commonpkgs:
+  pkg.installed:
+    - skip_suggestions: True
+    - pkgs:
       - wget
       - bind-utils
       - jq
@@ -87,33 +83,27 @@ commonpkgs:
       - curl
       - sqlite
       - argon2
-      - maridb-devel
+      - mariadb-devel
       - nmap-ncat
       - python3
-			-	python36-docker
-			-	python36-dateutil
-			-	python36-m2crypto
-			-	python36-mysql
-			-	yum-utils
-			-	device-mapper-persistent-data
-			-	lvm2
-			-	openssl
-      - salt-minion: 2019.2.5.el7
-        - hold: True
-        - update_holds: True
-      {% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-helix' or grains['role'] == 'so-mastersearch' or grains['role'] == 'so-standalone' %}
-      - salt-master: 2019.2.5.el7
-        - hold: True
-        - update_holds: True
-      {% endif %}
-      - containerd.io: 1.2.6-3.el7
-        - hold: True
-        - update_holds: True
-      - docker-ce: 19.03.9-3.el7
-        - hold: True
-        - update_holds: True
-      {% endif %}%}
-				
+      - python36-docker
+      - python36-dateutil
+      - python36-m2crypto
+      - python36-mysql
+      - yum-utils
+      - device-mapper-persistent-data
+      - lvm2
+      - openssl
+
+heldpackages:
+  pkg.installed:
+    - pkgs:
+      - containerd.io: 1.2.13-3.2.el7
+      - docker-ce: 3:19.03.9-3.el7
+    - hold: True
+    - update_holds: True
+{% endif %}
+
 # Always keep these packages up to date
 
 alwaysupdated:

From 2cb7464086c8208efa7184d935af78cd586c8686 Mon Sep 17 00:00:00 2001
From: Josh Brower <Josh@Defensivedepth.com>
Date: Tue, 26 May 2020 12:01:58 -0400
Subject: [PATCH 27/43] Add TheHive communityid link

---
 salt/elastalert/files/rules/so/nids2hive.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/elastalert/files/rules/so/nids2hive.yaml b/salt/elastalert/files/rules/so/nids2hive.yaml
index 68a173fcd..097511d56 100644
--- a/salt/elastalert/files/rules/so/nids2hive.yaml
+++ b/salt/elastalert/files/rules/so/nids2hive.yaml
@@ -40,7 +40,7 @@ hive_alert_config:
   title: '{match[rule][name]}'
   type: 'NIDS'
   source: 'SecurityOnion'
-  description: "`Hunting Pivot:` \n\n <https://{{MASTER}}/#/hunt?q=event.module%3A%20suricata%20AND%20rule.uuid%3A{match[rule][uuid]}%20%7C%20groupby%20source.ip%20destination.ip%20rule.name>  \n\n `Kibana Dashboard:` \n\n <https://{{MASTER}}/kibana/app/kibana#/dashboard/ed6f7e20-e060-11e9-8f0c-2ddbf5ed9290?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(_source),index:'*:logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:'sid:')),sort:!('@timestamp',desc))> \n\n `IPs: `{match[source][ip]}:{match[source][port]} --> {match[destination][ip]}:{match[destination][port]} \n\n `Signature:`{match[rule][rule]}"
+  description: "`Hunting Pivot:` \n\n <https://{{MASTER}}/#/hunt?q=event.module%3A%20suricata%20AND%20rule.uuid%3A{match[rule][uuid]}%20%7C%20groupby%20source.ip%20destination.ip%20rule.name>  \n\n `Kibana Dashboard - Signature Drilldown:` \n\n <https://{{MASTER}}/kibana/app/kibana#/dashboard/ed6f7e20-e060-11e9-8f0c-2ddbf5ed9290?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(_source),index:'*:logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:'sid:')),sort:!('@timestamp',desc))> \n\n `Kibana Dashboard - Community_ID:` \n\n <https://{{MASTER}}/kibana/app/kibana#/dashboard/30d0ac90-729f-11ea-8dd2-9d8795a1200b?_g=(filters:!(('$state':(store:globalState),meta:(alias:!n,disabled:!f,index:'*:so-*',key:network.community_id,negate:!f,params:(query:'{match[network][community_id]}'),type:phrase),query:(match_phrase:(network.community_id:'{match[network][community_id]}')))),refreshInterval:(pause:!t,value:0),time:(from:now-7d,to:now))>  \n\n `IPs: `{match[source][ip]}:{match[source][port]} --> {match[destination][ip]}:{match[destination][port]} \n\n `Signature:`{match[rule][rule]}"
   severity: 2
   tags: ['{match[rule][uuid]}','{match[source][ip]}','{match[destination][ip]}']
   tlp: 3

From 7a657d122989cdc4545ea01c3e1c49fd10a64f18 Mon Sep 17 00:00:00 2001
From: Doug Burks <doug.burks@gmail.com>
Date: Tue, 26 May 2020 12:58:01 -0400
Subject: [PATCH 28/43] add Community ID to default fields in bottom data table
 in Hunt

---
 salt/soc/files/soc/soc.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json
index 6b76e622c..76770e2bd 100644
--- a/salt/soc/files/soc/soc.json
+++ b/salt/soc/files/soc/soc.json
@@ -32,7 +32,7 @@
         "dateRangeMinutes": 1440,
         "mostRecentlyUsedLimit": 5,
         "eventFields": {
-          "default": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "log.id.uid" ],
+          "default": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "log.id.uid", "network.community_id" ],
           "bro_conn": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "protocol", "service", "log.id.uid" ],
           "bro_dce_rpc": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "endpoint", "named_pipe", "operation", "log.id.uid" ],
           "bro_dhcp": ["soc_timestamp", "source.ip", "destination.ip", "domain_name", "hostname", "message_types", "log.id.uid" ],

From 8723f8785ec2043f06945b3fd0b1d6ca424e5f11 Mon Sep 17 00:00:00 2001
From: Josh Brower <Josh@Defensivedepth.com>
Date: Tue, 26 May 2020 13:05:56 -0400
Subject: [PATCH 29/43] osquery pipeline fix and fail state if errors

---
 salt/elasticsearch/files/ingest/osquery.query_result | 2 +-
 salt/elasticsearch/files/so-elasticsearch-pipelines  | 5 ++++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/salt/elasticsearch/files/ingest/osquery.query_result b/salt/elasticsearch/files/ingest/osquery.query_result
index e9cdbe2d3..5b37655f9 100644
--- a/salt/elasticsearch/files/ingest/osquery.query_result
+++ b/salt/elasticsearch/files/ingest/osquery.query_result
@@ -31,7 +31,7 @@
     { "rename":      { "field": "message3.columns.remote_port",                             "target_field": "remote.port",              "ignore_missing": true  } },
     { "rename":      { "field": "message3.columns.process_name",                            "target_field": "process.name",              "ignore_missing": true  } },
     { "rename": 	   { "field": "message3.columns.eventid", 			                          "target_field": "event.code",		"ignore_missing": true 	} },
-    { "set":         { "if": "ctx.message3.columns.?data != null",   "field": "dataset", "value": "wel-{{message3.columns.source}}", "override": true }  },
+    { "set":         { "if": "ctx.message3.columns?.data != null",   "field": "dataset", "value": "wel-{{message3.columns.source}}", "override": true }  },
     { "rename": 	   { "field": "message3.columns.winlog.EventData.SubjectUserName", 			  "target_field": "user.name",		"ignore_missing": true 	} },
     { "rename": 	   { "field": "message3.columns.winlog.EventData.destinationHostname", 	  "target_field": "destination.hostname",	"ignore_missing": true 	} },
     { "rename": 	   { "field": "message3.columns.winlog.EventData.destinationIp", 		      "target_field": "destination.ip",	"ignore_missing": true 	} },
diff --git a/salt/elasticsearch/files/so-elasticsearch-pipelines b/salt/elasticsearch/files/so-elasticsearch-pipelines
index b1b6db158..514054359 100755
--- a/salt/elasticsearch/files/so-elasticsearch-pipelines
+++ b/salt/elasticsearch/files/so-elasticsearch-pipelines
@@ -15,6 +15,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+RETURN_CODE=0
 ELASTICSEARCH_HOST=$1
 ELASTICSEARCH_PORT=9200
 
@@ -46,7 +47,9 @@ fi
 cd ${ELASTICSEARCH_INGEST_PIPELINES}
 
 echo "Loading pipelines..."
-for i in *; do echo $i; curl ${ELASTICSEARCH_AUTH} -XPUT http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done
+for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -XPUT http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done
 echo
 
 cd - >/dev/null
+
+exit $RETURN_CODE
\ No newline at end of file

From d260224dfea4c7b8c384911a452d641d47a432a8 Mon Sep 17 00:00:00 2001
From: Mike Reeves <luke@geekempire.com>
Date: Tue, 26 May 2020 13:39:36 -0400
Subject: [PATCH 30/43] Update reboot language

---
 setup/so-whiptail | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup/so-whiptail b/setup/so-whiptail
index 559f791b7..72455fc9e 100755
--- a/setup/so-whiptail
+++ b/setup/so-whiptail
@@ -958,7 +958,7 @@ whiptail_setup_complete() {
 
 	[ -n "$TESTING" ] && return
 
-	whiptail --title "Security Onion Setup" --msgbox "Finished $install_type install. Press ENTER to reboot." 8 75
+	whiptail --title "Security Onion Setup" --msgbox "Finished $install_type install. Press Ok to reboot." 8 75
 	install_cleanup >> $setup_log 2>&1
 
 }
@@ -967,7 +967,7 @@ whiptail_setup_failed() {
 
 	[ -n "$TESTING" ] && return
 
-	whiptail --title "Security Onion Setup" --msgbox "Install had a problem. Please see $setup_log for details. Press ENTER to reboot." 8 75
+	whiptail --title "Security Onion Setup" --msgbox "Install had a problem. Please see $setup_log for details. Press Ok to reboot." 8 75
 	install_cleanup >> $setup_log 2>&1
 
 }

From 41935996d4384849fd4ffe93924d2c2fb6bd9ff7 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Tue, 26 May 2020 13:48:36 -0400
Subject: [PATCH 31/43] add ntpdate/ntp

---
 salt/common/init.sls | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/salt/common/init.sls b/salt/common/init.sls
index 77801dd1a..08ae7fa72 100644
--- a/salt/common/init.sls
+++ b/salt/common/init.sls
@@ -45,6 +45,7 @@ commonpkgs:
     - pkgs:
       - apache2-utils
       - wget
+      - ntp
       - jq
       - python3-docker
       - docker-ce
@@ -75,6 +76,7 @@ commonpkgs:
     - skip_suggestions: True
     - pkgs:
       - wget
+      - ntpdate
       - bind-utils
       - jq
       - tcpdump

From 21cd66d109c458b8aa335d3e93143fb9555a8d94 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Tue, 26 May 2020 14:05:47 -0400
Subject: [PATCH 32/43] add ntpdate/ntp

---
 salt/common/init.sls | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/common/init.sls b/salt/common/init.sls
index 08ae7fa72..09d71114b 100644
--- a/salt/common/init.sls
+++ b/salt/common/init.sls
@@ -45,7 +45,7 @@ commonpkgs:
     - pkgs:
       - apache2-utils
       - wget
-      - ntp
+      - ntpdate
       - jq
       - python3-docker
       - docker-ce

From 92c6a524965308d0b66c0abd52a9baccbf4a1706 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Tue, 26 May 2020 14:49:22 -0400
Subject: [PATCH 33/43] [feat][WIP] Add functionality to run setup through a
 proxy

---
 setup/so-functions | 56 +++++++++++++++++++++++++++++++---------------
 1 file changed, 38 insertions(+), 18 deletions(-)

diff --git a/setup/so-functions b/setup/so-functions
index 4e01c935c..fed162596 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -550,6 +550,9 @@ detect_os() {
 disable_onion_user() {
 	# Disable the default account cause security.
 	usermod -L onion
+
+	# Remove the automated setup script from crontab, if it exists
+	crontab -u onion -r
 }
 
 disable_misc_network_features() {
@@ -955,29 +958,47 @@ parse_options() {
 			if [[ $is_master || $is_helix ]]; then
 				local proxy
 				proxy=$(echo "$1" | tr -d '"' | awk -F'--turbo=' '{print $2}')
-				proxy_addr="http://$proxy"
-				use_proxy "$proxy_addr"
-				TURBO="$proxy_addr"
+				proxy_url="http://$proxy"
+				TURBO="$proxy_url"
+				use_turbo_proxy "$TURBO"
 			else
 				echo "turbo is not supported on this install type" >> $setup_log 2>&1
 			fi
 			;;
 		--proxy=*)
-			echo "Unimplimented"
-			return
+			local proxy
+			proxy=$(echo "$1" | tr -d '"' | awk -F'--proxy=' '{print $2}')
 
-			if [[ $2 != --proxy-user=* ]] || [[ $3 != --proxy-pass=* ]]; then
-				echo "Invalid options passed for proxy. Order is --proxy-user=<user> --proxy-pass=<password>"
-			else
-				local proxy
+			local proxy_protocol
+			proxy_protocol=$(echo "$proxy" |tr -d '"' | awk 'match($0, /http|https/) { print substr($0, RSTART, RLENGTH) }')
+
+			if [[ ! $proxy_protocol =~ ^(http|https) ]]; then
+				echo "Invalid proxy protocol"
+				echo "Ignoring proxy"
+				return
+			fi
+
+			if [[ $2 == --proxy-user=* && $3 == --proxy-pass=* ]]; then
 				local proxy_user
 				local proxy_password
-				proxy=$(echo "$1" | tr -d '"' | awk -F'--proxy=' '{print $2}')
 				proxy_user=$(echo "$2" | tr -d '"' | awk -F'--proxy-user=' '{print $2}')
 				proxy_password=$(echo "$3" | tr -d '"' | awk -F'--proxy-pass=' '{print $2}')
-				
-				use_proxy "$proxy" "$proxy_user" "$proxy_password"
+
+				local proxy_addr
+				proxy_addr=$(echo "$proxy" | tr -d '"' | awk -F'http\:\/\/|https\:\/\/' '{print $2}')
+
+				export http_proxy="${proxy_protocol}://${proxy_user}:${proxy_password}@${proxy_addr}"
+
+			elif [[ (-z $2 || -z $3) && (-n $2 || -n $3) || ($2 != --proxy-user=* || $3 != --proxy-pass=*) ]]; then
+				echo "Invalid options passed for proxy. Order is --proxy-user=<user> --proxy-pass=<password>"
+				echo "Ignoring proxy"
+				return
+
+			else
+				export http_proxy="$proxy"
 			fi
+
+			export {https,ftp,rsync,all}_proxy="$http_proxy"
 			;;
 		*)
 			echo "Invalid option"
@@ -1479,18 +1500,17 @@ update_packages() {
 	fi
 }
 
-use_proxy() {
-	local proxy_addr=$1
+use_turbo_proxy() {
+	local proxy_url=$1
 	#TODO: add options for username + pass
 
 	if [[ $OS == 'centos' ]]; then
-		printf '%s\n'\
-			"proxy=${proxy_addr}:3142" >> /etc/yum.conf
+		printf '%s\n' "proxy=${proxy_url}:3142" >> /etc/yum.conf
 	else
 		printf '%s\n'\
 			"Acquire {"\
-			"HTTP::proxy \"${proxy_addr}:3142\";"\
-			"HTTPS::proxy \"${proxy_addr}:3142\";"\
+			"  HTTP::proxy \"${proxy_url}:3142\";"\
+			"  HTTPS::proxy \"${proxy_url}:3142\";"\
 			"}" > /etc/apt/apt.conf.d/proxy.conf
 	fi
 }

From 84e3ce508e72ab106c7b7b69f846ae5eabb86947 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Tue, 26 May 2020 15:19:37 -0400
Subject: [PATCH 34/43] [fix] Only check for proxy-user & proxy-pass when
 needed

---
 setup/so-functions | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup/so-functions b/setup/so-functions
index fed162596..4b47d65d9 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -989,7 +989,7 @@ parse_options() {
 
 				export http_proxy="${proxy_protocol}://${proxy_user}:${proxy_password}@${proxy_addr}"
 
-			elif [[ (-z $2 || -z $3) && (-n $2 || -n $3) || ($2 != --proxy-user=* || $3 != --proxy-pass=*) ]]; then
+			elif [[ (-z $2 || -z $3) && (-n $2 || -n $3) || ( -n $2 && -n $3 && ($2 != --proxy-user=* || $3 != --proxy-pass=*) ) ]]; then
 				echo "Invalid options passed for proxy. Order is --proxy-user=<user> --proxy-pass=<password>"
 				echo "Ignoring proxy"
 				return

From 9e1ed6983f6d0dd0000fd2ea3efa4e2a1cdb8df6 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Tue, 26 May 2020 15:25:30 -0400
Subject: [PATCH 35/43] [fix] Parse options at beginning of setup

---
 setup/so-functions | 10 +++-------
 setup/so-setup     | 11 ++++++++---
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/setup/so-functions b/setup/so-functions
index 4b47d65d9..ae99d190e 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -960,7 +960,6 @@ parse_options() {
 				proxy=$(echo "$1" | tr -d '"' | awk -F'--turbo=' '{print $2}')
 				proxy_url="http://$proxy"
 				TURBO="$proxy_url"
-				use_turbo_proxy "$TURBO"
 			else
 				echo "turbo is not supported on this install type" >> $setup_log 2>&1
 			fi
@@ -1501,16 +1500,13 @@ update_packages() {
 }
 
 use_turbo_proxy() {
-	local proxy_url=$1
-	#TODO: add options for username + pass
-
 	if [[ $OS == 'centos' ]]; then
-		printf '%s\n' "proxy=${proxy_url}:3142" >> /etc/yum.conf
+		printf '%s\n' "proxy=${TURBO}:3142" >> /etc/yum.conf
 	else
 		printf '%s\n'\
 			"Acquire {"\
-			"  HTTP::proxy \"${proxy_url}:3142\";"\
-			"  HTTPS::proxy \"${proxy_url}:3142\";"\
+			"  HTTP::proxy \"${TURBO}:3142\";"\
+			"  HTTPS::proxy \"${TURBO}:3142\";"\
 			"}" > /etc/apt/apt.conf.d/proxy.conf
 	fi
 }
diff --git a/setup/so-setup b/setup/so-setup
index 4b0a4d6f6..6ce0a64ca 100755
--- a/setup/so-setup
+++ b/setup/so-setup
@@ -40,6 +40,12 @@ export PATH=$PATH:../salt/common/tools/sbin
 date -u > $setup_log 2>&1
 
 got_root
+
+if [[ $# -gt 1 ]]; then
+	set -- "${@:2}"
+	parse_options "$@" >> $setup_log 2>&1
+fi
+
 detect_os
 
 if [ "$OS" == ubuntu ]; then
@@ -241,9 +247,8 @@ fi
 
 whiptail_make_changes
 
-if [[ $# -gt 1 ]]; then
-	set -- "${@:2}"
-	parse_options "$@" >> $setup_log 2>&1
+if [[ -n "$TURBO" ]]; then
+	use_turbo_proxy
 fi
 
 if [[ "$setup_type" == 'iso' ]]; then

From 001f7c6694946c478b5ab6fffc82ab2ad9709a15 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Tue, 26 May 2020 16:01:44 -0400
Subject: [PATCH 36/43] [fix] export TURBO var so it can be used from so-setup

---
 setup/so-functions | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup/so-functions b/setup/so-functions
index ae99d190e..bf3f4d856 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -959,7 +959,7 @@ parse_options() {
 				local proxy
 				proxy=$(echo "$1" | tr -d '"' | awk -F'--turbo=' '{print $2}')
 				proxy_url="http://$proxy"
-				TURBO="$proxy_url"
+				export TURBO="$proxy_url"
 			else
 				echo "turbo is not supported on this install type" >> $setup_log 2>&1
 			fi

From 8e1bd32f4d5e9ecdd1b3bb8f3bdf334f1ad08a29 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Tue, 26 May 2020 16:11:31 -0400
Subject: [PATCH 37/43] Improve automated installs and remove sleep during
 progress updates

---
 setup/so-functions |  4 +---
 setup/so-setup     | 28 +++++++++++++++++++++++++---
 2 files changed, 26 insertions(+), 6 deletions(-)

diff --git a/setup/so-functions b/setup/so-functions
index bf6db26be..f8d41cfc7 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -968,7 +968,7 @@ parse_options() {
 			fi
 			;;
 		--proxy=*)
-			echo "Unimplimented"
+			echo "Unimplemented"
 			return
 
 			if [[ $2 != --proxy-user=* ]] || [[ $3 != --proxy-pass=* ]]; then
@@ -1268,8 +1268,6 @@ set_progress_str() {
 	'----'\
 	"$percentage% - ${progress_bar_text^^}"\
 	"----" >> "$setup_log" 2>&1
-	
-	sleep 5
 }
 
 sensor_pillar() {
diff --git a/setup/so-setup b/setup/so-setup
index 69c5763f9..03889606d 100755
--- a/setup/so-setup
+++ b/setup/so-setup
@@ -25,11 +25,33 @@ setup_type=$1
 export setup_type
 
 automation=$2
+
+automated=no
+
+function progress() {
+	if [ $automated == no ]; then
+		whiptail --title "Security Onion Install" --gauge 'Please wait while installing' 6 60 0
+	fi
+}
+
 if [[ -f automation/$automation && $(basename $automation) == $automation ]]; then
 	echo "Preselecting variable values based on automated setup: $automation"
-	exit 1
 	source automation/$automation
-	sleep 30 # Re-implement with network availability probe
+	automated=yes
+
+	attempt=1
+	attempts=60
+	ip a | grep "$MNIC:" | grep "state UP"
+	while [ $? -ne 0 ]; do
+		if [ $attempt -gt $attempts ]; then
+			echo "Network unavailable - setup cannot continue"
+			exit 1
+		fi
+		echo "Waiting for network to come up (attempt $attempt of $attempts)"
+		attempt=$((attempt + 1))
+		sleep 10;
+		ip a | grep "$MNIC:" | grep "state UP"
+	done
 fi
 
 case "$setup_type" in
@@ -498,7 +520,7 @@ fi
 	set_progress_str 95 'Verifying setup'
 	salt-call -l info  state.highstate >> $setup_log 2>&1
 
-} | whiptail --title "Hybrid Hunter Install" --gauge "Please wait while installing" 6 60 0
+} | progress
 
 success=$(tail -10 $setup_log | grep Failed | awk '{ print $2}')
 if [[ "$success" = 0 ]]; then

From ec0ec79470fe0bc822ad3e189aba63c79ae492ec Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Tue, 26 May 2020 16:23:06 -0400
Subject: [PATCH 38/43] [fix] Test install type after it has been set

---
 setup/so-functions | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/setup/so-functions b/setup/so-functions
index bf3f4d856..1f5df9678 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -955,14 +955,10 @@ node_pillar() {
 parse_options() {
 	case "$1" in
 		--turbo=*)
-			if [[ $is_master || $is_helix ]]; then
-				local proxy
-				proxy=$(echo "$1" | tr -d '"' | awk -F'--turbo=' '{print $2}')
-				proxy_url="http://$proxy"
-				export TURBO="$proxy_url"
-			else
-				echo "turbo is not supported on this install type" >> $setup_log 2>&1
-			fi
+			local proxy
+			proxy=$(echo "$1" | tr -d '"' | awk -F'--turbo=' '{print $2}')
+			proxy_url="http://$proxy"
+			TURBO="$proxy_url"
 			;;
 		--proxy=*)
 			local proxy
@@ -971,7 +967,7 @@ parse_options() {
 			local proxy_protocol
 			proxy_protocol=$(echo "$proxy" |tr -d '"' | awk 'match($0, /http|https/) { print substr($0, RSTART, RLENGTH) }')
 
-			if [[ ! $proxy_protocol =~ ^(http|https) ]]; then
+			if [[ ! $proxy_protocol =~ ^(http|https)$ ]]; then
 				echo "Invalid proxy protocol"
 				echo "Ignoring proxy"
 				return
@@ -1500,6 +1496,11 @@ update_packages() {
 }
 
 use_turbo_proxy() {
+	if [[ ! $install_type =~ ^(MASTER|EVAL|HELIXSENSOR|MASTERSEARCH|STANDALONE)$ ]]; then
+		echo "turbo is not supported on this install type" >> $setup_log 2>&1
+		return
+	fi
+
 	if [[ $OS == 'centos' ]]; then
 		printf '%s\n' "proxy=${TURBO}:3142" >> /etc/yum.conf
 	else

From 00681649bd7814144993071450cc89f231bfe7a1 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Tue, 26 May 2020 16:41:41 -0400
Subject: [PATCH 39/43] [fix] Don't run tr on a string again

---
 setup/so-functions | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup/so-functions b/setup/so-functions
index 4eabd8657..20fa67dfd 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -967,7 +967,7 @@ parse_options() {
 			proxy=$(echo "$1" | tr -d '"' | awk -F'--proxy=' '{print $2}')
 
 			local proxy_protocol
-			proxy_protocol=$(echo "$proxy" |tr -d '"' | awk 'match($0, /http|https/) { print substr($0, RSTART, RLENGTH) }')
+			proxy_protocol=$(echo "$proxy" | awk 'match($0, /http|https/) { print substr($0, RSTART, RLENGTH) }')
 
 			if [[ ! $proxy_protocol =~ ^(http|https)$ ]]; then
 				echo "Invalid proxy protocol"

From 377d8e6336c2f30e6f08831a8fe4c8036eeb1d4a Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Tue, 26 May 2020 16:54:51 -0400
Subject: [PATCH 40/43] [fix] Don't run tr on a string again

---
 setup/so-functions | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup/so-functions b/setup/so-functions
index 20fa67dfd..612b0147b 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -982,7 +982,7 @@ parse_options() {
 				proxy_password=$(echo "$3" | tr -d '"' | awk -F'--proxy-pass=' '{print $2}')
 
 				local proxy_addr
-				proxy_addr=$(echo "$proxy" | tr -d '"' | awk -F'http\:\/\/|https\:\/\/' '{print $2}')
+				proxy_addr=$(echo "$proxy" | awk -F'http\:\/\/|https\:\/\/' '{print $2}')
 
 				export http_proxy="${proxy_protocol}://${proxy_user}:${proxy_password}@${proxy_addr}"
 

From 5d0a7f99e91aee191266d0e122cd158e73bb6e30 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Tue, 26 May 2020 18:45:29 -0400
Subject: [PATCH 41/43] Improve logging of automated setup

---
 setup/so-setup | 40 ++++++++++++++++++++++++++++++----------
 1 file changed, 30 insertions(+), 10 deletions(-)

diff --git a/setup/so-setup b/setup/so-setup
index fdc69076b..0d309bfc3 100755
--- a/setup/so-setup
+++ b/setup/so-setup
@@ -28,6 +28,8 @@ automation=$2
 
 automated=no
 
+echo "---- Starting setup at $(date -u) ----" >> $setup_log 2>&1
+
 function progress() {
 	if [ $automated == no ]; then
 		whiptail --title "Security Onion Install" --gauge 'Please wait while installing' 6 60 0
@@ -35,31 +37,51 @@ function progress() {
 }
 
 if [[ -f automation/$automation && $(basename $automation) == $automation ]]; then
-	echo "Preselecting variable values based on automated setup: $automation"
+	echo "Preselecting variable values based on automated setup: $automation" >> $setup_log 2>&1
 	source automation/$automation
 	automated=yes
 
+	echo "Checking network configuration" >> $setup_log 2>&1g
+	ip a >> $setup_log 2>&1
+
 	attempt=1
 	attempts=60
-	ip a | grep "$MNIC:" | grep "state UP"
+	ip a | grep "$MNIC:" | grep "state UP" >> $setup_log 2>&1
 	while [ $? -ne 0 ]; do
+		ip a >> $setup_log 2>&1
 		if [ $attempt -gt $attempts ]; then
-			echo "Network unavailable - setup cannot continue"
+			echo "Network unavailable - setup cannot continue" >> $setup_log 2>&1
 			exit 1
 		fi
-		echo "Waiting for network to come up (attempt $attempt of $attempts)"
+		echo "Waiting for network to come up (attempt $attempt of $attempts)" >> $setup_log 2>&1
 		attempt=$((attempt + 1))
 		sleep 10;
-		ip a | grep "$MNIC:" | grep "state UP"
+		ip a | grep "$MNIC:" | grep "state UP" >> $setup_log 2>&1
 	done
+	echo "Network is up on $MNIC" >> $setup_log 2>&1
+
+	attempt=1
+	attempts=60
+	ping -c google.com >> $setup_log 2>&1
+	while [ $? -ne 0 ]; do
+		if [ $attempt -gt $attempts ]; then
+			echo "DNS unavailable - setup cannot continue" >> $setup_log 2>&1
+			exit 1
+		fi
+		echo "Waiting for DNS to become available (attempt $attempt of $attempts)" >> $setup_log 2>&1
+		attempt=$((attempt + 1))
+		sleep 10;
+		ping -c google.com >> $setup_log 2>&1
+	done
+	echo "DNS is available" >> $setup_log 2>&1
 fi
 
 case "$setup_type" in
 	iso | network) # Accepted values
-		echo "Beginning Security Onion $setup_type install"
+		echo "Beginning Security Onion $setup_type install" >> $setup_log 2>&1
 		;;
 	*)
-		echo "Invalid install type, must be 'iso' or 'network'"
+		echo "Invalid install type, must be 'iso' or 'network'" | tee $setup_log
 		exit 1
 		;;
 esac
@@ -67,8 +89,6 @@ esac
 # Allow execution of SO tools during setup
 export PATH=$PATH:../salt/common/tools/sbin
 
-date -u > $setup_log 2>&1
-
 got_root
 
 if [[ $# -gt 1 ]]; then
@@ -87,7 +107,7 @@ setterm -blank 0
 if [ "$setup_type" == 'iso' ] || (whiptail_you_sure); then
 	true
 else
-	echo "User cancelled setup." >> $setup_log 2>&1
+	echo "User cancelled setup." | tee $setup_log
 	whiptail_cancel
 fi
 

From 65431e0fccc7a82979637ac65cf5f1a353e15c1f Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Tue, 26 May 2020 22:39:18 -0400
Subject: [PATCH 42/43] Do not expect network access to be available prior to
 the NIC being configured for DHCP/Static

---
 setup/so-setup | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/setup/so-setup b/setup/so-setup
index 0d309bfc3..d18f60089 100755
--- a/setup/so-setup
+++ b/setup/so-setup
@@ -59,21 +59,6 @@ if [[ -f automation/$automation && $(basename $automation) == $automation ]]; th
 		ip a | grep "$MNIC:" | grep "state UP" >> $setup_log 2>&1
 	done
 	echo "Network is up on $MNIC" >> $setup_log 2>&1
-
-	attempt=1
-	attempts=60
-	ping -c google.com >> $setup_log 2>&1
-	while [ $? -ne 0 ]; do
-		if [ $attempt -gt $attempts ]; then
-			echo "DNS unavailable - setup cannot continue" >> $setup_log 2>&1
-			exit 1
-		fi
-		echo "Waiting for DNS to become available (attempt $attempt of $attempts)" >> $setup_log 2>&1
-		attempt=$((attempt + 1))
-		sleep 10;
-		ping -c google.com >> $setup_log 2>&1
-	done
-	echo "DNS is available" >> $setup_log 2>&1
 fi
 
 case "$setup_type" in

From f3809cb93d3871eabf722427499ab6d5aeac64e0 Mon Sep 17 00:00:00 2001
From: weslambert <wlambertts@gmail.com>
Date: Wed, 27 May 2020 08:31:14 -0400
Subject: [PATCH 43/43] Wrap with quotes

---
 salt/strelka/init.sls | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls
index a9842924d..5767531f4 100644
--- a/salt/strelka/init.sls
+++ b/salt/strelka/init.sls
@@ -112,5 +112,5 @@ strelka_filestream:
 strelka_zeek_extracted_sync:
   cron.present:
     - user: root
-    - name: [ -d /nsm/zeek/extracted/complete/ ] && mv /nsm/zeek/extracted/complete/* /nsm/strelka/ > /dev/null 2>&1
+    - name: '[ -d /nsm/zeek/extracted/complete/ ] && mv /nsm/zeek/extracted/complete/* /nsm/strelka/ > /dev/null 2>&1'
     - minute: '*'