Merge branch 'dev' into telegraf3

salt/common/tools/sbin/so-sensor-clean

@@ -115,7 +115,7 @@ clean() {
 }
 
 # Check to see if we are already running
-IS_RUNNING=$(ps aux | grep "sensor_clean" | grep -v grep | wc -l)
+IS_RUNNING=$(ps aux | grep "so-sensor-clean" | grep -v grep | wc -l)
 [ "$IS_RUNNING" -gt 2 ] && echo "$(date) - $IS_RUNNING sensor clean script processes running...exiting." >>$LOG && exit 0
 
 if [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; then

setup/automation/distributed-ami-manager

@@ -26,7 +26,7 @@ ALLOW_ROLE=a
 BASICZEEK=2
 BASICSURI=2
 # BLOGS=
-BNICS=ens6
+#BNICS=eth1
 ZEEKVERSION=ZEEK
 # CURCLOSEDAYS=
 # EVALADVANCED=BASIC
@@ -34,7 +34,7 @@ GRAFANA=1
 # HELIXAPIKEY=
 HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12
 HNSENSOR=inherit
-HOSTNAME=manager-aws
+HOSTNAME=distributed-manager
 install_type=MANAGER
 # LSINPUTBATCHCOUNT=
 # LSINPUTTHREADS=
@@ -46,7 +46,7 @@ MANAGERUPDATES=1
 # MGATEWAY=
 # MIP=
 # MMASK=
-MNIC=ens5
+MNIC=eth0
 # MSEARCH=
 # MSRV=
 # MTU=
@@ -62,8 +62,8 @@ OSQUERY=1
 # PATCHSCHEDULEHOURS=
 PATCHSCHEDULENAME=auto
 PLAYBOOK=1
-# REDIRECTHOST=
+REDIRECTHOST=$(curl http://169.254.169.254/latest/meta-data/public-ipv4)
-REDIRECTINFO=HOSTNAME
+REDIRECTINFO=OTHER
 RULESETUP=ETOPEN
 # SHARDCOUNT=
 SKIP_REBOOT=0

setup/automation/import-airgap

@@ -0,0 +1,78 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+TESTING=true
+
+address_type=DHCP
+ADMINUSER=onionuser
+ADMINPASS1=onionuser
+ADMINPASS2=onionuser
+ALLOW_CIDR=0.0.0.0/0
+ALLOW_ROLE=a
+BASICZEEK=2
+BASICSURI=2
+# BLOGS=
+#BNICS=eth1
+ZEEKVERSION=ZEEK
+# CURCLOSEDAYS=
+# EVALADVANCED=BASIC
+GRAFANA=1
+# HELIXAPIKEY=
+HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12
+HNSENSOR=inherit
+HOSTNAME=import
+install_type=IMPORT
+INTERWEBS=AIRGAP
+# LSINPUTBATCHCOUNT=
+# LSINPUTTHREADS=
+# LSPIPELINEBATCH=
+# LSPIPELINEWORKERS=
+MANAGERADV=BASIC
+MANAGERUPDATES=1
+# MDNS=
+# MGATEWAY=
+# MIP=
+# MMASK=
+MNIC=eth0
+# MSEARCH=
+# MSRV=
+# MTU=
+NIDS=Suricata
+# NODE_ES_HEAP_SIZE=
+# NODE_LS_HEAP_SIZE=
+NODESETUP=NODEBASIC
+NSMSETUP=BASIC
+NODEUPDATES=MANAGER
+# OINKCODE=
+OSQUERY=1
+# PATCHSCHEDULEDAYS=
+# PATCHSCHEDULEHOURS=
+PATCHSCHEDULENAME=auto
+# PLAYBOOK=1
+REDIRECTHOST=$(curl http://169.254.169.254/latest/meta-data/public-ipv4)
+REDIRECTINFO=OTHER
+RULESETUP=ETOPEN
+# SHARDCOUNT=
+# SKIP_REBOOT=
+# SOREMOTEPASS1=onionuser
+# SOREMOTEPASS2=onionuser
+STRELKA=1
+THEHIVE=1
+WAZUH=1
+WEBUSER=onionuser@somewhere.invalid
+WEBPASSWD1=0n10nus3r
+WEBPASSWD2=0n10nus3r

setup/automation/import-ami

@@ -0,0 +1,77 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+TESTING=true
+
+address_type=DHCP
+ADMINUSER=onionuser
+ADMINPASS1=onionuser
+ADMINPASS2=onionuser
+ALLOW_CIDR=0.0.0.0/0
+ALLOW_ROLE=a
+BASICZEEK=2
+BASICSURI=2
+# BLOGS=
+#BNICS=eth1
+ZEEKVERSION=ZEEK
+# CURCLOSEDAYS=
+# EVALADVANCED=BASIC
+GRAFANA=1
+# HELIXAPIKEY=
+HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12
+HNSENSOR=inherit
+HOSTNAME=import
+install_type=IMPORT
+# LSINPUTBATCHCOUNT=
+# LSINPUTTHREADS=
+# LSPIPELINEBATCH=
+# LSPIPELINEWORKERS=
+MANAGERADV=BASIC
+MANAGERUPDATES=1
+# MDNS=
+# MGATEWAY=
+# MIP=
+# MMASK=
+MNIC=eth0
+# MSEARCH=
+# MSRV=
+# MTU=
+NIDS=Suricata
+# NODE_ES_HEAP_SIZE=
+# NODE_LS_HEAP_SIZE=
+NODESETUP=NODEBASIC
+NSMSETUP=BASIC
+NODEUPDATES=MANAGER
+# OINKCODE=
+OSQUERY=1
+# PATCHSCHEDULEDAYS=
+# PATCHSCHEDULEHOURS=
+PATCHSCHEDULENAME=auto
+# PLAYBOOK=1
+REDIRECTHOST=$(curl http://169.254.169.254/latest/meta-data/public-ipv4)
+REDIRECTINFO=OTHER
+RULESETUP=ETOPEN
+# SHARDCOUNT=
+# SKIP_REBOOT=
+# SOREMOTEPASS1=onionuser
+# SOREMOTEPASS2=onionuser
+STRELKA=1
+THEHIVE=1
+WAZUH=1
+WEBUSER=onionuser@somewhere.invalid
+WEBPASSWD1=0n10nus3r
+WEBPASSWD2=0n10nus3r

setup/automation/import-iso

@@ -61,7 +61,7 @@ OSQUERY=1
 # PATCHSCHEDULEDAYS=
 # PATCHSCHEDULEHOURS=
 PATCHSCHEDULENAME=auto
-PLAYBOOK=1
+# PLAYBOOK=1
 # REDIRECTHOST=
 REDIRECTINFO=IP
 RULESETUP=ETOPEN

setup/automation/import-net-centos

@@ -0,0 +1,77 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+TESTING=true
+
+# address_type=DHCP
+ADMINUSER=onionuser
+ADMINPASS1=onionuser
+ADMINPASS2=onionuser
+ALLOW_CIDR=0.0.0.0/0
+ALLOW_ROLE=a
+BASICZEEK=2
+BASICSURI=2
+# BLOGS=
+#BNICS=eth1
+ZEEKVERSION=ZEEK
+# CURCLOSEDAYS=
+# EVALADVANCED=BASIC
+GRAFANA=1
+# HELIXAPIKEY=
+HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12
+HNSENSOR=inherit
+HOSTNAME=import
+install_type=IMPORT
+# LSINPUTBATCHCOUNT=
+# LSINPUTTHREADS=
+# LSPIPELINEBATCH=
+# LSPIPELINEWORKERS=
+MANAGERADV=BASIC
+MANAGERUPDATES=1
+# MDNS=
+# MGATEWAY=
+# MIP=
+# MMASK=
+MNIC=eth0
+# MSEARCH=
+# MSRV=
+# MTU=
+NIDS=Suricata
+# NODE_ES_HEAP_SIZE=
+# NODE_LS_HEAP_SIZE=
+NODESETUP=NODEBASIC
+NSMSETUP=BASIC
+NODEUPDATES=MANAGER
+# OINKCODE=
+OSQUERY=1
+# PATCHSCHEDULEDAYS=
+# PATCHSCHEDULEHOURS=
+PATCHSCHEDULENAME=auto
+#PLAYBOOK=1
+# REDIRECTHOST=
+REDIRECTINFO=IP
+RULESETUP=ETOPEN
+# SHARDCOUNT=
+# SKIP_REBOOT=
+# SOREMOTEPASS1=onionuser
+# SOREMOTEPASS2=onionuser
+STRELKA=1
+THEHIVE=1
+WAZUH=1
+WEBUSER=onionuser@somewhere.invalid
+WEBPASSWD1=0n10nus3r
+WEBPASSWD2=0n10nus3r

setup/automation/import-net-ubuntu

@@ -0,0 +1,77 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+TESTING=true
+
+# address_type=DHCP
+ADMINUSER=onionuser
+ADMINPASS1=onionuser
+ADMINPASS2=onionuser
+ALLOW_CIDR=0.0.0.0/0
+ALLOW_ROLE=a
+BASICZEEK=2
+BASICSURI=2
+# BLOGS=
+#BNICS=eth1
+ZEEKVERSION=ZEEK
+# CURCLOSEDAYS=
+# EVALADVANCED=BASIC
+GRAFANA=1
+# HELIXAPIKEY=
+HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12
+HNSENSOR=inherit
+HOSTNAME=import
+install_type=IMPORT
+# LSINPUTBATCHCOUNT=
+# LSINPUTTHREADS=
+# LSPIPELINEBATCH=
+# LSPIPELINEWORKERS=
+MANAGERADV=BASIC
+MANAGERUPDATES=1
+# MDNS=
+# MGATEWAY=
+# MIP=
+# MMASK=
+MNIC=ens18
+# MSEARCH=
+# MSRV=
+# MTU=
+NIDS=Suricata
+# NODE_ES_HEAP_SIZE=
+# NODE_LS_HEAP_SIZE=
+NODESETUP=NODEBASIC
+NSMSETUP=BASIC
+NODEUPDATES=MANAGER
+# OINKCODE=
+OSQUERY=1
+# PATCHSCHEDULEDAYS=
+# PATCHSCHEDULEHOURS=
+PATCHSCHEDULENAME=auto
+#PLAYBOOK=1
+# REDIRECTHOST=
+REDIRECTINFO=IP
+RULESETUP=ETOPEN
+# SHARDCOUNT=
+# SKIP_REBOOT=
+# SOREMOTEPASS1=onionuser
+# SOREMOTEPASS2=onionuser
+STRELKA=1
+THEHIVE=1
+WAZUH=1
+WEBUSER=onionuser@somewhere.invalid
+WEBPASSWD1=0n10nus3r
+WEBPASSWD2=0n10nus3r

setup/so-functions

@@ -913,6 +913,29 @@ detect_os() {
 			exit 1
 		fi
 
+	elif [ -f /etc/os-release ]; then
+		OS=ubuntu
+		if grep -q "UBUNTU_CODENAME=bionic" /etc/os-release; then
+			OSVER=bionic
+		elif grep -q "UBUNTU_CODENAME=xenial" /etc/os-release; then
+			OSVER=xenial
+		else
+			echo "We do not support your current version of Ubuntu."
+			exit 1
+		fi
+
+	else
+		echo "We were unable to determine if you are using a supported OS."
+		exit 1
+	fi
+
+	echo "Found OS: $OS $OSVER" >> "$setup_log" 2>&1
+
+}
+
+installer_prereq_packages() {
+
+	if [ "$OS" == centos ]; then
 		# Print message to stdout so the user knows setup is doing something
 		echo "Installing required packages to run installer..."
 		# Install bind-utils so the host command exists
@@ -937,17 +960,7 @@ detect_os() {
 		  logCmd "systemctl enable NetworkManager"
 		  logCmd "systemctl start NetworkManager"
         fi
-	elif [ -f /etc/os-release ]; then
+	elif [ "$OS" == ubuntu ]; then
-		OS=ubuntu
-		if grep -q "UBUNTU_CODENAME=bionic" /etc/os-release; then
-			OSVER=bionic
-		elif grep -q "UBUNTU_CODENAME=xenial" /etc/os-release; then
-			OSVER=xenial
-		else
-			echo "We do not support your current version of Ubuntu."
-			exit 1
-		fi
-
 		# Print message to stdout so the user knows setup is doing something
 		echo "Installing required packages to run installer..."
 		# Install network manager so we can do interface stuff
@@ -960,13 +973,6 @@ detect_os() {
 		fi
 		if wait_for_apt; then apt-get install -y bc curl >> "$setup_log" 2>&1; else exit 1; fi
 
-	else
-		echo "We were unable to determine if you are using a supported OS."
-		exit 1
-	fi
-
-	echo "Found OS: $OS $OSVER" >> "$setup_log" 2>&1
-
 }
 
 disable_auto_start() {

setup/so-setup

@@ -104,6 +104,8 @@ function progress() {
 	fi		
 }
 
+detect_os 
+
 if [[ -f automation/$automation && $(basename $automation) == $automation ]]; then
 	echo "Preselecting variable values based on automated setup: $automation" >> $setup_log 2>&1
 	source automation/$automation
@@ -128,7 +130,7 @@ if [[ -f automation/$automation && $(basename $automation) == $automation ]]; th
 	if [[ ! $is_iso ]]; then
 		echo "Installing sshpass for automated testing." >> $setup_log 2>&1
 		if [ "$OS" == ubuntu ]; then
-			apt-get -y install sshpass >> $setup_log 2>&1
+			if wait_for_apt; then apt-get -y install sshpass >> $setup_log 2>&1
 		else
 			yum -y install sshpass >> $setup_log 2>&1
 		fi
@@ -152,7 +154,7 @@ set_ssh_cmds $automated
 local_sbin="$(pwd)/../salt/common/tools/sbin"
 export PATH=$PATH:$local_sbin
 
-detect_os && detect_cloud
+installer_prereq_packages && detect_cloud
 set_network_dev_status_list
 
 if [ "$OS" == ubuntu ]; then