diff --git a/files/salt/master/master b/files/salt/master/master
index 42e7866d9..93e8ff938 100644
--- a/files/salt/master/master
+++ b/files/salt/master/master
@@ -13,6 +13,8 @@
# user: socore
log_file: /opt/so/log/salt/master
+log_level_logfile: info
+log_level: info
##### File Server settings #####
##########################################
diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common
index c71e9150c..0c18c4482 100755
--- a/salt/common/tools/sbin/so-common
+++ b/salt/common/tools/sbin/so-common
@@ -141,6 +141,34 @@ get_random_value() {
head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1
}
+retry() {
+ maxAttempts=$1
+ sleepDelay=$2
+ cmd=$3
+ expectedOutput=$4
+ attempt=0
+ while [[ $attempt -lt $maxAttempts ]]; do
+ attempt=$((attempt+1))
+ info "Executing command with retry support: $cmd"
+ output=$($cmd)
+ info "Results: $output"
+ exitcode=$?
+ if [ -n "$expectedOutput" ]; then
+ if [[ "$output" =~ "$expectedOutput" ]]; then
+ return $exitCode
+ else
+ info "Expected '$expectedOutput' but got '$output'"
+ fi
+ elif [[ $exitcode -eq 0 ]]; then
+ return $exitCode
+ fi
+ info "Command failed with exit code $exitcode; will retry in $sleepDelay seconds ($attempt / $maxAttempts)..."
+ sleep $sleepDelay
+ done
+ error "Command continues to fail; giving up."
+ return 1
+}
+
wait_for_apt() {
local progress_callback=$1
diff --git a/setup/automation/distributed-iso-manager b/setup/automation/distributed-iso-manager
new file mode 100644
index 000000000..cbf803dd2
--- /dev/null
+++ b/setup/automation/distributed-iso-manager
@@ -0,0 +1,77 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+TESTING=true
+
+address_type=DHCP
+ADMINUSER=onionuser
+ADMINPASS1=onionuser
+ADMINPASS2=onionuser
+ALLOW_CIDR=0.0.0.0/0
+ALLOW_ROLE=a
+BASICZEEK=7
+BASICSURI=7
+# BLOGS=
+#BNICS=eth1
+ZEEKVERSION=ZEEK
+# CURCLOSEDAYS=
+# EVALADVANCED=BASIC
+GRAFANA=1
+# HELIXAPIKEY=
+HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12
+HNSENSOR=inherit
+HOSTNAME=distributed-manager
+install_type=MANAGER
+# LSINPUTBATCHCOUNT=
+# LSINPUTTHREADS=
+# LSPIPELINEBATCH=
+# LSPIPELINEWORKERS=
+MANAGERADV=BASIC
+MANAGERUPDATES=1
+# MDNS=
+# MGATEWAY=
+# MIP=
+# MMASK=
+MNIC=eth0
+# MSEARCH=
+# MSRV=
+# MTU=
+NIDS=Suricata
+# NODE_ES_HEAP_SIZE=
+# NODE_LS_HEAP_SIZE=
+NODESETUP=NODEBASIC
+NSMSETUP=BASIC
+NODEUPDATES=MANAGER
+# OINKCODE=
+OSQUERY=1
+# PATCHSCHEDULEDAYS=
+# PATCHSCHEDULEHOURS=
+PATCHSCHEDULENAME=auto
+PLAYBOOK=1
+# REDIRECTHOST=
+REDIRECTINFO=IP
+RULESETUP=ETOPEN
+# SHARDCOUNT=
+# SKIP_REBOOT=
+SOREMOTEPASS1=onionuser
+SOREMOTEPASS2=onionuser
+STRELKA=1
+THEHIVE=1
+WAZUH=1
+WEBUSER=onionuser@somewhere.invalid
+WEBPASSWD1=0n10nus3r
+WEBPASSWD2=0n10nus3r
diff --git a/setup/automation/distributed-iso-search b/setup/automation/distributed-iso-search
new file mode 100644
index 000000000..d37a7d935
--- /dev/null
+++ b/setup/automation/distributed-iso-search
@@ -0,0 +1,78 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+TESTING=true
+
+address_type=DHCP
+ADMINUSER=onionuser
+ADMINPASS1=onionuser
+ADMINPASS2=onionuser
+# ALLOW_CIDR=0.0.0.0/0
+# ALLOW_ROLE=a
+# BASICZEEK=7
+# BASICSURI=7
+# BLOGS=
+# BNICS=eth1
+# ZEEKVERSION=ZEEK
+# CURCLOSEDAYS=
+# EVALADVANCED=BASIC
+# GRAFANA=1
+# HELIXAPIKEY=
+HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12
+HNSENSOR=inherit
+HOSTNAME=distributed-search
+install_type=SEARCHNODE
+# LSINPUTBATCHCOUNT=
+# LSINPUTTHREADS=
+# LSPIPELINEBATCH=
+# LSPIPELINEWORKERS=
+# MANAGERADV=BASIC
+MANAGERUPDATES=1
+# MDNS=
+# MGATEWAY=
+# MIP=
+# MMASK=
+MNIC=eth0
+# MSEARCH=
+MSRV=distributed-manager
+MSRVIP=10.66.166.42
+# MTU=
+# NIDS=Suricata
+# NODE_ES_HEAP_SIZE=
+# NODE_LS_HEAP_SIZE=
+NODESETUP=NODEBASIC
+NSMSETUP=BASIC
+NODEUPDATES=MANAGER
+# OINKCODE=
+# OSQUERY=1
+# PATCHSCHEDULEDAYS=
+# PATCHSCHEDULEHOURS=
+PATCHSCHEDULENAME=auto
+# PLAYBOOK=1
+# REDIRECTHOST=
+# REDIRECTINFO=IP
+# RULESETUP=ETOPEN
+# SHARDCOUNT=
+# SKIP_REBOOT=
+SOREMOTEPASS1=onionuser
+SOREMOTEPASS2=onionuser
+# STRELKA=1
+# THEHIVE=1
+WAZUH=1
+WEBUSER=onionuser@somewhere.invalid
+WEBPASSWD1=0n10nus3r
+WEBPASSWD2=0n10nus3r
diff --git a/setup/automation/distributed-iso-sensor b/setup/automation/distributed-iso-sensor
new file mode 100644
index 000000000..402049be9
--- /dev/null
+++ b/setup/automation/distributed-iso-sensor
@@ -0,0 +1,78 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+TESTING=true
+
+address_type=DHCP
+ADMINUSER=onionuser
+ADMINPASS1=onionuser
+ADMINPASS2=onionuser
+# ALLOW_CIDR=0.0.0.0/0
+# ALLOW_ROLE=a
+BASICZEEK=7
+BASICSURI=7
+# BLOGS=
+BNICS=eth1
+ZEEKVERSION=ZEEK
+# CURCLOSEDAYS=
+# EVALADVANCED=BASIC
+# GRAFANA=1
+# HELIXAPIKEY=
+HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12
+HNSENSOR=inherit
+HOSTNAME=distributed-sensor
+install_type=SENSOR
+# LSINPUTBATCHCOUNT=
+# LSINPUTTHREADS=
+# LSPIPELINEBATCH=
+# LSPIPELINEWORKERS=
+# MANAGERADV=BASIC
+MANAGERUPDATES=1
+# MDNS=
+# MGATEWAY=
+# MIP=
+# MMASK=
+MNIC=eth0
+# MSEARCH=
+MSRV=distributed-manager
+MSRVIP=10.66.166.42
+# MTU=
+# NIDS=Suricata
+# NODE_ES_HEAP_SIZE=
+# NODE_LS_HEAP_SIZE=
+# NODESETUP=NODEBASIC
+NSMSETUP=BASIC
+NODEUPDATES=MANAGER
+# OINKCODE=
+# OSQUERY=1
+# PATCHSCHEDULEDAYS=
+# PATCHSCHEDULEHOURS=
+PATCHSCHEDULENAME=auto
+# PLAYBOOK=1
+# REDIRECTHOST=
+# REDIRECTINFO=IP
+# RULESETUP=ETOPEN
+# SHARDCOUNT=
+# SKIP_REBOOT=
+SOREMOTEPASS1=onionuser
+SOREMOTEPASS2=onionuser
+STRELKA=1
+# THEHIVE=1
+WAZUH=1
+WEBUSER=onionuser@somewhere.invalid
+WEBPASSWD1=0n10nus3r
+WEBPASSWD2=0n10nus3r
diff --git a/setup/so-functions b/setup/so-functions
index 315334a18..e9de76233 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -467,6 +467,8 @@ configure_minion() {
printf '%s\n'\
"use_superseded:"\
" - module.run"\
+ "log_level: info"\
+ "log_level_logfile: info"\
"log_file: /opt/so/log/salt/minion" >> "$minion_config"
{
@@ -1968,6 +1970,8 @@ set_progress_str() {
echo -e "$percentage_str"
+ info "Progressing ($percentage%): $progress_bar_text"
+
printf '%s\n' \
'----'\
"$percentage% - ${progress_bar_text^^}"\
diff --git a/setup/so-setup b/setup/so-setup
index 15189746f..e62dc434f 100755
--- a/setup/so-setup
+++ b/setup/so-setup
@@ -637,12 +637,12 @@ set_redirect >> $setup_log 2>&1
if [[ $is_minion ]]; then
set_progress_str 20 'Accepting Salt key on manager'
- accept_salt_key_remote >> $setup_log 2>&1
+ retry 20 10 accept_salt_key_remote "going to be accepted"
fi
if [[ $is_manager || $is_import || $is_helix ]]; then
set_progress_str 20 'Accepting Salt key'
- salt-key -ya "$MINION_ID" >> $setup_log 2>&1
+ retry 20 10 "salt-key -ya $MINION_ID" "going to be accepted"
fi
set_progress_str 21 'Copying minion pillars to manager'