CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2025-12-06 17:22:49 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2825 from Security-Onion-Solutions/foxtrot

Browse Source 
Setup: dpkg retry, whiptail changes, fix zeek state condition
This commit is contained in:
William Wernert
2021-02-02 14:41:48 -05:00
committed by GitHub
parent ef2fe2bb61 46581c0528
commit 8f0b0ac936
3 changed files with 69 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
salt/common/tools/sbin/so-common
View File

@@ -229,7 +229,7 @@ retry() {
while [[ $attempt -lt $maxAttempts ]]; do while [[ $attempt -lt $maxAttempts ]]; do
attempt=$((attempt+1)) attempt=$((attempt+1))
echo "Executing command with retry support: $cmd" echo "Executing command with retry support: $cmd"
output=$($cmd) output=$(eval "$cmd")
exitcode=$? exitcode=$?
echo "Results: $output ($exitcode)" echo "Results: $output ($exitcode)"
if [ -n "$expectedOutput" ]; then if [ -n "$expectedOutput" ]; then
@@ -248,51 +248,6 @@ retry() {
return 1 return 1
} }
wait_for_apt() {
local progress_callback=$1
local retry_count=30
local retry_timeout='10s'
local lock_msg="Could not acquire dpkg lock, waiting $retry_timeout for lock to release."
if [[ -z $progress_callback ]]; then
if [[ -z $progress_bar_text ]]; then
local old_text="Installing..."
else
local old_text="$progress_bar_text"
fi
fi
local count=0
while [[ "$count" -lt "$retry_count" ]]; do
((count++))
[[ -z $progress_callback ]] && echo "Attempting to acquire dpkg lock to run apt command... (Attempt $count/$retry_count)"
if __check_apt_lock; then
if [[ -z $progress_callback ]]; then
echo " $lock_msg" | tee -a "$setup_log"
else
$progress_callback "Could not acquire dpkg lock, waiting $retry_timeout ($count/$retry_count)"
fi
else
[[ -z $progress_callback ]] || $progress_callback "$old_text"
return 0
fi
sleep "$retry_timeout"
done
if __check_apt_lock; then
[[ -z $progress_callback ]] && echo "Could not acquire lock after $retry_count attempts, aborting."
return 1
else
return 0
fi
}
__check_apt_lock() {
lsof /var/lib/dpkg/lock &> /dev/null
local lock=$?
return $lock
}
valid_cidr() { valid_cidr() {
# Verify there is a backslash in the string # Verify there is a backslash in the string
echo "$1" | grep -qP "^[^/]+/[^/]+$" || return 1 echo "$1" | grep -qP "^[^/]+/[^/]+$" || return 1

132
setup/so-functions
View File

@@ -760,7 +760,7 @@ compare_versions() {
if [[ $manager_ver == '' ]]; then if [[ $manager_ver == '' ]]; then
echo "Could not determine version of Security Onion running on manager $MSRV. Please check your network settings and run setup again." | tee -a "$setup_log" echo "Could not determine version of Security Onion running on manager $MSRV. Please check your network settings and run setup again." | tee -a "$setup_log"
kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 exit 1
fi fi
[[ "$manager_ver" == "$SOVERSION" ]] [[ "$manager_ver" == "$SOVERSION" ]]
@@ -965,13 +965,13 @@ installer_prereq_packages() {
echo "Installing required packages to run installer..." echo "Installing required packages to run installer..."
# Install network manager so we can do interface stuff # Install network manager so we can do interface stuff
if ! command -v nmcli > /dev/null 2>&1; then if ! command -v nmcli > /dev/null 2>&1; then
if wait_for_apt; then apt-get install -y network-manager >> "$setup_log" 2<&1; else exit 1; fi retry 50 10 "apt-get install -y network-manager" >> "$setup_log" 2>&1 || exit 1
{ {
systemctl enable NetworkManager systemctl enable NetworkManager
systemctl start NetworkManager systemctl start NetworkManager
} >> "$setup_log" 2<&1 } >> "$setup_log" 2<&1
fi fi
if wait_for_apt; then apt-get install -y bc curl >> "$setup_log" 2>&1; else exit 1; fi retry 50 10 "apt-get install -y bc curl" >> "$setup_log" 2>&1 || exit 1
fi fi
} }
@@ -1041,28 +1041,18 @@ docker_install() {
else else
case "$install_type" in case "$install_type" in
'MANAGER' | 'EVAL' | 'STANDALONE' | 'MANAGERSEARCH' | 'IMPORT') 'MANAGER' | 'EVAL' | 'STANDALONE' | 'MANAGERSEARCH' | 'IMPORT')
if wait_for_apt 'whiptail_prog_new_message'; then apt-get update >> "$setup_log" 2>&1; else kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1; fi retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1
;; ;;
*) *)
if wait_for_apt 'whiptail_prog_new_message'; then retry 50 10 "apt-key add $temp_install_dir/gpg/docker.pub" >> "$setup_log" 2>&1 || exit 1
{ add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" >> "$setup_log" 2>&1
apt-key add "$temp_install_dir"/gpg/docker.pub; retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1
add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable";
apt-get update;
} >> "$setup_log" 2>&1
else
kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
fi
;; ;;
esac esac
if wait_for_apt 'whiptail_prog_new_message'; then if [ $OSVER != "xenial" ]; then
if [ $OSVER != "xenial" ]; then retry 50 10 "apt-get -y install docker-ce python3-docker" >> "$setup_log" 2>&1 || exit 1
apt-get -y install docker-ce python3-docker >> "$setup_log" 2>&1
else
apt-get -y install docker-ce python-docker >> "$setup_log" 2>&1
fi
else else
kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 retry 50 10 "apt-get -y install docker-ce python-docker" >> "$setup_log" 2>&1 || exit 1
fi fi
fi fi
docker_registry docker_registry
@@ -1145,7 +1135,7 @@ download_repo_tarball() {
rm -rf $install_opt_file rm -rf $install_opt_file
local message="Could not download $manager_ver.tar.gz from manager, please check your network settings and verify the file /opt/so/repo/$manager_ver.tar.gz exists on the manager." local message="Could not download $manager_ver.tar.gz from manager, please check your network settings and verify the file /opt/so/repo/$manager_ver.tar.gz exists on the manager."
echo "$message" | tee -a "$setup_log" echo "$message" | tee -a "$setup_log"
kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 exit 1
fi fi
mkdir -p /root/manager_setup/securityonion mkdir -p /root/manager_setup/securityonion
@@ -1821,7 +1811,7 @@ reinstall_init() {
# Stop the systemctl process trying to kill the service, show user a message, then exit setup # Stop the systemctl process trying to kill the service, show user a message, then exit setup
kill -9 $pid kill -9 $pid
kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 exit 1
fi fi
sleep 5 sleep 5
@@ -1873,11 +1863,7 @@ remove_package() {
fi fi
else else
if dpkg -l | grep -q "$package_name"; then if dpkg -l | grep -q "$package_name"; then
if wait_for_apt 'whiptail_prog_new_message'; then retry 50 10 "apt purge -y \"$package_name\""
apt purge -y "$package_name"
else
exit 1
fi
fi fi
fi fi
} }
@@ -1964,28 +1950,26 @@ saltify() {
} >> "$setup_log" 2>&1 } >> "$setup_log" 2>&1
yum versionlock salt* yum versionlock salt*
else else
if wait_for_apt 'whiptail_prog_new_message'; then if ! (DEBIAN_FRONTEND=noninteractive retry 50 10 "apt-get -y -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" upgrade" >> "$setup_log" 2>&1); then
DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" upgrade >> "$setup_log" 2>&1 exit 1
else
kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
fi fi
if [ $OSVER != "xenial" ]; then if [ $OSVER != "xenial" ]; then
# Switch to Python 3 as default if this is not xenial # Switch to Python 3 as default if this is not xenial
update-alternatives --install /usr/bin/python python /usr/bin/python3.6 10 >> "$setup_log" 2>&1 update-alternatives --install /usr/bin/python python /usr/bin/python3.6 10 >> "$setup_log" 2>&1
fi fi
if wait_for_apt 'whiptail_prog_new_message'; then
# Add the pre-requisites for installing docker-ce local pkg_arr=(
apt-get -y install ca-certificates\ 'ca-certificates'
curl\ 'curl'
software-properties-common\ 'software-properties-common'
apt-transport-https\ 'apt-transport-https'
openssl\ 'openssl'
netcat\ 'netcat'
jq >> "$setup_log" 2>&1 'jq'
else )
kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 retry 50 10 "apt-get -y install ${pkg_arr[*]}" >> "$setup_log" 2>&1 || exit 1
fi
# Grab the version from the os-release file # Grab the version from the os-release file
local ubuntu_version local ubuntu_version
ubuntu_version=$(grep VERSION_ID /etc/os-release | awk -F '[ "]' '{print $2}') ubuntu_version=$(grep VERSION_ID /etc/os-release | awk -F '[ "]' '{print $2}')
@@ -1993,10 +1977,10 @@ saltify() {
case "$install_type" in case "$install_type" in
'FLEET') 'FLEET')
if wait_for_apt 'whiptail_prog_new_message'; then if [[ $OSVER != 'xenial' ]]; then
if [ "$OSVER" != 'xenial' ]; then apt-get -y install python3-mysqldb >> "$setup_log" 2>&1; else apt-get -y install python-mysqldb >> "$setup_log" 2>&1; fi retry 50 10 "apt-get -y install python3-mysqldb" >> "$setup_log" 2>&1 || exit 1
else else
kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 retry 50 10 "apt-get -y install python-mysqldb" >> "$setup_log" 2>&1 || exit 1
fi fi
;; ;;
'MANAGER' | 'EVAL' | 'MANAGERSEARCH' | 'STANDALONE' | 'IMPORT' | 'HELIXSENSOR') 'MANAGER' | 'EVAL' | 'MANAGERSEARCH' | 'STANDALONE' | 'IMPORT' | 'HELIXSENSOR')
@@ -2020,17 +2004,12 @@ saltify() {
# Add repo # Add repo
echo "deb https://packages.wazuh.com/3.x/apt/ stable main" > /etc/apt/sources.list.d/wazuh.list 2>> "$setup_log" echo "deb https://packages.wazuh.com/3.x/apt/ stable main" > /etc/apt/sources.list.d/wazuh.list 2>> "$setup_log"
if wait_for_apt 'whiptail_prog_new_message'; then retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1
# Initialize the new repos set_progress_str 6 'Installing various dependencies'
apt-get update >> "$setup_log" 2>&1 retry 50 10 "apt-get -y install sqlite3 argon2 libssl-dev" >> "$setup_log" 2>&1 || exit 1
set_progress_str 6 'Installing various dependencies' set_progress_str 7 'Installing salt-master'
apt-get -y install sqlite3 argon2 libssl-dev >> "$setup_log" 2>&1 retry 50 10 "apt-get -y install salt-master=3002.2+ds-1" >> "$setup_log" 2>&1 || exit 1
set_progress_str 7 'Installing salt-master' retry 50 10 "apt-mark hold salt-master" >> "$setup_log" 2>&1 || exit 1
apt-get -y install salt-master=3002.2+ds-1 >> "$setup_log" 2>&1
apt-mark hold salt-master >> "$setup_log" 2>&1
else
kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
fi
;; ;;
*) *)
# Copy down the gpg keys and install them from the manager # Copy down the gpg keys and install them from the manager
@@ -2044,19 +2023,15 @@ saltify() {
echo "deb https://packages.wazuh.com/3.x/apt/ stable main" > /etc/apt/sources.list.d/wazuh.list 2>> "$setup_log" echo "deb https://packages.wazuh.com/3.x/apt/ stable main" > /etc/apt/sources.list.d/wazuh.list 2>> "$setup_log"
;; ;;
esac esac
if wait_for_apt 'whiptail_prog_new_message'; then
apt-get update >> "$setup_log" 2>&1 retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1
set_progress_str 8 'Installing salt-minion & python modules' set_progress_str 8 'Installing salt-minion & python modules'
apt-get -y install salt-minion=3002.2+ds-1\ retry 50 10 "apt-get -y install salt-minion=3002.2+ds-1 salt-common=3002.2+ds-1" >> "$setup_log" 2>&1 || exit 1
salt-common=3002.2+ds-1 >> "$setup_log" 2>&1 retry 50 10 "apt-mark hold salt-minion salt-common" >> "$setup_log" 2>&1 || exit 1
apt-mark hold salt-minion salt-common >> "$setup_log" 2>&1 if [[ $OSVER != 'xenial' ]]; then
if [ "$OSVER" != 'xenial' ]; then retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb" >> "$setup_log" 2>&1 || exit 1
apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb >> "$setup_log" 2>&1
else
apt-get -y install python-pip python-dateutil python-m2crypto python-mysqldb >> "$setup_log" 2>&1
fi
else else
kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 retry 50 10 "apt-get -y install python-pip python-dateutil python-m2crypto python-mysqldb" >> "$setup_log" 2>&1 || exit 1
fi fi
fi fi
} }
@@ -2097,7 +2072,7 @@ salt_checkin() {
if [ $count -gt 12 ]; then if [ $count -gt 12 ]; then
echo "$service could not be restarted in 120 seconds, exiting" >> "$setup_log" 2>&1 echo "$service could not be restarted in 120 seconds, exiting" >> "$setup_log" 2>&1
kill -9 "$pid" kill -9 "$pid"
kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 exit 1
fi fi
sleep 10; sleep 10;
((count++)) ((count++))
@@ -2109,7 +2084,7 @@ salt_checkin() {
echo "salt minion cannot talk to salt master" >> "$setup_log" 2>&1 echo "salt minion cannot talk to salt master" >> "$setup_log" 2>&1
if [ $count -gt 30 ]; then if [ $count -gt 30 ]; then
echo "salt minion could not talk to salt master after 30 attempts, exiting" >> "$setup_log" 2>&1 echo "salt minion could not talk to salt master after 30 attempts, exiting" >> "$setup_log" 2>&1
kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 exit 1
fi fi
sleep 1; sleep 1;
((count++)) ((count++))
@@ -2120,7 +2095,7 @@ salt_checkin() {
echo "salt master did not get a job response from salt minion" >> "$setup_log" 2>&1 echo "salt master did not get a job response from salt minion" >> "$setup_log" 2>&1
if [ $count -gt 30 ]; then if [ $count -gt 30 ]; then
echo "salt master did not get a job response from salt minion after 30 attempts, exiting" >> "$setup_log" 2>&1 echo "salt master did not get a job response from salt minion after 30 attempts, exiting" >> "$setup_log" 2>&1
kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 exit 1
fi fi
systemctl kill salt-minion systemctl kill salt-minion
systemctl start salt-minion systemctl start salt-minion
@@ -2522,12 +2497,8 @@ update_packages() {
if [ "$OS" = 'centos' ]; then if [ "$OS" = 'centos' ]; then
yum -y update >> "$setup_log" yum -y update >> "$setup_log"
else else
if wait_for_apt 'whiptail_prog_new_message'; then retry 50 10 "apt-get -y update" >> "$setup_log" 2>&1 || exit 1
apt-get -y update >> "$setup_log" retry 50 10 "apt-get -y upgrade" >> "$setup_log" 2>&1 || exit 1
apt-get -y upgrade >> "$setup_log"
else
kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
fi
fi fi
} }
@@ -2572,11 +2543,6 @@ wait_for_file() {
return 1 return 1
} }
whiptail_prog_new_message() {
local message=$1
set_progress_str "$percentage" "$message"
}
# Enable Zeek Logs # Enable Zeek Logs
zeek_logs_enabled() { zeek_logs_enabled() {
echo "Enabling Zeek Logs" >> "$setup_log" 2>&1 echo "Enabling Zeek Logs" >> "$setup_log" 2>&1

25
setup/so-setup
View File

@@ -130,7 +130,7 @@ if [[ -f automation/$automation && $(basename $automation) == $automation ]]; th
if [[ ! $is_iso ]]; then if [[ ! $is_iso ]]; then
echo "Installing sshpass for automated testing." >> $setup_log 2>&1 echo "Installing sshpass for automated testing." >> $setup_log 2>&1
if [ "$OS" == ubuntu ]; then if [ "$OS" == ubuntu ]; then
if wait_for_apt; then apt-get -y install sshpass >> $setup_log 2>&1; else exit 1; fi retry 50 10 "apt-get -y install sshpass" >> $setup_log 2>&1 || exit 1
else else
yum -y install sshpass >> $setup_log 2>&1 yum -y install sshpass >> $setup_log 2>&1
fi fi
@@ -397,7 +397,9 @@ if [[ ! $is_import ]]; then
collect_patch_schedule collect_patch_schedule
fi fi
collect_homenet_mngr if [[ $is_helix || $is_manager || $is_import ]]; then
collect_homenet_mngr
fi
if [[ $is_helix || $is_manager || $is_node || $is_import ]]; then if [[ $is_helix || $is_manager || $is_node || $is_import ]]; then
set_base_heapsizes set_base_heapsizes
@@ -456,10 +458,15 @@ if [[ $is_distmanager ]]; then
fi fi
if [[ $is_sensor && ! $is_eval ]]; then if [[ $is_sensor && ! $is_eval ]]; then
collect_homenet_snsr [[ $is_manager ]] || collect_homenet_snsr
whiptail_sensor_config whiptail_sensor_config
if [ $NSMSETUP == 'ADVANCED' ]; then if [ $NSMSETUP == 'ADVANCED' ]; then
[[ $ZEEKVERSION == "ZEEK" ]] && whiptail_zeek_pins if [[ $is_manager ]]; then
[[ $ZEEKVERSION == "ZEEK" ]] && whiptail_zeek_pins
else
whiptail_zeek_pins
fi
whiptail_suricata_pins whiptail_suricata_pins
collect_mtu collect_mtu
else else
@@ -469,7 +476,13 @@ if [[ $is_sensor && ! $is_eval ]]; then
else else
PROCS=$lb_procs PROCS=$lb_procs
fi fi
[[ $ZEEKVERSION == "ZEEK" ]] && collect_zeek
if [[ $is_manager ]]; then
[[ $ZEEKVERSION == "ZEEK" ]] && collect_zeek
else
collect_zeek
fi
collect_suri collect_suri
fi fi
fi fi
@@ -730,7 +743,7 @@ set_redirect >> $setup_log 2>&1
set_progress_str 66 "$(print_salt_state_apply 'suricata')" set_progress_str 66 "$(print_salt_state_apply 'suricata')"
salt-call state.apply -l info suricata >> $setup_log 2>&1 salt-call state.apply -l info suricata >> $setup_log 2>&1
if [[ $ZEEKVERSION == 'ZEEK' ]]; then if [[ $(lookup_pillar "mdengine") == 'ZEEK' ]]; then
set_progress_str 67 "$(print_salt_state_apply 'zeek')" set_progress_str 67 "$(print_salt_state_apply 'zeek')"
salt-call state.apply -l info zeek >> $setup_log 2>&1 salt-call state.apply -l info zeek >> $setup_log 2>&1
fi fi