CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2025-12-06 17:22:49 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2825 from Security-Onion-Solutions/foxtrot

Browse Source 
Setup: dpkg retry, whiptail changes, fix zeek state condition
This commit is contained in:
William Wernert
2021-02-02 14:41:48 -05:00
committed by GitHub
parent ef2fe2bb61 46581c0528
commit 8f0b0ac936
3 changed files with 69 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
salt/common/tools/sbin/so-common
View File

@@ -229,7 +229,7 @@ retry() {
while [[ $attempt -lt $maxAttempts ]]; do
attempt=$((attempt+1))
echo "Executing command with retry support: $cmd"
output=$($cmd)
output=$(eval "$cmd")
exitcode=$?
echo "Results: $output ($exitcode)"
if [ -n "$expectedOutput" ]; then
@@ -248,51 +248,6 @@ retry() {
return 1
}
wait_for_apt() {
local progress_callback=$1
local retry_count=30
local retry_timeout='10s'
local lock_msg="Could not acquire dpkg lock, waiting $retry_timeout for lock to release."
if [[ -z $progress_callback ]]; then
if [[ -z $progress_bar_text ]]; then
local old_text="Installing..."
else
local old_text="$progress_bar_text"
fi
fi
local count=0
while [[ "$count" -lt "$retry_count" ]]; do
((count++))
[[ -z $progress_callback ]] && echo "Attempting to acquire dpkg lock to run apt command... (Attempt $count/$retry_count)"
if __check_apt_lock; then
if [[ -z $progress_callback ]]; then
echo " $lock_msg" | tee -a "$setup_log"
else
$progress_callback "Could not acquire dpkg lock, waiting $retry_timeout ($count/$retry_count)"
fi
else
[[ -z $progress_callback ]] || $progress_callback "$old_text"
return 0
fi
sleep "$retry_timeout"
done
if __check_apt_lock; then
[[ -z $progress_callback ]] && echo "Could not acquire lock after $retry_count attempts, aborting."
return 1
else
return 0
fi
}
__check_apt_lock() {
lsof /var/lib/dpkg/lock &> /dev/null
local lock=$?
return $lock
}
valid_cidr() {
# Verify there is a backslash in the string
echo "$1" | grep -qP "^[^/]+/[^/]+$" || return 1

124
setup/so-functions
View File

@@ -760,7 +760,7 @@ compare_versions() {
if [[ $manager_ver == '' ]]; then
echo "Could not determine version of Security Onion running on manager $MSRV. Please check your network settings and run setup again." | tee -a "$setup_log"
kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
exit 1
fi
[[ "$manager_ver" == "$SOVERSION" ]]
@@ -965,13 +965,13 @@ installer_prereq_packages() {
echo "Installing required packages to run installer..."
# Install network manager so we can do interface stuff
if ! command -v nmcli > /dev/null 2>&1; then
if wait_for_apt; then apt-get install -y network-manager >> "$setup_log" 2<&1; else exit 1; fi
retry 50 10 "apt-get install -y network-manager" >> "$setup_log" 2>&1 || exit 1
{
systemctl enable NetworkManager
systemctl start NetworkManager
} >> "$setup_log" 2<&1
fi
if wait_for_apt; then apt-get install -y bc curl >> "$setup_log" 2>&1; else exit 1; fi
retry 50 10 "apt-get install -y bc curl" >> "$setup_log" 2>&1 || exit 1
fi
}
@@ -1041,28 +1041,18 @@ docker_install() {
else
case "$install_type" in
'MANAGER' | 'EVAL' | 'STANDALONE' | 'MANAGERSEARCH' | 'IMPORT')
if wait_for_apt 'whiptail_prog_new_message'; then apt-get update >> "$setup_log" 2>&1; else kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1; fi
retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1
;;
*)
if wait_for_apt 'whiptail_prog_new_message'; then
{
apt-key add "$temp_install_dir"/gpg/docker.pub;
add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable";
apt-get update;
} >> "$setup_log" 2>&1
else
kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
fi
retry 50 10 "apt-key add $temp_install_dir/gpg/docker.pub" >> "$setup_log" 2>&1 || exit 1
add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" >> "$setup_log" 2>&1
retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1
;;
esac
if wait_for_apt 'whiptail_prog_new_message'; then
if [ $OSVER != "xenial" ]; then
apt-get -y install docker-ce python3-docker >> "$setup_log" 2>&1
retry 50 10 "apt-get -y install docker-ce python3-docker" >> "$setup_log" 2>&1 || exit 1
else
apt-get -y install docker-ce python-docker >> "$setup_log" 2>&1
fi
else
kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
retry 50 10 "apt-get -y install docker-ce python-docker" >> "$setup_log" 2>&1 || exit 1
fi
fi
docker_registry
@@ -1145,7 +1135,7 @@ download_repo_tarball() {
rm -rf $install_opt_file
local message="Could not download $manager_ver.tar.gz from manager, please check your network settings and verify the file /opt/so/repo/$manager_ver.tar.gz exists on the manager."
echo "$message" | tee -a "$setup_log"
kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
exit 1
fi
mkdir -p /root/manager_setup/securityonion
@@ -1821,7 +1811,7 @@ reinstall_init() {
# Stop the systemctl process trying to kill the service, show user a message, then exit setup
kill -9 $pid
kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
exit 1
fi
sleep 5
@@ -1873,11 +1863,7 @@ remove_package() {
fi
else
if dpkg -l | grep -q "$package_name"; then
if wait_for_apt 'whiptail_prog_new_message'; then
apt purge -y "$package_name"
else
exit 1
fi
retry 50 10 "apt purge -y \"$package_name\""
fi
fi
}
@@ -1964,28 +1950,26 @@ saltify() {
} >> "$setup_log" 2>&1
yum versionlock salt*
else
if wait_for_apt 'whiptail_prog_new_message'; then
DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" upgrade >> "$setup_log" 2>&1
else
kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
if ! (DEBIAN_FRONTEND=noninteractive retry 50 10 "apt-get -y -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" upgrade" >> "$setup_log" 2>&1); then
exit 1
fi
if [ $OSVER != "xenial" ]; then
# Switch to Python 3 as default if this is not xenial
update-alternatives --install /usr/bin/python python /usr/bin/python3.6 10 >> "$setup_log" 2>&1
fi
if wait_for_apt 'whiptail_prog_new_message'; then
# Add the pre-requisites for installing docker-ce
apt-get -y install ca-certificates\
curl\
software-properties-common\
apt-transport-https\
openssl\
netcat\
jq >> "$setup_log" 2>&1
else
kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
fi
local pkg_arr=(
'ca-certificates'
'curl'
'software-properties-common'
'apt-transport-https'
'openssl'
'netcat'
'jq'
)
retry 50 10 "apt-get -y install ${pkg_arr[*]}" >> "$setup_log" 2>&1 || exit 1
# Grab the version from the os-release file
local ubuntu_version
ubuntu_version=$(grep VERSION_ID /etc/os-release | awk -F '[ "]' '{print $2}')
@@ -1993,10 +1977,10 @@ saltify() {
case "$install_type" in
'FLEET')
if wait_for_apt 'whiptail_prog_new_message'; then
if [ "$OSVER" != 'xenial' ]; then apt-get -y install python3-mysqldb >> "$setup_log" 2>&1; else apt-get -y install python-mysqldb >> "$setup_log" 2>&1; fi
if [[ $OSVER != 'xenial' ]]; then
retry 50 10 "apt-get -y install python3-mysqldb" >> "$setup_log" 2>&1 || exit 1
else
kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
retry 50 10 "apt-get -y install python-mysqldb" >> "$setup_log" 2>&1 || exit 1
fi
;;
'MANAGER' | 'EVAL' | 'MANAGERSEARCH' | 'STANDALONE' | 'IMPORT' | 'HELIXSENSOR')
@@ -2020,17 +2004,12 @@ saltify() {
# Add repo
echo "deb https://packages.wazuh.com/3.x/apt/ stable main" > /etc/apt/sources.list.d/wazuh.list 2>> "$setup_log"
if wait_for_apt 'whiptail_prog_new_message'; then
# Initialize the new repos
apt-get update >> "$setup_log" 2>&1
retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1
set_progress_str 6 'Installing various dependencies'
apt-get -y install sqlite3 argon2 libssl-dev >> "$setup_log" 2>&1
retry 50 10 "apt-get -y install sqlite3 argon2 libssl-dev" >> "$setup_log" 2>&1 || exit 1
set_progress_str 7 'Installing salt-master'
apt-get -y install salt-master=3002.2+ds-1 >> "$setup_log" 2>&1
apt-mark hold salt-master >> "$setup_log" 2>&1
else
kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
fi
retry 50 10 "apt-get -y install salt-master=3002.2+ds-1" >> "$setup_log" 2>&1 || exit 1
retry 50 10 "apt-mark hold salt-master" >> "$setup_log" 2>&1 || exit 1
;;
*)
# Copy down the gpg keys and install them from the manager
@@ -2044,19 +2023,15 @@ saltify() {
echo "deb https://packages.wazuh.com/3.x/apt/ stable main" > /etc/apt/sources.list.d/wazuh.list 2>> "$setup_log"
;;
esac
if wait_for_apt 'whiptail_prog_new_message'; then
apt-get update >> "$setup_log" 2>&1
retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1
set_progress_str 8 'Installing salt-minion & python modules'
apt-get -y install salt-minion=3002.2+ds-1\
salt-common=3002.2+ds-1 >> "$setup_log" 2>&1
apt-mark hold salt-minion salt-common >> "$setup_log" 2>&1
if [ "$OSVER" != 'xenial' ]; then
apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb >> "$setup_log" 2>&1
retry 50 10 "apt-get -y install salt-minion=3002.2+ds-1 salt-common=3002.2+ds-1" >> "$setup_log" 2>&1 || exit 1
retry 50 10 "apt-mark hold salt-minion salt-common" >> "$setup_log" 2>&1 || exit 1
if [[ $OSVER != 'xenial' ]]; then
retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb" >> "$setup_log" 2>&1 || exit 1
else
apt-get -y install python-pip python-dateutil python-m2crypto python-mysqldb >> "$setup_log" 2>&1
fi
else
kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
retry 50 10 "apt-get -y install python-pip python-dateutil python-m2crypto python-mysqldb" >> "$setup_log" 2>&1 || exit 1
fi
fi
}
@@ -2097,7 +2072,7 @@ salt_checkin() {
if [ $count -gt 12 ]; then
echo "$service could not be restarted in 120 seconds, exiting" >> "$setup_log" 2>&1
kill -9 "$pid"
kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
exit 1
fi
sleep 10;
((count++))
@@ -2109,7 +2084,7 @@ salt_checkin() {
echo "salt minion cannot talk to salt master" >> "$setup_log" 2>&1
if [ $count -gt 30 ]; then
echo "salt minion could not talk to salt master after 30 attempts, exiting" >> "$setup_log" 2>&1
kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
exit 1
fi
sleep 1;
((count++))
@@ -2120,7 +2095,7 @@ salt_checkin() {
echo "salt master did not get a job response from salt minion" >> "$setup_log" 2>&1
if [ $count -gt 30 ]; then
echo "salt master did not get a job response from salt minion after 30 attempts, exiting" >> "$setup_log" 2>&1
kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
exit 1
fi
systemctl kill salt-minion
systemctl start salt-minion
@@ -2522,12 +2497,8 @@ update_packages() {
if [ "$OS" = 'centos' ]; then
yum -y update >> "$setup_log"
else
if wait_for_apt 'whiptail_prog_new_message'; then
apt-get -y update >> "$setup_log"
apt-get -y upgrade >> "$setup_log"
else
kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
fi
retry 50 10 "apt-get -y update" >> "$setup_log" 2>&1 || exit 1
retry 50 10 "apt-get -y upgrade" >> "$setup_log" 2>&1 || exit 1
fi
}
@@ -2572,11 +2543,6 @@ wait_for_file() {
return 1
}
whiptail_prog_new_message() {
local message=$1
set_progress_str "$percentage" "$message"
}
# Enable Zeek Logs
zeek_logs_enabled() {
echo "Enabling Zeek Logs" >> "$setup_log" 2>&1

21
setup/so-setup
View File

@@ -130,7 +130,7 @@ if [[ -f automation/$automation && $(basename $automation) == $automation ]]; th
if [[ ! $is_iso ]]; then
echo "Installing sshpass for automated testing." >> $setup_log 2>&1
if [ "$OS" == ubuntu ]; then
if wait_for_apt; then apt-get -y install sshpass >> $setup_log 2>&1; else exit 1; fi
retry 50 10 "apt-get -y install sshpass" >> $setup_log 2>&1 || exit 1
else
yum -y install sshpass >> $setup_log 2>&1
fi
@@ -397,7 +397,9 @@ if [[ ! $is_import ]]; then
collect_patch_schedule
fi
collect_homenet_mngr
if [[ $is_helix || $is_manager || $is_import ]]; then
collect_homenet_mngr
fi
if [[ $is_helix || $is_manager || $is_node || $is_import ]]; then
set_base_heapsizes
@@ -456,10 +458,15 @@ if [[ $is_distmanager ]]; then
fi
if [[ $is_sensor && ! $is_eval ]]; then
collect_homenet_snsr
[[ $is_manager ]] || collect_homenet_snsr
whiptail_sensor_config
if [ $NSMSETUP == 'ADVANCED' ]; then
if [[ $is_manager ]]; then
[[ $ZEEKVERSION == "ZEEK" ]] && whiptail_zeek_pins
else
whiptail_zeek_pins
fi
whiptail_suricata_pins
collect_mtu
else
@@ -469,7 +476,13 @@ if [[ $is_sensor && ! $is_eval ]]; then
else
PROCS=$lb_procs
fi
if [[ $is_manager ]]; then
[[ $ZEEKVERSION == "ZEEK" ]] && collect_zeek
else
collect_zeek
fi
collect_suri
fi
fi
@@ -730,7 +743,7 @@ set_redirect >> $setup_log 2>&1
set_progress_str 66 "$(print_salt_state_apply 'suricata')"
salt-call state.apply -l info suricata >> $setup_log 2>&1
if [[ $ZEEKVERSION == 'ZEEK' ]]; then
if [[ $(lookup_pillar "mdengine") == 'ZEEK' ]]; then
set_progress_str 67 "$(print_salt_state_apply 'zeek')"
salt-call state.apply -l info zeek >> $setup_log 2>&1
fi