diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 86763dda8..c71e9150c 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -17,8 +17,8 @@ # Check for prerequisites if [ "$(id -u)" -ne 0 ]; then - echo "This script must be run using sudo!" - exit 1 + echo "This script must be run using sudo!" + exit 1 fi # Define a banner to separate sections @@ -26,46 +26,46 @@ banner="======================================================================== header() { echo - printf '%s\n' "$banner" "$*" "$banner" + printf '%s\n' "$banner" "$*" "$banner" } lookup_salt_value() { - key=$1 - group=$2 - kind=$3 + key=$1 + group=$2 + kind=$3 - if [ -z "$kind" ]; then - kind=pillar - fi + if [ -z "$kind" ]; then + kind=pillar + fi - if [ -n "$group" ]; then - group=${group}: - fi + if [ -n "$group" ]; then + group=${group}: + fi - salt-call --no-color ${kind}.get ${group}${key} --out=newline_values_only + salt-call --no-color ${kind}.get ${group}${key} --out=newline_values_only } lookup_pillar() { - key=$1 - pillar=$2 - if [ -z "$pillar" ]; then - pillar=global - fi - lookup_salt_value "$key" "$pillar" "pillar" + key=$1 + pillar=$2 + if [ -z "$pillar" ]; then + pillar=global + fi + lookup_salt_value "$key" "$pillar" "pillar" } lookup_pillar_secret() { - lookup_pillar "$1" "secrets" + lookup_pillar "$1" "secrets" } lookup_grain() { - lookup_salt_value "$1" "" "grains" + lookup_salt_value "$1" "" "grains" } lookup_role() { - id=$(lookup_grain id) - pieces=($(echo $id | tr '_' ' ')) - echo ${pieces[1]} + id=$(lookup_grain id) + pieces=($(echo $id | tr '_' ' ')) + echo ${pieces[1]} } check_container() { @@ -74,98 +74,143 @@ check_container() { } check_password() { - local password=$1 - echo "$password" | egrep -v "'|\"|\\$|\\\\" > /dev/null 2>&1 - return $? + local password=$1 + echo "$password" | egrep -v "'|\"|\\$|\\\\" > /dev/null 2>&1 + return $? } set_os() { - if [ -f /etc/redhat-release ]; then - OS=centos - else - OS=ubuntu - fi + if [ -f /etc/redhat-release ]; then + OS=centos + else + OS=ubuntu + fi } set_minionid() { - MINIONID=$(lookup_grain id) + MINIONID=$(lookup_grain id) } set_version() { - CURRENTVERSION=0.0.0 - if [ -f /etc/soversion ]; then - CURRENTVERSION=$(cat /etc/soversion) - fi - if [ -z "$VERSION" ]; then - if [ -z "$NEWVERSION" ]; then - if [ "$CURRENTVERSION" == "0.0.0" ]; then - echo "ERROR: Unable to detect Security Onion version; terminating script." - exit 1 - else - VERSION=$CURRENTVERSION - fi - else - VERSION="$NEWVERSION" - fi - fi + CURRENTVERSION=0.0.0 + if [ -f /etc/soversion ]; then + CURRENTVERSION=$(cat /etc/soversion) + fi + if [ -z "$VERSION" ]; then + if [ -z "$NEWVERSION" ]; then + if [ "$CURRENTVERSION" == "0.0.0" ]; then + echo "ERROR: Unable to detect Security Onion version; terminating script." + exit 1 + else + VERSION=$CURRENTVERSION + fi + else + VERSION="$NEWVERSION" + fi + fi } require_manager() { - # Check to see if this is a manager - MANAGERCHECK=$(cat /etc/salt/grains | grep role | awk '{print $2}') - if [ $MANAGERCHECK == 'so-eval' ] || [ $MANAGERCHECK == 'so-manager' ] || [ $MANAGERCHECK == 'so-managersearch' ] || [ $MANAGERCHECK == 'so-standalone' ] || [ $MANAGERCHECK == 'so-helix' ] || [ $MANAGERCHECK == 'so-import' ]; then - echo "This is a manager, We can proceed." - else - echo "Please run this command on the manager; the manager controls the grid." - exit 1 - fi + # Check to see if this is a manager + MANAGERCHECK=$(cat /etc/salt/grains | grep role | awk '{print $2}') + if [ $MANAGERCHECK == 'so-eval' ] || [ $MANAGERCHECK == 'so-manager' ] || [ $MANAGERCHECK == 'so-managersearch' ] || [ $MANAGERCHECK == 'so-standalone' ] || [ $MANAGERCHECK == 'so-helix' ] || [ $MANAGERCHECK == 'so-import' ]; then + echo "This is a manager, We can proceed." + else + echo "Please run this command on the manager; the manager controls the grid." + exit 1 + fi } is_single_node_grid() { - role=$(lookup_role) - if [ "$role" != "eval" ] && [ "$role" != "standalone" ] && [ "$role" != "import" ]; then - return 1 - fi - return 0 + role=$(lookup_role) + if [ "$role" != "eval" ] && [ "$role" != "standalone" ] && [ "$role" != "import" ]; then + return 1 + fi + return 0 } fail() { - msg=$1 - echo "ERROR: $msg" - echo "Exiting." - exit 1 + msg=$1 + echo "ERROR: $msg" + echo "Exiting." + exit 1 } get_random_value() { - length=${1:-20} - head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1 + length=${1:-20} + head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1 +} + +wait_for_apt() { + local progress_callback=$1 + + local retry_count=30 + local retry_timeout='10s' + local lock_msg="Could not acquire dpkg lock, waiting $retry_timeout for lock to release." + if [[ -z $progress_callback ]]; then + if [[ -z $progress_bar_text ]]; then + local old_text="Installing..." + else + local old_text="$progress_bar_text" + fi + fi + local count=0 + while [[ "$count" -lt "$retry_count" ]]; do + ((count++)) + [[ -z $progress_callback ]] && echo "Attempting to acquire dpkg lock to run apt command... (Attempt $count/$retry_count)" + if __check_apt_lock; then + if [[ -z $progress_callback ]]; then + echo " $lock_msg" | tee -a "$setup_log" + else + $progress_callback "Could not acquire dpkg lock, waiting $retry_timeout ($count/$retry_count)" + fi + else + [[ -z $progress_callback ]] || $progress_callback "$old_text" + return 0 + fi + sleep "$retry_timeout" + done + + if __check_apt_lock; then + [[ -z $progress_callback ]] && echo "Could not acquire lock after $retry_count attempts, aborting." + return 1 + else + return 0 + fi +} + +__check_apt_lock() { + lsof /var/lib/dpkg/lock &> /dev/null + local lock=$? + + return $lock } wait_for_web_response() { - url=$1 - expected=$2 - maxAttempts=${3:-300} - logfile=/root/wait_for_web_response.log - attempt=0 - while [[ $attempt -lt $maxAttempts ]]; do - attempt=$((attempt+1)) - echo "Waiting for value '$expected' at '$url' ($attempt/$maxAttempts)" - result=$(curl -ks -L $url) - exitcode=$? + url=$1 + expected=$2 + maxAttempts=${3:-300} + logfile=/root/wait_for_web_response.log + attempt=0 + while [[ $attempt -lt $maxAttempts ]]; do + attempt=$((attempt+1)) + echo "Waiting for value '$expected' at '$url' ($attempt/$maxAttempts)" + result=$(curl -ks -L $url) + exitcode=$? - echo "--------------------------------------------------" >> $logfile - echo "$(date) - Checking web URL: $url ($attempt/$maxAttempts)" >> $logfile - echo "$result" >> $logfile - echo "exit code=$exitcode" >> $logfile - echo "" >> $logfile + echo "--------------------------------------------------" >> $logfile + echo "$(date) - Checking web URL: $url ($attempt/$maxAttempts)" >> $logfile + echo "$result" >> $logfile + echo "exit code=$exitcode" >> $logfile + echo "" >> $logfile - if [[ $exitcode -eq 0 && "$result" =~ $expected ]]; then - echo "Received expected response; proceeding." - return 0 - fi - echo "Server is not ready" - sleep 1 - done - echo "Server still not ready after $maxAttempts attempts; giving up." - return 1 -} \ No newline at end of file + if [[ $exitcode -eq 0 && "$result" =~ $expected ]]; then + echo "Received expected response; proceeding." + return 0 + fi + echo "Server is not ready" + sleep 1 + done + echo "Server still not ready after $maxAttempts attempts; giving up." + return 1 +} diff --git a/setup/so-functions b/setup/so-functions index e723a89c8..06dcd10de 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -84,7 +84,7 @@ calculate_useable_cores() { lb_procs_round=$(printf "%.0f\n" $cores_for_zeek) if [ "$lb_procs_round" -lt 1 ]; then lb_procs=1; else lb_procs=$lb_procs_round; fi - export lb_procs + export lb_procs } airgap_rules() { @@ -799,13 +799,13 @@ detect_os() { echo "Installing required packages to run installer..." # Install network manager so we can do interface stuff if ! command -v nmcli > /dev/null 2>&1; then + if wait_for_apt; then apt-get install -y network-manager >> "$setup_log" 2<&1; else exit 1; fi { - apt-get install -y network-manager; - systemctl enable NetworkManager; - systemctl start NetworkManager; + systemctl enable NetworkManager + systemctl start NetworkManager } >> "$setup_log" 2<&1 fi - apt-get install -y bc curl >> "$setup_log" 2>&1 + if wait_for_apt; then apt-get install -y bc curl >> "$setup_log" 2>&1; else exit 1; fi else echo "We were unable to determine if you are using a supported OS." @@ -882,21 +882,28 @@ docker_install() { else case "$install_type" in 'MANAGER' | 'EVAL' | 'STANDALONE' | 'MANAGERSEARCH' | 'IMPORT') - apt-get update >> "$setup_log" 2>&1 + if wait_for_apt 'whiptail_prog_new_message'; then apt-get update >> "$setup_log" 2>&1; else kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1; fi ;; *) - { - apt-key add "$temp_install_dir"/gpg/docker.pub; - add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"; - apt-get update; - } >> "$setup_log" 2>&1 + if wait_for_apt 'whiptail_prog_new_message'; then + { + apt-key add "$temp_install_dir"/gpg/docker.pub; + add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"; + apt-get update; + } >> "$setup_log" 2>&1 + else + kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 + fi ;; esac - - if [ $OSVER != "xenial" ]; then - apt-get -y install docker-ce python3-docker >> "$setup_log" 2>&1 + if wait_for_apt 'whiptail_prog_new_message'; then + if [ $OSVER != "xenial" ]; then + apt-get -y install docker-ce python3-docker >> "$setup_log" 2>&1 + else + apt-get -y install docker-ce python-docker >> "$setup_log" 2>&1 + fi else - apt-get -y install docker-ce python-docker >> "$setup_log" 2>&1 + kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 fi fi docker_registry @@ -939,7 +946,7 @@ docker_seed_update() { if [ "$install_type" == 'HELIXSENSOR' ]; then percent_delta=6 fi - ((docker_seed_update_percent=docker_seed_update_percent+percent_delta)) + ((docker_seed_update_percent+=percent_delta)) set_progress_str "$docker_seed_update_percent" "Downloading $name" } @@ -1572,7 +1579,11 @@ remove_package() { fi else if dpkg -l | grep -q "$package_name"; then - apt purge -y "$package_name" + if wait_for_apt 'whiptail_prog_new_message'; then + apt purge -y "$package_name" + else + exit 1 + fi fi fi } @@ -1644,36 +1655,43 @@ saltify() { yum -y install epel-release yum -y install salt-minion-3002.2\ python3\ - python36-docker\ - python36-dateutil\ - python36-m2crypto\ - python36-mysql\ - yum-utils\ - device-mapper-persistent-data\ - lvm2\ - openssl\ - jq; - yum -y update --exclude=salt*; + python36-docker\ + python36-dateutil\ + python36-m2crypto\ + python36-mysql\ + yum-utils\ + device-mapper-persistent-data\ + lvm2\ + openssl\ + jq; + yum -y update --exclude=salt*; fi systemctl enable salt-minion; } >> "$setup_log" 2>&1 yum versionlock salt* else - DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" upgrade >> "$setup_log" 2>&1 + if wait_for_apt 'whiptail_prog_new_message'; then + DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" upgrade >> "$setup_log" 2>&1 + else + kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 + fi if [ $OSVER != "xenial" ]; then # Switch to Python 3 as default if this is not xenial update-alternatives --install /usr/bin/python python /usr/bin/python3.6 10 >> "$setup_log" 2>&1 fi - # Add the pre-requisites for installing docker-ce - apt-get -y install ca-certificates\ - curl\ - software-properties-common\ - apt-transport-https\ - openssl\ - netcat\ - jq >> "$setup_log" 2>&1 - + if wait_for_apt 'whiptail_prog_new_message'; then + # Add the pre-requisites for installing docker-ce + apt-get -y install ca-certificates\ + curl\ + software-properties-common\ + apt-transport-https\ + openssl\ + netcat\ + jq >> "$setup_log" 2>&1 + else + kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 + fi # Grab the version from the os-release file local ubuntu_version ubuntu_version=$(grep VERSION_ID /etc/os-release | awk -F '[ "]' '{print $2}') @@ -1681,7 +1699,11 @@ saltify() { case "$install_type" in 'FLEET') - if [ "$OSVER" != 'xenial' ]; then apt-get -y install python3-mysqldb >> "$setup_log" 2>&1; else apt-get -y install python-mysqldb >> "$setup_log" 2>&1; fi + if wait_for_apt 'whiptail_prog_new_message'; then + if [ "$OSVER" != 'xenial' ]; then apt-get -y install python3-mysqldb >> "$setup_log" 2>&1; else apt-get -y install python-mysqldb >> "$setup_log" 2>&1; fi + else + kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 + fi ;; 'MANAGER' | 'EVAL' | 'MANAGERSEARCH' | 'STANDALONE' | 'IMPORT' | 'HELIXSENSOR') @@ -1703,13 +1725,18 @@ saltify() { curl -s https://packages.wazuh.com/key/GPG-KEY-WAZUH | apt-key add - >> "$setup_log" 2>&1 # Add repo echo "deb https://packages.wazuh.com/3.x/apt/ stable main" > /etc/apt/sources.list.d/wazuh.list 2>> "$setup_log" - # Initialize the new repos - apt-get update >> "$setup_log" 2>&1 - set_progress_str 6 'Installing various dependencies' - apt-get -y install sqlite3 argon2 libssl-dev >> "$setup_log" 2>&1 - set_progress_str 7 'Installing salt-master' - apt-get -y install salt-master=3002.2+ds-1 >> "$setup_log" 2>&1 - apt-mark hold salt-master >> "$setup_log" 2>&1 + + if wait_for_apt 'whiptail_prog_new_message'; then + # Initialize the new repos + apt-get update >> "$setup_log" 2>&1 + set_progress_str 6 'Installing various dependencies' + apt-get -y install sqlite3 argon2 libssl-dev >> "$setup_log" 2>&1 + set_progress_str 7 'Installing salt-master' + apt-get -y install salt-master=3002.2+ds-1 >> "$setup_log" 2>&1 + apt-mark hold salt-master >> "$setup_log" 2>&1 + else + kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 + fi ;; *) # Copy down the gpg keys and install them from the manager @@ -1723,18 +1750,21 @@ saltify() { echo "deb https://packages.wazuh.com/3.x/apt/ stable main" > /etc/apt/sources.list.d/wazuh.list 2>> "$setup_log" ;; esac - apt-get update >> "$setup_log" 2>&1 - set_progress_str 8 'Installing salt-minion & python modules' - apt-get -y install salt-minion=3002.2+ds-1\ - salt-common=3002.2+ds-1 >> "$setup_log" 2>&1 - apt-mark hold salt-minion salt-common >> "$setup_log" 2>&1 - if [ "$OSVER" != 'xenial' ]; then - apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb >> "$setup_log" 2>&1 - else - apt-get -y install python-pip python-dateutil python-m2crypto python-mysqldb >> "$setup_log" 2>&1 + if wait_for_apt 'whiptail_prog_new_message'; then + apt-get update >> "$setup_log" 2>&1 + set_progress_str 8 'Installing salt-minion & python modules' + apt-get -y install salt-minion=3002.2+ds-1\ + salt-common=3002.2+ds-1 >> "$setup_log" 2>&1 + apt-mark hold salt-minion salt-common >> "$setup_log" 2>&1 + if [ "$OSVER" != 'xenial' ]; then + apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb >> "$setup_log" 2>&1 + else + apt-get -y install python-pip python-dateutil python-m2crypto python-mysqldb >> "$setup_log" 2>&1 + fi + else + kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 fi fi - } salt_checkin() { @@ -1897,7 +1927,8 @@ setup_salt_master_dirs() { set_progress_str() { local percentage_input=$1 - local progress_bar_text=$2 + progress_bar_text=$2 + export progress_bar_text if (( "$percentage_input" >= "$percentage" )); then percentage="$percentage_input" @@ -2154,8 +2185,12 @@ update_packages() { if [ "$OS" = 'centos' ]; then yum -y update >> "$setup_log" else - apt-get -y update >> "$setup_log" - apt-get -y upgrade >> "$setup_log" + if wait_for_apt 'whiptail_prog_new_message'; then + apt-get -y update >> "$setup_log" + apt-get -y upgrade >> "$setup_log" + else + kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 + fi fi } @@ -2226,6 +2261,12 @@ es_heapsize() { fi } + +whiptail_prog_new_message() { + local message=$1 + set_progress_str "$percentage" "$message" +} + # Enable Zeek Logs zeek_logs_enabled() { echo "Enabling Zeek Logs" >> "$setup_log" 2>&1