From 8a8885e14f439579d360e638d1c53130b0c42457 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 30 Nov 2020 16:53:02 -0500 Subject: [PATCH 01/71] [feat] Verify that main ip = mngmt ip * Add a check to check whether the src ip in the routing table is also the ip assigned to the management nic --- setup/so-functions | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/setup/so-functions b/setup/so-functions index 4ba639fa5..e685d6940 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1710,6 +1710,17 @@ set_network_dev_status_list() { set_main_ip() { MAINIP=$(ip route get 1 | awk '{print $7;exit}') + MNIC_IP=$(ip a s "$MNIC" | grep -oE 'inet [0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' | cut -d' ' -f) + + if [[ $MAINIP != $MNIC_IP ]]; then + read -r -d '' message <<- EOM + The IP being routed by Linux is not the IP address assigned to the management interface ($MNIC). + + This is not a supported configuration, please remediate and rerun setup. + EOM + whiptail --title "Security Onion Setup" --msgbox "$message" 10 75 + exit 1 + fi } # Add /usr/sbin to everyone's path From 9c919f3c925b0fb878e8b245743d18cd7fb70228 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 2 Dec 2020 17:07:49 -0500 Subject: [PATCH 02/71] [reafactor] systemctl stop -> kill --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 6aa30f89c..4103f0988 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1400,7 +1400,7 @@ reinstall_init() { # Kill any salt processes (safely) for service in "${salt_services[@]}"; do # Stop the service in the background so we can exit after a certain amount of time - systemctl stop "$service" & + systemctl kill "$service" & local pid=$! local count=0 From 786665d8cf5b624384b4095037b24cbd22ee77f1 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 3 Dec 2020 10:18:44 -0500 Subject: [PATCH 03/71] [fix] Correct logic for service check + bash trap --- setup/so-functions | 2 +- setup/so-setup | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 4103f0988..76e579765 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1404,7 +1404,7 @@ reinstall_init() { local pid=$! local count=0 - while check_service_status "$service"; do + while ! (check_service_status "$service"); do if [[ $count -gt $service_retry_count ]]; then echo "Could not stop $service after 1 minute, exiting setup." diff --git a/setup/so-setup b/setup/so-setup index 79ba916a9..924bdf307 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -485,14 +485,12 @@ fi # Exit parent script if -trap 'catch $? $LINENO' SIGUSR1 +trap 'catch $LINENO' SIGUSR1 catch() { - if [ "$1" != 0 ]; then - info "Fatal error occurred at $2 in so-setup, failing setup." - whiptail_setup_failed - exit - fi + info "Fatal error occurred at $2 in so-setup, failing setup." + whiptail_setup_failed + exit } # Begin install From f410c451cd35162ed3948ad66e11ab6d0f8fcd53 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 3 Dec 2020 10:31:45 -0500 Subject: [PATCH 04/71] [fix] kill -> stop, add indent to service check, revert incorrect logic --- setup/so-functions | 8 ++++---- setup/so-setup | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 76e579765..767ca6288 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -259,10 +259,10 @@ check_service_status() { systemctl status $service_name > /dev/null 2>&1 local status=$? if [ $status -gt 0 ]; then - echo "$service_name is not running" >> "$setup_log" 2>&1 + echo " $service_name is not running" >> "$setup_log" 2>&1 return 1; else - echo "$service_name is running" >> "$setup_log" 2>&1 + echo " $service_name is running" >> "$setup_log" 2>&1 return 0; fi @@ -1400,11 +1400,11 @@ reinstall_init() { # Kill any salt processes (safely) for service in "${salt_services[@]}"; do # Stop the service in the background so we can exit after a certain amount of time - systemctl kill "$service" & + systemctl stop "$service" & local pid=$! local count=0 - while ! (check_service_status "$service"); do + while check_service_status "$service"; do if [[ $count -gt $service_retry_count ]]; then echo "Could not stop $service after 1 minute, exiting setup." diff --git a/setup/so-setup b/setup/so-setup index 924bdf307..3bec2bb87 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -499,11 +499,11 @@ catch() { export percentage=0 set_path - if [[ $is_manager && $is_airgap ]]; then - info "Creating airgap repo" - create_repo >> $setup_log 2>&1 + if [[ $is_manager && $is_airgap ]]; then + info "Creating airgap repo" + create_repo >> $setup_log 2>&1 airgap_rules >> $setup_log 2>&1 - fi + fi if [[ $is_minion ]]; then set_progress_str 1 'Configuring firewall' From ddcf5dec5bdef24ad43184a9517eddf3f46b0d58 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 3 Dec 2020 13:59:25 -0500 Subject: [PATCH 05/71] [refactor] Run all changes inside whiptail progress, use grep -q --- setup/so-functions | 4 +- setup/so-setup | 117 +++++++++++++++++++++++---------------------- 2 files changed, 62 insertions(+), 59 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 767ca6288..8c23441ed 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -766,12 +766,12 @@ detect_os() { disable_auto_start() { - if crontab -l -u $INSTALLUSERNAME 2>&1 | grep so-setup > /dev/null 2>&1; then + if crontab -l -u $INSTALLUSERNAME 2>&1 | grep -q so-setup; then # Remove the automated setup script from crontab, if it exists logCmd "crontab -u $INSTALLUSERNAME -r" fi - if grep so-setup /home/$INSTALLUSERNAME/.bash_profile > /dev/null 2>&1; then + if grep -q so-setup /home/$INSTALLUSERNAME/.bash_profile; then # Truncate last line of the bash profile info "Removing auto-run of setup from bash profile" sed -i '$ d' /home/$INSTALLUSERNAME/.bash_profile >> "$setup_log" 2>&1 diff --git a/setup/so-setup b/setup/so-setup index 3bec2bb87..73363959c 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -428,63 +428,7 @@ whiptail_make_changes # From here on changes will be made. echo "1" > /root/accept_changes -if [[ $is_reinstall ]]; then - reinstall_init -fi - -if [[ -n "$TURBO" ]]; then - use_turbo_proxy -fi - -if [[ "$setup_type" == 'iso' ]]; then - # Init networking so rest of install works - set_hostname - set_management_interface -fi - -disable_ipv6 -disable_auto_start - -if [[ "$setup_type" != 'iso' ]]; then - set_hostname -fi - -if [[ $is_minion ]]; then - add_mngr_ip_to_hosts -fi - -{ - mark_version; - clear_manager; -} >> $setup_log 2>&1 - - -if [[ $is_manager || $is_import ]]; then - { - generate_passwords; - secrets_pillar; - add_socore_user_manager; - } >> $setup_log 2>&1 -fi - -if [[ $is_manager && ! $is_eval ]]; then - add_soremote_user_manager >> $setup_log 2>&1 -fi - -{ - set_main_ip; - set_redirect; -} >> $setup_log 2>&1 - -host_pillar >> $setup_log 2>&1 - -if [[ $is_minion || $is_import ]]; then - set_updates >> $setup_log 2>&1 - [ "$automated" == no ] && copy_ssh_key >> $setup_log 2>&1 -fi - - -# Exit parent script if +# Set up handler for setup to exit early (use `kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1` in child scripts) trap 'catch $LINENO' SIGUSR1 catch() { @@ -497,8 +441,67 @@ catch() { { # Set initial percentage to 0 export percentage=0 + + # Show initial progress message + set_progress_str 0 'Running initial configuration steps' + set_path + if [[ $is_reinstall ]]; then + reinstall_init + fi + + if [[ -n "$TURBO" ]]; then + use_turbo_proxy + fi + + if [[ "$setup_type" == 'iso' ]]; then + # Init networking so rest of install works + set_hostname >> $setup_log 2>&1 + set_management_interface + fi + + disable_ipv6 + disable_auto_start + + if [[ "$setup_type" != 'iso' ]]; then + set_hostname >> $setup_log 2>&1 + fi + + if [[ $is_minion ]]; then + add_mngr_ip_to_hosts + fi + + { + mark_version; + clear_manager; + } >> $setup_log 2>&1 + + + if [[ $is_manager || $is_import ]]; then + { + generate_passwords; + secrets_pillar; + add_socore_user_manager; + } >> $setup_log 2>&1 + fi + + if [[ $is_manager && ! $is_eval ]]; then + add_soremote_user_manager >> $setup_log 2>&1 + fi + + { + set_main_ip; + set_redirect; + } >> $setup_log 2>&1 + + host_pillar >> $setup_log 2>&1 + + if [[ $is_minion || $is_import ]]; then + set_updates >> $setup_log 2>&1 + [ "$automated" == no ] && copy_ssh_key >> $setup_log 2>&1 + fi + if [[ $is_manager && $is_airgap ]]; then info "Creating airgap repo" create_repo >> $setup_log 2>&1 From 2e516629f9fdf1e853e416cb8f809c2644363555 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 3 Dec 2020 14:42:13 -0500 Subject: [PATCH 06/71] [fix] Kill + start salt-minion if it isn't responding --- setup/so-functions | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 8c23441ed..b42e03bb7 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1382,10 +1382,12 @@ reserve_group_ids() { reinstall_init() { info "Putting system in state to run setup again" - local salt_services=( - "salt-master" - "salt-minion" - ) + if [[ $install_type =~ ^(MANAGER|EVAL|HELIXSENSOR|MANAGERSEARCH|STANDALONE|FLEET|IMPORT)$ ]]; then + local salt_services=( "salt-master" "salt-minion" ) + else + local salt_services=( "salt-minion" ) + fi + local service_retry_count=20 { @@ -1412,6 +1414,7 @@ reinstall_init() { kill -9 $pid kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 fi + sleep 5 ((count++)) done @@ -1671,6 +1674,8 @@ salt_checkin() { count=0 while ! (check_salt_minion_status); do echo "salt master did not get a job response from salt minion" >> "$setup_log" 2>&1 + systemctl kill salt-minion + systemctl start salt-minion if [ $count -gt 30 ]; then echo "salt master did not get a job response from salt minion after 30 attempts, exiting" >> "$setup_log" 2>&1 kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 From 916db4acec020ce92aeb918b3821a1fe40b26fd5 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 3 Dec 2020 14:55:23 -0500 Subject: [PATCH 07/71] [fix] kill/start after if statement --- setup/so-functions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index b42e03bb7..30399170f 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1674,12 +1674,12 @@ salt_checkin() { count=0 while ! (check_salt_minion_status); do echo "salt master did not get a job response from salt minion" >> "$setup_log" 2>&1 - systemctl kill salt-minion - systemctl start salt-minion if [ $count -gt 30 ]; then echo "salt master did not get a job response from salt minion after 30 attempts, exiting" >> "$setup_log" 2>&1 kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 fi + systemctl kill salt-minion + systemctl start salt-minion sleep 1; ((count++)) done From 39dce13cf6d061748975b5ce860479cdc798a322 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 3 Dec 2020 15:10:41 -0500 Subject: [PATCH 08/71] [fix] Move set_redirect out of sub-shell --- setup/so-setup | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 73363959c..8dcce0e9b 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -437,6 +437,12 @@ catch() { exit } +# This block sets REDIRECTIT which is used by a function outside the below subshell +{ + set_main_ip; + set_redirect; +} >> $setup_log 2>&1 + # Begin install { # Set initial percentage to 0 @@ -490,11 +496,6 @@ catch() { add_soremote_user_manager >> $setup_log 2>&1 fi - { - set_main_ip; - set_redirect; - } >> $setup_log 2>&1 - host_pillar >> $setup_log 2>&1 if [[ $is_minion || $is_import ]]; then From 7458313d3d5c324cf2ed8c110f194a7e38362e9c Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 3 Dec 2020 15:20:33 -0500 Subject: [PATCH 09/71] [fix] Also kill+start while trying to restart service initially --- setup/so-functions | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 30399170f..e17fa23ce 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1650,7 +1650,8 @@ salt_checkin() { count=0 while ! (check_service_status "$service"); do - echo "$service still not running" >> "$setup_log" 2>&1 + systemctl kill "$service" + systemctl start "$service" if [ $count -gt 120 ]; then echo "$service could not be restarted in 120 seconds, exiting" >> "$setup_log" 2>&1 kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 From ff1cfb578f87ced958a3fb719fea534558cf481d Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 3 Dec 2020 15:26:59 -0500 Subject: [PATCH 10/71] Only kill+start on final loop and increase time between status checks --- setup/so-functions | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index e17fa23ce..d6c309431 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1650,13 +1650,16 @@ salt_checkin() { count=0 while ! (check_service_status "$service"); do - systemctl kill "$service" - systemctl start "$service" - if [ $count -gt 120 ]; then + if [ $count -eq 12 ]; then + systemctl kill "$service" + systemctl start "$service" + fi + + if [ $count -gt 12 ]; then echo "$service could not be restarted in 120 seconds, exiting" >> "$setup_log" 2>&1 kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 fi - sleep 1; + sleep 10; ((count++)) done done From 7b43c2955e2c550caf7154f8325d297a956673b4 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 3 Dec 2020 15:35:50 -0500 Subject: [PATCH 11/71] [fix] kill old restart pid and assign new pid for start --- setup/so-functions | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index d6c309431..67cbb7c24 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1650,13 +1650,19 @@ salt_checkin() { count=0 while ! (check_service_status "$service"); do + # On final loop, kill the pid trying to restart service and try to manually kill then start it if [ $count -eq 12 ]; then - systemctl kill "$service" - systemctl start "$service" + { + kill -9 "$pid" + systemctl kill "$service" + systemctl start "$service" & + local pid=$! + } >> "$setup_log" 2>&1 fi if [ $count -gt 12 ]; then echo "$service could not be restarted in 120 seconds, exiting" >> "$setup_log" 2>&1 + kill -9 "$pid" kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 fi sleep 10; From 6fc3232637663c95b442f93215ad8a41bfc7c987 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 7 Dec 2020 14:16:06 -0500 Subject: [PATCH 12/71] [fix] Set INSTALLUSERNAME to the user running the script Resolves #2243 --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index bd16f9cd2..38077269f 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1321,7 +1321,7 @@ elasticsearch_pillar() { parse_install_username() { # parse out the install username so things copy correctly - INSTALLUSERNAME=$(pwd | sed -E 's/\// /g' | awk '{ print $2 }') + INSTALLUSERNAME=${SUDO_USER:-${USER}} } patch_pillar() { From 08ab36927d91063f4981162c15a8d610e73e4f3d Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 7 Dec 2020 14:16:54 -0500 Subject: [PATCH 13/71] [refactor] Kill parent script on exit --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 38077269f..dffc52b4f 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1752,7 +1752,7 @@ set_main_ip() { This is not a supported configuration, please remediate and rerun setup. EOM whiptail --title "Security Onion Setup" --msgbox "$message" 10 75 - exit 1 + kill -SIGKILL "$(ps --pid $$ -oppid=)"; exit 1 fi } From d88364c9fde41a6a3199bcc2e23ed3447c47aeeb Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 7 Dec 2020 14:18:01 -0500 Subject: [PATCH 14/71] [feat] Create error log for easy copy/paste Resolves #2165 --- setup/so-setup | 18 +++++++++--------- setup/so-variables | 3 +++ setup/so-whiptail | 15 ++++++++++++++- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 8ee236bf1..4260f813e 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -59,6 +59,7 @@ if [[ -f /root/accept_changes ]]; then # Move last setup log to backup mv "$setup_log" "$setup_log.bak" + mv "$error_log" "$error_log.bak" fi # Begin Installation pre-processing @@ -72,14 +73,6 @@ analyze_system automated=no function progress() { local title='Security Onion Install' - if grep -q -E "ERROR|Result: False" $setup_log || [[ -s /var/spool/mail/root ]]; then - if [[ -s /var/spool/mail/root ]]; then - echo '[ ERROR ] /var/spool/mail/root grew unexpectedly' >> $setup_log 2>&1 - fi - - export SO_ERROR=1 - title="Error found, please check $setup_log" - fi if [ $automated == no ]; then whiptail --title "$title" --gauge 'Please wait while installing...' 6 60 0 # append to text @@ -433,6 +426,7 @@ trap 'catch $LINENO' SIGUSR1 catch() { info "Fatal error occurred at $1 in so-setup, failing setup." + grep --color=never "ERROR" "$setup_log" > "$error_log" whiptail_setup_failed exit } @@ -780,12 +774,18 @@ success=$(tail -10 $setup_log | grep Failed | awk '{ print $2}') if [[ $success != 0 ]]; then SO_ERROR=1; fi # Check entire setup log for errors or unexpected salt states and ensure cron jobs are not reporting errors to root's mailbox -if grep -q -E "ERROR|Result: False" $setup_log || [[ -s /var/spool/mail/root && "$setup_type" == "iso" ]]; then SO_ERROR=1; fi +if grep -q -E "ERROR|Result: False" $setup_log || [[ -s /var/spool/mail/root && "$setup_type" == "iso" ]]; then + SO_ERROR=1 + + grep --color=never "ERROR" "$setup_log" > "$error_log" +fi if [[ -n $SO_ERROR ]]; then echo "Errors detected during setup; skipping post-setup steps to allow for analysis of failures." >> $setup_log 2>&1 + SKIP_REBOOT=1 whiptail_setup_failed + else echo "Successfully completed setup! Continuing with post-installation steps" >> $setup_log 2>&1 { diff --git a/setup/so-variables b/setup/so-variables index 83b9b4325..2223fe106 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -21,6 +21,9 @@ export node_es_port setup_log="/root/sosetup.log" export setup_log +error_log="/root/errors.log" +export error_log + filesystem_root=$(df / | awk '$3 ~ /[0-9]+/ { print $2 * 1000 }') export filesystem_root diff --git a/setup/so-whiptail b/setup/so-whiptail index 11d968910..444260907 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1259,7 +1259,20 @@ whiptail_setup_failed() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --msgbox "Install had a problem. Please see $setup_log for details. Press Ok to exit." 8 75 + local check_err_msg + local height + + [ -f "$error_log" ] && check_err_msg="A summary of errors can be found in $error_log.\n" + + if [[ -n $check_err_msg ]]; then height=11; else height=10; fi + + read -r -d '' message <<- EOM + Install had a problem. Please see $setup_log for details.\n + $check_err_msg + Press Ok to exit. + EOM + + whiptail --title "Security Onion Setup" --msgbox "$message" $height 75 } whiptail_shard_count() { From 997e2735e3a119b83945af9fd0c4b687360863cb Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 8 Dec 2020 13:59:42 -0500 Subject: [PATCH 15/71] [refactor] Press -> select --- setup/so-whiptail | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 444260907..5bc84eecf 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -345,7 +345,7 @@ whiptail_requirements_error() { if [[ $(echo "$requirement_needed" | tr '[:upper:]' '[:lower:]') == 'nics' ]]; then whiptail --title "Security Onion Setup" \ - --msgbox "This machine currently has $current_val $requirement_needed, but needs $needed_val to meet minimum requirements. Press OK to exit setup and reconfigure the machine." 10 75 + --msgbox "This machine currently has $current_val $requirement_needed, but needs $needed_val to meet minimum requirements. Select OK to exit setup and reconfigure the machine." 10 75 # Same as whiptail_cancel, but changed the wording to exit instead of cancel. whiptail --title "Security Onion Setup" --msgbox "Exiting Setup. No changes have been made." 8 75 @@ -359,7 +359,7 @@ whiptail_requirements_error() { exit else whiptail --title "Security Onion Setup" \ - --yesno "This machine currently has $current_val $requirement_needed, but needs $needed_val to meet minimum requirements. Press YES to continue anyway, or press NO to cancel." 10 75 + --yesno "This machine currently has $current_val $requirement_needed, but needs $needed_val to meet minimum requirements. Select YES to continue anyway, or select NO to cancel." 10 75 local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -380,7 +380,7 @@ whiptail_storage_requirements() { Visit https://docs.securityonion.net/en/2.1/hardware.html for more information. - Press YES to continue anyway, or press NO to cancel. + Select YES to continue anyway, or select NO to cancel. EOM whiptail \ @@ -441,7 +441,7 @@ whiptail_dhcp_warn() { [ -n "$TESTING" ] && return if [[ $setup_type == "iso" ]]; then - local interaction_text="Press YES to keep DHCP or NO to go back." + local interaction_text="Select YES to keep DHCP or NO to go back." local window_type="yesno" else local interaction_text="Press ENTER to continue." @@ -795,7 +795,7 @@ whiptail_make_changes() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --yesno "We are going to set this machine up as a $install_type. Please press YES to make changes or NO to cancel." 8 75 + whiptail --title "Security Onion Setup" --yesno "We are going to set this machine up as a $install_type. Please select YES to make changes or NO to cancel." 8 75 local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -909,7 +909,7 @@ whiptail_network_notice() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --yesno "Since this is a network install we assume the management interface, DNS, Hostname, etc are already set up. Press YES to continue." 8 75 + whiptail --title "Security Onion Setup" --yesno "Since this is a network install we assume the management interface, DNS, Hostname, etc are already set up. Select YES to continue." 8 75 local exitstatus=$? whiptail_check_exitstatus $exitstatus From 65d994a2f8ec2a0f784ef198855978d58c14dc65 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 8 Dec 2020 14:02:45 -0500 Subject: [PATCH 16/71] [feat] Generate gzipped tarball of repo during setup and soup --- salt/common/tools/sbin/soup | 9 +++++++++ setup/so-functions | 4 ++++ setup/so-setup | 7 ++++++- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 27439a137..da534281e 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -155,6 +155,13 @@ copy_new_files() { cd /tmp } +generate_and_clean_tarballs() { + local new_version + new_version=$(cat $UPDATE_DIR/VERSION) + tar -cxf "/opt/so/repo/$new_version.tar.gz" "$UPDATE_DIR" + find "/opt/so/repo" -type f -not -name "$new_version.tar.gz" -exec rm -rf {} \; +} + highstate() { # Run a highstate. salt-call state.highstate -l info queue=True @@ -417,6 +424,8 @@ else echo "Cloning Security Onion github repo into $UPDATE_DIR." clone_to_tmp fi +echo "Generating new repo archive" +generate_and_clean_tarballs if [ -f /usr/sbin/so-image-common ]; then . /usr/sbin/so-image-common else diff --git a/setup/so-functions b/setup/so-functions index dffc52b4f..a95fe55b1 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -972,6 +972,10 @@ generate_passwords(){ KRATOSKEY=$(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 20 | head -n 1) } +generate_repo_tarball() { + tar -czf /opt/so/repo/"$SOVERSION".tar.gz ../. +} + get_redirect() { whiptail_set_redirect if [ "$REDIRECTINFO" = "OTHER" ]; then diff --git a/setup/so-setup b/setup/so-setup index 4260f813e..b60b99eca 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -791,10 +791,15 @@ else { export percentage=95 # set to last percentage used in previous subshell if [[ -n $ALLOW_ROLE && -n $ALLOW_CIDR ]]; then - set_progress_str 98 "Running so-allow -${ALLOW_ROLE} for ${ALLOW_CIDR}" + set_progress_str 97 "Running so-allow -${ALLOW_ROLE} for ${ALLOW_CIDR}" IP=$ALLOW_CIDR so-allow -$ALLOW_ROLE >> $setup_log 2>&1 fi + if [[ $is_manager ]]; then + set_progress_str 98 "Generating archive for setup directory" + generate_repo_tarball + fi + if [[ $THEHIVE == 1 ]]; then set_progress_str 99 'Waiting for TheHive to start up' check_hive_init >> $setup_log 2>&1 From 4210d25fae1235a70792d95d73fac7da7fc6c4e9 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 8 Dec 2020 14:03:21 -0500 Subject: [PATCH 17/71] [feat] Init network + soremote key early --- setup/so-setup | 48 ++++++++++++++++++++++++----------------------- setup/so-whiptail | 16 ++++++++++++++++ 2 files changed, 41 insertions(+), 23 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index b60b99eca..f9ae6fe50 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -205,13 +205,11 @@ fi # Check if this is an airgap install -if [[ $is_manager ]]; then - if [[ $is_iso ]]; then - whiptail_airgap - if [[ "$INTERWEBS" == 'AIRGAP' ]]; then - is_airgap=true - fi - fi +if [[ $is_manager && $is_iso ]]; then + whiptail_airgap + if [[ "$INTERWEBS" == 'AIRGAP' ]]; then + is_airgap=true + fi fi if [[ $is_manager && $is_sensor ]]; then @@ -226,10 +224,6 @@ elif [[ $is_import ]]; then check_requirements "import" fi -if [[ ! $is_import ]]; then - whiptail_patch_schedule -fi - case "$setup_type" in 'iso') whiptail_set_hostname @@ -243,8 +237,6 @@ case "$setup_type" in whiptail_management_interface_dns whiptail_management_interface_dns_search fi - - #collect_adminuser_inputs ;; 'network') whiptail_network_notice @@ -254,6 +246,22 @@ case "$setup_type" in ;; esac +if [[ $is_minion ]]; then + whiptail_management_server +fi + +whiptail_management_interface_setup + +if [[ "$setup_type" == 'iso' ]]; then + # Init networking so rest of install works + set_hostname + set_management_interface +fi + +if [[ $is_minion || $is_import ]]; then + [ "$automated" == no ] && copy_ssh_key >> $setup_log 2>&1 +fi + short_name=$(echo "$HOSTNAME" | awk -F. '{print $1}') MINION_ID=$(echo "${short_name}_${install_type}" | tr '[:upper:]' '[:lower:]') @@ -320,6 +328,10 @@ if [[ $is_helix || $is_sensor || $is_import ]]; then calculate_useable_cores fi +if [[ ! $is_import ]]; then + whiptail_patch_schedule +fi + whiptail_homenet_manager whiptail_dockernet_check @@ -365,10 +377,6 @@ if [[ $is_distmanager || ( $is_sensor || $is_node || $is_fleet_standalone ) && ! fi fi -if [[ $is_minion ]]; then - whiptail_management_server -fi - if [[ $is_distmanager ]]; then collect_soremote_inputs fi @@ -436,11 +444,6 @@ if [[ -n "$TURBO" ]]; then use_turbo_proxy fi -if [[ "$setup_type" == 'iso' ]]; then - set_hostname >> $setup_log 2>&1 - set_management_interface -fi - disable_ipv6 if [[ "$setup_type" != 'iso' ]]; then @@ -495,7 +498,6 @@ fi if [[ $is_minion || $is_import ]]; then set_updates >> $setup_log 2>&1 - [ "$automated" == no ] && copy_ssh_key >> $setup_log 2>&1 fi if [[ $is_manager && $is_airgap ]]; then diff --git a/setup/so-whiptail b/setup/so-whiptail index 5bc84eecf..edbc19c0b 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -765,6 +765,22 @@ whiptail_management_nic() { } +whiptail_management_interface_setup() { + [ -n "$TESTING" ] && return + + local minion_msg + + if [[ $is_minion || $is_import ]]; then + minion_msg=" and copy the ssh key for soremote to the manager" + else + minion_msg="" + fi + + whiptail --title "Security Onion Setup" --yesno "Setup will now initialize networking$minion_msg. Select YES to continue or NO to cancel." 8 75 + local exitstatus=$? + whiptail_check_exitstatus $exitstatus +} + whiptail_nids() { [ -n "$TESTING" ] && return From 4899ea23f882eaa900af5286e1790c8013dd37a1 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 8 Dec 2020 14:03:59 -0500 Subject: [PATCH 18/71] [fix] Put conditions in install_cleanup function --- setup/so-functions | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index a95fe55b1..fd7a02858 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1011,18 +1011,22 @@ host_pillar() { } install_cleanup() { - echo "Installer removing the following files:" - ls -lR "$temp_install_dir" + if [ -f "$temp_install_dir" ]; then + echo "Installer removing the following files:" + ls -lR "$temp_install_dir" - # Clean up after ourselves - rm -rf "$temp_install_dir" + # Clean up after ourselves + rm -rf "$temp_install_dir" + fi # All cleanup prior to this statement must be compatible with automated testing. Cleanup # that will disrupt automated tests should be placed beneath this statement. [ -n "$TESTING" ] && return # If Mysql is running stop it - /usr/sbin/so-mysql-stop + if docker ps --format "{{.Names}}" 2>&1 | grep -q "so-mysql"; then + /usr/sbin/so-mysql-stop + fi if [[ $setup_type == 'iso' ]]; then info "Removing so-setup permission entry from sudoers file" From b41ba1ea3ce2b951213d6cbd0d178ce6ee4ba66a Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 8 Dec 2020 15:29:04 -0500 Subject: [PATCH 19/71] [feat] Compare setup version to manager, dl tarball + exec on mismatch --- setup/so-functions | 20 +++++++ setup/so-setup | 132 ++++++++++++++++++++++++++------------------- 2 files changed, 97 insertions(+), 55 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index fd7a02858..5375d725c 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -518,6 +518,14 @@ check_requirements() { fi } +compare_versions() { + manager_ver=$(ssh -i /root/.ssh/so.key soremote@"$MSRV" cat /etc/soversion) + export manager_ver + + [[ "$manager_ver" == "$SOVERSION" ]] + return +} + configure_network_sensor() { echo "Setting up sensor interface" >> "$setup_log" 2>&1 local nic_error=0 @@ -913,6 +921,18 @@ docker_seed_registry() { } +download_repo_tarball() { + scp -i /root/.ssh/so.key soremote@"$MSRV":/opt/so/repo/"$manager_ver".tar.gz /root/new_setup + + # Fail if the file doesn't download + if ! [ -f /root/new_setup/"$manager_ver".tar.gz ]; then + kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 + fi + + tar -xzf /root/new_setup/"$manager_ver".tar.gz -C /root/new_setup/securityonion + rm -rf /root/new_setup/"$manager_ver".tar.gz +} + fireeye_pillar() { local fireeye_pillar_path=$local_salt_dir/pillar/fireeye diff --git a/setup/so-setup b/setup/so-setup index f9ae6fe50..8ea110e96 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -54,6 +54,10 @@ while [[ $# -gt 0 ]]; do esac done +if ! [ -f /root/install_opt ] && [ -d /root/manager_setup/securityonion ] && [[ $(pwd) != /root/manager_setup/securityonion/setup ]]; then + exec bash /root/manager_setup/securityonion/setup/so-setup "$@" +fi + if [[ -f /root/accept_changes ]]; then is_reinstall=true @@ -62,13 +66,16 @@ if [[ -f /root/accept_changes ]]; then mv "$error_log" "$error_log.bak" fi -# Begin Installation pre-processing -parse_install_username -title "Initializing Setup" -info "Installing as the $INSTALLUSERNAME user" +if ! [ -f /root/install_opt ]; then + # Begin Installation pre-processing + parse_install_username -analyze_system + title "Initializing Setup" + info "Installing as the $INSTALLUSERNAME user" + + analyze_system +fi automated=no function progress() { @@ -145,14 +152,18 @@ if [ "$automated" == no ]; then fi fi -if (whiptail_you_sure); then - true -else - echo "User cancelled setup." | tee $setup_log - whiptail_cancel -fi +if ! [ -f /root/install_opt ]; then + if (whiptail_you_sure); then + true + else + echo "User cancelled setup." | tee "$setup_log" + whiptail_cancel + fi -whiptail_install_type + whiptail_install_type +else + install_type=$(cat /root/install_opt) +fi if [ "$install_type" = 'EVAL' ]; then is_node=true @@ -204,7 +215,6 @@ if [[ "$setup_type" == 'iso' ]]; then fi # Check if this is an airgap install - if [[ $is_manager && $is_iso ]]; then whiptail_airgap if [[ "$INTERWEBS" == 'AIRGAP' ]]; then @@ -212,54 +222,66 @@ if [[ $is_manager && $is_iso ]]; then fi fi -if [[ $is_manager && $is_sensor ]]; then - check_requirements "standalone" -elif [[ $is_fleet_standalone ]]; then - check_requirements "dist" "fleet" -elif [[ $is_sensor && ! $is_eval ]]; then - check_requirements "dist" "sensor" -elif [[ $is_distmanager || $is_minion ]] && [[ ! $is_import ]]; then - check_requirements "dist" -elif [[ $is_import ]]; then - check_requirements "import" -fi +if ! [ -f /root/install_opt ]; then + if [[ $is_manager && $is_sensor ]]; then + check_requirements "standalone" + elif [[ $is_fleet_standalone ]]; then + check_requirements "dist" "fleet" + elif [[ $is_sensor && ! $is_eval ]]; then + check_requirements "dist" "sensor" + elif [[ $is_distmanager || $is_minion ]] && [[ ! $is_import ]]; then + check_requirements "dist" + elif [[ $is_import ]]; then + check_requirements "import" + fi -case "$setup_type" in - 'iso') - whiptail_set_hostname - whiptail_management_nic - whiptail_dhcp_or_static + case "$setup_type" in + 'iso') + whiptail_set_hostname + whiptail_management_nic + whiptail_dhcp_or_static - if [ "$address_type" != 'DHCP' ]; then - whiptail_management_interface_ip - whiptail_management_interface_mask - whiptail_management_interface_gateway - whiptail_management_interface_dns - whiptail_management_interface_dns_search - fi - ;; - 'network') - whiptail_network_notice - whiptail_dhcp_warn - whiptail_set_hostname - whiptail_management_nic - ;; -esac + if [ "$address_type" != 'DHCP' ]; then + whiptail_management_interface_ip + whiptail_management_interface_mask + whiptail_management_interface_gateway + whiptail_management_interface_dns + whiptail_management_interface_dns_search + fi + ;; + 'network') + whiptail_network_notice + whiptail_dhcp_warn + whiptail_set_hostname + whiptail_management_nic + ;; + esac -if [[ $is_minion ]]; then - whiptail_management_server -fi + if [[ $is_minion ]]; then + whiptail_management_server + fi -whiptail_management_interface_setup + whiptail_management_interface_setup -if [[ "$setup_type" == 'iso' ]]; then - # Init networking so rest of install works - set_hostname - set_management_interface -fi + if [[ "$setup_type" == 'iso' ]]; then + # Init networking so rest of install works + set_hostname + set_management_interface + fi -if [[ $is_minion || $is_import ]]; then - [ "$automated" == no ] && copy_ssh_key >> $setup_log 2>&1 + if [[ $is_minion ]]; then + [ "$automated" == no ] && copy_ssh_key >> $setup_log 2>&1 + fi + + if [[ $is_minion ]] && ! (compare_versions); then + info "Installer version mismatch, downloading correct version from manager" + echo "$install_type" > /root/install_opt + download_repo_tarball >> "$setup_log" 2>&1 + exec bash /root/manager_setup/securityonion/setup/so-setup "$@" + fi + +else + rm -rf /root/install_opt >> "$setup_log" 2>&1 fi short_name=$(echo "$HOSTNAME" | awk -F. '{print $1}') From 813fe77582e98c56d3d4d41f801eb43861995f09 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 8 Dec 2020 15:29:31 -0500 Subject: [PATCH 20/71] [feat] Run so-analyst-install after network init --- setup/so-setup | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 8ea110e96..17118800a 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -204,9 +204,7 @@ elif [ "$install_type" = 'HELIXSENSOR' ]; then elif [ "$install_type" = 'IMPORT' ]; then is_import=true elif [ "$install_type" = 'ANALYST' ]; then - cd .. || exit 255 - ./so-analyst-install - exit 0 + is_analyst=true fi # Say yes to the dress if its an ISO install @@ -280,6 +278,11 @@ if ! [ -f /root/install_opt ]; then exec bash /root/manager_setup/securityonion/setup/so-setup "$@" fi + if [[ $is_analyst ]]; then + cd .. || exit 255 + exec bash so-analyst-install + fi + else rm -rf /root/install_opt >> "$setup_log" 2>&1 fi From a8f1ec37a3ff87cc4c2073be1e43ce942a96d1da Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 8 Dec 2020 15:29:48 -0500 Subject: [PATCH 21/71] [refactor] Remove is_smooshed var --- setup/so-setup | 2 -- setup/so-whiptail | 14 +++++++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 17118800a..d0efafc07 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -176,7 +176,6 @@ elif [ "$install_type" = 'STANDALONE' ]; then is_distmanager=true is_node=true is_sensor=true - is_smooshed=true elif [ "$install_type" = 'MANAGERSEARCH' ]; then is_manager=true is_distmanager=true @@ -194,7 +193,6 @@ elif [ "$install_type" = 'HEAVYNODE' ]; then is_node=true is_minion=true is_sensor=true - is_smooshed=true elif [ "$install_type" = 'FLEET' ]; then is_minion=true is_fleet_standalone=true diff --git a/setup/so-whiptail b/setup/so-whiptail index edbc19c0b..fdf318e06 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -35,7 +35,7 @@ whiptail_basic_zeek() { [ -n "$TESTING" ] && return - if [[ $is_smooshed ]]; then + if [[ $is_node && $is_sensor && ! $is_eval ]]; then local PROCS=$(expr $lb_procs / 2) if [ "$PROCS" -lt 1 ]; then PROCS=1; else PROCS=$PROCS; fi else @@ -53,7 +53,7 @@ whiptail_basic_suri() { [ -n "$TESTING" ] && return - if [[ $is_smooshed ]]; then + if [[ $is_node && $is_sensor && ! $is_eval ]];; then local PROCS=$(expr $lb_procs / 2) if [ "$PROCS" -lt 1 ]; then PROCS=1; else PROCS=$PROCS; fi else @@ -77,7 +77,7 @@ whiptail_zeek_pins() { cpu_core_list_whiptail+=("$item" "OFF") done - if [[ $is_smooshed ]]; then + if [[ $is_node && $is_sensor && ! $is_eval ]];; then local PROCS=$(expr $lb_procs / 2) if [ "$PROCS" -lt 1 ]; then PROCS=1; else PROCS=$PROCS; fi else @@ -1354,11 +1354,11 @@ whiptail_suricata_pins() { readarray -t filtered_core_list <<< "$(echo "${cpu_core_list[@]}" "${ZEEKPINS[@]}" | xargs -n1 | sort | uniq -u | awk '{print $1}')" local filtered_core_str=() - for item in "${filtered_core_list[@]}"; do - filtered_core_str+=("$item" "") - done + for item in "${filtered_core_list[@]}"; do + filtered_core_str+=("$item" "") + done - if [[ $is_smooshed ]]; then + if [[ $is_node && $is_sensor && ! $is_eval ]];; then local PROCS=$(expr $lb_procs / 2) if [ "$PROCS" -lt 1 ]; then PROCS=1; else PROCS=$PROCS; fi else From 652c4d49c90494ce80349fa5c212fe4e2a7ad1d2 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 9 Dec 2020 09:47:10 -0500 Subject: [PATCH 22/71] [fix] Remove extra semicolon --- setup/so-whiptail | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index fdf318e06..68734c1c5 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -53,7 +53,7 @@ whiptail_basic_suri() { [ -n "$TESTING" ] && return - if [[ $is_node && $is_sensor && ! $is_eval ]];; then + if [[ $is_node && $is_sensor && ! $is_eval ]]; then local PROCS=$(expr $lb_procs / 2) if [ "$PROCS" -lt 1 ]; then PROCS=1; else PROCS=$PROCS; fi else @@ -77,7 +77,7 @@ whiptail_zeek_pins() { cpu_core_list_whiptail+=("$item" "OFF") done - if [[ $is_node && $is_sensor && ! $is_eval ]];; then + if [[ $is_node && $is_sensor && ! $is_eval ]]; then local PROCS=$(expr $lb_procs / 2) if [ "$PROCS" -lt 1 ]; then PROCS=1; else PROCS=$PROCS; fi else @@ -1358,7 +1358,7 @@ whiptail_suricata_pins() { filtered_core_str+=("$item" "") done - if [[ $is_node && $is_sensor && ! $is_eval ]];; then + if [[ $is_node && $is_sensor && ! $is_eval ]]; then local PROCS=$(expr $lb_procs / 2) if [ "$PROCS" -lt 1 ]; then PROCS=1; else PROCS=$PROCS; fi else From 950c05e53da9719259909b6f209a6fe486d09469 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 9 Dec 2020 09:50:30 -0500 Subject: [PATCH 23/71] [fix] Only move error log if present --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index d0efafc07..5109365da 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -63,7 +63,7 @@ if [[ -f /root/accept_changes ]]; then # Move last setup log to backup mv "$setup_log" "$setup_log.bak" - mv "$error_log" "$error_log.bak" + [ -f "$error_log" ] && mv "$error_log" "$error_log.bak" fi From 51650147ef1b398d8841512531f08a036bc54d0d Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 9 Dec 2020 09:59:44 -0500 Subject: [PATCH 24/71] [fix] Only show network init message if valid --- setup/so-setup | 4 +++- setup/so-whiptail | 15 ++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 5109365da..fe11d5361 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -257,7 +257,9 @@ if ! [ -f /root/install_opt ]; then whiptail_management_server fi - whiptail_management_interface_setup + if [[ $is_minion || $is_iso ]]; then + whiptail_management_interface_setup + fi if [[ "$setup_type" == 'iso' ]]; then # Init networking so rest of install works diff --git a/setup/so-whiptail b/setup/so-whiptail index 68734c1c5..a37340764 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -769,14 +769,23 @@ whiptail_management_interface_setup() { [ -n "$TESTING" ] && return local minion_msg + local msg - if [[ $is_minion || $is_import ]]; then - minion_msg=" and copy the ssh key for soremote to the manager" + if [[ $is_minion ]]; then + minion_msg="copy the ssh key for soremote to the manager" else minion_msg="" fi - whiptail --title "Security Onion Setup" --yesno "Setup will now initialize networking$minion_msg. Select YES to continue or NO to cancel." 8 75 + if [[ $is_iso ]]; then + if [[ $minion_msg != "" ]]; then + msg="initialize networking and $minion_msg" + else + msg="initialize networking" + fi + fi + + whiptail --title "Security Onion Setup" --yesno "Setup will now $msg. Select YES to continue or NO to cancel." 8 75 local exitstatus=$? whiptail_check_exitstatus $exitstatus } From f3ce2fc71e91b1027956fff51bebafcf6acdc176 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 9 Dec 2020 10:06:02 -0500 Subject: [PATCH 25/71] [fix] new_setup -> manager_setup --- setup/so-functions | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 5375d725c..83fed9c66 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -922,15 +922,15 @@ docker_seed_registry() { } download_repo_tarball() { - scp -i /root/.ssh/so.key soremote@"$MSRV":/opt/so/repo/"$manager_ver".tar.gz /root/new_setup + scp -i /root/.ssh/so.key soremote@"$MSRV":/opt/so/repo/"$manager_ver".tar.gz /root/manager_setup # Fail if the file doesn't download - if ! [ -f /root/new_setup/"$manager_ver".tar.gz ]; then + if ! [ -f /root/manager_setup/"$manager_ver".tar.gz ]; then kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 fi - tar -xzf /root/new_setup/"$manager_ver".tar.gz -C /root/new_setup/securityonion - rm -rf /root/new_setup/"$manager_ver".tar.gz + tar -xzf /root/manager_setup/"$manager_ver".tar.gz -C /root/manager_setup/securityonion + rm -rf /root/manager_setup/"$manager_ver".tar.gz } fireeye_pillar() { From 795cacecf3c26fca5329b60afc66f2a0fa39822d Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 9 Dec 2020 10:06:14 -0500 Subject: [PATCH 26/71] [fix] Fix cut command options --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 83fed9c66..d143a79ff 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1771,7 +1771,7 @@ set_network_dev_status_list() { set_main_ip() { MAINIP=$(ip route get 1 | awk '{print $7;exit}') - MNIC_IP=$(ip a s "$MNIC" | grep -oE 'inet [0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' | cut -d' ' -f) + MNIC_IP=$(ip a s "$MNIC" | grep -oE 'inet [0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' | cut -d' ' -f2) if [[ $MAINIP != $MNIC_IP ]]; then read -r -d '' message <<- EOM From 223856c0b9d226a80817f54da42833657164064b Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 9 Dec 2020 10:16:42 -0500 Subject: [PATCH 27/71] [fix] Don't redirect whiptail message, use SIGINT instead of SIGKILL --- setup/so-functions | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index d143a79ff..cd9b63ce2 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1772,15 +1772,17 @@ set_network_dev_status_list() { set_main_ip() { MAINIP=$(ip route get 1 | awk '{print $7;exit}') MNIC_IP=$(ip a s "$MNIC" | grep -oE 'inet [0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' | cut -d' ' -f2) +} - if [[ $MAINIP != $MNIC_IP ]]; then +compare_main_nic_ip() { + if [[ "$MAINIP" != "$MNIC_IP" ]]; then read -r -d '' message <<- EOM The IP being routed by Linux is not the IP address assigned to the management interface ($MNIC). This is not a supported configuration, please remediate and rerun setup. EOM whiptail --title "Security Onion Setup" --msgbox "$message" 10 75 - kill -SIGKILL "$(ps --pid $$ -oppid=)"; exit 1 + kill -SIGINT "$(ps --pid $$ -oppid=)"; exit 1 fi } From 282b4090ce3768c8ffdff5b5e2bf0babc5ec3e6d Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 9 Dec 2020 11:51:07 -0500 Subject: [PATCH 28/71] [fix] Actually call nic comparison function, redirect tarball gen to setup_log --- setup/so-setup | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index fe11d5361..5b751d124 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -480,10 +480,9 @@ if [[ $is_minion ]]; then fi # This block sets REDIRECTIT which is used by a function outside the below subshell -{ - set_main_ip; - set_redirect; -} >> $setup_log 2>&1 + set_main_ip >> $setup_log 2>&1 + compare_main_nic_ip + set_redirect >> $setup_log 2>&1 # Begin install { @@ -824,7 +823,7 @@ else if [[ $is_manager ]]; then set_progress_str 98 "Generating archive for setup directory" - generate_repo_tarball + generate_repo_tarball >> "$setup_log" 2>&1 fi if [[ $THEHIVE == 1 ]]; then @@ -837,6 +836,6 @@ else echo "Post-installation steps have completed." >> $setup_log 2>&1 fi -install_cleanup >> $setup_log 2>&1 +install_cleanup >> "$setup_log" 2>&1 if [[ -z $SKIP_REBOOT ]]; then shutdown -r now; else exit; fi From 987008811c837707170e7ca33c1a28fb18b6a229 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 9 Dec 2020 12:47:35 -0500 Subject: [PATCH 29/71] [fix] Make repo directory before using it --- setup/so-functions | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-functions b/setup/so-functions index cd9b63ce2..d98b6c82c 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -993,6 +993,7 @@ generate_passwords(){ } generate_repo_tarball() { + mkdir /opt/so/repo tar -czf /opt/so/repo/"$SOVERSION".tar.gz ../. } From a2e48f91b2f73e3e58672ac1589fb2f1a0bfa0ae Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 9 Dec 2020 13:13:51 -0500 Subject: [PATCH 30/71] [fix] Add manager to hosts before attempting ssh --- setup/so-setup | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 5b751d124..5334b49f2 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -261,12 +261,21 @@ if ! [ -f /root/install_opt ]; then whiptail_management_interface_setup fi + # Init networking so rest of install works + disable_ipv6 if [[ "$setup_type" == 'iso' ]]; then - # Init networking so rest of install works set_hostname set_management_interface fi + if [[ -n "$TURBO" ]]; then + use_turbo_proxy + fi + + if [[ $is_minion ]]; then + add_mngr_ip_to_hosts + fi + if [[ $is_minion ]]; then [ "$automated" == no ] && copy_ssh_key >> $setup_log 2>&1 fi @@ -464,25 +473,10 @@ catch() { exit } -# Init networking so rest of install works -if [[ -n "$TURBO" ]]; then - use_turbo_proxy -fi - -disable_ipv6 - -if [[ "$setup_type" != 'iso' ]]; then - set_hostname >> $setup_log 2>&1 -fi - -if [[ $is_minion ]]; then - add_mngr_ip_to_hosts -fi - # This block sets REDIRECTIT which is used by a function outside the below subshell - set_main_ip >> $setup_log 2>&1 - compare_main_nic_ip - set_redirect >> $setup_log 2>&1 +set_main_ip >> $setup_log 2>&1 +compare_main_nic_ip +set_redirect >> $setup_log 2>&1 # Begin install { From 86313796a5fa569c82fc8bdbb16e18d744dd55be Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 10 Dec 2020 11:00:52 -0500 Subject: [PATCH 31/71] [fix] Set manager_ver in download function --- setup/so-functions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index d98b6c82c..b03a96c97 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -519,8 +519,7 @@ check_requirements() { } compare_versions() { - manager_ver=$(ssh -i /root/.ssh/so.key soremote@"$MSRV" cat /etc/soversion) - export manager_ver + manager_ver=$(ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo cat /etc/soversion) [[ "$manager_ver" == "$SOVERSION" ]] return @@ -922,6 +921,7 @@ docker_seed_registry() { } download_repo_tarball() { + manager_ver=$(ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo cat /etc/soversion) scp -i /root/.ssh/so.key soremote@"$MSRV":/opt/so/repo/"$manager_ver".tar.gz /root/manager_setup # Fail if the file doesn't download From 21e107f2e8c667bea24d23b79432b8f965690123 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 10 Dec 2020 13:13:45 -0500 Subject: [PATCH 32/71] [fix] Remove sudo from version check, only remove known_hosts entry if exists --- setup/so-functions | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index b03a96c97..bedbb43b5 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -519,7 +519,7 @@ check_requirements() { } compare_versions() { - manager_ver=$(ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo cat /etc/soversion) + manager_ver=$(ssh -i /root/.ssh/so.key soremote@"$MSRV" cat /etc/soversion) [[ "$manager_ver" == "$SOVERSION" ]] return @@ -660,7 +660,7 @@ copy_ssh_key() { chown -R "$SUDO_USER":"$SUDO_USER" /root/.ssh echo "Removing old entry for manager from known_hosts if it exists" - sed -i "/${MSRV}/d" /root/.ssh/known_hosts + grep -q "$MSRV" /root/.ssh/known_hosts && sed -i "/${MSRV}/d" /root/.ssh/known_hosts echo "Copying the SSH key to the manager" #Copy the key over to the manager @@ -921,7 +921,7 @@ docker_seed_registry() { } download_repo_tarball() { - manager_ver=$(ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo cat /etc/soversion) + manager_ver=$(ssh -i /root/.ssh/so.key soremote@"$MSRV" cat /etc/soversion) scp -i /root/.ssh/so.key soremote@"$MSRV":/opt/so/repo/"$manager_ver".tar.gz /root/manager_setup # Fail if the file doesn't download From 830211975655c54bb9955c40c89e45a6751549f2 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 10 Dec 2020 13:26:19 -0500 Subject: [PATCH 33/71] [fix] Don't redirect entire download function to setup log --- setup/so-functions | 21 +++++++++++++++++---- setup/so-setup | 2 +- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index bedbb43b5..a87a36b55 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -521,6 +521,11 @@ check_requirements() { compare_versions() { manager_ver=$(ssh -i /root/.ssh/so.key soremote@"$MSRV" cat /etc/soversion) + if [[ $manager_ver == "" ]]; then + echo "Could not determine version of Security Onion running on manager $MSRV. Please check your network settings and run setup again." | tee "$setup_log" + kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 + fi + [[ "$manager_ver" == "$SOVERSION" ]] return } @@ -921,16 +926,24 @@ docker_seed_registry() { } download_repo_tarball() { - manager_ver=$(ssh -i /root/.ssh/so.key soremote@"$MSRV" cat /etc/soversion) - scp -i /root/.ssh/so.key soremote@"$MSRV":/opt/so/repo/"$manager_ver".tar.gz /root/manager_setup + { + local manager_ver + manager_ver=$(ssh -i /root/.ssh/so.key soremote@"$MSRV" cat /etc/soversion) + scp -i /root/.ssh/so.key soremote@"$MSRV":/opt/so/repo/"$manager_ver".tar.gz /root/manager_setup + } >> "$setup_log" 2>&1 + # Fail if the file doesn't download if ! [ -f /root/manager_setup/"$manager_ver".tar.gz ]; then + local message="Could not download $manager_ver.tar.gz from manager, please check your network settings and verify the file /opt/so/repo/$manager_ver.tar.gz exists on the manager." + echo "$message" | tee "$setup_log" kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 fi - tar -xzf /root/manager_setup/"$manager_ver".tar.gz -C /root/manager_setup/securityonion - rm -rf /root/manager_setup/"$manager_ver".tar.gz + { + tar -xzf /root/manager_setup/"$manager_ver".tar.gz -C /root/manager_setup/securityonion + rm -rf /root/manager_setup/"$manager_ver".tar.gz + } >> "$setup_log" 2>&1 } fireeye_pillar() { diff --git a/setup/so-setup b/setup/so-setup index 5334b49f2..7bf768791 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -283,7 +283,7 @@ if ! [ -f /root/install_opt ]; then if [[ $is_minion ]] && ! (compare_versions); then info "Installer version mismatch, downloading correct version from manager" echo "$install_type" > /root/install_opt - download_repo_tarball >> "$setup_log" 2>&1 + download_repo_tarball exec bash /root/manager_setup/securityonion/setup/so-setup "$@" fi From bc6a0c1e6f4590e90f6d37a72133c8b61c0af247 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 10 Dec 2020 13:54:41 -0500 Subject: [PATCH 34/71] [fix] Add missing append flags to tee --- setup/so-functions | 6 +++--- setup/so-setup | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index a87a36b55..311b82225 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -522,7 +522,7 @@ compare_versions() { manager_ver=$(ssh -i /root/.ssh/so.key soremote@"$MSRV" cat /etc/soversion) if [[ $manager_ver == "" ]]; then - echo "Could not determine version of Security Onion running on manager $MSRV. Please check your network settings and run setup again." | tee "$setup_log" + echo "Could not determine version of Security Onion running on manager $MSRV. Please check your network settings and run setup again." | tee -a "$setup_log" kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 fi @@ -936,7 +936,7 @@ download_repo_tarball() { # Fail if the file doesn't download if ! [ -f /root/manager_setup/"$manager_ver".tar.gz ]; then local message="Could not download $manager_ver.tar.gz from manager, please check your network settings and verify the file /opt/so/repo/$manager_ver.tar.gz exists on the manager." - echo "$message" | tee "$setup_log" + echo "$message" | tee -a "$setup_log" kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 fi @@ -1655,7 +1655,7 @@ saltify() { apt-key add "$temp_install_dir"/gpg/GPG-KEY-WAZUH >> "$setup_log" 2>&1 echo "deb http://repo.saltstack.com$py_ver_url_path/ubuntu/$ubuntu_version/amd64/archive/3002.2/ $OSVER main" > /etc/apt/sources.list.d/saltstack.list 2>> "$setup_log" echo "deb https://packages.wazuh.com/3.x/apt/ stable main" > /etc/apt/sources.list.d/wazuh.list 2>> "$setup_log" - ;; + ;; esac apt-get update >> "$setup_log" 2>&1 set_progress_str 8 'Installing salt-minion & python modules' diff --git a/setup/so-setup b/setup/so-setup index 7bf768791..292cbf6f4 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -115,7 +115,7 @@ case "$setup_type" in echo "Beginning Security Onion $setup_type install" >> $setup_log 2>&1 ;; *) - echo "Invalid install type, must be 'iso' or 'network'" | tee $setup_log + echo "Invalid install type, must be 'iso' or 'network'" | tee -a $setup_log exit 1 ;; esac @@ -156,7 +156,7 @@ if ! [ -f /root/install_opt ]; then if (whiptail_you_sure); then true else - echo "User cancelled setup." | tee "$setup_log" + echo "User cancelled setup." | tee -a "$setup_log" whiptail_cancel fi From 1f1cfde3acd4a82bdcceb9d33b44f630809e4b44 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 10 Dec 2020 14:03:54 -0500 Subject: [PATCH 35/71] [fix] Make directory for new setup download --- setup/so-functions | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-functions b/setup/so-functions index 311b82225..890b1e8e0 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -929,6 +929,7 @@ download_repo_tarball() { { local manager_ver manager_ver=$(ssh -i /root/.ssh/so.key soremote@"$MSRV" cat /etc/soversion) + mkdir -p /root/manager_setup scp -i /root/.ssh/so.key soremote@"$MSRV":/opt/so/repo/"$manager_ver".tar.gz /root/manager_setup } >> "$setup_log" 2>&1 From 58bcc79c542961c2d3144865c29ddabeb92116de Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 10 Dec 2020 14:17:47 -0500 Subject: [PATCH 36/71] [fix] Create full dir structure, rm /root/install_opt on failure --- setup/so-functions | 5 +++-- setup/so-whiptail | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 890b1e8e0..aef409397 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -522,6 +522,7 @@ compare_versions() { manager_ver=$(ssh -i /root/.ssh/so.key soremote@"$MSRV" cat /etc/soversion) if [[ $manager_ver == "" ]]; then + rm /root/install_opt echo "Could not determine version of Security Onion running on manager $MSRV. Please check your network settings and run setup again." | tee -a "$setup_log" kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 fi @@ -926,16 +927,16 @@ docker_seed_registry() { } download_repo_tarball() { + mkdir -p /root/manager_setup/securityonion { local manager_ver manager_ver=$(ssh -i /root/.ssh/so.key soremote@"$MSRV" cat /etc/soversion) - mkdir -p /root/manager_setup scp -i /root/.ssh/so.key soremote@"$MSRV":/opt/so/repo/"$manager_ver".tar.gz /root/manager_setup } >> "$setup_log" 2>&1 - # Fail if the file doesn't download if ! [ -f /root/manager_setup/"$manager_ver".tar.gz ]; then + rm /root/install_opt local message="Could not download $manager_ver.tar.gz from manager, please check your network settings and verify the file /opt/so/repo/$manager_ver.tar.gz exists on the manager." echo "$message" | tee -a "$setup_log" kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 diff --git a/setup/so-whiptail b/setup/so-whiptail index a37340764..922f47b5e 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -783,6 +783,8 @@ whiptail_management_interface_setup() { else msg="initialize networking" fi + else + msg=$minion_msg fi whiptail --title "Security Onion Setup" --yesno "Setup will now $msg. Select YES to continue or NO to cancel." 8 75 From ab856532e641aafabc6cad82a7af6ed089db4e8b Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 10 Dec 2020 14:20:48 -0500 Subject: [PATCH 37/71] [fix] Show airgap option on import install --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 292cbf6f4..e175a834d 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -211,7 +211,7 @@ if [[ "$setup_type" == 'iso' ]]; then fi # Check if this is an airgap install -if [[ $is_manager && $is_iso ]]; then +if [[ ( $is_manager || $is_import ) && $is_iso ]]; then whiptail_airgap if [[ "$INTERWEBS" == 'AIRGAP' ]]; then is_airgap=true From 5c4103681cb9c135f2c0856163497d680d85fb35 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 10 Dec 2020 14:45:24 -0500 Subject: [PATCH 38/71] [fix] Save original argument array to use later --- setup/so-setup | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index e175a834d..96322b983 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -22,6 +22,9 @@ if [ "$uid" -ne 0 ]; then exit 1 fi +# Save the original argument array since we modify it +readarraay -t original_args <<< "$@" + cd "$(dirname "$0")" || exit 255 source ./so-functions @@ -55,7 +58,7 @@ while [[ $# -gt 0 ]]; do done if ! [ -f /root/install_opt ] && [ -d /root/manager_setup/securityonion ] && [[ $(pwd) != /root/manager_setup/securityonion/setup ]]; then - exec bash /root/manager_setup/securityonion/setup/so-setup "$@" + exec bash /root/manager_setup/securityonion/setup/so-setup "${original_args[@]}" fi if [[ -f /root/accept_changes ]]; then @@ -284,7 +287,7 @@ if ! [ -f /root/install_opt ]; then info "Installer version mismatch, downloading correct version from manager" echo "$install_type" > /root/install_opt download_repo_tarball - exec bash /root/manager_setup/securityonion/setup/so-setup "$@" + exec bash /root/manager_setup/securityonion/setup/so-setup "${original_args[@]}" fi if [[ $is_analyst ]]; then From b6a0e692c67ed3bbd3f21953aa8faac442bd1209 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 11 Dec 2020 09:38:35 -0500 Subject: [PATCH 39/71] [refactor] Use command -v for netplan check --- setup/so-functions | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index aef409397..9e35c5958 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1324,12 +1324,10 @@ network_setup() { disable_misc_network_features; echo "... Setting ONBOOT for management interface"; - if ! netplan > /dev/null 2>&1; then - nmcli con mod "$MNIC" connection.autoconnect "yes"; - fi + command -v netplan &> /dev/null || nmcli con mod "$MNIC" connection.autoconnect "yes" - echo "... Copying 99-so-checksum-offload-disable"; - cp ./install_scripts/99-so-checksum-offload-disable /etc/NetworkManager/dispatcher.d/pre-up.d/99-so-checksum-offload-disable ; + echo "... Copying 99-so-checksum-offload-disable"; + cp ./install_scripts/99-so-checksum-offload-disable /etc/NetworkManager/dispatcher.d/pre-up.d/99-so-checksum-offload-disable ; echo "... Modifying 99-so-checksum-offload-disable"; sed -i "s/\$MNIC/${MNIC}/g" /etc/NetworkManager/dispatcher.d/pre-up.d/99-so-checksum-offload-disable; From 3c7a8fe92f79af476b07827ae6c900727a65cfb0 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 11 Dec 2020 09:39:00 -0500 Subject: [PATCH 40/71] [fix] Don't cd in so-variables --- setup/so-variables | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-variables b/setup/so-variables index 2223fe106..266dba11e 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -62,5 +62,5 @@ mkdir -p "$default_salt_dir" export local_salt_dir=/opt/so/saltstack/local mkdir -p "$local_salt_dir" -SCRIPTDIR=$(cd "$(dirname "$0")" && pwd) +SCRIPTDIR=$(pwd) export SCRIPTDIR From 870cc6b79b929884a252540a8f803da8f6826bae Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 11 Dec 2020 09:39:22 -0500 Subject: [PATCH 41/71] [fix][typo] readaraay -> readarray --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 96322b983..43c9f36a6 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -23,7 +23,7 @@ if [ "$uid" -ne 0 ]; then fi # Save the original argument array since we modify it -readarraay -t original_args <<< "$@" +readarray -t original_args <<< "$@" cd "$(dirname "$0")" || exit 255 From 75c5abef30f2ece33266768791a5f08d419370e4 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 11 Dec 2020 10:16:00 -0500 Subject: [PATCH 42/71] [fix] Add all selected options to install_opts --- setup/so-setup | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 43c9f36a6..087e05172 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -165,7 +165,7 @@ if ! [ -f /root/install_opt ]; then whiptail_install_type else - install_type=$(cat /root/install_opt) + source /root/install_opt fi if [ "$install_type" = 'EVAL' ]; then @@ -285,7 +285,12 @@ if ! [ -f /root/install_opt ]; then if [[ $is_minion ]] && ! (compare_versions); then info "Installer version mismatch, downloading correct version from manager" - echo "$install_type" > /root/install_opt + printf '%s\n' \ + "install_type=$install_type" \ + "MNIC=$MNIC" \ + "HOSTNAME=$HOSTNAME" \ + "MSRV=$MSRV"\ + "MSRVIP=$MSRVIP" > /root/install_opt download_repo_tarball exec bash /root/manager_setup/securityonion/setup/so-setup "${original_args[@]}" fi @@ -388,9 +393,9 @@ if [[ $is_manager && ! $is_eval ]]; then whiptail_oinkcode fi - if [[ "$STRELKA" == 1 ]]; then - STRELKARULES=1 - fi + if [[ "$STRELKA" = 1 ]]; then + STRELKARULES=1 + fi if [ "$MANAGERADV" = 'ADVANCED' ] && [ "$ZEEKVERSION" != 'SURICATA' ]; then whiptail_manager_adv_service_zeeklogs @@ -758,7 +763,7 @@ set_redirect >> $setup_log 2>&1 set_progress_str 81 "$(print_salt_state_apply 'strelka')" salt-call state.apply -l info strelka >> $setup_log 2>&1 fi - if [[ $STRELKARULES == 1 ]]; then + if [[ "$STRELKARULES" = 1 ]]; then /usr/sbin/so-yara-update >> $setup_log 2>&1 fi fi From db276d902026584488a5e20d3d95539cf7ba377f Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 11 Dec 2020 11:02:27 -0500 Subject: [PATCH 43/71] [fix] Always set hostname --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 087e05172..d45f400a1 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -266,8 +266,8 @@ if ! [ -f /root/install_opt ]; then # Init networking so rest of install works disable_ipv6 + set_hostname if [[ "$setup_type" == 'iso' ]]; then - set_hostname set_management_interface fi From 66495e6bae63b1c93c00b30b3db0f161afe22626 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 11 Dec 2020 17:38:42 -0500 Subject: [PATCH 44/71] Swap localhost for 127.0.0.1 --- salt/common/tools/sbin/so-fleet-setup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-fleet-setup b/salt/common/tools/sbin/so-fleet-setup index 21aebc966..96644576f 100755 --- a/salt/common/tools/sbin/so-fleet-setup +++ b/salt/common/tools/sbin/so-fleet-setup @@ -15,8 +15,8 @@ if [ ! "$(docker ps -q -f name=so-fleet)" ]; then salt-call state.apply redis queue=True >> /root/fleet-setup.log fi -docker exec so-fleet fleetctl config set --address https://localhost:8080 --tls-skip-verify --url-prefix /fleet -docker exec -it so-fleet bash -c 'while [[ "$(curl -s -o /dev/null --insecure -w ''%{http_code}'' https://localhost:8080/fleet)" != "301" ]]; do sleep 5; done' +docker exec so-fleet fleetctl config set --address https://127.0.0.1:8080 --tls-skip-verify --url-prefix /fleet +docker exec -it so-fleet bash -c 'while [[ "$(curl -s -o /dev/null --insecure -w ''%{http_code}'' https://127.0.0.1:8080/fleet)" != "301" ]]; do sleep 5; done' docker exec so-fleet fleetctl setup --email $1 --password $2 docker exec so-fleet fleetctl apply -f /packs/palantir/Fleet/Endpoints/MacOS/osquery.yaml From a533e6fa3530f4d649cd7d31f3065d312ad71d00 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 14 Dec 2020 11:42:34 -0500 Subject: [PATCH 45/71] [fix] Always set INSTALLUSERNAME var --- setup/so-setup | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index d45f400a1..2fad47e3e 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -69,11 +69,10 @@ if [[ -f /root/accept_changes ]]; then [ -f "$error_log" ] && mv "$error_log" "$error_log.bak" fi +parse_install_username if ! [ -f /root/install_opt ]; then # Begin Installation pre-processing - parse_install_username - title "Initializing Setup" info "Installing as the $INSTALLUSERNAME user" From aa479b9c8ebd74eb09531fe4de79a11307f7f325 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 14 Dec 2020 10:43:01 -0500 Subject: [PATCH 46/71] Move node address/desc into the minion pillar --- setup/so-functions | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 3ff66be30..ad0587f31 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1042,6 +1042,9 @@ host_pillar() { printf '%s\n'\ "host:"\ " mainint: '$MNIC'"\ + "sensoroni:"\ + " node_address: '$MAINIP'"\ + " node_description: '$NODE_DESCRIPTION'"\ "" > "$pillar_file" } @@ -1205,8 +1208,6 @@ manager_global() { " imagerepo: '$IMAGEREPO'"\ " pipeline: 'redis'"\ "sensoroni:"\ - " node_address: '$MAINIP'"\ - " node_description: '$NODE_DESCRIPTION'"\ " node_checkin_interval_ms: $NODE_CHECKIN_INTERVAL_MS"\ "strelka:"\ " enabled: $STRELKA"\ From aa281f849febff6594e8d3797deddc339419f67f Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 14 Dec 2020 15:31:25 -0500 Subject: [PATCH 47/71] [feat] Add message about dropping to command line when setting up ssh key --- setup/so-whiptail | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 0a2a7e4fc..94b359574 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -748,11 +748,14 @@ whiptail_management_interface_setup() { local minion_msg local msg + local line_count if [[ $is_minion ]]; then - minion_msg="copy the ssh key for soremote to the manager" + line_count=11 + minion_msg="copy the ssh key for soremote to the manager. This will bring you to the command line temporarily to accept the manager's ECDSA certificate and enter the password for soremote" else - minion_msg="" + line_count=9 + minion_msg="" fi if [[ $is_iso ]]; then @@ -765,7 +768,13 @@ whiptail_management_interface_setup() { msg=$minion_msg fi - whiptail --title "Security Onion Setup" --yesno "Setup will now $msg. Select YES to continue or NO to cancel." 8 75 + read -r -d '' message <<- EOM + Setup will now $msg. + + Select OK to continue. + EOM + + whiptail --title "Security Onion Setup" --msgbox "$message" $line_count 75 local exitstatus=$? whiptail_check_exitstatus $exitstatus } From cbd59ed86a5815d17e6d5205e75f668b9e31f146 Mon Sep 17 00:00:00 2001 From: TOoSmOotH Date: Mon, 14 Dec 2020 20:46:31 -0500 Subject: [PATCH 48/71] SOUP Changes --- salt/common/files/daemon.json | 12 +++++++++++ salt/common/init.sls | 9 ++++++++ salt/common/tools/sbin/soup | 39 +++++++++++++++++++++++++++++++++++ salt/docker_clean/init.sls | 2 +- setup/so-functions | 1 + setup/so-setup | 2 +- setup/so-whiptail | 2 +- 7 files changed, 64 insertions(+), 3 deletions(-) create mode 100644 salt/common/files/daemon.json diff --git a/salt/common/files/daemon.json b/salt/common/files/daemon.json new file mode 100644 index 000000000..bc047bc80 --- /dev/null +++ b/salt/common/files/daemon.json @@ -0,0 +1,12 @@ +{%- set DOCKERRANGE = salt['pillar.get']('docker:range') %} +{%- set DOCKERBIND = salt['pillar.get']('docker:bip') %} +{ + "registry-mirrors": [ "https://:5000" ], + "bip": "{{ DOCKERBIND }}", + "default-address-pools": [ + { + "base" : "{{ DOCKERRANGE }}", + "size" : 24 + } + ] +} \ No newline at end of file diff --git a/salt/common/init.sls b/salt/common/init.sls index 1192923b7..337103fd9 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -244,10 +244,19 @@ commonlogrotateconf: - dayweek: '*' {% endif %} +# Manager daemon.json +docker_daemon: + file.managed: + - source: salt://common/files/daemon.json + - name: /etc/docker/daemon.json + - template: jinja + # Make sure Docker is always running docker: service.running: - enable: True + - watch: + - file: docker_daemon {% else %} diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index ca840de59..21076ba3d 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -197,6 +197,7 @@ pillar_changes() { [[ "$INSTALLEDVERSION" =~ rc.1 ]] && rc1_to_rc2 [[ "$INSTALLEDVERSION" =~ rc.2 ]] && rc2_to_rc3 [[ "$INSTALLEDVERSION" =~ rc.3 ]] && rc3_to_2.3.0 + [[ "$INSTALLEDVERSION" == 2.3.0 ]] || [[ "$INSTALLEDVERSION" == 2.3.1 ]] || [[ "$INSTALLEDVERSION" == 2.3.2 ]] || [[ "$INSTALLEDVERSION" == 2.3.10 ]] && 2.3.0_to_2.3.20 } rc1_to_rc2() { @@ -278,6 +279,44 @@ rc3_to_2.3.0() { echo "playbook_admin: $(get_random_value)" echo "playbook_automation: $(get_random_value)" } >> /opt/so/saltstack/local/pillar/secrets.sls + + INSTALLEDVERSION=2.3.0 +} + +2.3.0_to_2.3.20(){ + # Remove PCAP from global + sed '/pcap:/d' /opt/so/saltstack/local/pillar/global.sls + sed '/sensor_checkin_interval_ms:/d' /opt/so/saltstack/local/pillar/global.sls + + # Add checking interval to glbal + echo "sensoroni:" >> /opt/so/saltstack/local/pillar/global.sls + echo " node_checkin_interval_ms: 10000" >> /opt/so/saltstack/local/pillar/global.sls + + # Update pillar fiels for new sensoroni functionality + for file in /opt/so/saltstack/local/pillat/minions/*; do + echo "sensoroni:" >> $file + echo " node_description:" >> $file + local SOMEADDRESS=$(cat $file | grep mainip | tail -n 1 | awk '{print $2'}) + echo " node_address: $SOMEADDRESS" >> $file + done + + # Remove old firewall config to reduce confusion + rm -f /opt/so/saltstack/default/pillar/firewall/ports.sls + + # Fix daemon.json by managing it + echo "docker:" >> /opt/so/saltstack/local/pillar/global.sls + DOCKERGREP=$(cat /etc/docker/daemon.json | grep base | awk {'print $3'} | cut -f1 -d"/") + if [ -z "$DOCKERGREP" ]; then + echo " range: '172.17.0.0/24'" >> /opt/so/saltstack/local/pillar/global.sls + echo " bip: '172.17.0.1/24'" >> /opt/so/saltstack/local/pillar/global.sls + else + DOCKERSTUFF="${DOCKERGREP//\"}" + DOCKERSTUFFBIP=$(echo $DOCKERSTUFF | awk -F'.' '{print $1,$2,$3,1}' OFS='.')/24 + echo " range: '$DOCKERSTUFF'/24" >> /opt/so/saltstack/local/pillar/global.sls + echo " bip: '$DOCKERSTUFFBIP'" >> /opt/so/saltstack/local/pillar/global.sls + + fi + } space_check() { diff --git a/salt/docker_clean/init.sls b/salt/docker_clean/init.sls index 61499cdb5..9c5ce0d17 100644 --- a/salt/docker_clean/init.sls +++ b/salt/docker_clean/init.sls @@ -1,6 +1,6 @@ {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} -{% set OLDVERSIONS = ['2.0.0-rc.1','2.0.1-rc.1','2.0.2-rc.1','2.0.3-rc.1','2.1.0-rc.2','2.2.0-rc.3','2.3.0','2.3.1']%} +{% set OLDVERSIONS = ['2.0.0-rc.1','2.0.1-rc.1','2.0.2-rc.1','2.0.3-rc.1','2.1.0-rc.2','2.2.0-rc.3','2.3.0','2.3.1','2.3.2']%} {% for VERSION in OLDVERSIONS %} remove_images_{{ VERSION }}: diff --git a/setup/so-functions b/setup/so-functions index 3ff66be30..83d9525f3 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -898,6 +898,7 @@ docker_registry() { echo "Setting up Docker Registry" >> "$setup_log" 2>&1 mkdir -p /etc/docker >> "$setup_log" 2>&1 + # This will get applied so docker can attempt to start if [ -z "$DOCKERNET" ]; then DOCKERNET=172.17.0.0 fi diff --git a/setup/so-setup b/setup/so-setup index 73e66d058..d6566bdd3 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -338,7 +338,6 @@ if [[ $is_helix || $is_sensor || $is_import ]]; then fi whiptail_homenet_manager -whiptail_dockernet_check if [[ $is_helix || $is_manager || $is_node || $is_import ]]; then set_base_heapsizes @@ -373,6 +372,7 @@ fi if [[ $is_manager ]]; then whiptail_components_adv_warning whiptail_enable_components + whiptail_dockernet_check fi if [[ $is_manager || $is_import ]]; then diff --git a/setup/so-whiptail b/setup/so-whiptail index 791cceb76..63acadc90 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -426,7 +426,7 @@ whiptail_dockernet_net() { [ -n "$TESTING" ] && return DOCKERNET=$(whiptail --title "Security Onion Setup" --inputbox \ - "\nEnter a /24 network range for docker to use: \nThe same range MUST be used on ALL nodes \n(Default value is pre-populated.)" 10 75 172.17.0.0 3>&1 1>&2 2>&3) + "\nEnter a /24 size network range for docker to use WITHOUT the /24 notation: \nThis range will be used on ALL nodes \n(Default value is pre-populated.)" 10 75 172.17.0.0 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus From 80a61d33164b9200902bbcb6aceef3290c4c7013 Mon Sep 17 00:00:00 2001 From: TOoSmOotH Date: Tue, 15 Dec 2020 12:06:30 -0500 Subject: [PATCH 49/71] SOUP Features --- salt/common/tools/sbin/so-image-common | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 3449158c0..f0fd8d691 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -17,6 +17,7 @@ # NOTE: This script depends on so-common IMAGEREPO=securityonion +FEATURESCHECK=$(lookup_pillar elastic features) container_list() { MANAGERCHECK=$1 @@ -46,17 +47,24 @@ container_list() { "so-curator" \ "so-domainstats" \ "so-elastalert" \ - "so-elasticsearch" \ - "so-filebeat" \ + if [[ "$FEATURESCHECK" == "True" ]]; then + "so-elasticsearch-features" \ + "so-filebeat-features" \ + "so-logstash-features" \ + "so-kibana-features" \ + else + "so-elasticsearch" \ + "so-filebeat" \ + "so-logstash" \ + "so-kibana" \ + fi "so-fleet" \ "so-fleet-launcher" \ "so-freqserver" \ "so-grafana" \ "so-idstools" \ "so-influxdb" \ - "so-kibana" \ "so-kratos" \ - "so-logstash" \ "so-minio" \ "so-mysql" \ "so-nginx" \ From 3da7a26e885e4259d9e09dc1ddafa5cd24b4ad12 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 15 Dec 2020 12:37:01 -0500 Subject: [PATCH 50/71] Remove jinja whitespace trimming to avoid syntax error in bash --- salt/common/tools/sbin/so-yara-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-yara-update b/salt/common/tools/sbin/so-yara-update index a2a633957..ddddb87eb 100755 --- a/salt/common/tools/sbin/so-yara-update +++ b/salt/common/tools/sbin/so-yara-update @@ -165,6 +165,6 @@ else echo "No connectivity to Github...exiting..." exit 1 fi -{%- endif -%} +{% endif %} echo "Finished rule updates at $(date)..." From f70d828aa633b68860339cac5c002c6149cacd60 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 15 Dec 2020 13:04:09 -0500 Subject: [PATCH 51/71] [fix] Create array correctly --- salt/common/tools/sbin/so-image-common | 131 +++++++++++++------------ 1 file changed, 69 insertions(+), 62 deletions(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index f0fd8d691..6f5095aa3 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -29,73 +29,80 @@ container_list() { fi if [ $MANAGERCHECK == 'so-import' ]; then - TRUSTED_CONTAINERS=( \ - "so-elasticsearch" \ - "so-filebeat" \ - "so-idstools" \ - "so-kibana" \ - "so-kratos" \ - "so-nginx" \ - "so-pcaptools" \ - "so-soc" \ - "so-steno" \ - "so-suricata" \ - "so-zeek" ) + TRUSTED_CONTAINERS=( + "so-elasticsearch" + "so-filebeat" + "so-idstools" + "so-kibana" + "so-kratos" + "so-nginx" + "so-pcaptools" + "so-soc" + "so-steno" + "so-suricata" + "so-zeek" + ) elif [ $MANAGERCHECK != 'so-helix' ]; then - TRUSTED_CONTAINERS=( \ - "so-acng" \ - "so-curator" \ - "so-domainstats" \ - "so-elastalert" \ + TRUSTED_CONTAINERS=( + "so-acng" + "so-curator" + "so-domainstats" + "so-elastalert" + "so-fleet" + "so-fleet-launcher" + "so-freqserver" + "so-grafana" + "so-idstools" + "so-influxdb" + "so-kratos" + "so-minio" + "so-mysql" + "so-nginx" + "so-pcaptools" + "so-playbook" + "so-redis" + "so-soc" + "so-soctopus" + "so-steno" + "so-strelka-backend" + "so-strelka-filestream" + "so-strelka-frontend" + "so-strelka-manager" + "so-suricata" + "so-telegraf" + "so-thehive" + "so-thehive-cortex" + "so-thehive-es" + "so-wazuh" + "so-zeek" + ) if [[ "$FEATURESCHECK" == "True" ]]; then - "so-elasticsearch-features" \ - "so-filebeat-features" \ - "so-logstash-features" \ - "so-kibana-features" \ + TRUSTED_CONTAINERS=( "${TRUSTED_CONTAINERS[@]}" + "so-elasticsearch-features" + "so-filebeat-features" + "so-logstash-features" + "so-kibana-features" + ) else - "so-elasticsearch" \ - "so-filebeat" \ - "so-logstash" \ - "so-kibana" \ + TRUSTED_CONTAINERS=( "${TRUSTED_CONTAINERS[@]}" + "so-elasticsearch" + "so-filebeat" + "so-logstash" + "so-kibana" + ) fi - "so-fleet" \ - "so-fleet-launcher" \ - "so-freqserver" \ - "so-grafana" \ - "so-idstools" \ - "so-influxdb" \ - "so-kratos" \ - "so-minio" \ - "so-mysql" \ - "so-nginx" \ - "so-pcaptools" \ - "so-playbook" \ - "so-redis" \ - "so-soc" \ - "so-soctopus" \ - "so-steno" \ - "so-strelka-backend" \ - "so-strelka-filestream" \ - "so-strelka-frontend" \ - "so-strelka-manager" \ - "so-suricata" \ - "so-telegraf" \ - "so-thehive" \ - "so-thehive-cortex" \ - "so-thehive-es" \ - "so-wazuh" \ - "so-zeek" ) else - TRUSTED_CONTAINERS=( \ - "so-filebeat" \ - "so-idstools" \ - "so-logstash" \ - "so-nginx" \ - "so-redis" \ - "so-steno" \ - "so-suricata" \ - "so-telegraf" \ - "so-zeek" ) + TRUSTED_CONTAINERS=( + "so-filebeat" + "so-idstools" + "so-logstash" + "so-nginx" + "so-redis" + "so-steno" + "so-suricata" + "so-telegraf" + "so-zeek" + ) fi } From f7d02763e8f8e48aa90b1babfdeaaf3c6e540fa4 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 15 Dec 2020 13:07:21 -0500 Subject: [PATCH 52/71] [fix] Move FEATURESCHECK var assignment, fix indentation --- salt/common/tools/sbin/so-image-common | 159 +++++++++++++------------ 1 file changed, 80 insertions(+), 79 deletions(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 6f5095aa3..dd4cfc979 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -17,93 +17,94 @@ # NOTE: This script depends on so-common IMAGEREPO=securityonion -FEATURESCHECK=$(lookup_pillar elastic features) container_list() { - MANAGERCHECK=$1 - if [ -z "$MANAGERCHECK" ]; then - MANAGERCHECK=so-unknown - if [ -f /etc/salt/grains ]; then - MANAGERCHECK=$(cat /etc/salt/grains | grep role | awk '{print $2}') - fi - fi + MANAGERCHECK=$1 + FEATURESCHECK=$(lookup_pillar elastic features) - if [ $MANAGERCHECK == 'so-import' ]; then - TRUSTED_CONTAINERS=( + if [ -z "$MANAGERCHECK" ]; then + MANAGERCHECK=so-unknown + if [ -f /etc/salt/grains ]; then + MANAGERCHECK=$(cat /etc/salt/grains | grep role | awk '{print $2}') + fi + fi + + if [ $MANAGERCHECK == 'so-import' ]; then + TRUSTED_CONTAINERS=( + "so-elasticsearch" + "so-filebeat" + "so-idstools" + "so-kibana" + "so-kratos" + "so-nginx" + "so-pcaptools" + "so-soc" + "so-steno" + "so-suricata" + "so-zeek" + ) + elif [ $MANAGERCHECK != 'so-helix' ]; then + TRUSTED_CONTAINERS=( + "so-acng" + "so-curator" + "so-domainstats" + "so-elastalert" + "so-fleet" + "so-fleet-launcher" + "so-freqserver" + "so-grafana" + "so-idstools" + "so-influxdb" + "so-kratos" + "so-minio" + "so-mysql" + "so-nginx" + "so-pcaptools" + "so-playbook" + "so-redis" + "so-soc" + "so-soctopus" + "so-steno" + "so-strelka-backend" + "so-strelka-filestream" + "so-strelka-frontend" + "so-strelka-manager" + "so-suricata" + "so-telegraf" + "so-thehive" + "so-thehive-cortex" + "so-thehive-es" + "so-wazuh" + "so-zeek" + ) + if [[ "$FEATURESCHECK" == "True" ]]; then + TRUSTED_CONTAINERS=( "${TRUSTED_CONTAINERS[@]}" + "so-elasticsearch-features" + "so-filebeat-features" + "so-logstash-features" + "so-kibana-features" + ) + else + TRUSTED_CONTAINERS=( "${TRUSTED_CONTAINERS[@]}" "so-elasticsearch" "so-filebeat" - "so-idstools" - "so-kibana" - "so-kratos" - "so-nginx" - "so-pcaptools" - "so-soc" - "so-steno" - "so-suricata" - "so-zeek" - ) - elif [ $MANAGERCHECK != 'so-helix' ]; then - TRUSTED_CONTAINERS=( - "so-acng" - "so-curator" - "so-domainstats" - "so-elastalert" - "so-fleet" - "so-fleet-launcher" - "so-freqserver" - "so-grafana" - "so-idstools" - "so-influxdb" - "so-kratos" - "so-minio" - "so-mysql" - "so-nginx" - "so-pcaptools" - "so-playbook" - "so-redis" - "so-soc" - "so-soctopus" - "so-steno" - "so-strelka-backend" - "so-strelka-filestream" - "so-strelka-frontend" - "so-strelka-manager" - "so-suricata" - "so-telegraf" - "so-thehive" - "so-thehive-cortex" - "so-thehive-es" - "so-wazuh" - "so-zeek" - ) - if [[ "$FEATURESCHECK" == "True" ]]; then - TRUSTED_CONTAINERS=( "${TRUSTED_CONTAINERS[@]}" - "so-elasticsearch-features" - "so-filebeat-features" - "so-logstash-features" - "so-kibana-features" - ) - else - TRUSTED_CONTAINERS=( "${TRUSTED_CONTAINERS[@]}" - "so-elasticsearch" - "so-filebeat" - "so-logstash" - "so-kibana" - ) - fi - else - TRUSTED_CONTAINERS=( - "so-filebeat" - "so-idstools" "so-logstash" - "so-nginx" - "so-redis" - "so-steno" - "so-suricata" - "so-telegraf" - "so-zeek" + "so-kibana" ) fi + else + TRUSTED_CONTAINERS=( + "so-filebeat" + "so-idstools" + "so-logstash" + "so-nginx" + "so-redis" + "so-steno" + "so-suricata" + "so-telegraf" + "so-zeek" + ) + fi } update_docker_containers() { From e89c06f71b2e5471aad57c49a998ffa2882fdf8b Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 15 Dec 2020 13:37:21 -0500 Subject: [PATCH 53/71] [fix] Add backslash for newline --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 7982420a5..164aa74b2 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -996,8 +996,8 @@ fireeye_pillar() { printf '%s\n'\ "fireeye:"\ " helix:"\ - " api_key: '$HELIXAPIKEY'" "" > "$fireeye_pillar_path"/init.sls + " api_key: '$HELIXAPIKEY'" \ } From 343e9f8b2c416d9cfcb5be29cdeeac73eb5574fc Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 15 Dec 2020 13:37:46 -0500 Subject: [PATCH 54/71] [fix] Only try to stop/remove containers if at least one exists --- setup/so-functions | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 164aa74b2..41d673e60 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1525,8 +1525,10 @@ reinstall_init() { if command -v docker &> /dev/null; then # Stop and remove all so-* containers so files can be changed with more safety - docker stop $(docker ps -a -q --filter "name=so-") - docker rm -f $(docker ps -a -q --filter "name=so-") + if [ $(docker ps -a -q --filter "name=so-") -gt 0 ]; then + docker stop $(docker ps -a -q --filter "name=so-") + docker rm -f $(docker ps -a -q --filter "name=so-") + fi fi local date_string From 7ba10ee6989dadab17d9c12adf05cbe332d7cebe Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 15 Dec 2020 13:38:00 -0500 Subject: [PATCH 55/71] [fix] Add HELIXSENSOR to case for Ubuntu --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 41d673e60..2cf1b28cf 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1675,7 +1675,7 @@ saltify() { 'FLEET') if [ "$OSVER" != 'xenial' ]; then apt-get -y install python3-mysqldb >> "$setup_log" 2>&1; else apt-get -y install python-mysqldb >> "$setup_log" 2>&1; fi ;; - 'MANAGER' | 'EVAL' | 'MANAGERSEARCH' | 'STANDALONE' | 'IMPORT') # TODO: should this also be HELIXSENSOR? + 'MANAGER' | 'EVAL' | 'MANAGERSEARCH' | 'STANDALONE' | 'IMPORT' | 'HELIXSENSOR') # Add saltstack repo(s) wget -q --inet4-only -O - https://repo.saltstack.com"$py_ver_url_path"/ubuntu/"$ubuntu_version"/amd64/archive/3002.2/SALTSTACK-GPG-KEY.pub | apt-key add - >> "$setup_log" 2>&1 From 951556902c25f2af38baebfee5aaace3f59a4b13 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 15 Dec 2020 13:41:00 -0500 Subject: [PATCH 56/71] [fix] Accept salt key on Helix Sensor install --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 6d2bd60c1..f29162852 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -624,7 +624,7 @@ set_redirect >> $setup_log 2>&1 accept_salt_key_remote >> $setup_log 2>&1 fi - if [[ $is_manager || $is_import ]]; then + if [[ $is_manager || $is_import || $is_helix ]]; then set_progress_str 20 'Accepting Salt key' salt-key -ya "$MINION_ID" >> $setup_log 2>&1 fi From 18257762716e1544be9bdb6ffdbdb9617c4c02f0 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 15 Dec 2020 13:58:36 -0500 Subject: [PATCH 57/71] [fix] helix -> helixsensor --- salt/top.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/top.sls b/salt/top.sls index 9d41481fe..c98123c7e 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -47,7 +47,7 @@ base: - sensoroni - salt.lasthighstate - '*_helix and G@saltversion:{{saltversion}}': + '*_helixsensor and G@saltversion:{{saltversion}}': - match: compound - salt.master - ca From c7c3d004ca2d42ca4139210a642464198dcb9aaa Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 15 Dec 2020 14:01:19 -0500 Subject: [PATCH 58/71] [fix] More helix -> helixsensor --- pillar/top.sls | 4 ++-- salt/common/tools/sbin/soup | 4 ++-- salt/suricata/suricata_config.map.jinja | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pillar/top.sls b/pillar/top.sls index 627fed80b..a795e03c1 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -3,7 +3,7 @@ base: - patch.needs_restarting - logrotate - '*_eval or *_helix or *_heavynode or *_sensor or *_standalone or *_import': + '*_eval or *_helixsensor or *_heavynode or *_sensor or *_standalone or *_import': - match: compound - zeek @@ -62,7 +62,7 @@ base: - global - minions.{{ grains.id }} - '*_helix': + '*_helixsensor': - fireeye - zeeklogs - logstash diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index e399780d5..d874c6e31 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -585,9 +585,9 @@ if [ "$UPGRADESALT" == "1" ]; then echo "" echo "Upgrading Salt on the remaining Security Onion nodes from $INSTALLEDSALTVERSION to $NEWSALTVERSION." if [ $is_airgap -eq 0 ]; then - salt -C 'not *_eval and not *_helix and not *_manager and not *_managersearch and not *_standalone' cmd.run "yum clean all" + salt -C 'not *_eval and not *_helixsensor and not *_manager and not *_managersearch and not *_standalone' cmd.run "yum clean all" fi - salt -C 'not *_eval and not *_helix and not *_manager and not *_managersearch and not *_standalone' -b $BATCHSIZE state.apply salt.minion queue=True + salt -C 'not *_eval and not *_helixsensor and not *_manager and not *_managersearch and not *_standalone' -b $BATCHSIZE state.apply salt.minion queue=True echo "" fi diff --git a/salt/suricata/suricata_config.map.jinja b/salt/suricata/suricata_config.map.jinja index d8669c231..8c11901d0 100644 --- a/salt/suricata/suricata_config.map.jinja +++ b/salt/suricata/suricata_config.map.jinja @@ -20,7 +20,7 @@ HOME_NET: "[{{salt['pillar.get']('global:hnmanager', '')}}]" '*_eval': { 'default-packet-size': salt['pillar.get']('sensor:mtu', 1500) + hardware_header, }, - '*_helix': { + '*_helixsensor': { 'default-packet-size': salt['pillar.get']('sensor:mtu', 9000) + hardware_header, }, '*': { From 15347d1209cbaf85140c4724cb3fca16035d2728 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 15 Dec 2020 15:08:33 -0500 Subject: [PATCH 59/71] [fix] More condition changes for Helix --- salt/logstash/init.sls | 2 ++ salt/ssl/init.sls | 2 +- salt/top.sls | 2 -- setup/so-setup | 13 +++++++++---- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index e23e4eef2..d332f737a 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -45,8 +45,10 @@ {% set DOCKER_OPTIONS = salt['pillar.get']('logstash:docker_options', {}) %} {% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %} +{% if grains['role'] != 'so-helix' %} include: - elasticsearch +{% endif %} # Create the logstash group logstashgroup: diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 49e87f784..221c58c93 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -12,7 +12,7 @@ {% set MAINIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] %} {% set CUSTOM_FLEET_HOSTNAME = salt['pillar.get']('global:fleet_custom_hostname', None) %} -{% if grains.id.split('_')|last in ['manager', 'eval', 'standalone', 'import'] %} +{% if grains.id.split('_')|last in ['manager', 'eval', 'standalone', 'import', 'helixsensor'] %} {% set trusttheca_text = salt['cp.get_file_str']('/etc/pki/ca.crt')|replace('\n', '') %} {% set ca_server = grains.id %} {% else %} diff --git a/salt/top.sls b/salt/top.sls index c98123c7e..b6913895d 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -61,9 +61,7 @@ base: - suricata - zeek - redis - {%- if LOGSTASH %} - logstash - {%- endif %} {%- if FILEBEAT %} - filebeat {%- endif %} diff --git a/setup/so-setup b/setup/so-setup index f29162852..7b8621aa9 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -526,10 +526,13 @@ set_redirect >> $setup_log 2>&1 { generate_passwords; secrets_pillar; - add_socore_user_manager; } >> $setup_log 2>&1 fi + if [[ $is_manager || $is_import || $is_helix ]]; then + add_socore_user_manager >> $setup_log 2>&1 + fi + if [[ $is_manager && ! $is_eval ]]; then add_soremote_user_manager >> $setup_log 2>&1 fi @@ -680,8 +683,10 @@ set_redirect >> $setup_log 2>&1 set_progress_str 63 "$(print_salt_state_apply 'common')" salt-call state.apply -l info common >> $setup_log 2>&1 - set_progress_str 64 "$(print_salt_state_apply 'nginx')" - salt-call state.apply -l info nginx >> $setup_log 2>&1 + if [[ ! $is_helix ]]; then + set_progress_str 64 "$(print_salt_state_apply 'nginx')" + salt-call state.apply -l info nginx >> $setup_log 2>&1 + fi if [[ $is_manager || $is_node || $is_import ]]; then set_progress_str 64 "$(print_salt_state_apply 'elasticsearch')" @@ -782,7 +787,7 @@ set_redirect >> $setup_log 2>&1 fi fi - if [[ $is_manager || $is_helix || $is_import ]]; then + if [[ $is_manager || $is_import ]]; then set_progress_str 82 "$(print_salt_state_apply 'utility')" salt-call state.apply -l info utility >> $setup_log 2>&1 fi From e58ca9389682a2dbeadbbe33ce54ea085dd54b5a Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 15 Dec 2020 15:46:55 -0500 Subject: [PATCH 60/71] Add logging for strelka configuration during setup --- setup/so-setup | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 7b8621aa9..70df60feb 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -407,7 +407,10 @@ if [[ $is_manager && ! $is_eval ]]; then fi if [[ "$STRELKA" = 1 ]]; then + info "Enabling Strelka rules" STRELKARULES=1 + else + info "Disabling Strelka rules: STRELKA='$STRELKA'" fi if [ "$MANAGERADV" = 'ADVANCED' ] && [ "$ZEEKVERSION" != 'SURICATA' ]; then @@ -783,7 +786,9 @@ set_redirect >> $setup_log 2>&1 salt-call state.apply -l info strelka >> $setup_log 2>&1 fi if [[ "$STRELKARULES" = 1 ]]; then - /usr/sbin/so-yara-update >> $setup_log 2>&1 + logCmd /usr/sbin/so-yara-update + else + info "Skipping running yara update: STRELKARULES='$STRELKARULES'" fi fi From 6cab65a548c21c8e0488b613e9fdf9a18c2d5aa8 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 15 Dec 2020 16:06:21 -0500 Subject: [PATCH 61/71] Update so-image-common --- salt/common/tools/sbin/so-image-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index dd4cfc979..426f42c02 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -20,7 +20,7 @@ IMAGEREPO=securityonion container_list() { MANAGERCHECK=$1 - FEATURESCHECK=$(lookup_pillar elastic features) + FEATURESCHECK=$(lookup_pillar features elastic) if [ -z "$MANAGERCHECK" ]; then MANAGERCHECK=so-unknown From 06dd3432f8350d9147d44747fee45c8755fdac14 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 15 Dec 2020 16:13:51 -0500 Subject: [PATCH 62/71] Copy the correct files over that soup needs --- salt/common/tools/sbin/soup | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index d874c6e31..342a07c7d 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -422,6 +422,8 @@ verify_latest_update_script() { else echo "You are not running the latest soup version. Updating soup." cp $UPDATE_DIR/salt/common/tools/sbin/soup $DEFAULT_SALT_DIR/salt/common/tools/sbin/ + cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ + cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ salt-call state.apply common queue=True echo "" echo "soup has been updated. Please run soup again." From 7909834722b91757fa65b85ae1b94d75e968580c Mon Sep 17 00:00:00 2001 From: TOoSmOotH Date: Tue, 15 Dec 2020 16:23:49 -0500 Subject: [PATCH 63/71] Clean up previous upgrade dirs in temp --- salt/common/tools/sbin/soup | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 342a07c7d..25a9e633c 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -463,6 +463,8 @@ if [ $is_airgap -eq 0 ]; then airgap_mounted else echo "Cloning Security Onion github repo into $UPDATE_DIR." + echo "Removing previous upgrade sources." + rm -rf $UPDATE_DIR clone_to_tmp fi echo "Generating new repo archive" From e3c8018824d1c7a7d7378b26f3de89d3c4c0ccdd Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 15 Dec 2020 16:44:47 -0500 Subject: [PATCH 64/71] Toggle strelka rules after the user is prompted it strelka should be installed to ensure strelka rules are updated later during the setup process --- setup/so-setup | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 70df60feb..3c59c59cb 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -406,13 +406,6 @@ if [[ $is_manager && ! $is_eval ]]; then whiptail_oinkcode fi - if [[ "$STRELKA" = 1 ]]; then - info "Enabling Strelka rules" - STRELKARULES=1 - else - info "Disabling Strelka rules: STRELKA='$STRELKA'" - fi - if [ "$MANAGERADV" = 'ADVANCED' ] && [ "$ZEEKVERSION" != 'SURICATA' ]; then whiptail_manager_adv_service_zeeklogs fi @@ -421,6 +414,14 @@ fi if [[ $is_manager ]]; then whiptail_components_adv_warning whiptail_enable_components + + if [[ "$STRELKA" = 1 ]]; then + info "Enabling Strelka rules" + STRELKARULES=1 + else + info "Disabling Strelka rules: STRELKA='$STRELKA'" + fi + whiptail_dockernet_check fi From 04a26df4f7777e71f0c78b61ab107edadff6d1df Mon Sep 17 00:00:00 2001 From: TOoSmOotH Date: Tue, 15 Dec 2020 17:05:33 -0500 Subject: [PATCH 65/71] Fix the features suffix --- salt/common/tools/sbin/so-image-common | 19 ++++--------------- salt/common/tools/sbin/soup | 20 +++++++++++++++----- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 426f42c02..31e5c04fb 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -49,13 +49,17 @@ container_list() { "so-curator" "so-domainstats" "so-elastalert" + "so-elasticsearch" + "so-filebeat" "so-fleet" "so-fleet-launcher" "so-freqserver" "so-grafana" "so-idstools" "so-influxdb" + "so-kibana" "so-kratos" + "so-logstash" "so-minio" "so-mysql" "so-nginx" @@ -77,21 +81,6 @@ container_list() { "so-wazuh" "so-zeek" ) - if [[ "$FEATURESCHECK" == "True" ]]; then - TRUSTED_CONTAINERS=( "${TRUSTED_CONTAINERS[@]}" - "so-elasticsearch-features" - "so-filebeat-features" - "so-logstash-features" - "so-kibana-features" - ) - else - TRUSTED_CONTAINERS=( "${TRUSTED_CONTAINERS[@]}" - "so-elasticsearch" - "so-filebeat" - "so-logstash" - "so-kibana" - ) - fi else TRUSTED_CONTAINERS=( "so-filebeat" diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 25a9e633c..314a86b20 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -467,6 +467,12 @@ else rm -rf $UPDATE_DIR clone_to_tmp fi + +echo "" +echo "Verifying we have the latest soup script." +verify_latest_update_script +echo "" + echo "Generating new repo archive" generate_and_clean_tarballs if [ -f /usr/sbin/so-image-common ]; then @@ -475,11 +481,6 @@ else add_common fi -echo "" -echo "Verifying we have the latest soup script." -verify_latest_update_script -echo "" - echo "Let's see if we need to update Security Onion." upgrade_check space_check @@ -496,6 +497,15 @@ if [ $is_airgap -eq 0 ]; then else update_registry update_docker_containers "soup" + FEATURESCHECK=$(lookup_pillar features elastic) + if [[ "$FEATURESCHECK" == "True" ]]; then + TRUSTED_CONTAINERS=( \ + "so-elasticsearch" \ + "so-filebeat" \ + "so-kibana" \ + "so-logstash" ) + update_docker_containers "features" "-features" + fi fi echo "" echo "Stopping Salt Minion service." From 082fd51b0545f8522ad6bcdf33e42a916f3faa32 Mon Sep 17 00:00:00 2001 From: TOoSmOotH Date: Tue, 15 Dec 2020 17:07:40 -0500 Subject: [PATCH 66/71] Remove extra variable --- salt/common/tools/sbin/so-image-common | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 31e5c04fb..767f9d21c 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -20,8 +20,7 @@ IMAGEREPO=securityonion container_list() { MANAGERCHECK=$1 - FEATURESCHECK=$(lookup_pillar features elastic) - + if [ -z "$MANAGERCHECK" ]; then MANAGERCHECK=so-unknown if [ -f /etc/salt/grains ]; then From 87882b4d91e73beb53bef7be822b8f8b34a2f03e Mon Sep 17 00:00:00 2001 From: TOoSmOotH Date: Tue, 15 Dec 2020 18:18:26 -0500 Subject: [PATCH 67/71] Fix upgrade function --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 314a86b20..d9619a1fa 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -319,7 +319,7 @@ rc3_to_2.3.0() { else DOCKERSTUFF="${DOCKERGREP//\"}" DOCKERSTUFFBIP=$(echo $DOCKERSTUFF | awk -F'.' '{print $1,$2,$3,1}' OFS='.')/24 - echo " range: '$DOCKERSTUFF'/24" >> /opt/so/saltstack/local/pillar/global.sls + echo " range: '$DOCKERSTUFF/24'" >> /opt/so/saltstack/local/pillar/global.sls echo " bip: '$DOCKERSTUFFBIP'" >> /opt/so/saltstack/local/pillar/global.sls fi From e30d7a8d8e7435a06432738e792bda01812288f6 Mon Sep 17 00:00:00 2001 From: TOoSmOotH Date: Tue, 15 Dec 2020 18:25:41 -0500 Subject: [PATCH 68/71] Fix upgrade docker variable --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index d9619a1fa..5ad2b87b8 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -312,7 +312,7 @@ rc3_to_2.3.0() { # Fix daemon.json by managing it echo "docker:" >> /opt/so/saltstack/local/pillar/global.sls - DOCKERGREP=$(cat /etc/docker/daemon.json | grep base | awk {'print $3'} | cut -f1 -d"/") + DOCKERGREP=$(cat /etc/docker/daemon.json | grep base | awk {'print $3'} | cut -f1 -d"," | tr -d '"') if [ -z "$DOCKERGREP" ]; then echo " range: '172.17.0.0/24'" >> /opt/so/saltstack/local/pillar/global.sls echo " bip: '172.17.0.1/24'" >> /opt/so/saltstack/local/pillar/global.sls From f1be6cc259c36d11108b8cfe971e514ea48f2f7d Mon Sep 17 00:00:00 2001 From: TOoSmOotH Date: Tue, 15 Dec 2020 18:32:07 -0500 Subject: [PATCH 69/71] Check MD5 of all components --- salt/common/tools/sbin/soup | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 5ad2b87b8..11074f7cf 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -417,10 +417,15 @@ verify_latest_update_script() { # Check to see if the update scripts match. If not run the new one. CURRENTSOUP=$(md5sum /opt/so/saltstack/default/salt/common/tools/sbin/soup | awk '{print $1}') GITSOUP=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/soup | awk '{print $1}') - if [[ "$CURRENTSOUP" == "$GITSOUP" ]]; then + CURRENTCMN=$(md5sum /opt/so/saltstack/default/salt/common/tools/sbin/so-common | awk '{print $1}') + GITCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-common | awk '{print $1}') + CURRENTIMGCMN=$(md5sum /opt/so/saltstack/default/salt/common/tools/sbin/so-image-common | awk '{print $1}') + GITIMGCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-image-common | awk '{print $1}') + + if [[ "$CURRENTSOUP" == "$GITSOUP" ]] && [[ "$CURRENTCMN" == "$GITCMN" ]] && [[ "$CURRENTIMGCMN" == "$GITIMGCMN" ]]; then echo "This version of the soup script is up to date. Proceeding." else - echo "You are not running the latest soup version. Updating soup." + echo "You are not running the latest soup version. Updating soup and its components. Might take multiple runs to complete" cp $UPDATE_DIR/salt/common/tools/sbin/soup $DEFAULT_SALT_DIR/salt/common/tools/sbin/ cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ From 4ca4141819a7773d43068ef0a7432636871f21e8 Mon Sep 17 00:00:00 2001 From: TOoSmOotH Date: Tue, 15 Dec 2020 19:29:35 -0500 Subject: [PATCH 70/71] Fix conditional statement --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 11074f7cf..6c5897d11 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -422,7 +422,7 @@ verify_latest_update_script() { CURRENTIMGCMN=$(md5sum /opt/so/saltstack/default/salt/common/tools/sbin/so-image-common | awk '{print $1}') GITIMGCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-image-common | awk '{print $1}') - if [[ "$CURRENTSOUP" == "$GITSOUP" ]] && [[ "$CURRENTCMN" == "$GITCMN" ]] && [[ "$CURRENTIMGCMN" == "$GITIMGCMN" ]]; then + if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" ]]; then echo "This version of the soup script is up to date. Proceeding." else echo "You are not running the latest soup version. Updating soup and its components. Might take multiple runs to complete" From 805e25f495195fddb588e26934b9f62b4492285d Mon Sep 17 00:00:00 2001 From: TOoSmOotH Date: Tue, 15 Dec 2020 20:40:59 -0500 Subject: [PATCH 71/71] Fix typeo --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 6c5897d11..4d168c077 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -300,7 +300,7 @@ rc3_to_2.3.0() { echo " node_checkin_interval_ms: 10000" >> /opt/so/saltstack/local/pillar/global.sls # Update pillar fiels for new sensoroni functionality - for file in /opt/so/saltstack/local/pillat/minions/*; do + for file in /opt/so/saltstack/local/pillar/minions/*; do echo "sensoroni:" >> $file echo " node_description:" >> $file local SOMEADDRESS=$(cat $file | grep mainip | tail -n 1 | awk '{print $2'})