From e1757926cf79174b09201ab17b97e52f6c8fd329 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 20 Jan 2022 15:26:03 -0500 Subject: [PATCH 1/5] start cron and reenable highstate on soup exit --- salt/common/tools/sbin/so-common | 10 ++++++++++ salt/common/tools/sbin/soup | 25 ++++++------------------- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 584e57926..53fe16480 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -405,6 +405,16 @@ set_version() { fi } +systemctl_func() { + local action=$1 + local service_name=$2 + + echo "" + echo "${action^}ing $service_name service at $(date +"%T.%6N")" + systemctl $action $service_name + echo "" +} + has_uppercase() { local string=$1 diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 2c49ce444..5c15ea453 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -483,10 +483,7 @@ stop_salt_master() { echo "Storing salt-master pid." MASTERPID=$(pgrep salt-master | head -1) echo "Found salt-master PID $MASTERPID" - echo "" - echo "Stopping Salt Master service at $(date +"%T.%6N")" - systemctl stop salt-master - echo "" + systemctl_func "stop" "salt-master" timeout 30 tail --pid=$MASTERPID -f /dev/null || echo "salt-master still running at $(date +"%T.%6N") after waiting 30s. We cannot kill due to systemd restart option." } @@ -505,8 +502,7 @@ stop_salt_minion() { echo "Storing salt-minion pid." MINIONPID=$(pgrep salt-minion | head -1) echo "Found salt-minion PID $MINIONPID" - echo "Stopping Salt Minion service at $(date +"%T.%6N")." - systemctl stop salt-minion + systemctl_func "stop" "salt-minion" set +e timeout 30 tail --pid=$MINIONPID -f /dev/null || echo "Killing salt-minion at $(date +"%T.%6N") after waiting 30s" && pkill -9 -ef /usr/bin/salt-minion @@ -958,7 +954,7 @@ fix_wazuh() { } main() { - trap 'check_err $?' EXIT + trap 'trap_exit_code=$?; systemctl_func "start" "$cron_service_name"; salt-call state.enable highstate -l info --local; check_err $trap_exit_code' EXIT if [ -n "$BRANCH" ]; then echo "SOUP will use the $BRANCH branch." @@ -1033,9 +1029,7 @@ main() { echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION." echo "" - echo "Stopping $cron_service_name service at $(date +"%T.%6N")." - echo "" - systemctl stop "$cron_service_name" + systemctl_func "stop" "$cron_service_name" # update mine items prior to stopping salt-minion and salt-master update_salt_mine @@ -1104,9 +1098,7 @@ main() { echo "Locking down Salt Master for upgrade at $(date +"%T.%6N")." masterlock - echo "" - echo "Starting Salt Master service at $(date +"%T.%6N")." - systemctl start salt-master + systemctl_func "start" "salt-master" # Testing that salt-master is up by checking that is it connected to itself set +e @@ -1150,9 +1142,7 @@ main() { masterunlock - echo "" - echo "Starting Salt Master service at $(date +"%T.%6N") ." - systemctl start salt-master + systemctl_func "start" "salt-master" set +e echo "Waiting on the Salt Master service to be ready." @@ -1207,9 +1197,6 @@ main() { esac fi - echo "Starting $cron_service_name service at $(date +"%T.%6N")." - systemctl start "$cron_service_name" - if [[ $NUM_MINIONS -gt 1 ]]; then cat << EOF From 01cb5053389fc139f2e9baaddc738f4225d8dca1 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 20 Jan 2022 16:31:01 -0500 Subject: [PATCH 2/5] start cron and enable highstate if soup exits on error --- salt/common/tools/sbin/so-common | 9 +++++++-- salt/common/tools/sbin/soup | 6 +++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 53fe16480..26411e381 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -407,11 +407,16 @@ set_version() { systemctl_func() { local action=$1 + local echo_action=$1 local service_name=$2 + if [[ "$echo_action" == "stop" ]]; then + $echo_action = "stopp" + fi + echo "" - echo "${action^}ing $service_name service at $(date +"%T.%6N")" - systemctl $action $service_name + echo "${echo_action^}ing $service_name service at $(date +"%T.%6N")" + systemctl $action $service_name && echo "Successfully ${echo_action}ed $service_name." || echo "Failed to $action $service_name." echo "" } diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 5c15ea453..9c2379a8c 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -92,6 +92,10 @@ check_err() { if [[ $exit_code -ge 64 && $exit_code -le 113 ]]; then echo "$err_msg" fi + set +e + systemctl_func "start" "$cron_service_name" + echo "Ensuring highstate is enabled." + salt-call state.enable highstate --local exit $exit_code fi @@ -954,7 +958,7 @@ fix_wazuh() { } main() { - trap 'trap_exit_code=$?; systemctl_func "start" "$cron_service_name"; salt-call state.enable highstate -l info --local; check_err $trap_exit_code' EXIT + trap 'check_err $?' EXIT if [ -n "$BRANCH" ]; then echo "SOUP will use the $BRANCH branch." From 00d0eb1ce562471598e06cfad350cf5754839d97 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 20 Jan 2022 16:37:33 -0500 Subject: [PATCH 3/5] fix setting var --- salt/common/tools/sbin/so-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 26411e381..d50ec0672 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -411,7 +411,7 @@ systemctl_func() { local service_name=$2 if [[ "$echo_action" == "stop" ]]; then - $echo_action = "stopp" + echo_action="stopp" fi echo "" From baa93301b51ba9a46b228f69d87ec51e3152dd0a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 20 Jan 2022 16:53:33 -0500 Subject: [PATCH 4/5] enable cron at the end of soup --- salt/common/tools/sbin/soup | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 9c2379a8c..9b305fc76 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -1185,6 +1185,8 @@ main() { echo "Checking for necessary user migrations." so-user migrate + systemctl_func "start" "$cron_service_name" + if [[ -n $lsl_msg ]]; then case $lsl_msg in 'distributed') From 281e5d9b2582ae3f616d2f8fe73ef2fd49df46ac Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 21 Jan 2022 08:09:04 -0500 Subject: [PATCH 5/5] remove salt.enable_higstate state --- salt/salt/enable_highstate.sls | 7 ------- salt/top.sls | 1 - 2 files changed, 8 deletions(-) delete mode 100644 salt/salt/enable_highstate.sls diff --git a/salt/salt/enable_highstate.sls b/salt/salt/enable_highstate.sls deleted file mode 100644 index 72e5c1410..000000000 --- a/salt/salt/enable_highstate.sls +++ /dev/null @@ -1,7 +0,0 @@ -enable_highstate: - module.run: - - state.enable: - - states: - - highstate - - unless: pgrep soup - \ No newline at end of file diff --git a/salt/top.sls b/salt/top.sls index 513439255..0efc73d02 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -20,7 +20,6 @@ base: '*': - - salt.enable_highstate - cron.running 'not G@saltversion:{{saltversion}}':