From fcfd3e3758d725a455b194eda81d25409e79f7db Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 17 Nov 2020 16:09:30 -0500 Subject: [PATCH 01/16] change location yum/apt verison locks https://github.com/Security-Onion-Solutions/securityonion/issues/1961 --- salt/salt/map.jinja | 6 +++--- salt/salt/minion.sls | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index cf62f6db3..f238c3eae 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -12,12 +12,12 @@ {% if grains.saltversion|string != SALTVERSION|string %} {% if grains.os|lower in ['centos', 'redhat'] %} {% if ISAIRGAP is sameas true %} - {% set UPGRADECOMMAND = 'yum clean all && yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -r -F -x python3 stable ' ~ SALTVERSION ~ ' && systemctl restart salt-minion' %} + {% set UPGRADECOMMAND = 'yum clean all && yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -r -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && systemctl restart salt-minion' %} {% else %} - {% set UPGRADECOMMAND = 'yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -F -x python3 stable ' ~ SALTVERSION ~ ' && systemctl restart salt-minion' %} + {% set UPGRADECOMMAND = 'yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && systemctl restart salt-minion' %} {% endif %} {% elif grains.os|lower == 'ubuntu' %} - {% set UPGRADECOMMAND = 'apt-mark unhold salt-common && apt-mark unhold salt-minion && /usr/sbin/bootstrap-salt.sh -F -x python3 stable ' ~ SALTVERSION ~ ' && systemctl restart salt-minion' %} + {% set UPGRADECOMMAND = 'apt-mark unhold salt-common && apt-mark unhold salt-minion && /usr/sbin/bootstrap-salt.sh -F -x python3 stable ' ~ SALTVERSION ~ ' && apt-mark hold salt-common && apt-mark hold salt-minion && systemctl restart salt-minion' %} {% endif %} {% else %} {% set UPGRADECOMMAND = 'echo Already running Salt Minon version ' ~ SALTVERSION %} diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 841762d37..4a92c8d80 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -21,6 +21,7 @@ salt_minion_package: - {{ COMMON }} - salt-minion - hold: True + - onlyif: "[[ '{{INSTALLEDSALTVERSION}}' == '{{SALTVERSION}}' ]]" salt_minion_service: service.running: From 1fd2196dd5bdebd767a184b2363067b925fd2208 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 17 Nov 2020 16:18:50 -0500 Subject: [PATCH 02/16] fix check of salt was upgraded during soup for ubuntu and centos --- salt/common/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 0453ea29d..df2776fee 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -460,8 +460,8 @@ fi echo "Checking if Salt was upgraded." echo "" -# Check that Salt was upgraded, should be 3 'salt' packages on a manager node. salt-minion, salt-master and salt or salt-common depending on Ubuntu or CentOS. we could add salt-syndic in the future so checking that there are at least 3 packages -if [[ `rpm -qa | grep salt | grep $NEWSALTVERSION | wc -l` < 3 ]]; then +# Check that Salt was upgraded +if [[ `salt --versions-report | grep Salt: | awk {'print $2'}` == "$NEWSALTVERSION" ]]; then echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG." echo "Once the issue is resolved, run soup again." echo "Exiting." From 7d1cf56160cc37bbfb92dd7683611566f710f8a1 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 17 Nov 2020 16:29:35 -0500 Subject: [PATCH 03/16] change check of salt was upgraded during soup for ubuntu and centos --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index df2776fee..73432d5f1 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -461,7 +461,7 @@ fi echo "Checking if Salt was upgraded." echo "" # Check that Salt was upgraded -if [[ `salt --versions-report | grep Salt: | awk {'print $2'}` == "$NEWSALTVERSION" ]]; then +if [[ $(salt --versions-report | grep Salt: | awk {'print $2'}) -eq "$NEWSALTVERSION" ]]; then echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG." echo "Once the issue is resolved, run soup again." echo "Exiting." From 2bfc48be35f1c7e6fcdde30af04caa16211eb199 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 17 Nov 2020 16:31:11 -0500 Subject: [PATCH 04/16] change check of salt was upgraded during soup for ubuntu and centos --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 73432d5f1..622e06eed 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -461,7 +461,7 @@ fi echo "Checking if Salt was upgraded." echo "" # Check that Salt was upgraded -if [[ $(salt --versions-report | grep Salt: | awk {'print $2'}) -eq "$NEWSALTVERSION" ]]; then +if [[ $(salt --versions-report | grep Salt: | awk {'print $2'}) == "$NEWSALTVERSION" ]]; then echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG." echo "Once the issue is resolved, run soup again." echo "Exiting." From 42126f125bcbf2e7a33a0d98e753ac3b338d26f6 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 17 Nov 2020 17:00:59 -0500 Subject: [PATCH 05/16] change verison check to != --- salt/common/tools/sbin/soup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 622e06eed..d8e3ee0bb 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -461,7 +461,7 @@ fi echo "Checking if Salt was upgraded." echo "" # Check that Salt was upgraded -if [[ $(salt --versions-report | grep Salt: | awk {'print $2'}) == "$NEWSALTVERSION" ]]; then +if [[ $(salt --versions-report | grep Salt: | awk {'print $2'}) != "$NEWSALTVERSION" ]]; then echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG." echo "Once the issue is resolved, run soup again." echo "Exiting." @@ -510,6 +510,7 @@ masterunlock echo "" echo "Starting Salt Master service." systemctl start salt-master +echo "Running a highstate. This could take several minutes." highstate playbook unmount_update From 695cce0b509586ba90c7fdeddc6937a1ce6eeade Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 17 Nov 2020 17:54:56 -0500 Subject: [PATCH 06/16] upgrad command changes https://github.com/Security-Onion-Solutions/securityonion/issues/1961 --- salt/salt/map.jinja | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index f238c3eae..422fa76d0 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -12,12 +12,12 @@ {% if grains.saltversion|string != SALTVERSION|string %} {% if grains.os|lower in ['centos', 'redhat'] %} {% if ISAIRGAP is sameas true %} - {% set UPGRADECOMMAND = 'yum clean all && yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -r -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && systemctl restart salt-minion' %} + {% set UPGRADECOMMAND = 'yum clean all && yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -r -F -x python3 stable ' ~ SALTVERSION ~ ' && salt-call --local service.restart salt-minion && yum versionlock add "salt-*"' %} {% else %} - {% set UPGRADECOMMAND = 'yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && systemctl restart salt-minion' %} + {% set UPGRADECOMMAND = 'yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -F -x python3 stable ' ~ SALTVERSION ~ ' && salt-call --local service.restart salt-minion && yum versionlock add "salt-*"' %} {% endif %} {% elif grains.os|lower == 'ubuntu' %} - {% set UPGRADECOMMAND = 'apt-mark unhold salt-common && apt-mark unhold salt-minion && /usr/sbin/bootstrap-salt.sh -F -x python3 stable ' ~ SALTVERSION ~ ' && apt-mark hold salt-common && apt-mark hold salt-minion && systemctl restart salt-minion' %} + {% set UPGRADECOMMAND = 'apt-mark unhold salt-common && apt-mark unhold salt-minion && /usr/sbin/bootstrap-salt.sh -F -x python3 stable ' ~ SALTVERSION ~ ' && salt-call --local service.restart salt-minion && apt-mark hold salt-common && apt-mark hold salt-minion' %} {% endif %} {% else %} {% set UPGRADECOMMAND = 'echo Already running Salt Minon version ' ~ SALTVERSION %} From 7291d64e8202db4bc27f6a50758eba5787f3de9b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 17 Nov 2020 18:38:01 -0500 Subject: [PATCH 07/16] pkill salt-minion before restartiong salt-minion service https://github.com/Security-Onion-Solutions/securityonion/issues/1961 --- salt/salt/map.jinja | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 422fa76d0..42f9260ad 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -12,12 +12,12 @@ {% if grains.saltversion|string != SALTVERSION|string %} {% if grains.os|lower in ['centos', 'redhat'] %} {% if ISAIRGAP is sameas true %} - {% set UPGRADECOMMAND = 'yum clean all && yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -r -F -x python3 stable ' ~ SALTVERSION ~ ' && salt-call --local service.restart salt-minion && yum versionlock add "salt-*"' %} + {% set UPGRADECOMMAND = 'yum clean all && yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -r -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && pkill -9 -ef /usr/bin/salt-minion && systemctl restart salt-minion' %} {% else %} - {% set UPGRADECOMMAND = 'yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -F -x python3 stable ' ~ SALTVERSION ~ ' && salt-call --local service.restart salt-minion && yum versionlock add "salt-*"' %} + {% set UPGRADECOMMAND = 'yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && pkill -9 -ef /usr/bin/salt-minion && systemctl restart salt-minion' %} {% endif %} {% elif grains.os|lower == 'ubuntu' %} - {% set UPGRADECOMMAND = 'apt-mark unhold salt-common && apt-mark unhold salt-minion && /usr/sbin/bootstrap-salt.sh -F -x python3 stable ' ~ SALTVERSION ~ ' && salt-call --local service.restart salt-minion && apt-mark hold salt-common && apt-mark hold salt-minion' %} + {% set UPGRADECOMMAND = 'apt-mark unhold salt-common && apt-mark unhold salt-minion && /usr/sbin/bootstrap-salt.sh -F -x python3 stable ' ~ SALTVERSION ~ ' && apt-mark hold salt-common && apt-mark hold salt-minion && pkill -9 -ef /usr/bin/salt-minion && systemctl restart salt-minion' %} {% endif %} {% else %} {% set UPGRADECOMMAND = 'echo Already running Salt Minon version ' ~ SALTVERSION %} From 5c6e9e0e3a7eee3f3fae7a8e5de5bf82ab214458 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 17 Nov 2020 19:40:42 -0500 Subject: [PATCH 08/16] run a highstate and let that start the salt-minion back up https://github.com/Security-Onion-Solutions/securityonion/issues/1961 --- salt/salt/map.jinja | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 42f9260ad..711701b69 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -12,12 +12,12 @@ {% if grains.saltversion|string != SALTVERSION|string %} {% if grains.os|lower in ['centos', 'redhat'] %} {% if ISAIRGAP is sameas true %} - {% set UPGRADECOMMAND = 'yum clean all && yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -r -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && pkill -9 -ef /usr/bin/salt-minion && systemctl restart salt-minion' %} + {% set UPGRADECOMMAND = 'yum clean all && yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -s 90 -r -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && pkill -9 -ef /usr/bin/salt-minion && salt-call state.highstate --log-file-level info' %} {% else %} - {% set UPGRADECOMMAND = 'yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && pkill -9 -ef /usr/bin/salt-minion && systemctl restart salt-minion' %} + {% set UPGRADECOMMAND = 'yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -s 90 -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && pkill -9 -ef /usr/bin/salt-minion && salt-call state.highstate --log-file-level info' %} {% endif %} {% elif grains.os|lower == 'ubuntu' %} - {% set UPGRADECOMMAND = 'apt-mark unhold salt-common && apt-mark unhold salt-minion && /usr/sbin/bootstrap-salt.sh -F -x python3 stable ' ~ SALTVERSION ~ ' && apt-mark hold salt-common && apt-mark hold salt-minion && pkill -9 -ef /usr/bin/salt-minion && systemctl restart salt-minion' %} + {% set UPGRADECOMMAND = 'apt-mark unhold salt-common && apt-mark unhold salt-minion && /usr/sbin/bootstrap-salt.sh -s 90 -F -x python3 stable ' ~ SALTVERSION ~ ' && apt-mark hold salt-common && apt-mark hold salt-minion && pkill -9 -ef /usr/bin/salt-minion && salt-call state.highstate --log-file-level info' %} {% endif %} {% else %} {% set UPGRADECOMMAND = 'echo Already running Salt Minon version ' ~ SALTVERSION %} From c95619d335b095d018ca8634c1fc60dc52ad4b68 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 18 Nov 2020 08:35:56 -0500 Subject: [PATCH 09/16] change upgradecommand order https://github.com/Security-Onion-Solutions/securityonion/issues/1961 --- salt/salt/map.jinja | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 711701b69..d11491046 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -12,12 +12,12 @@ {% if grains.saltversion|string != SALTVERSION|string %} {% if grains.os|lower in ['centos', 'redhat'] %} {% if ISAIRGAP is sameas true %} - {% set UPGRADECOMMAND = 'yum clean all && yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -s 90 -r -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && pkill -9 -ef /usr/bin/salt-minion && salt-call state.highstate --log-file-level info' %} + {% set UPGRADECOMMAND = 'yum clean all && yum versionlock delete "salt-*" && systemctl stop salt-minion && pkill -9 -ef /usr/bin/salt-minion && /usr/sbin/bootstrap-salt.sh -s 90 -r -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && salt-call state.highstate --l info' %} {% else %} - {% set UPGRADECOMMAND = 'yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -s 90 -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && pkill -9 -ef /usr/bin/salt-minion && salt-call state.highstate --log-file-level info' %} + {% set UPGRADECOMMAND = 'yum versionlock delete "salt-*" && systemctl stop salt-minion && pkill -9 -ef /usr/bin/salt-minion && /usr/sbin/bootstrap-salt.sh -s 90 -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && salt-call state.highstate --l info' %} {% endif %} {% elif grains.os|lower == 'ubuntu' %} - {% set UPGRADECOMMAND = 'apt-mark unhold salt-common && apt-mark unhold salt-minion && /usr/sbin/bootstrap-salt.sh -s 90 -F -x python3 stable ' ~ SALTVERSION ~ ' && apt-mark hold salt-common && apt-mark hold salt-minion && pkill -9 -ef /usr/bin/salt-minion && salt-call state.highstate --log-file-level info' %} + {% set UPGRADECOMMAND = 'apt-mark unhold salt-common && apt-mark unhold salt-minion && systemctl stop salt-minion && pkill -9 -ef /usr/bin/salt-minion && /usr/sbin/bootstrap-salt.sh -s 90 -F -x python3 stable ' ~ SALTVERSION ~ ' && apt-mark hold salt-common && apt-mark hold salt-minion && salt-call state.highstate --l info' %} {% endif %} {% else %} {% set UPGRADECOMMAND = 'echo Already running Salt Minon version ' ~ SALTVERSION %} From 4bb1ad9799059a856f1eafbcf4bd1bffb81a7ee2 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 18 Nov 2020 09:29:09 -0500 Subject: [PATCH 10/16] dont restart or kill salt-minon in upgrade command https://github.com/Security-Onion-Solutions/securityonion/issues/1961 --- salt/salt/map.jinja | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index d11491046..bd1f6b1ff 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -12,12 +12,12 @@ {% if grains.saltversion|string != SALTVERSION|string %} {% if grains.os|lower in ['centos', 'redhat'] %} {% if ISAIRGAP is sameas true %} - {% set UPGRADECOMMAND = 'yum clean all && yum versionlock delete "salt-*" && systemctl stop salt-minion && pkill -9 -ef /usr/bin/salt-minion && /usr/sbin/bootstrap-salt.sh -s 90 -r -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && salt-call state.highstate --l info' %} + {% set UPGRADECOMMAND = 'yum clean all && yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -s 120 -r -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && salt-call state.highstate -l info' %} {% else %} - {% set UPGRADECOMMAND = 'yum versionlock delete "salt-*" && systemctl stop salt-minion && pkill -9 -ef /usr/bin/salt-minion && /usr/sbin/bootstrap-salt.sh -s 90 -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && salt-call state.highstate --l info' %} + {% set UPGRADECOMMAND = 'yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && salt-call state.highstate -l info' %} {% endif %} {% elif grains.os|lower == 'ubuntu' %} - {% set UPGRADECOMMAND = 'apt-mark unhold salt-common && apt-mark unhold salt-minion && systemctl stop salt-minion && pkill -9 -ef /usr/bin/salt-minion && /usr/sbin/bootstrap-salt.sh -s 90 -F -x python3 stable ' ~ SALTVERSION ~ ' && apt-mark hold salt-common && apt-mark hold salt-minion && salt-call state.highstate --l info' %} + {% set UPGRADECOMMAND = 'apt-mark unhold salt-common && apt-mark unhold salt-minion && /usr/sbin/bootstrap-salt.sh -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' && apt-mark hold salt-common && apt-mark hold salt-minion && salt-call state.highstate -l info' %} {% endif %} {% else %} {% set UPGRADECOMMAND = 'echo Already running Salt Minon version ' ~ SALTVERSION %} From bddc3d6df9cae3f87821d312afc55c30f1ba781b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 18 Nov 2020 10:40:23 -0500 Subject: [PATCH 11/16] kill all salt-minion again since they hang and redirect highstate to a logfile --- salt/salt/map.jinja | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index bd1f6b1ff..102c4ee44 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -12,12 +12,12 @@ {% if grains.saltversion|string != SALTVERSION|string %} {% if grains.os|lower in ['centos', 'redhat'] %} {% if ISAIRGAP is sameas true %} - {% set UPGRADECOMMAND = 'yum clean all && yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -s 120 -r -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && salt-call state.highstate -l info' %} + {% set UPGRADECOMMAND = 'yum clean all && yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -s 120 -r -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && pkill -9 -ef /usr/bin/salt-minion && salt-call state.highstate -l info > /opt/so/log/salt/salt-upgrade-highstate 2>&1' %} {% else %} - {% set UPGRADECOMMAND = 'yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && salt-call state.highstate -l info' %} + {% set UPGRADECOMMAND = 'yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && pkill -9 -ef /usr/bin/salt-minion && salt-call state.highstate -l info > /opt/so/log/salt/salt-upgrade-highstate 2>&1' %} {% endif %} {% elif grains.os|lower == 'ubuntu' %} - {% set UPGRADECOMMAND = 'apt-mark unhold salt-common && apt-mark unhold salt-minion && /usr/sbin/bootstrap-salt.sh -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' && apt-mark hold salt-common && apt-mark hold salt-minion && salt-call state.highstate -l info' %} + {% set UPGRADECOMMAND = 'apt-mark unhold salt-common && apt-mark unhold salt-minion && /usr/sbin/bootstrap-salt.sh -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' && apt-mark hold salt-common && apt-mark hold salt-minion && pkill -9 -ef /usr/bin/salt-minion && salt-call state.highstate -l info > /opt/so/log/salt/salt-upgrade-highstate 2>&1' %} {% endif %} {% else %} {% set UPGRADECOMMAND = 'echo Already running Salt Minon version ' ~ SALTVERSION %} From ce70e0a61f23d5c56f9a5faa9d7f9ac7af7ae291 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 18 Nov 2020 11:51:28 -0500 Subject: [PATCH 12/16] changes to upgradecommand https://github.com/Security-Onion-Solutions/securityonion/issues/1961 --- salt/salt/map.jinja | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 102c4ee44..bb4a01fa9 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -12,12 +12,12 @@ {% if grains.saltversion|string != SALTVERSION|string %} {% if grains.os|lower in ['centos', 'redhat'] %} {% if ISAIRGAP is sameas true %} - {% set UPGRADECOMMAND = 'yum clean all && yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -s 120 -r -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && pkill -9 -ef /usr/bin/salt-minion && salt-call state.highstate -l info > /opt/so/log/salt/salt-upgrade-highstate 2>&1' %} + {% set UPGRADECOMMAND = 'yum clean all && yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -s 120 -r -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && salt-call state.highstate && pkill -9 -ef /usr/bin/salt-minion && systemctl restart salt-minion' %} {% else %} - {% set UPGRADECOMMAND = 'yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && pkill -9 -ef /usr/bin/salt-minion && salt-call state.highstate -l info > /opt/so/log/salt/salt-upgrade-highstate 2>&1' %} + {% set UPGRADECOMMAND = 'yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && salt-call state.highstate && pkill -9 -ef /usr/bin/salt-minion && systemctl restart salt-minion' %} {% endif %} {% elif grains.os|lower == 'ubuntu' %} - {% set UPGRADECOMMAND = 'apt-mark unhold salt-common && apt-mark unhold salt-minion && /usr/sbin/bootstrap-salt.sh -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' && apt-mark hold salt-common && apt-mark hold salt-minion && pkill -9 -ef /usr/bin/salt-minion && salt-call state.highstate -l info > /opt/so/log/salt/salt-upgrade-highstate 2>&1' %} + {% set UPGRADECOMMAND = 'apt-mark unhold salt-common && apt-mark unhold salt-minion && /usr/sbin/bootstrap-salt.sh -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' && apt-mark hold salt-common && apt-mark hold salt-minion && salt-call state.highstate && pkill -9 -ef /usr/bin/salt-minion && systemctl restart salt-minion' %} {% endif %} {% else %} {% set UPGRADECOMMAND = 'echo Already running Salt Minon version ' ~ SALTVERSION %} From d0e7b5b55ae6d13d0386bc8d99ab3564a105330e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 18 Nov 2020 12:32:21 -0500 Subject: [PATCH 13/16] only ensure salt-minion service is running if salt is on right verison https://github.com/Security-Onion-Solutions/securityonion/issues/1961 --- salt/salt/minion.sls | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 4a92c8d80..f7d2126c1 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -26,4 +26,5 @@ salt_minion_package: salt_minion_service: service.running: - name: salt-minion - - enable: True \ No newline at end of file + - enable: True + - onlyif: "[[ '{{INSTALLEDSALTVERSION}}' == '{{SALTVERSION}}' ]]" \ No newline at end of file From 276c011a4f8b6727567a7f05d3dc47f6744a3bf4 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 18 Nov 2020 13:22:11 -0500 Subject: [PATCH 14/16] queue state and change upgrade command https://github.com/Security-Onion-Solutions/securityonion/issues/1961 --- salt/common/tools/sbin/soup | 2 +- salt/salt/map.jinja | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index d8e3ee0bb..a256bb30d 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -521,7 +521,7 @@ if [ "$UPGRADESALT" == "1" ]; then if [ $is_airgap -eq 0 ]; then salt -C 'not *_eval and not *_helix and not *_manager and not *_managersearch and not *_standalone' cmd.run "yum clean all" fi - salt -C 'not *_eval and not *_helix and not *_manager and not *_managersearch and not *_standalone' -b $BATCHSIZE state.apply salt.minion + salt -C 'not *_eval and not *_helix and not *_manager and not *_managersearch and not *_standalone' -b $BATCHSIZE state.apply salt.minion queue=True echo "" fi diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index bb4a01fa9..1688e562a 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -12,12 +12,12 @@ {% if grains.saltversion|string != SALTVERSION|string %} {% if grains.os|lower in ['centos', 'redhat'] %} {% if ISAIRGAP is sameas true %} - {% set UPGRADECOMMAND = 'yum clean all && yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -s 120 -r -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && salt-call state.highstate && pkill -9 -ef /usr/bin/salt-minion && systemctl restart salt-minion' %} + {% set UPGRADECOMMAND = 'yum clean all && yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -s 120 -r -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && pkill -9 -ef /usr/bin/salt-minion && salt-call state.highstate && sleep 300 && salt-call state.apply salt.minion queue=True'%} {% else %} - {% set UPGRADECOMMAND = 'yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && salt-call state.highstate && pkill -9 -ef /usr/bin/salt-minion && systemctl restart salt-minion' %} + {% set UPGRADECOMMAND = 'yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && pkill -9 -ef /usr/bin/salt-minion && salt-call state.highstate && sleep 300 && salt-call state.apply salt.minion queue=True' %} {% endif %} {% elif grains.os|lower == 'ubuntu' %} - {% set UPGRADECOMMAND = 'apt-mark unhold salt-common && apt-mark unhold salt-minion && /usr/sbin/bootstrap-salt.sh -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' && apt-mark hold salt-common && apt-mark hold salt-minion && salt-call state.highstate && pkill -9 -ef /usr/bin/salt-minion && systemctl restart salt-minion' %} + {% set UPGRADECOMMAND = 'apt-mark unhold salt-common && apt-mark unhold salt-minion && /usr/sbin/bootstrap-salt.sh -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' && apt-mark hold salt-common && apt-mark hold salt-minion && pkill -9 -ef /usr/bin/salt-minion && salt-call state.highstate && sleep 300 && salt-call state.apply salt.minion queue=True' %} {% endif %} {% else %} {% set UPGRADECOMMAND = 'echo Already running Salt Minon version ' ~ SALTVERSION %} From 1e2b4048362caa90251c5c4dececa37c6a1817d2 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 18 Nov 2020 13:29:42 -0500 Subject: [PATCH 15/16] remove -s --- salt/salt/map.jinja | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 1688e562a..f6cf42570 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -12,12 +12,12 @@ {% if grains.saltversion|string != SALTVERSION|string %} {% if grains.os|lower in ['centos', 'redhat'] %} {% if ISAIRGAP is sameas true %} - {% set UPGRADECOMMAND = 'yum clean all && yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -s 120 -r -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && pkill -9 -ef /usr/bin/salt-minion && salt-call state.highstate && sleep 300 && salt-call state.apply salt.minion queue=True'%} + {% set UPGRADECOMMAND = 'yum clean all && yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -r -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && pkill -9 -ef /usr/bin/salt-minion && salt-call state.highstate && sleep 300 && salt-call state.apply salt.minion queue=True'%} {% else %} - {% set UPGRADECOMMAND = 'yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && pkill -9 -ef /usr/bin/salt-minion && salt-call state.highstate && sleep 300 && salt-call state.apply salt.minion queue=True' %} + {% set UPGRADECOMMAND = 'yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && pkill -9 -ef /usr/bin/salt-minion && salt-call state.highstate && sleep 300 && salt-call state.apply salt.minion queue=True' %} {% endif %} {% elif grains.os|lower == 'ubuntu' %} - {% set UPGRADECOMMAND = 'apt-mark unhold salt-common && apt-mark unhold salt-minion && /usr/sbin/bootstrap-salt.sh -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' && apt-mark hold salt-common && apt-mark hold salt-minion && pkill -9 -ef /usr/bin/salt-minion && salt-call state.highstate && sleep 300 && salt-call state.apply salt.minion queue=True' %} + {% set UPGRADECOMMAND = 'apt-mark unhold salt-common && apt-mark unhold salt-minion && /usr/sbin/bootstrap-salt.sh -F -x python3 stable ' ~ SALTVERSION ~ ' && apt-mark hold salt-common && apt-mark hold salt-minion && pkill -9 -ef /usr/bin/salt-minion && salt-call state.highstate && sleep 300 && salt-call state.apply salt.minion queue=True' %} {% endif %} {% else %} {% set UPGRADECOMMAND = 'echo Already running Salt Minon version ' ~ SALTVERSION %} From ceef07b74b8e500e1e5a115668ec5f514808c983 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 18 Nov 2020 14:00:01 -0500 Subject: [PATCH 16/16] remove pkill --- salt/salt/map.jinja | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index f6cf42570..5cb2c76e2 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -12,12 +12,12 @@ {% if grains.saltversion|string != SALTVERSION|string %} {% if grains.os|lower in ['centos', 'redhat'] %} {% if ISAIRGAP is sameas true %} - {% set UPGRADECOMMAND = 'yum clean all && yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -r -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && pkill -9 -ef /usr/bin/salt-minion && salt-call state.highstate && sleep 300 && salt-call state.apply salt.minion queue=True'%} + {% set UPGRADECOMMAND = 'yum clean all && yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -r -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && salt-call state.highstate && sleep 300 && salt-call state.apply salt.minion queue=True'%} {% else %} - {% set UPGRADECOMMAND = 'yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && pkill -9 -ef /usr/bin/salt-minion && salt-call state.highstate && sleep 300 && salt-call state.apply salt.minion queue=True' %} + {% set UPGRADECOMMAND = 'yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && salt-call state.highstate && sleep 300 && salt-call state.apply salt.minion queue=True' %} {% endif %} {% elif grains.os|lower == 'ubuntu' %} - {% set UPGRADECOMMAND = 'apt-mark unhold salt-common && apt-mark unhold salt-minion && /usr/sbin/bootstrap-salt.sh -F -x python3 stable ' ~ SALTVERSION ~ ' && apt-mark hold salt-common && apt-mark hold salt-minion && pkill -9 -ef /usr/bin/salt-minion && salt-call state.highstate && sleep 300 && salt-call state.apply salt.minion queue=True' %} + {% set UPGRADECOMMAND = 'apt-mark unhold salt-common && apt-mark unhold salt-minion && /usr/sbin/bootstrap-salt.sh -F -x python3 stable ' ~ SALTVERSION ~ ' && apt-mark hold salt-common && apt-mark hold salt-minion && salt-call state.highstate && sleep 300 && salt-call state.apply salt.minion queue=True' %} {% endif %} {% else %} {% set UPGRADECOMMAND = 'echo Already running Salt Minon version ' ~ SALTVERSION %}