From 0615d635ebafb7edc3bfebea6069e217999b7772 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 17 Jun 2021 16:12:39 -0400 Subject: [PATCH 1/9] let the first highstate pass --- salt/common/tools/sbin/soup | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 32280f707..ba32cc3ca 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -904,7 +904,9 @@ main() { echo "" echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes." + set +e salt-call state.highstate -l info queue=True + set -e echo "" echo "Upgrade from $INSTALLEDVERSION to $NEWVERSION complete." From 9fa83d1ceeee9585d7088e7616c8cad0ca5a0360 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 22 Jun 2021 09:54:25 -0400 Subject: [PATCH 2/9] change to client_timeout --- salt/registry/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/registry/init.sls b/salt/registry/init.sls index b59465fb7..eb0c2df0c 100644 --- a/salt/registry/init.sls +++ b/salt/registry/init.sls @@ -43,7 +43,7 @@ so-dockerregistry: - /nsm/docker-registry/docker:/var/lib/registry/docker:rw - /etc/pki/registry.crt:/etc/pki/registry.crt:ro - /etc/pki/registry.key:/etc/pki/registry.key:ro - - timeout: 180 + - client_timeout: 180 - retry: attempts: 5 interval: 30 From 85bb234cf94e1d7fd035fd02efdbb5c626d7b28b Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 22 Jun 2021 10:14:33 -0400 Subject: [PATCH 3/9] Fix Retry Spam --- salt/common/tools/sbin/soup | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index ba32cc3ca..e9313ad9e 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -884,7 +884,8 @@ main() { # Testing that salt-master is up by checking that is it connected to itself set +e - retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check $SOUP_LOG for details." + echo "Waiting on the Salt Master service to be ready." + salt-call state.show_top -l error queue=True || fail "salt-master could not be reached. Check $SOUP_LOG for details." set -e echo "" @@ -920,9 +921,9 @@ main() { echo "Starting Salt Master service." systemctl start salt-master - # Testing that salt-master is up by checking that is it connected to itself set +e - retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check $SOUP_LOG for details." + echo "Waiting on the Salt Master service to be ready." + salt-call state.show_top -l error queue=True || fail "salt-master could not be reached. Check $SOUP_LOG for details." set -e echo "Running a highstate. This could take several minutes." @@ -997,4 +998,4 @@ EOF read -r input main "$@" | tee -a $SOUP_LOG - \ No newline at end of file + From 9fad0876c57cf2b6f3c9fd8a5fe11fe93a288bc2 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 22 Jun 2021 11:31:31 -0400 Subject: [PATCH 4/9] only attempt to upgrade salt on minions if the minion count it > 1 --- salt/common/tools/sbin/soup | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index ba32cc3ca..1a8b7cef5 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -931,7 +931,8 @@ main() { [[ $is_airgap -eq 0 ]] && unmount_update thehive_maint - if [[ $UPGRADESALT -eq 1 ]]; then + NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l) + if [[ $UPGRADESALT -eq 1 ]] && [[ $NUM_MINIONS -gt 1 ]]; then if [[ $is_airgap -eq 0 ]]; then echo "" echo "Cleaning repos on remote Security Onion nodes." @@ -958,8 +959,6 @@ main() { esac fi - NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l) - if [[ $NUM_MINIONS -gt 1 ]]; then cat << EOF From 26bb6cc0110d1cb87612dbc8778fba64a28dd7ed Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 22 Jun 2021 15:29:36 -0400 Subject: [PATCH 5/9] Require either true | false in parameter to so-elastic-auth and ensure all minions are fully updated with the new auth setting --- salt/common/tools/sbin/so-elastic-auth | 22 ++++++++++------------ setup/so-setup | 2 +- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/salt/common/tools/sbin/so-elastic-auth b/salt/common/tools/sbin/so-elastic-auth index 6631badf2..715f55fe4 100755 --- a/salt/common/tools/sbin/so-elastic-auth +++ b/salt/common/tools/sbin/so-elastic-auth @@ -22,7 +22,7 @@ fi ES_AUTH_PILLAR=${ELASTIC_AUTH_PILLAR:-/opt/so/saltstack/local/pillar/elasticsearch/auth.sls} ES_USERS_FILE=${ELASTIC_USERS_FILE:-/opt/so/saltstack/local/salt/elasticsearch/files/users} -authEnable=${1:-true} +authEnable=$1 if ! grep -q "enabled: " "$ES_AUTH_PILLAR"; then echo "Elastic auth pillar file is invalid. Unable to proceed." @@ -30,17 +30,18 @@ if ! grep -q "enabled: " "$ES_AUTH_PILLAR"; then fi function restart() { - so-elastic-stop - salt-call state.highstate queue=True + if [[ -z "$ELASTIC_AUTH_SKIP_HIGHSTATE" ]]; then + echo "Elasticsearch on all affected minions will now be stopped and then restarted..." + salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-node or G@role:so-heavynode' cmd.run so-elastic-stop + echo "Applying highstate to all affected minions..." + salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-node or G@role:so-heavynode' state.highstate queue=True + fi } if [[ "$authEnable" == "true" ]]; then if grep -q "enabled: False" "$ES_AUTH_PILLAR"; then sed -i 's/enabled: False/enabled: True/g' "$ES_AUTH_PILLAR" - if [[ -z "$ELASTIC_AUTH_SKIP_HIGHSTATE" ]]; then - echo "Applying highstate - this may take a few minutes..." - restart - fi + restart echo "Elastic auth is now enabled." if grep -q "argon" "$ES_USERS_FILE"; then echo "" @@ -53,10 +54,7 @@ if [[ "$authEnable" == "true" ]]; then elif [[ "$authEnable" == "false" ]]; then if grep -q "enabled: True" "$ES_AUTH_PILLAR"; then sed -i 's/enabled: True/enabled: False/g' "$ES_AUTH_PILLAR" - if [[ -z "$ELASTIC_AUTH_SKIP_HIGHSTATE" ]]; then - echo "Applying highstate - this may take a few minutes..." - restart - fi + restart echo "Elastic auth is now disabled." else echo "Auth is already disabled." @@ -64,6 +62,6 @@ elif [[ "$authEnable" == "false" ]]; then else echo "Usage: $0 " echo "" - echo "Enables Elastic authentication. Defaults to true." + echo "Toggles Elastic authentication. Elasticsearch will be restarted on each affected minion." echo "" fi diff --git a/setup/so-setup b/setup/so-setup index 34802fef9..68490657f 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -753,7 +753,7 @@ echo "1" > /root/accept_changes salt-call state.apply -l info manager >> $setup_log 2>&1 echo "Executing so-elastic-auth..." >> $setup_log 2>&1 - ELASTIC_AUTH_SKIP_HIGHSTATE=true bash /opt/so/saltstack/default/salt/common/tools/sbin/so-elastic-auth >> $setup_log 2>&1 + ELASTIC_AUTH_SKIP_HIGHSTATE=true bash /opt/so/saltstack/default/salt/common/tools/sbin/so-elastic-auth true >> $setup_log 2>&1 echo "Finished so-elastic-auth..." >> $setup_log 2>&1 fi From 3fbc850774c007b40399363f4fa4fba691b78b05 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 23 Jun 2021 09:17:37 -0400 Subject: [PATCH 6/9] Fix output so that it can be redirected to local file with appropriate syntax --- salt/common/tools/sbin/so-elasticsearch-pipeline-view | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-elasticsearch-pipeline-view b/salt/common/tools/sbin/so-elasticsearch-pipeline-view index 9f799c07f..3239aec0a 100755 --- a/salt/common/tools/sbin/so-elasticsearch-pipeline-view +++ b/salt/common/tools/sbin/so-elasticsearch-pipeline-view @@ -21,5 +21,5 @@ if [ "$1" == "" ]; then {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq . else - {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq . + {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq .[] fi From caa14e0cade92be5c481d65688178da37584af88 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 22 Jun 2021 10:14:33 -0400 Subject: [PATCH 7/9] Fix Retry Spam --- salt/common/tools/sbin/soup | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 1a8b7cef5..c1e25b208 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -884,7 +884,8 @@ main() { # Testing that salt-master is up by checking that is it connected to itself set +e - retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check $SOUP_LOG for details." + echo "Waiting on the Salt Master service to be ready." + salt-call state.show_top -l error queue=True || fail "salt-master could not be reached. Check $SOUP_LOG for details." set -e echo "" @@ -920,9 +921,9 @@ main() { echo "Starting Salt Master service." systemctl start salt-master - # Testing that salt-master is up by checking that is it connected to itself set +e - retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check $SOUP_LOG for details." + echo "Waiting on the Salt Master service to be ready." + salt-call state.show_top -l error queue=True || fail "salt-master could not be reached. Check $SOUP_LOG for details." set -e echo "Running a highstate. This could take several minutes." @@ -996,4 +997,4 @@ EOF read -r input main "$@" | tee -a $SOUP_LOG - \ No newline at end of file + From c5b81f2f4b57a67f577f632ea961251dd72b85b3 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 23 Jun 2021 09:17:37 -0400 Subject: [PATCH 8/9] Fix output so that it can be redirected to local file with appropriate syntax --- salt/common/tools/sbin/so-elasticsearch-pipeline-view | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-elasticsearch-pipeline-view b/salt/common/tools/sbin/so-elasticsearch-pipeline-view index 9f799c07f..3239aec0a 100755 --- a/salt/common/tools/sbin/so-elasticsearch-pipeline-view +++ b/salt/common/tools/sbin/so-elasticsearch-pipeline-view @@ -21,5 +21,5 @@ if [ "$1" == "" ]; then {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq . else - {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq . + {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq .[] fi From 243e888717a9a8bd5746e0f8e2196055c74b0e89 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 23 Jun 2021 14:41:34 -0400 Subject: [PATCH 9/9] Add queue=True -- needed for all salt commands, not just state changes --- salt/common/tools/sbin/so-elastic-auth | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-elastic-auth b/salt/common/tools/sbin/so-elastic-auth index 715f55fe4..6157cb4b4 100755 --- a/salt/common/tools/sbin/so-elastic-auth +++ b/salt/common/tools/sbin/so-elastic-auth @@ -32,7 +32,7 @@ fi function restart() { if [[ -z "$ELASTIC_AUTH_SKIP_HIGHSTATE" ]]; then echo "Elasticsearch on all affected minions will now be stopped and then restarted..." - salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-node or G@role:so-heavynode' cmd.run so-elastic-stop + salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-node or G@role:so-heavynode' cmd.run so-elastic-stop queue=True echo "Applying highstate to all affected minions..." salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-node or G@role:so-heavynode' state.highstate queue=True fi