From 291ac7d361f375793e3aed069a6d540cd021e612 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 10 Jan 2022 10:36:42 -0500 Subject: [PATCH 01/20] https://github.com/Security-Onion-Solutions/securityonion/issues/6811 --- salt/common/init.sls | 9 +++++++++ salt/manager/elasticsearch.sls | 8 ++++++++ salt/manager/init.sls | 9 +-------- 3 files changed, 18 insertions(+), 8 deletions(-) create mode 100644 salt/manager/elasticsearch.sls diff --git a/salt/common/init.sls b/salt/common/init.sls index 17cea3480..8824a2df9 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -4,6 +4,11 @@ {% set role = grains.id.split('_') | last %} {% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %} +{% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %} +include: + - manager.elasticsearch # needed for elastic_curl_config state +{% endif %} + # Remove variables.txt from /tmp - This is temp rmvariablesfile: file.absent: @@ -189,6 +194,10 @@ elastic_curl_config: - mode: 600 - show_changes: False - makedirs: True +{% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %} + - require: + - file: elastic_curl_config_distributed +{% endif %} # Sync some Utilities utilsyncscripts: diff --git a/salt/manager/elasticsearch.sls b/salt/manager/elasticsearch.sls new file mode 100644 index 000000000..63f2dccdc --- /dev/null +++ b/salt/manager/elasticsearch.sls @@ -0,0 +1,8 @@ + +elastic_curl_config_distributed: + file.managed: + - name: /opt/so/saltstack/local/salt/elasticsearch/curl.config + - source: salt://elasticsearch/files/curl.config.template + - template: jinja + - mode: 600 + - show_changes: False diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 990eda3d3..3604f3cf6 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -25,6 +25,7 @@ include: - kibana.secrets - salt.minion - kratos + - manager.elasticsearch socore_own_saltstack: file.directory: @@ -110,14 +111,6 @@ strelka_yara_update: - hour: '7' - minute: '1' -elastic_curl_config_distributed: - file.managed: - - name: /opt/so/saltstack/local/salt/elasticsearch/curl.config - - source: salt://elasticsearch/files/curl.config.template - - template: jinja - - mode: 600 - - show_changes: False - # Must run before elasticsearch docker container is started! syncesusers: cmd.run: From beb9a33628d66ca883b6e2fd0a0fc5b30548d125 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 10 Jan 2022 11:48:16 -0500 Subject: [PATCH 02/20] only include curl.config if elasticsearch:auth is enabled --- salt/common/init.sls | 4 +++- salt/elasticsearch/files/curl.config.template | 2 +- salt/manager/elasticsearch.sls | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 8824a2df9..e511308a7 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -187,6 +187,7 @@ alwaysupdated: Etc/UTC: timezone.system +{% if salt['pillar.get']('elasticsearch:auth:enabled', False) %} elastic_curl_config: file.managed: - name: /opt/so/conf/elasticsearch/curl.config @@ -194,9 +195,10 @@ elastic_curl_config: - mode: 600 - show_changes: False - makedirs: True -{% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %} + {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %} - require: - file: elastic_curl_config_distributed + {% endif %} {% endif %} # Sync some Utilities diff --git a/salt/elasticsearch/files/curl.config.template b/salt/elasticsearch/files/curl.config.template index 14f5a2a1d..514eeaf65 100644 --- a/salt/elasticsearch/files/curl.config.template +++ b/salt/elasticsearch/files/curl.config.template @@ -1 +1 @@ -user = "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user') }}:{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass') }}" \ No newline at end of file +user = "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user') }}:{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass') }}" diff --git a/salt/manager/elasticsearch.sls b/salt/manager/elasticsearch.sls index 63f2dccdc..24c509fb4 100644 --- a/salt/manager/elasticsearch.sls +++ b/salt/manager/elasticsearch.sls @@ -1,4 +1,4 @@ - +{% if salt['pillar.get']('elasticsearch:auth:enabled', False) %} elastic_curl_config_distributed: file.managed: - name: /opt/so/saltstack/local/salt/elasticsearch/curl.config @@ -6,3 +6,4 @@ elastic_curl_config_distributed: - template: jinja - mode: 600 - show_changes: False +{% endif %} From 716c98ec61beff9a8e3ef89d705ee46ca095b116 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 10 Jan 2022 14:39:00 -0500 Subject: [PATCH 03/20] requires and ordering for socusersroles state --- salt/elasticsearch/files/curl.config.template | 2 +- salt/manager/init.sls | 20 ++---------- salt/manager/sync_es_users.sls | 31 +++++++++++++++++++ salt/soc/init.sls | 13 +++----- 4 files changed, 39 insertions(+), 27 deletions(-) create mode 100644 salt/manager/sync_es_users.sls diff --git a/salt/elasticsearch/files/curl.config.template b/salt/elasticsearch/files/curl.config.template index 514eeaf65..9c057cabf 100644 --- a/salt/elasticsearch/files/curl.config.template +++ b/salt/elasticsearch/files/curl.config.template @@ -1 +1 @@ -user = "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user') }}:{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass') }}" +user = "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', 'NO_USER_SET') }}:{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', 'NO_PW_SET') }}" diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 3604f3cf6..c913383b0 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -21,10 +21,9 @@ {% set STRELKA_RULES = salt['pillar.get']('strelka:rules', '1') %} include: - - elasticsearch.auth - - kibana.secrets - salt.minion - - kratos + - kibana.secrets + - manager.sync_es_users - manager.elasticsearch socore_own_saltstack: @@ -111,21 +110,6 @@ strelka_yara_update: - hour: '7' - minute: '1' -# Must run before elasticsearch docker container is started! -syncesusers: - cmd.run: - - name: so-user sync - - env: - - SKIP_STATE_APPLY: 'true' - - creates: - - /opt/so/saltstack/local/salt/elasticsearch/files/users - - /opt/so/saltstack/local/salt/elasticsearch/files/users_roles - - /opt/so/conf/soc/soc_users_roles - - show_changes: False - - require: - - docker_container: so-kratos - - http: wait_for_kratos - {% else %} {{sls}}_state_not_allowed: diff --git a/salt/manager/sync_es_users.sls b/salt/manager/sync_es_users.sls new file mode 100644 index 000000000..4546fc52f --- /dev/null +++ b/salt/manager/sync_es_users.sls @@ -0,0 +1,31 @@ +include: + - elasticsearch.auth + - kratos + +so-user.lock: + file.missing: + - name: /var/tmp/so-user.lock + +# Must run before elasticsearch docker container is started! +sync_es_users: + cmd.run: + - name: so-user sync + - env: + - SKIP_STATE_APPLY: 'true' + - creates: + - /opt/so/saltstack/local/salt/elasticsearch/files/users + - /opt/so/saltstack/local/salt/elasticsearch/files/users_roles + - /opt/so/conf/soc/soc_users_roles + - show_changes: False + - require: + - docker_container: so-kratos + - http: wait_for_kratos + - file: so-user.lock # require so-user.lock file to be missing + +# we dont want this added too early in setup, so we add the onlyif to verify 'startup_states: highstate' +# is in the minion config. That line is added before the final highstate during setup +sosyncusers: + cron.present: + - user: root + - name: 'PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin /usr/sbin/so-user sync &>> /opt/so/log/soc/sync.log' + - onlyif: "grep 'startup_states: highstate' /etc/salt/minion" diff --git a/salt/soc/init.sls b/salt/soc/init.sls index 856f929bd..bfb6ea4d9 100644 --- a/salt/soc/init.sls +++ b/salt/soc/init.sls @@ -5,6 +5,9 @@ {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} +include: + - manager.sync_es_users + socdir: file.directory: - name: /opt/so/conf/soc @@ -84,14 +87,8 @@ soccustomroles: socusersroles: file.exists: - name: /opt/so/conf/soc/soc_users_roles - -# we dont want this added too early in setup, so we add the onlyif to verify 'startup_states: highstate' -# is in the minion config. That line is added before the final highstate during setup -sosyncusers: - cron.present: - - user: root - - name: 'PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin /usr/sbin/so-user sync &>> /opt/so/log/soc/sync.log' - - onlyif: "grep 'startup_states: highstate' /etc/salt/minion" + - require: + - sls: manager.sync_es_users so-soc: docker_container.running: From 86c8fc6c1ce26857076e2ee3cefa694715adfd99 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 11 Jan 2022 08:56:38 -0500 Subject: [PATCH 04/20] need to update mine after salt-master starts --- salt/common/tools/sbin/soup | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 14c803faf..eb57ca441 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -665,10 +665,6 @@ up_to_2.3.90() { up_to_2.3.100() { echo "Updating to Security Onion to 2.3.100" - echo "Populating the mine with network.ip_addrs pillar.host.mainint for each host." - set +e - salt \* cmd.run cmd='MAININT=$(salt-call pillar.get host:mainint --out=newline_values_only) && salt-call mine.send name=network.ip_addrs interface="$MAININT"' - set -e fix_wazuh } @@ -1060,6 +1056,11 @@ main() { salt-call state.apply salt.python3-influxdb -l info queue=True echo "" + echo "Populating the mine with network.ip_addrs pillar.host.mainint for each host." + set +e + salt \* cmd.run cmd='MAININT=$(salt-call pillar.get host:mainint --out=newline_values_only) && salt-call mine.send name=network.ip_addrs interface="$MAININT"' + set -e + # Only regenerate osquery packages if Fleet is enabled FLEET_MANAGER=$(lookup_pillar fleet_manager) FLEET_NODE=$(lookup_pillar fleet_node) From a8d1b9eb90452d1e47518965c4f82cd12b765eb4 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 11 Jan 2022 09:29:12 -0500 Subject: [PATCH 05/20] restart salt-minion at end of run if mine_functions changes --- salt/salt/minion.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 38f8889c3..d8bebf0b5 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -84,6 +84,8 @@ mine_functions: - name: /etc/salt/minion.d/mine_functions.conf - source: salt://salt/etc/minion.d/mine_functions.conf - template: jinja + - listen_in: + - service: salt_minion_service # this has to be outside the if statement above since there are _in calls to this state salt_minion_service: @@ -91,8 +93,6 @@ salt_minion_service: - name: salt-minion - enable: True - onlyif: test "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}" - - watch: - - file: mine_functions patch_pkg: pkg.installed: From 91ef9b936611b0ee8bfc2db0023fd4fcb51b6f26 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 11 Jan 2022 10:57:48 -0500 Subject: [PATCH 06/20] update salt mine before salt-master and salt-minion get stopped --- salt/common/tools/sbin/soup | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index eb57ca441..5afc83cce 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -731,6 +731,13 @@ update_centos_repo() { createrepo /nsm/repo } +update_salt_mine() { + echo "Populating the mine with network.ip_addrs pillar.host.mainint for each host." + set +e + salt \* cmd.run cmd='MAININT=$(salt-call pillar.get host:mainint --out=newline_values_only) && salt-call mine.send name=network.ip_addrs interface="$MAININT"' + set -e +} + update_version() { # Update the version to the latest echo "Updating the Security Onion version file." @@ -969,6 +976,9 @@ main() { echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION." echo "" + # update mine items prior to stopping salt-minion and salt-master + update_salt_mine + echo "Updating dockers to $NEWVERSION." if [[ $is_airgap -eq 0 ]]; then airgap_update_dockers @@ -1056,11 +1066,6 @@ main() { salt-call state.apply salt.python3-influxdb -l info queue=True echo "" - echo "Populating the mine with network.ip_addrs pillar.host.mainint for each host." - set +e - salt \* cmd.run cmd='MAININT=$(salt-call pillar.get host:mainint --out=newline_values_only) && salt-call mine.send name=network.ip_addrs interface="$MAININT"' - set -e - # Only regenerate osquery packages if Fleet is enabled FLEET_MANAGER=$(lookup_pillar fleet_manager) FLEET_NODE=$(lookup_pillar fleet_node) From 14eed8e5b9cd54814ed9fc8e1a2409f5cb791cf1 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 11 Jan 2022 11:20:30 -0500 Subject: [PATCH 07/20] redirect to setup_log --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index ff7904e61..6bc7bed70 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2144,7 +2144,7 @@ restore_file() { if [ -f "$src" ]; then [ ! -d "$dst" ] && mkdir -v -p "$dst" echo "Restoring $src to $dst." >> "$setup_log" 2>&1 - cp -v "$src" "$dst" + cp -v "$src" "$dst" >> "$setup_log" 2>&1 fi } From 9d19cba6007808d66d639a1b0c086759fd0c8f26 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 11 Jan 2022 13:09:05 -0500 Subject: [PATCH 08/20] log time when salt services stopped and started --- salt/common/tools/sbin/soup | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 71891a57e..73c1dd943 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -993,14 +993,14 @@ main() { fi echo "" - echo "Stopping Salt Minion service." + echo "Stopping Salt Minion service at $(date +"%T.%6N")." systemctl stop salt-minion - echo "Killing any remaining Salt Minion processes." + echo "Killing any remaining Salt Minion processes at $(date +"%T.%6N")" set +e pkill -9 -ef /usr/bin/salt-minion set -e echo "" - echo "Stopping Salt Master service." + echo "Stopping Salt Master service at $(date +"%T.%6N")" systemctl stop salt-master echo "" @@ -1048,11 +1048,11 @@ main() { update_version echo "" - echo "Locking down Salt Master for upgrade" + echo "Locking down Salt Master for upgrade at $(date +"%T.%6N")." masterlock echo "" - echo "Starting Salt Master service." + echo "Starting Salt Master service at $(date +"%T.%6N")." systemctl start salt-master # Testing that salt-master is up by checking that is it connected to itself @@ -1083,13 +1083,13 @@ main() { set -e echo "" - echo "Stopping Salt Master to remove ACL" + echo "Stopping Salt Master to remove ACL at $(date +"%T.%6N")." systemctl stop salt-master masterunlock echo "" - echo "Starting Salt Master service." + echo "Starting Salt Master service at $(date +"%T.%6N") ." systemctl start salt-master set +e From e33a9eb45cbf476127ebdaec905347db1956a673 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 11 Jan 2022 13:11:25 -0500 Subject: [PATCH 09/20] bootstrap.sh, dont start salt services after salt upgrade, allow soup to do it --- salt/common/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 73c1dd943..87ad8abda 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -788,7 +788,7 @@ upgrade_salt() { echo "" set +e run_check_net_err \ - "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \ + "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -X -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \ "Could not update salt, please check $SOUP_LOG for details." set -e echo "Applying yum versionlock for Salt." @@ -805,7 +805,7 @@ upgrade_salt() { echo "" set +e run_check_net_err \ - "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable \"$NEWSALTVERSION\"" \ + "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -X -F -M -x python3 stable \"$NEWSALTVERSION\"" \ "Could not update salt, please check $SOUP_LOG for details." set -e echo "Applying apt hold for Salt." From 0ef130bd38c3744da23880bce4a7d14264858ad2 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 11 Jan 2022 13:12:07 -0500 Subject: [PATCH 10/20] bootstrap.sh, dont start salt services after salt upgrade, allow soup to do it --- salt/common/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 87ad8abda..1a99ea859 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -784,7 +784,7 @@ upgrade_salt() { echo "Removing yum versionlock for Salt." echo "" yum versionlock delete "salt-*" - echo "Updating Salt packages and restarting services." + echo "Updating Salt packages." echo "" set +e run_check_net_err \ @@ -801,7 +801,7 @@ upgrade_salt() { apt-mark unhold "salt-common" apt-mark unhold "salt-master" apt-mark unhold "salt-minion" - echo "Updating Salt packages and restarting services." + echo "Updating Salt packages." echo "" set +e run_check_net_err \ From 5ade8193f0326de692972251a034e54496cf8892 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 11 Jan 2022 13:41:51 -0500 Subject: [PATCH 11/20] move highstate messages for more accurate final highstate message --- salt/common/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 1a99ea859..241011e94 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -1077,7 +1077,7 @@ main() { fi echo "" - echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes." + echo "Running a highstate. This could take several minutes." set +e salt-call state.highstate -l info queue=True set -e @@ -1097,7 +1097,7 @@ main() { salt-call state.show_top -l error queue=True || fail "salt-master could not be reached. Check $SOUP_LOG for details." set -e - echo "Running a highstate. This could take several minutes." + echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes." salt-call state.highstate -l info queue=True postupgrade_changes [[ $is_airgap -eq 0 ]] && unmount_update From ae0f392035067db7708c8fd8df6a578550080878 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 11 Jan 2022 16:57:29 -0500 Subject: [PATCH 12/20] wait for salt-master and salt-minin to exit. disable highstate before stopping salt-minion. apply salt-minion state before first highstate to update configs --- salt/common/tools/sbin/soup | 40 ++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 241011e94..e6e878163 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -992,17 +992,34 @@ main() { set -e fi + echo "Disabling highstate to prevent from running if salt-minion restarts." + salt-call state.disable highstate -l info --local echo "" + + echo "Storing salt-minion pid." + MINIONPID=$(pgrep salt-minion | head -1) + echo "Found salt-minion PID $MINIONPID" echo "Stopping Salt Minion service at $(date +"%T.%6N")." systemctl stop salt-minion - echo "Killing any remaining Salt Minion processes at $(date +"%T.%6N")" + set +e - pkill -9 -ef /usr/bin/salt-minion + timeout 30 tail --pid=$MINIONPID -f /dev/null || echo "Killing salt-minion at $(date +"%T.%6N") after waiting 30s" && pkill -9 -ef /usr/bin/salt-minion set -e + + #echo "Killing any remaining Salt Minion processes at $(date +"%T.%6N")" + #set +e + #pkill -9 -ef /usr/bin/salt-minion + #set -e + + echo "" + echo "Storing salt-master pid." + MASTERPID=$(pgrep salt-master | head -1) + echo "Found salt-master PID $MASTERPID" echo "" echo "Stopping Salt Master service at $(date +"%T.%6N")" systemctl stop salt-master echo "" + timeout 30 tail --pid=$MASTERPID -f /dev/null || echo "salt-master still running at $(date +"%T.%6N") after waiting 30s. We cannot kill due to systemd restart option." upgrade_to_2.3.50_repo @@ -1066,6 +1083,13 @@ main() { salt-call state.apply salt.python3-influxdb -l info queue=True echo "" + # update the salt-minion configs here and start the minion + # since highstate are disabled above, minion start should not trigger a highstate + echo "" + echo "Ensuring salt-minion configs are up-to-date." + salt-call state.apply salt.minion -l info queue=True + echo "" + # Only regenerate osquery packages if Fleet is enabled FLEET_MANAGER=$(lookup_pillar fleet_manager) FLEET_NODE=$(lookup_pillar fleet_node) @@ -1076,6 +1100,10 @@ main() { echo "" fi + echo "Enabling highstate." + salt-call state.enable highstate -l info --local + echo "" + echo "" echo "Running a highstate. This could take several minutes." set +e @@ -1083,8 +1111,14 @@ main() { set -e echo "" - echo "Stopping Salt Master to remove ACL at $(date +"%T.%6N")." + echo "Storing salt-master pid." + MASTERPID=$(pgrep salt-master | head -1) + echo "Found salt-master PID $MASTERPID" + echo "" + echo "Stopping Salt Master service to remove ACL(masterunlock) at $(date +"%T.%6N")" systemctl stop salt-master + echo "" + timeout 30 tail --pid=$MASTERPID -f /dev/null || echo "salt-master still running at $(date +"%T.%6N") after waiting 30s. We cannot kill due to systemd restart option." masterunlock From abf3a9401b29cfe905d3dd139d5c9e877f67c605 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 11 Jan 2022 18:31:35 -0500 Subject: [PATCH 13/20] listen instead to not start service if not running then restart if changes to files --- salt/salt/minion.sls | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index d8bebf0b5..a9320defb 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -62,8 +62,6 @@ set_log_levels: - text: - "log_level: error" - "log_level_logfile: error" - - listen_in: - - service: salt_minion_service salt_minion_service_unit_file: file.managed: @@ -74,8 +72,6 @@ salt_minion_service_unit_file: service_start_delay: {{ service_start_delay }} - onchanges_in: - module: systemd_reload - - listen_in: - - service: salt_minion_service {% endif %} @@ -84,8 +80,6 @@ mine_functions: - name: /etc/salt/minion.d/mine_functions.conf - source: salt://salt/etc/minion.d/mine_functions.conf - template: jinja - - listen_in: - - service: salt_minion_service # this has to be outside the if statement above since there are _in calls to this state salt_minion_service: @@ -93,6 +87,13 @@ salt_minion_service: - name: salt-minion - enable: True - onlyif: test "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}" + - listen: + - file: mine_functions +{% if INSTALLEDSALTVERSION|string == SALTVERSION|string %} + - file: set_log_levels + - file: salt_minion_service_unit_file +{% endif %} + patch_pkg: pkg.installed: From 494737549d1843816d1e2f3b31b57038ff0fb19d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 12 Jan 2022 10:20:05 -0500 Subject: [PATCH 14/20] move some es script to src elasticsearch/tools/sbin and dst /usr/sbin. set requires --- salt/elasticsearch/init.sls | 52 +++++++++++++------ .../{files/scripts => tools/sbin}/so-catrust | 0 .../sbin}/so-elasticsearch-pipelines | 0 .../tools/sbin/so-elasticsearch-roles-load | 0 .../sbin/so-elasticsearch-templates-load | 0 5 files changed, 36 insertions(+), 16 deletions(-) rename salt/elasticsearch/{files/scripts => tools/sbin}/so-catrust (100%) rename salt/elasticsearch/{files => tools/sbin}/so-elasticsearch-pipelines (100%) rename salt/{common => elasticsearch}/tools/sbin/so-elasticsearch-roles-load (100%) rename salt/{common => elasticsearch}/tools/sbin/so-elasticsearch-templates-load (100%) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index e24eab25e..78a586428 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -53,7 +53,7 @@ vm.max_map_count: cascriptsync: file.managed: - name: /usr/sbin/so-catrust - - source: salt://elasticsearch/files/scripts/so-catrust + - source: salt://elasticsearch/tools/sbin/so-catrust - user: 939 - group: 939 - mode: 750 @@ -63,9 +63,37 @@ cascriptsync: cascriptfun: cmd.run: - name: /usr/sbin/so-catrust - + - require: + - file: cascriptsync {% endif %} +# Sync some es scripts +es_sync_scripts: + file.recurse: + - name: /usr/sbin + - user: root + - group: root + - file_mode: 755 + - template: jinja + - source: salt://elasticsearch/tools/sbin + - defaults: + ELASTICCURL: 'curl' + - context: + ELASTICCURL: {{ ELASTICAUTH.elasticcurl }} + - exclude_pat: + - so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state + +so-elasticsearch-pipelines-script: + file.managed: + - name: /usr/sbin/so-elasticsearch-pipelines + - source: salt://elasticsearch/tools/sbin/so-elasticsearch-pipelines + - user: 930 + - group: 939 + - mode: 754 + - template: jinja + - defaults: + ELASTICCURL: {{ ELASTICAUTH.elasticcurl }} + # Move our new CA over so Elastic and Logstash can use SSL with the internal CA catrustdir: file.directory: @@ -297,7 +325,7 @@ so-elasticsearch: - file: esyml - file: esingestconf - file: esingestdynamicconf - - file: so-elasticsearch-pipelines-file + - file: so-elasticsearch-pipelines-script - require: - file: esyml - file: eslog4jfile @@ -322,27 +350,17 @@ append_so-elasticsearch_so-status.conf: - name: /opt/so/conf/so-status/so-status.conf - text: so-elasticsearch -so-elasticsearch-pipelines-file: - file.managed: - - name: /opt/so/conf/elasticsearch/so-elasticsearch-pipelines - - source: salt://elasticsearch/files/so-elasticsearch-pipelines - - user: 930 - - group: 939 - - mode: 754 - - template: jinja - - defaults: - ELASTICCURL: {{ ELASTICAUTH.elasticcurl }} - so-elasticsearch-pipelines: cmd.run: - - name: /opt/so/conf/elasticsearch/so-elasticsearch-pipelines {{ grains.host }} + - name: /usr/sbin/so-elasticsearch-pipelines {{ grains.host }} - onchanges: - file: esingestconf - file: esingestdynamicconf - file: esyml - - file: so-elasticsearch-pipelines-file + - file: so-elasticsearch-pipelines-script - require: - docker_container: so-elasticsearch + - file: so-elasticsearch-pipelines-script {% if TEMPLATES %} so-elasticsearch-templates: @@ -352,6 +370,7 @@ so-elasticsearch-templates: - template: jinja - require: - docker_container: so-elasticsearch + - file: es_sync_scripts {% endif %} so-elasticsearch-roles-load: @@ -361,6 +380,7 @@ so-elasticsearch-roles-load: - template: jinja - require: - docker_container: so-elasticsearch + - file: es_sync_scripts {% endif %} {# if grains['role'] != 'so-helix' #} diff --git a/salt/elasticsearch/files/scripts/so-catrust b/salt/elasticsearch/tools/sbin/so-catrust similarity index 100% rename from salt/elasticsearch/files/scripts/so-catrust rename to salt/elasticsearch/tools/sbin/so-catrust diff --git a/salt/elasticsearch/files/so-elasticsearch-pipelines b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines similarity index 100% rename from salt/elasticsearch/files/so-elasticsearch-pipelines rename to salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines diff --git a/salt/common/tools/sbin/so-elasticsearch-roles-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-roles-load rename to salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load diff --git a/salt/common/tools/sbin/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-templates-load rename to salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load From 0388912ba7a75f47b69e6319f5aa856f6091a8c2 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 12 Jan 2022 11:05:47 -0500 Subject: [PATCH 15/20] kill all salt jobs across grid before stopping salt-master. kill all salt jobs on manager before stopping salt-minion. --- salt/common/tools/sbin/soup | 81 ++++++++++++++++++++----------------- 1 file changed, 45 insertions(+), 36 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index e6e878163..607396195 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -468,6 +468,48 @@ post_to_2.3.100() { echo "Post Processing for .100" } +stop_salt_master() { + # kill all salt jobs across the grid because the hang indefinitely if they are queued and salt-master restarts + set +e + echo "" + echo "Killing all Salt jobs across the grid." + salt \* saltutil.kill_all_jobs + set -e + + echo "" + echo "Storing salt-master pid." + MASTERPID=$(pgrep salt-master | head -1) + echo "Found salt-master PID $MASTERPID" + echo "" + echo "Stopping Salt Master service at $(date +"%T.%6N")" + systemctl stop salt-master + echo "" + timeout 30 tail --pid=$MASTERPID -f /dev/null || echo "salt-master still running at $(date +"%T.%6N") after waiting 30s. We cannot kill due to systemd restart option." +} + +stop_salt_minion() { + echo "Disabling highstate to prevent from running if salt-minion restarts." + salt-call state.disable highstate -l info --local + echo "" + + # kill all salt jobs before stopping salt-minion + set +e + echo "" + echo "Killing Salt jobs on this node." + salt-call saltutil.kill_all_jobs --local + set -e + + echo "Storing salt-minion pid." + MINIONPID=$(pgrep salt-minion | head -1) + echo "Found salt-minion PID $MINIONPID" + echo "Stopping Salt Minion service at $(date +"%T.%6N")." + systemctl stop salt-minion + + set +e + timeout 30 tail --pid=$MINIONPID -f /dev/null || echo "Killing salt-minion at $(date +"%T.%6N") after waiting 30s" && pkill -9 -ef /usr/bin/salt-minion + set -e +} + up_to_2.3.20(){ DOCKERSTUFFBIP=$(echo $DOCKERSTUFF | awk -F'.' '{print $1,$2,$3,1}' OFS='.')/24 # Remove PCAP from global @@ -992,34 +1034,9 @@ main() { set -e fi - echo "Disabling highstate to prevent from running if salt-minion restarts." - salt-call state.disable highstate -l info --local - echo "" + stop_salt_minion - echo "Storing salt-minion pid." - MINIONPID=$(pgrep salt-minion | head -1) - echo "Found salt-minion PID $MINIONPID" - echo "Stopping Salt Minion service at $(date +"%T.%6N")." - systemctl stop salt-minion - - set +e - timeout 30 tail --pid=$MINIONPID -f /dev/null || echo "Killing salt-minion at $(date +"%T.%6N") after waiting 30s" && pkill -9 -ef /usr/bin/salt-minion - set -e - - #echo "Killing any remaining Salt Minion processes at $(date +"%T.%6N")" - #set +e - #pkill -9 -ef /usr/bin/salt-minion - #set -e - - echo "" - echo "Storing salt-master pid." - MASTERPID=$(pgrep salt-master | head -1) - echo "Found salt-master PID $MASTERPID" - echo "" - echo "Stopping Salt Master service at $(date +"%T.%6N")" - systemctl stop salt-master - echo "" - timeout 30 tail --pid=$MASTERPID -f /dev/null || echo "salt-master still running at $(date +"%T.%6N") after waiting 30s. We cannot kill due to systemd restart option." + stop_salt_master upgrade_to_2.3.50_repo @@ -1110,15 +1127,7 @@ main() { salt-call state.highstate -l info queue=True set -e - echo "" - echo "Storing salt-master pid." - MASTERPID=$(pgrep salt-master | head -1) - echo "Found salt-master PID $MASTERPID" - echo "" - echo "Stopping Salt Master service to remove ACL(masterunlock) at $(date +"%T.%6N")" - systemctl stop salt-master - echo "" - timeout 30 tail --pid=$MASTERPID -f /dev/null || echo "salt-master still running at $(date +"%T.%6N") after waiting 30s. We cannot kill due to systemd restart option." + stop_salt_master masterunlock From 0cf877f169bc30448a09cc3460cc1d42b9b6e128 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 12 Jan 2022 12:27:19 -0500 Subject: [PATCH 16/20] kill any possible queued salt jobs before stopping salt-master --- salt/common/tools/sbin/soup | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 607396195..ade009676 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -474,6 +474,9 @@ stop_salt_master() { echo "" echo "Killing all Salt jobs across the grid." salt \* saltutil.kill_all_jobs + echo "" + echo "Killing any queued Salt jobs on the manager." + pkill -9 -ef "/usr/bin/python3 /bin/salt" set -e echo "" From 03b9b74ace8b021d9a5d6e7a3318ae213cd5d086 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 12 Jan 2022 16:04:10 -0500 Subject: [PATCH 17/20] stop cron before soup upgrades the manager, start cron at the end. add cron state that is in included in common --- salt/common/init.sls | 3 ++- salt/common/tools/sbin/so-common | 7 +++++++ salt/common/tools/sbin/soup | 8 ++++++++ salt/cron/dead.sls | 6 ++++++ salt/cron/map.jinja | 8 ++++++++ salt/cron/running.sls | 7 +++++++ 6 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 salt/cron/dead.sls create mode 100644 salt/cron/map.jinja create mode 100644 salt/cron/running.sls diff --git a/salt/common/init.sls b/salt/common/init.sls index e511308a7..da781e0ef 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -4,8 +4,9 @@ {% set role = grains.id.split('_') | last %} {% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %} -{% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %} include: + - cron.running +{% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %} - manager.elasticsearch # needed for elastic_curl_config state {% endif %} diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 66c91aa7b..584e57926 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -360,6 +360,13 @@ run_check_net_err() { exit $exit_code fi } +set_cron_service_name() { + if [[ "$OS" == "centos" ]]; then + cron_service_name="crond" + else + cron_service_name="cron" + fi +} set_os() { if [ -f /etc/redhat-release ]; then diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index ade009676..d06a868b4 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -988,6 +988,7 @@ main() { verify_latest_update_script echo "" set_os + set_cron_service_name set_palette check_elastic_license echo "" @@ -1021,6 +1022,10 @@ main() { echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION." echo "" + echo "Stopping $cron_service_name service at $(date +"%T.%6N")." + echo "" + systemctl stop "$cron_service_name" + # update mine items prior to stopping salt-minion and salt-master update_salt_mine @@ -1191,6 +1196,9 @@ main() { esac fi + echo "Starting $cron_service_name service at $(date +"%T.%6N")." + systemctl start "$cron_service_name" + if [[ $NUM_MINIONS -gt 1 ]]; then cat << EOF diff --git a/salt/cron/dead.sls b/salt/cron/dead.sls new file mode 100644 index 000000000..8ddb79cbf --- /dev/null +++ b/salt/cron/dead.sls @@ -0,0 +1,6 @@ +{% from "cron/map.jinja" import cronmap with context %} + +crond_service: + service.dead: + - name: {{ cronmap.service }} + - enable: True diff --git a/salt/cron/map.jinja b/salt/cron/map.jinja new file mode 100644 index 000000000..5c5bed04f --- /dev/null +++ b/salt/cron/map.jinja @@ -0,0 +1,8 @@ +{% set cronmap = salt['grains.filter_by']({ + 'Ubuntu': { + 'service': 'cron', + }, + 'CentOS': { + 'service': 'crond', + }, +}) %} diff --git a/salt/cron/running.sls b/salt/cron/running.sls new file mode 100644 index 000000000..bcd5e7ef9 --- /dev/null +++ b/salt/cron/running.sls @@ -0,0 +1,7 @@ +{% from "cron/map.jinja" import cronmap with context %} + +crond_service: + service.running: + - name: {{ cronmap.service }} + - enable: True + - unless: pgrep soup From 443dc6ebaa7e0dc08b71526ffa471d8e6b33b64b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 12 Jan 2022 16:14:49 -0500 Subject: [PATCH 18/20] move branch echo to main so it is in the log --- salt/common/tools/sbin/soup | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index d06a868b4..8e2a7d332 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -958,7 +958,18 @@ fix_wazuh() { main() { trap 'check_err $?' EXIT + if [ -n "$BRANCH" ]; then + echo "SOUP will use the $BRANCH branch." + echo "" + fi + echo "### Preparing soup at $(date) ###" + echo "" + + echo "Checking to see if this is a manager." + echo "" + require_manager + check_pillar_items echo "Checking to see if this is an airgap install." @@ -968,9 +979,7 @@ main() { echo "Missing file argument (-f ) for unattended airgap upgrade." exit 0 fi - echo "Checking to see if this is a manager." - echo "" - require_manager + set_minionid echo "Found that Security Onion $INSTALLEDVERSION is currently installed." echo "" @@ -1277,12 +1286,6 @@ https://blog.securityonion.net EOF - if [ -n "$BRANCH" ]; then - cat << EOF -SOUP will use the $BRANCH branch. - -EOF - fi cat << EOF Press Enter to continue or Ctrl-C to cancel. EOF From a28bb23d2070382fb3b7c139079c7bf7d951c975 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 12 Jan 2022 17:27:47 -0500 Subject: [PATCH 19/20] fix os_family for cron state map --- salt/cron/map.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/cron/map.jinja b/salt/cron/map.jinja index 5c5bed04f..4f5b78823 100644 --- a/salt/cron/map.jinja +++ b/salt/cron/map.jinja @@ -1,8 +1,8 @@ {% set cronmap = salt['grains.filter_by']({ - 'Ubuntu': { + 'Debian': { 'service': 'cron', }, - 'CentOS': { + 'RedHat': { 'service': 'crond', }, }) %} From 389ff1a46de312028f333a04a26f98b1a8195132 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 13 Jan 2022 09:39:46 -0500 Subject: [PATCH 20/20] create enable_highstate state to reenable highstate following minion restart if it was previously disabled. same with cron --- salt/common/init.sls | 3 +-- salt/salt/enable_highstate.sls | 7 +++++++ salt/top.sls | 4 ++++ 3 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 salt/salt/enable_highstate.sls diff --git a/salt/common/init.sls b/salt/common/init.sls index da781e0ef..e511308a7 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -4,9 +4,8 @@ {% set role = grains.id.split('_') | last %} {% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %} -include: - - cron.running {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %} +include: - manager.elasticsearch # needed for elastic_curl_config state {% endif %} diff --git a/salt/salt/enable_highstate.sls b/salt/salt/enable_highstate.sls new file mode 100644 index 000000000..72e5c1410 --- /dev/null +++ b/salt/salt/enable_highstate.sls @@ -0,0 +1,7 @@ +enable_highstate: + module.run: + - state.enable: + - states: + - highstate + - unless: pgrep soup + \ No newline at end of file diff --git a/salt/top.sls b/salt/top.sls index 4fd8c1fd3..513439255 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -19,6 +19,10 @@ base: + '*': + - salt.enable_highstate + - cron.running + 'not G@saltversion:{{saltversion}}': - match: compound - salt.minion-state-apply-test