From 59aa55f9bced286186b4fdd6ffa5588d166113c9 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 18 Aug 2020 15:29:41 -0400 Subject: [PATCH 01/12] Add playsecrets --- salt/common/tools/sbin/soup | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index a93a000b7..41dccb7c6 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -103,6 +103,12 @@ pillar_changes() { sed -i '/^ url_base:/d' /opt/so/saltstack/local/pillar/minions/$MINIONID.sls; sed -i "/^global:/a \\$line" /opt/so/saltstack/local/pillar/global.sls; + # Adding play values to the global.sls + local HIVEPLAYSECRET=$(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 20 | head -n 1) + local CORTEXPLAYSECRET=$(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 20 | head -n 1) + sed -i "/^global:/a \\ hiveplaysecret: $HIVEPLAYSECRET" /opt/so/saltstack/local/pillar/global.sls; + sed -i "/^global:/a \\ cortexplaysecret: $CORTEXPLAYSECRET" /opt/so/saltstack/local/pillar/global.sls; + fi } From 294a197cbfe3ac96b32936b41635364817b20043 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 18 Aug 2020 16:57:38 -0400 Subject: [PATCH 02/12] Add cross cluster for SSL --- salt/common/tools/sbin/soup | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 41dccb7c6..5d5196b97 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -109,6 +109,25 @@ pillar_changes() { sed -i "/^global:/a \\ hiveplaysecret: $HIVEPLAYSECRET" /opt/so/saltstack/local/pillar/global.sls; sed -i "/^global:/a \\ cortexplaysecret: $CORTEXPLAYSECRET" /opt/so/saltstack/local/pillar/global.sls; + # Move storage nodes to hostname for SSL + # Get a list we can use: + grep -A1 searchnode /opt/so/saltstack/local/pillar/data/nodestab.sls | grep -v '\-\-' | sed '$!N;s/\n/ /' | awk '{print $1,$3}' | awk '/_searchnode:/{gsub(/\_searchnode:/, "_searchnode"); print}' >/tmp/nodes.txt + # Remove the nodes from cluster settings + while read p; do + local NAME=$(echo $p | awk '{print $1}') + local IP=$(echo $p | awk '{print $2}') + echo "Removing the old cross cluster config for $NAME" + curl -XPUT -H 'Content-Type: application/json' http://localhost:9200/_cluster/settings -d '{"persistent":{"cluster":{"remote":{"'$NAME'":{"skip_unavailable":null,"seeds":null}}}}}' + done Date: Tue, 18 Aug 2020 17:38:35 -0400 Subject: [PATCH 03/12] Add cross cluster for SSL --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 5d5196b97..45f018b7f 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -124,7 +124,7 @@ pillar_changes() { local NAME=$(echo $p | awk '{print $1}') local EHOSTNAME=$(echo $p | awk -F"_" '{print $1}') echo "Adding the new cross cluster config for $NAME" - curl -XPUT http://localhost:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"$NAME": {"skip_unavailable": "true", "seeds": ["$EHOSTNAME:9300"]}}}}}' + curl -XPUT http://localhost:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"'$NAME'": {"skip_unavailable": "true", "seeds": ["$EHOSTNAME:9300"]}}}}}' done Date: Tue, 18 Aug 2020 17:45:14 -0400 Subject: [PATCH 04/12] Add cross cluster for SSL --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 45f018b7f..6134a8900 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -124,7 +124,7 @@ pillar_changes() { local NAME=$(echo $p | awk '{print $1}') local EHOSTNAME=$(echo $p | awk -F"_" '{print $1}') echo "Adding the new cross cluster config for $NAME" - curl -XPUT http://localhost:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"'$NAME'": {"skip_unavailable": "true", "seeds": ["$EHOSTNAME:9300"]}}}}}' + curl -XPUT http://localhost:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"'$NAME'": {"skip_unavailable": "true", "seeds": ["'$EHOSTNAME':9300"]}}}}}' done Date: Wed, 19 Aug 2020 10:08:11 -0400 Subject: [PATCH 05/12] Salt ACL --- salt/common/tools/sbin/soup | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 6134a8900..f06b085b4 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -87,6 +87,28 @@ highstate() { salt-call state.highstate -l info } +masterlock() { + # Lock the ACL to just the manager + cp -v /etc/salt/master /etc/salt/master.upgrade + echo "peer:" >> /etc/salt/master + echo " *_manager:" >> /etc/salt/master + echo " - .*" >> /etc/salt/master + echo " *_standalone:" >> /etc/salt/master + echo " - .*" >> /etc/salt/master + echo " *_managersearch:" >> /etc/salt/master + echo " - .*" >> /etc/salt/master + echo " *_eval:" >> /etc/salt/master + echo " - .*" >> /etc/salt/master + echo " *_helix:" >> /etc/salt/master + echo " - .*" >> /etc/salt/master + echo " *_import:" >> /etc/salt/master + echo " - .*" >> /etc/salt/master +} + +masterunlock() { + mv /etc/salt/master.upgrade /etc/salt/master +} + pillar_changes() { # This function is to add any new pillar items if needed. echo "Checking to see if pillar changes are needed." @@ -343,6 +365,19 @@ copy_new_files echo "" update_version +echo "" +echo "Locking down Salt Master for upgrade" +masterlock + +echo "" +echo "Starting Salt Master service." +systemctl start salt-master + +echo "" +echo "Stopping Salt Master to remove ACL" +systemctl stop salt-master + +masterunlock echo "" echo "Starting Salt Master service." From f57e0fbc56511f2e03a9fcbf6853bea4476630b8 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 19 Aug 2020 10:33:26 -0400 Subject: [PATCH 06/12] Salt ACL --- salt/common/tools/sbin/soup | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index f06b085b4..1f09f20b2 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -373,6 +373,12 @@ echo "" echo "Starting Salt Master service." systemctl start salt-master +echo "" +echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes." +highstate +echo "" +echo "Upgrade from $INSTALLEDVERSION to $NEWVERSION complete." + echo "" echo "Stopping Salt Master to remove ACL" systemctl stop salt-master @@ -383,13 +389,6 @@ echo "" echo "Starting Salt Master service." systemctl start salt-master - -echo "" -echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes." -highstate -echo "" -echo "Upgrade from $INSTALLEDVERSION to $NEWVERSION complete." - SALTUPGRADED="True" if [[ "$SALTUPGRADED" == "True" ]]; then echo "" From 2f0ffffca472576a963e2e3e48edd7b279b725b4 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 19 Aug 2020 11:46:29 -0400 Subject: [PATCH 07/12] lock and unlock master during soup --- salt/common/tools/sbin/soup | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 1f09f20b2..53b7a2baa 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -88,25 +88,19 @@ highstate() { } masterlock() { - # Lock the ACL to just the manager - cp -v /etc/salt/master /etc/salt/master.upgrade - echo "peer:" >> /etc/salt/master - echo " *_manager:" >> /etc/salt/master - echo " - .*" >> /etc/salt/master - echo " *_standalone:" >> /etc/salt/master - echo " - .*" >> /etc/salt/master - echo " *_managersearch:" >> /etc/salt/master - echo " - .*" >> /etc/salt/master - echo " *_eval:" >> /etc/salt/master - echo " - .*" >> /etc/salt/master - echo " *_helix:" >> /etc/salt/master - echo " - .*" >> /etc/salt/master - echo " *_import:" >> /etc/salt/master - echo " - .*" >> /etc/salt/master + TOPFILE=/opt/so/saltstack/default/salt/top.sls + BACKUPTOPFILE=/opt/so/saltstack/default/salt/top.sls.backup + mv -v $TOPFILE $BACKUPTOPFILE + echo "base:" > $TOPFILE + echo " $MINIONID:" >> $TOPFILE + echo " - ca" >> $TOPFILE + echo " - ssl" >> $TOPFILE + echo " - elasticsearch" >> $TOPFILE + } masterunlock() { - mv /etc/salt/master.upgrade /etc/salt/master + mv -v $BACKUPTOPFILE $TOPFILE } pillar_changes() { From 9280dbb9d973128c35660e929e9966d0d417f104 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 19 Aug 2020 12:00:25 -0400 Subject: [PATCH 08/12] Update soup --- salt/common/tools/sbin/soup | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 53b7a2baa..f0d30e8fa 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -88,19 +88,24 @@ highstate() { } masterlock() { - TOPFILE=/opt/so/saltstack/default/salt/top.sls - BACKUPTOPFILE=/opt/so/saltstack/default/salt/top.sls.backup - mv -v $TOPFILE $BACKUPTOPFILE - echo "base:" > $TOPFILE - echo " $MINIONID:" >> $TOPFILE - echo " - ca" >> $TOPFILE - echo " - ssl" >> $TOPFILE - echo " - elasticsearch" >> $TOPFILE - + echo "Locking Salt Master" + if [[ "$INSTALLEDVERSION" =~ rc.1 ]]; then + TOPFILE=/opt/so/saltstack/default/salt/top.sls + BACKUPTOPFILE=/opt/so/saltstack/default/salt/top.sls.backup + mv -v $TOPFILE $BACKUPTOPFILE + echo "base:" > $TOPFILE + echo " $MINIONID:" >> $TOPFILE + echo " - ca" >> $TOPFILE + echo " - ssl" >> $TOPFILE + echo " - elasticsearch" >> $TOPFILE + fi } masterunlock() { - mv -v $BACKUPTOPFILE $TOPFILE + echo "Unlocking Salt Master" + if [[ "$INSTALLEDVERSION" =~ rc.1 ]]; then + mv -v $BACKUPTOPFILE $TOPFILE + fi } pillar_changes() { @@ -393,4 +398,4 @@ fi } -main "$@" | tee /dev/fd/3 \ No newline at end of file +main "$@" | tee /dev/fd/3 From 3d48c1f99baa181ad679081bcac0f8495a88c0f5 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 19 Aug 2020 12:14:11 -0400 Subject: [PATCH 09/12] Add playbook updates --- salt/common/tools/sbin/soup | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index f0d30e8fa..058a1d507 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -108,6 +108,15 @@ masterunlock() { fi } +playbook() { + echo "Applying playbook settings" + if [[ "$INSTALLEDVERSION" =~ rc.1 ]]; then + salt-call state.apply playbook.db_init + rm -f /opt/so/rules/elastalert/playbook/*.yaml + so-playbook-ruleupdate >> /root/soup_playbook_rule_update.log 2>&1 & + fi +} + pillar_changes() { # This function is to add any new pillar items if needed. echo "Checking to see if pillar changes are needed." @@ -387,6 +396,8 @@ masterunlock echo "" echo "Starting Salt Master service." systemctl start salt-master +highstate +playbook SALTUPGRADED="True" if [[ "$SALTUPGRADED" == "True" ]]; then From bf84822d36785763849f9427745f710393bda1c8 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 19 Aug 2020 13:04:10 -0400 Subject: [PATCH 10/12] fix if logic --- salt/ssl/init.sls | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 82512068c..393d3a2b7 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -518,6 +518,7 @@ fleetkeyperms: {% if salt['file.file_exists']('/etc/pki/elasticsearch.key') -%} - prereq: - x509: /etc/pki/elasticsearch.crt + {%- endif %} /etc/pki/elasticsearch.crt: x509.certificate_managed: @@ -550,6 +551,4 @@ elastickeyperms: - name: /etc/pki/elasticsearch.key - mode: 640 - group: 930 - - {%- endif %} {%- endif %} From 6edf1c14f8bc6f97c6a7ce01f4c2d73b5e33a6bc Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 19 Aug 2020 13:35:58 -0400 Subject: [PATCH 11/12] Fix filebeat certs --- salt/ssl/init.sls | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 393d3a2b7..a2c1d6e39 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -368,7 +368,18 @@ fleetkeyperms: - group: 939 {% endif %} -{% if grains['role'] in ['so-sensor', 'so-manager', 'so-node', 'so-eval', 'so-helix', 'so-managersearch', 'so-heavynode', 'so-fleet', 'so-standalone', 'so-import'] %} +{% if grains['role'] in ['so-sensor', 'so-manager', 'so-searchnode', 'so-eval', 'so-helix', 'so-managersearch', 'so-heavynode', 'so-fleet', 'so-standalone', 'so-import'] %} + +removefbcertdir: + file.absent: + - name: /etc/pki/filebeat.crt + - onlyif: "[ -d /etc/pki/filebeat.crt ]" + +removefbcertdir: + file.absent: + - name: /etc/pki/filebeat.p8 + - onlyif: "[ -d /etc/pki/filebeat.p8 ]" + fbcertdir: file.directory: @@ -505,7 +516,7 @@ fleetkeyperms: {% endif %} -{% if grains['role'] in ['so-node', 'so-heavynode'] %} +{% if grains['role'] in ['so-searchnode', 'so-heavynode'] %} # Create a cert for elasticsearch /etc/pki/elasticsearch.key: x509.private_key_managed: @@ -551,4 +562,5 @@ elastickeyperms: - name: /etc/pki/elasticsearch.key - mode: 640 - group: 930 + {%- endif %} From db2cc5f7a711c3a530d317d48a4c47c1dcffea1a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 19 Aug 2020 15:43:51 -0400 Subject: [PATCH 12/12] Update init.sls --- salt/ssl/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index a2c1d6e39..f535a8257 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -375,7 +375,7 @@ removefbcertdir: - name: /etc/pki/filebeat.crt - onlyif: "[ -d /etc/pki/filebeat.crt ]" -removefbcertdir: +removefbp8dir: file.absent: - name: /etc/pki/filebeat.p8 - onlyif: "[ -d /etc/pki/filebeat.p8 ]"