Merge pull request #6974 from Security-Onion-Solutions/issue/6599

https://github.com/Security-Onion-Solutions/securityonion/issues/6599

salt/common/init.sls

@@ -4,8 +4,9 @@
 {% set role = grains.id.split('_') | last %}
 {% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %}
 
-{% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %}
 include:
+  - common.soup_scripts
+{% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %}
   - manager.elasticsearch # needed for elastic_curl_config state
 {% endif %}
 
@@ -214,6 +215,11 @@ utilsyncscripts:
         ELASTICCURL: 'curl'
     - context:
         ELASTICCURL: {{ ELASTICAUTH.elasticcurl }}
+    - exclude_pat:
+        - so-common
+        - so-firewall
+        - so-image-common
+        - soup
 
 {% if role in ['eval', 'standalone', 'sensor', 'heavynode'] %}
 # Add sensor cleanup

salt/common/soup_scripts.sls

@@ -0,0 +1,13 @@
+# Sync some Utilities
+soup_scripts:
+  file.recurse:
+    - name: /usr/sbin
+    - user: root
+    - group: root
+    - file_mode: 755
+    - source: salt://common/tools/sbin
+    - include_pat:
+        - so-common
+        - so-firewall
+        - so-image-common
+        - soup

salt/common/tools/sbin/soup

@@ -16,7 +16,6 @@
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 . /usr/sbin/so-common
-export LC_CTYPE="en_US.UTF-8"
 
 UPDATE_DIR=/tmp/sogh/securityonion
 DEFAULT_SALT_DIR=/opt/so/saltstack/default
@@ -663,9 +662,6 @@ up_to_2.3.90() {
     fi
   done
   
-  # There was a bug in 2.3.0 so-firewall addhostgroup that was resolved in 2.3.1 - commit 32294eb2ed30ac74b15bb4bfab687084a928daf2
-  echo "Verify so-firewall is up to date"
-  verify_latest_so-firewall_script
   # Create Endgame Hostgroup
   echo "Adding endgame hostgroup with so-firewall"
   if so-firewall addhostgroup endgame 2>&1 | grep -q 'Already exists'; then
@@ -686,8 +682,8 @@ up_to_2.3.90() {
   if [[ -f "/opt/so/saltstack/local/salt/elasticsearch/files/ingest/common" ]]; then
     mv -v /opt/so/saltstack/local/salt/elasticsearch/files/ingest/common /opt/so/saltstack/local/salt/elasticsearch/files/ingest-dynamic/common
     # since json file, we need to wrap with raw
-    sed -i '1s/^/{{'{% raw %}'}}\n/' /opt/so/saltstack/local/salt/elasticsearch/files/ingest-dynamic/common
+    sed -i '1s/^/{% raw %}\n/' /opt/so/saltstack/local/salt/elasticsearch/files/ingest-dynamic/common
-    sed -i -e '$a{{'{% endraw %}'}}\n' /opt/so/saltstack/local/salt/elasticsearch/files/ingest-dynamic/common
+    sed -i -e '$a{% endraw %}\n' /opt/so/saltstack/local/salt/elasticsearch/files/ingest-dynamic/common
   fi
 
   # Generate FleetDM Service Account creds if they do not exist
@@ -889,50 +885,31 @@ update_repo() {
 }
 
 verify_latest_update_script() {
-  #we need to render soup and so-common first since they contain jinja
-  salt-call slsutil.renderer $UPDATE_DIR/salt/common/tools/sbin/soup default_renderer='jinja' --local --out=newline_values_only --out-indent=-4 --out-file=/tmp/soup
-  sed -i -e '$a\' /tmp/soup
-  salt-call slsutil.renderer $UPDATE_DIR/salt/common/tools/sbin/so-common default_renderer='jinja' --local --out=newline_values_only --out-indent=-4 --out-file=/tmp/so-common
-  sed -i -e '$a\' /tmp/so-common
   # Check to see if the update scripts match. If not run the new one.
   CURRENTSOUP=$(md5sum /usr/sbin/soup | awk '{print $1}')
-  GITSOUP=$(md5sum /tmp/soup | awk '{print $1}')
+  GITSOUP=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/soup | awk '{print $1}')
   CURRENTCMN=$(md5sum /usr/sbin/so-common | awk '{print $1}')
-  GITCMN=$(md5sum /tmp/so-common | awk '{print $1}')
+  GITCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-common | awk '{print $1}')
   CURRENTIMGCMN=$(md5sum /usr/sbin/so-image-common | awk '{print $1}')
   GITIMGCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-image-common | awk '{print $1}')
+  CURRENTSOFIREWALL=$(md5sum /usr/sbin/so-firewall | awk '{print $1}')
+  GITSOFIREWALL=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-firewall | awk '{print $1}')
 
-  if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" ]]; then
+  if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" ]]; then
     echo "This version of the soup script is up to date. Proceeding."
-    rm -f /tmp/soup /tmp/so-common
   else
     echo "You are not running the latest soup version. Updating soup and its components. Might take multiple runs to complete"
     cp $UPDATE_DIR/salt/common/tools/sbin/soup $DEFAULT_SALT_DIR/salt/common/tools/sbin/
     cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
     cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
-    salt-call state.apply -l info common queue=True
+    cp $UPDATE_DIR/salt/common/tools/sbin/so-firewall $DEFAULT_SALT_DIR/salt/common/tools/sbin/
+    salt-call state.apply common.soup_scripts queue=True -linfo --file-root=$UPDATE_DIR/salt --local
     echo ""
     echo "soup has been updated. Please run soup again."
     exit 0
   fi
 }
 
-verify_latest_so-firewall_script() {
-  # Check to see if the so-firewall script matches. If not run the new one.
-  CURRENTSOFIREWALL=$(md5sum /usr/sbin/so-firewall | awk '{print $1}')
-  GITSOFIREWALL=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-firewall | awk '{print $1}')
-
-  if [[ "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" ]]; then
-    echo "This version of the so-firewall script is up to date. Proceeding."
-  else
-    echo "You are not running the latest version of so-firewall. Updating so-firewall."
-    cp $UPDATE_DIR/salt/common/tools/sbin/so-firewall $DEFAULT_SALT_DIR/salt/common/tools/sbin/
-    cp $UPDATE_DIR/salt/common/tools/sbin/so-firewall /usr/sbin/
-    echo ""
-    echo "so-firewall has been updated."
-  fi
-}
-
 apply_hotfix() {
   if [[ "$INSTALLEDVERSION" == "2.3.90" ]] ; then
     fix_wazuh