Merge pull request #6856 from Security-Onion-Solutions/issue/6811

Issue/6811

salt/common/init.sls

@@ -4,6 +4,11 @@
 {% set role = grains.id.split('_') | last %}
 {% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %}
 
+{% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %}
+include:
+  - manager.elasticsearch # needed for elastic_curl_config state
+{% endif %}
+
 # Remove variables.txt from /tmp - This is temp
 rmvariablesfile:
   file.absent:
@@ -182,6 +187,7 @@ alwaysupdated:
 Etc/UTC:
   timezone.system
 
+{% if salt['pillar.get']('elasticsearch:auth:enabled', False) %}
 elastic_curl_config:
   file.managed:
     - name: /opt/so/conf/elasticsearch/curl.config
@@ -189,6 +195,11 @@ elastic_curl_config:
     - mode: 600
     - show_changes: False
     - makedirs: True
+  {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %}
+    - require:
+      - file: elastic_curl_config_distributed
+  {% endif %}
+{% endif %}
 
 # Sync some Utilities
 utilsyncscripts:

salt/common/tools/sbin/so-common

@@ -360,6 +360,13 @@ run_check_net_err() {
 		exit $exit_code
 	fi
 }
+set_cron_service_name() {
+  if [[ "$OS" == "centos" ]]; then
+    cron_service_name="crond"
+  else
+    cron_service_name="cron"
+  fi
+}
 
 set_os() {
 	if [ -f /etc/redhat-release ]; then

salt/common/tools/sbin/soup

@@ -468,6 +468,51 @@ post_to_2.3.100() {
   echo "Post Processing for .100"
 }
 
+stop_salt_master() {
+    # kill all salt jobs across the grid because the hang indefinitely if they are queued and salt-master restarts
+    set +e
+    echo ""
+    echo "Killing all Salt jobs across the grid."
+    salt \* saltutil.kill_all_jobs
+    echo ""
+    echo "Killing any queued Salt jobs on the manager."
+    pkill -9 -ef "/usr/bin/python3 /bin/salt"
+    set -e
+
+    echo ""
+    echo "Storing salt-master pid."
+    MASTERPID=$(pgrep salt-master | head -1)
+    echo "Found salt-master PID $MASTERPID"
+    echo ""
+    echo "Stopping Salt Master service at $(date +"%T.%6N")"
+    systemctl stop salt-master
+    echo ""
+    timeout 30 tail --pid=$MASTERPID -f /dev/null || echo "salt-master still running at $(date +"%T.%6N") after waiting 30s. We cannot kill due to systemd restart option."
+}
+
+stop_salt_minion() {
+    echo "Disabling highstate to prevent from running if salt-minion restarts."
+    salt-call state.disable highstate -l info --local
+    echo ""
+
+    # kill all salt jobs before stopping salt-minion
+    set +e
+    echo ""
+    echo "Killing Salt jobs on this node."
+    salt-call saltutil.kill_all_jobs --local
+    set -e
+
+    echo "Storing salt-minion pid."
+    MINIONPID=$(pgrep salt-minion | head -1)
+    echo "Found salt-minion PID $MINIONPID"
+    echo "Stopping Salt Minion service at $(date +"%T.%6N")."
+    systemctl stop salt-minion
+
+    set +e
+    timeout 30 tail --pid=$MINIONPID -f /dev/null || echo "Killing salt-minion at $(date +"%T.%6N") after waiting 30s" && pkill -9 -ef /usr/bin/salt-minion
+    set -e
+}
+
 up_to_2.3.20(){
   DOCKERSTUFFBIP=$(echo $DOCKERSTUFF | awk -F'.' '{print $1,$2,$3,1}' OFS='.')/24
   # Remove PCAP from global
@@ -784,11 +829,11 @@ upgrade_salt() {
     echo "Removing yum versionlock for Salt."
     echo ""
     yum versionlock delete "salt-*"
-    echo "Updating Salt packages and restarting services."
+    echo "Updating Salt packages."
     echo ""
     set +e
     run_check_net_err \
-    "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \
+    "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -X -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \
     "Could not update salt, please check $SOUP_LOG for details."
     set -e
     echo "Applying yum versionlock for Salt."
@@ -801,11 +846,11 @@ upgrade_salt() {
     apt-mark unhold "salt-common"
     apt-mark unhold "salt-master"
     apt-mark unhold "salt-minion"
-    echo "Updating Salt packages and restarting services."
+    echo "Updating Salt packages."
     echo ""
     set +e
     run_check_net_err \
-    "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable \"$NEWSALTVERSION\"" \
+    "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -X -F -M -x python3 stable \"$NEWSALTVERSION\"" \
     "Could not update salt, please check $SOUP_LOG for details."
     set -e
     echo "Applying apt hold for Salt."
@@ -913,7 +958,18 @@ fix_wazuh() {
 main() {
   trap 'check_err $?' EXIT
   
+  if [ -n "$BRANCH" ]; then
+    echo "SOUP will use the $BRANCH branch."
+    echo ""
+  fi
+
   echo "### Preparing soup at $(date) ###"
+  echo ""
+
+  echo "Checking to see if this is a manager."
+  echo ""
+  require_manager
+
   check_pillar_items
 
   echo "Checking to see if this is an airgap install."
@@ -923,9 +979,7 @@ main() {
     echo "Missing file argument (-f <FILENAME>) for unattended airgap upgrade."
     exit 0
   fi
-  echo "Checking to see if this is a manager."
+
-  echo ""
-  require_manager
   set_minionid
   echo "Found that Security Onion $INSTALLEDVERSION is currently installed."
   echo ""
@@ -943,6 +997,7 @@ main() {
   verify_latest_update_script
   echo ""
   set_os
+  set_cron_service_name
   set_palette
   check_elastic_license
   echo ""
@@ -976,6 +1031,10 @@ main() {
     echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION."
     echo ""
 
+    echo "Stopping $cron_service_name service at $(date +"%T.%6N")."
+    echo ""
+    systemctl stop "$cron_service_name"
+
     # update mine items prior to stopping salt-minion and salt-master
     update_salt_mine
 
@@ -992,17 +1051,9 @@ main() {
       set -e
     fi
 
-    echo ""
+    stop_salt_minion
-    echo "Stopping Salt Minion service."
+
-    systemctl stop salt-minion
+    stop_salt_master
-    echo "Killing any remaining Salt Minion processes."
-    set +e
-    pkill -9 -ef /usr/bin/salt-minion
-    set -e
-    echo ""
-    echo "Stopping Salt Master service."
-    systemctl stop salt-master
-    echo ""
 
     upgrade_to_2.3.50_repo
 
@@ -1048,11 +1099,11 @@ main() {
     update_version
 
     echo ""
-    echo "Locking down Salt Master for upgrade"
+    echo "Locking down Salt Master for upgrade at $(date +"%T.%6N")."
     masterlock
 
     echo ""
-    echo "Starting Salt Master service."
+    echo "Starting Salt Master service at $(date +"%T.%6N")."
     systemctl start salt-master
 
     # Testing that salt-master is up by checking that is it connected to itself
@@ -1066,6 +1117,13 @@ main() {
     salt-call state.apply salt.python3-influxdb -l info queue=True
     echo ""
 
+    # update the salt-minion configs here and start the minion
+    # since highstate are disabled above, minion start should not trigger a highstate
+    echo ""
+    echo "Ensuring salt-minion configs are up-to-date."
+    salt-call state.apply salt.minion -l info queue=True
+    echo ""
+
     # Only regenerate osquery packages if Fleet is enabled
     FLEET_MANAGER=$(lookup_pillar fleet_manager)
     FLEET_NODE=$(lookup_pillar fleet_node)
@@ -1076,20 +1134,22 @@ main() {
       echo ""
     fi
 
+    echo "Enabling highstate."
+    salt-call state.enable highstate -l info --local
     echo ""
-    echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes."
+
+    echo ""
+    echo "Running a highstate. This could take several minutes."
     set +e
     salt-call state.highstate -l info queue=True
     set -e
 
-    echo ""
+    stop_salt_master
-    echo "Stopping Salt Master to remove ACL"
-    systemctl stop salt-master
 
     masterunlock
 
     echo ""
-    echo "Starting Salt Master service."
+    echo "Starting Salt Master service at $(date +"%T.%6N") ."
     systemctl start salt-master
 
     set +e
@@ -1097,7 +1157,7 @@ main() {
     salt-call state.show_top -l error queue=True || fail "salt-master could not be reached. Check $SOUP_LOG for details."
     set -e
     
-    echo "Running a highstate. This could take several minutes."
+    echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes."
     salt-call state.highstate -l info queue=True
     postupgrade_changes
     [[ $is_airgap -eq 0 ]] && unmount_update
@@ -1145,6 +1205,9 @@ main() {
       esac
     fi
 
+    echo "Starting $cron_service_name service at $(date +"%T.%6N")."
+    systemctl start "$cron_service_name"
+
     if [[ $NUM_MINIONS -gt 1 ]]; then
 
       cat << EOF
@@ -1223,12 +1286,6 @@ https://blog.securityonion.net
 
 EOF
 
-  if [ -n "$BRANCH" ]; then
-    cat << EOF
-SOUP will use the $BRANCH branch.
-
-EOF
-  fi
   cat << EOF
 Press Enter to continue or Ctrl-C to cancel.
 EOF

salt/cron/dead.sls

@@ -0,0 +1,6 @@
+{% from "cron/map.jinja" import cronmap with context %}
+
+crond_service:
+  service.dead:
+    - name: {{ cronmap.service }}
+    - enable: True

salt/cron/map.jinja

@@ -0,0 +1,8 @@
+{% set cronmap = salt['grains.filter_by']({
+    'Debian': {
+        'service': 'cron',
+    },
+    'RedHat': {
+        'service': 'crond',
+    },
+}) %}

salt/cron/running.sls

@@ -0,0 +1,7 @@
+{% from "cron/map.jinja" import cronmap with context %}
+
+crond_service:
+  service.running:
+    - name: {{ cronmap.service }}
+    - enable: True
+    - unless: pgrep soup

salt/elasticsearch/files/curl.config.template

@@ -1 +1 @@
-user = "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user') }}:{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass') }}"
+user = "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', 'NO_USER_SET') }}:{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', 'NO_PW_SET') }}"

salt/elasticsearch/init.sls

@@ -53,7 +53,7 @@ vm.max_map_count:
 cascriptsync:
   file.managed:
     - name: /usr/sbin/so-catrust
-    - source: salt://elasticsearch/files/scripts/so-catrust
+    - source: salt://elasticsearch/tools/sbin/so-catrust
     - user: 939
     - group: 939
     - mode: 750
@@ -63,9 +63,37 @@ cascriptsync:
 cascriptfun:
   cmd.run:
     - name: /usr/sbin/so-catrust
-
+    - require:
+        - file: cascriptsync
 {% endif %}
 
+# Sync some es scripts
+es_sync_scripts:
+  file.recurse:
+    - name: /usr/sbin
+    - user: root
+    - group: root
+    - file_mode: 755
+    - template: jinja
+    - source: salt://elasticsearch/tools/sbin
+    - defaults:
+        ELASTICCURL: 'curl'
+    - context:
+        ELASTICCURL: {{ ELASTICAUTH.elasticcurl }}
+    - exclude_pat:
+        - so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state
+
+so-elasticsearch-pipelines-script:
+  file.managed:
+    - name: /usr/sbin/so-elasticsearch-pipelines
+    - source: salt://elasticsearch/tools/sbin/so-elasticsearch-pipelines
+    - user: 930
+    - group: 939
+    - mode: 754
+    - template: jinja
+    - defaults:
+        ELASTICCURL: {{ ELASTICAUTH.elasticcurl }}
+
 # Move our new CA over so Elastic and Logstash can use SSL with the internal CA
 catrustdir:
   file.directory:
@@ -297,7 +325,7 @@ so-elasticsearch:
       - file: esyml
       - file: esingestconf
       - file: esingestdynamicconf
-      - file: so-elasticsearch-pipelines-file
+      - file: so-elasticsearch-pipelines-script
     - require:
       - file: esyml
       - file: eslog4jfile
@@ -322,27 +350,17 @@ append_so-elasticsearch_so-status.conf:
     - name: /opt/so/conf/so-status/so-status.conf
     - text: so-elasticsearch
 
-so-elasticsearch-pipelines-file:
-  file.managed:
-    - name: /opt/so/conf/elasticsearch/so-elasticsearch-pipelines
-    - source: salt://elasticsearch/files/so-elasticsearch-pipelines
-    - user: 930
-    - group: 939
-    - mode: 754
-    - template: jinja
-    - defaults:
-        ELASTICCURL: {{ ELASTICAUTH.elasticcurl }}
-
 so-elasticsearch-pipelines:
   cmd.run:
-    - name: /opt/so/conf/elasticsearch/so-elasticsearch-pipelines {{ grains.host }}
+    - name: /usr/sbin/so-elasticsearch-pipelines {{ grains.host }}
     - onchanges:
       - file: esingestconf
       - file: esingestdynamicconf
       - file: esyml
-      - file: so-elasticsearch-pipelines-file
+      - file: so-elasticsearch-pipelines-script
     - require:
       - docker_container: so-elasticsearch
+      - file: so-elasticsearch-pipelines-script
 
 {% if TEMPLATES %}
 so-elasticsearch-templates:
@@ -352,6 +370,7 @@ so-elasticsearch-templates:
     - template: jinja
     - require:
       - docker_container: so-elasticsearch
+      - file: es_sync_scripts
 {% endif %}
 
 so-elasticsearch-roles-load:
@@ -361,6 +380,7 @@ so-elasticsearch-roles-load:
     - template: jinja
     - require:
       - docker_container: so-elasticsearch
+      - file: es_sync_scripts
 
 {% endif %} {# if grains['role'] != 'so-helix' #}
 

salt/manager/elasticsearch.sls

@@ -0,0 +1,9 @@
+{% if salt['pillar.get']('elasticsearch:auth:enabled', False) %}
+elastic_curl_config_distributed:
+  file.managed:
+    - name: /opt/so/saltstack/local/salt/elasticsearch/curl.config
+    - source: salt://elasticsearch/files/curl.config.template
+    - template: jinja
+    - mode: 600
+    - show_changes: False
+{% endif %}

salt/manager/init.sls

@@ -21,10 +21,10 @@
 {% set STRELKA_RULES = salt['pillar.get']('strelka:rules', '1') %}
 
 include:
-  - elasticsearch.auth
-  - kibana.secrets
   - salt.minion
-  - kratos
+  - kibana.secrets
+  - manager.sync_es_users
+  - manager.elasticsearch
 
 socore_own_saltstack:
   file.directory:
@@ -110,29 +110,6 @@ strelka_yara_update:
     - hour: '7'
     - minute: '1'
 
-elastic_curl_config_distributed:
-  file.managed:
-    - name: /opt/so/saltstack/local/salt/elasticsearch/curl.config
-    - source: salt://elasticsearch/files/curl.config.template
-    - template: jinja
-    - mode: 600
-    - show_changes: False
-
-# Must run before elasticsearch docker container is started!
-syncesusers:
-  cmd.run:
-    - name: so-user sync
-    - env:
-      - SKIP_STATE_APPLY: 'true'
-    - creates:
-      - /opt/so/saltstack/local/salt/elasticsearch/files/users
-      - /opt/so/saltstack/local/salt/elasticsearch/files/users_roles
-      - /opt/so/conf/soc/soc_users_roles
-    - show_changes: False
-    - require:
-      - docker_container: so-kratos
-      - http: wait_for_kratos
-
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/manager/sync_es_users.sls

@@ -0,0 +1,31 @@
+include:
+  - elasticsearch.auth
+  - kratos
+
+so-user.lock:
+  file.missing:
+    - name: /var/tmp/so-user.lock
+
+# Must run before elasticsearch docker container is started!
+sync_es_users:
+  cmd.run:
+    - name: so-user sync
+    - env:
+      - SKIP_STATE_APPLY: 'true'
+    - creates:
+      - /opt/so/saltstack/local/salt/elasticsearch/files/users
+      - /opt/so/saltstack/local/salt/elasticsearch/files/users_roles
+      - /opt/so/conf/soc/soc_users_roles
+    - show_changes: False
+    - require:
+      - docker_container: so-kratos
+      - http: wait_for_kratos
+      - file: so-user.lock # require so-user.lock file to be missing
+
+# we dont want this added too early in setup, so we add the onlyif to verify 'startup_states: highstate'
+# is in the minion config. That line is added before the final highstate during setup
+sosyncusers:
+  cron.present:
+    - user: root
+    - name: 'PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin /usr/sbin/so-user sync &>> /opt/so/log/soc/sync.log'
+    - onlyif: "grep 'startup_states: highstate' /etc/salt/minion"

salt/salt/enable_highstate.sls

@@ -0,0 +1,7 @@
+enable_highstate:
+  module.run:
+    - state.enable:
+      - states:
+        - highstate
+    - unless: pgrep soup
+  

salt/salt/minion.sls

@@ -62,8 +62,6 @@ set_log_levels:
     - text:
       - "log_level: error"
       - "log_level_logfile: error"
-    - listen_in:
-      - service: salt_minion_service
 
 salt_minion_service_unit_file:
   file.managed:
@@ -74,8 +72,6 @@ salt_minion_service_unit_file:
         service_start_delay: {{ service_start_delay }}
     - onchanges_in:
       - module: systemd_reload
-    - listen_in:
-      - service: salt_minion_service
 
 {% endif %}
 
@@ -91,8 +87,13 @@ salt_minion_service:
     - name: salt-minion
     - enable: True
     - onlyif: test "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}"
-    - watch:
+    - listen:
       - file: mine_functions
+{% if INSTALLEDSALTVERSION|string == SALTVERSION|string %}
+      - file: set_log_levels
+      - file: salt_minion_service_unit_file
+{% endif %}
+
 
 patch_pkg:
   pkg.installed:

salt/soc/init.sls

@@ -5,6 +5,9 @@
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 {% set MANAGER = salt['grains.get']('master') %}
 
+include:
+  - manager.sync_es_users
+
 socdir:
   file.directory:
     - name: /opt/so/conf/soc
@@ -84,14 +87,8 @@ soccustomroles:
 socusersroles:
   file.exists:
     - name: /opt/so/conf/soc/soc_users_roles
-
+    - require:
-# we dont want this added too early in setup, so we add the onlyif to verify 'startup_states: highstate'
+      - sls: manager.sync_es_users
-# is in the minion config. That line is added before the final highstate during setup
-sosyncusers:
-  cron.present:
-    - user: root
-    - name: 'PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin /usr/sbin/so-user sync &>> /opt/so/log/soc/sync.log'
-    - onlyif: "grep 'startup_states: highstate' /etc/salt/minion"
 
 so-soc:
   docker_container.running:

salt/top.sls

@@ -19,6 +19,10 @@
 
 base:
 
+  '*':
+    - salt.enable_highstate
+    - cron.running
+
   'not G@saltversion:{{saltversion}}':
     - match: compound
     - salt.minion-state-apply-test

setup/so-functions

@@ -2144,7 +2144,7 @@ restore_file() {
 	if [ -f "$src" ]; then
 		[ ! -d "$dst" ] && mkdir -v -p "$dst"
 		echo "Restoring $src to $dst." >> "$setup_log" 2>&1
-		cp -v "$src" "$dst"
+		cp -v "$src" "$dst" >> "$setup_log" 2>&1
 	fi
 }