merge with 2.3.40

salt/common/init.sls

@@ -90,7 +90,6 @@ commonpkgs:
       - ntpdate
       - jq
       - python3-docker
-      - docker-ce
       - curl
       - ca-certificates
       - software-properties-common
@@ -104,14 +103,17 @@ commonpkgs:
       - python3-dateutil
       - python3-m2crypto
       - python3-mysqldb
+      - python3-packaging
       - git
       - patch
 
 heldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 1.2.13-2
-      - docker-ce: 5:19.03.14~3-0~ubuntu-bionic
+      - containerd.io: 1.4.4-1
+      - docker-ce: 5:20.10.5~3-0~ubuntu-bionic
+      - docker-ce-cli: 5:20.10.5~3-0~ubuntu-bionic
+      - docker-ce-rootless-extras: 5:20.10.5~3-0~ubuntu-bionic
     - hold: True
     - update_holds: True
 
@@ -137,6 +139,7 @@ commonpkgs:
       - python36-dateutil
       - python36-m2crypto
       - python36-mysql
+      - python36-packaging
       - yum-utils
       - device-mapper-persistent-data
       - lvm2
@@ -147,8 +150,10 @@ commonpkgs:
 heldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 1.2.13-3.2.el7
-      - docker-ce: 3:19.03.14-3.el7
+      - containerd.io: 1.4.4-3.1.el7
+      - docker-ce: 3:20.10.5-3.el7
+      - docker-ce-cli: 1:20.10.5-3.el7
+      - docker-ce-rootless-extras: 20.10.5-3.el7
     - hold: True
     - update_holds: True
 {% endif %}

salt/common/tools/sbin/so-common

@@ -86,6 +86,19 @@ add_interface_bond0() {
 	fi
 }
 
+check_airgap() {
+  # See if this is an airgap install
+  AIRGAP=$(cat /opt/so/saltstack/local/pillar/global.sls | grep airgap: | awk '{print $2}')
+  if [[ "$AIRGAP" == "True" ]]; then
+      is_airgap=0
+      UPDATE_DIR=/tmp/soagupdate/SecurityOnion
+      AGDOCKER=/tmp/soagupdate/docker
+      AGREPO=/tmp/soagupdate/Packages
+  else 
+      is_airgap=1
+  fi
+}
+
 check_container() {
 	docker ps | grep "$1:" > /dev/null 2>&1
 	return $?
@@ -97,6 +110,46 @@ check_password() {
 	return $?
 }
 
+check_elastic_license() {
+
+	[ -n "$TESTING" ] && return
+
+	# See if the user has already accepted the license
+	if [ ! -f /opt/so/state/yeselastic.txt ]; then
+	  elastic_license
+	else
+	  echo "Elastic License has already been accepted"
+	fi  
+}
+
+elastic_license() {
+
+read -r -d '' message <<- EOM
+\n
+Starting in Elastic Stack version 7.11, the Elastic Stack binaries are only available under the Elastic License:
+https://securityonion.net/elastic-license
+
+Please review the Elastic License:
+https://www.elastic.co/licensing/elastic-license
+
+Do you agree to the terms of the Elastic License?
+
+If so, type AGREE to accept the Elastic License and continue.  Otherwise, press Enter to exit this program without making any changes.
+EOM
+
+AGREED=$(whiptail --title "Security Onion Setup" --inputbox \
+"$message" 20 75 3>&1 1>&2 2>&3)
+
+if [ "${AGREED^^}" = 'AGREE' ]; then
+  mkdir -p /opt/so/state
+  touch /opt/so/state/yeselastic.txt 
+else
+  echo "Starting in 2.3.40 you must accept the Elastic license if you want to run Security Onion."
+  exit 1
+fi
+
+}
+
 fail() {
 	msg=$1
 	echo "ERROR: $msg"
@@ -250,6 +303,12 @@ set_minionid() {
 	MINIONID=$(lookup_grain id)
 }
 
+set_palette() {
+	if [ "$OS" == ubuntu ]; then
+	    update-alternatives --set newt-palette /etc/newt/palette.original
+    fi
+}
+
 set_version() {
 	CURRENTVERSION=0.0.0
 	if [ -f /etc/soversion ]; then
@@ -340,6 +399,26 @@ valid_int() {
 
 # {% raw %}
 
+valid_proxy() {
+	local proxy=$1
+	local url_prefixes=( 'http://' 'https://' )
+
+	local has_prefix=false
+	for prefix in "${url_prefixes[@]}"; do
+		echo "$proxy" | grep -q "$prefix" && has_prefix=true && proxy=${proxy#"$prefix"} && break
+	done
+	
+	local url_arr
+	mapfile -t url_arr <<< "$(echo "$proxy" | tr ":" "\n")"
+
+	local valid_url=true
+	if ! valid_ip4 "${url_arr[0]}" && ! valid_fqdn "${url_arr[0]}" && ! valid_hostname "${url_arr[0]}"; then
+		valid_url=false
+	fi
+
+	[[ $has_prefix == true ]] && [[ $valid_url == true ]] && return 0 || return 1
+}
+
 valid_string() {
 	local str=$1
 	local min_length=${2:-1}

salt/common/tools/sbin/so-cortex-user-add

@@ -30,7 +30,7 @@ fi
 
 USER=$1
 
-CORTEX_KEY=$(lookup_pillar cortexkey)
+CORTEX_KEY=$(lookup_pillar cortexorguserkey)
 CORTEX_API_URL="$(lookup_pillar url_base)/cortex/api"
 CORTEX_ORG_NAME=$(lookup_pillar cortexorgname)
 CORTEX_USER=$USER

salt/common/tools/sbin/so-cortex-user-enable

@@ -30,7 +30,7 @@ fi
 
 USER=$1
 
-CORTEX_KEY=$(lookup_pillar cortexkey)
+CORTEX_KEY=$(lookup_pillar cortexorguserkey)
 CORTEX_API_URL="$(lookup_pillar url_base)/cortex/api"
 CORTEX_USER=$USER
 

salt/common/tools/sbin/so-docker-prune

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import sys, argparse, re, docker
+from packaging.version import Version, InvalidVersion
+from itertools import groupby, chain
+
+
+def get_image_name(string) -> str:
+  return ':'.join(string.split(':')[:-1])
+
+
+def get_so_image_basename(string) -> str:
+  return get_image_name(string).split('/so-')[-1]
+
+
+def get_image_version(string) -> str:
+  ver = string.split(':')[-1]
+  if ver == 'latest':
+    # Version doesn't like "latest", so use a high semver
+    return '999999.9.9'
+  else:
+    try:
+      Version(ver)
+    except InvalidVersion:
+      # Strip the last substring following a hyphen for automated branches
+      ver = '-'.join(ver.split('-')[:-1])  
+    return ver
+
+
+def main(quiet):
+  client = docker.from_env()
+
+  image_list = client.images.list(filters={ 'dangling': False })
+
+  # Map list of image objects to flattened list of tags (format: "name:version")
+  tag_list = list(chain.from_iterable(list(map(lambda x: x.attrs.get('RepoTags'), image_list))))
+
+  # Filter to only SO images (base name begins with "so-")
+  tag_list = list(filter(lambda x: re.match(r'^.*\/so-[^\/]*$', get_image_name(x)), tag_list))
+
+  # Group tags into lists by base name (sort by same projection first)
+  tag_list.sort(key=lambda x: get_so_image_basename(x))
+  grouped_tag_lists = [ list(it) for _, it in groupby(tag_list, lambda x: get_so_image_basename(x)) ]
+
+  no_prunable = True
+  for t_list in grouped_tag_lists:
+    try:
+      # Keep the 2 most current images
+      t_list.sort(key=lambda x: Version(get_image_version(x)), reverse=True)
+      if len(t_list) <= 2:
+        continue
+      else:
+        no_prunable = False
+        for tag in t_list[2:]:
+          if not quiet: print(f'Removing image {tag}')
+          client.images.remove(tag)
+    except InvalidVersion as e:
+      print(f'so-{get_so_image_basename(t_list[0])}: {e.args[0]}', file=sys.stderr)
+      exit(1)
+
+  if no_prunable and not quiet:
+    print('No Security Onion images to prune')
+
+
+if __name__ == "__main__":
+  main_parser = argparse.ArgumentParser(add_help=False)
+  main_parser.add_argument('-q', '--quiet', action='store_const', const=True, required=False)
+  args = main_parser.parse_args(sys.argv[1:])
+ 
+  main(args.quiet)

salt/common/tools/sbin/so-kibana-space-defaults

@@ -0,0 +1,13 @@
+. /usr/sbin/so-common
+
+wait_for_web_response "http://localhost:5601/app/kibana" "Elastic"
+## This hackery will be removed if using Elastic Auth ##
+
+# Let's snag a cookie from Kibana
+THECOOKIE=$(curl -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
+
+# Disable certain Features from showing up in the Kibana UI
+echo
+echo "Setting up default Space:"
+curl -b "sid=$THECOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","siem","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","fleet"]} ' >> /opt/so/log/kibana/misc.log
+echo

salt/common/tools/sbin/so-rule

@@ -37,11 +37,9 @@ def print_err(string: str):
 
 
 def check_apply(args: dict, prompt: bool = True):
-  cmd_arr = ['salt-call', 'state.apply', 'idstools', 'queue=True']
-
   if args.apply:
-    print('Configuration updated. Applying idstools state...')
-    return subprocess.run(cmd_arr)
+    print('Configuration updated. Applying changes:')
+    return apply()
   else:
     if prompt:
       message = 'Configuration updated. Would you like to apply your changes now? (y/N) '
@@ -51,12 +49,24 @@ def check_apply(args: dict, prompt: bool = True):
       if answer.lower() in [ 'n', '' ]:
         return 0
       else:
-        print('Applying idstools state...')
-        return subprocess.run(cmd_arr)
+        print('Applying changes:')
+        return apply()
     else:
       return 0
 
 
+def apply():
+  salt_cmd = ['salt-call', 'state.apply', '-l', 'quiet', 'idstools.sync_files', 'queue=True']
+  update_cmd = ['so-rule-update']
+  print('Syncing config files...')
+  cmd = subprocess.run(salt_cmd, stdout=subprocess.DEVNULL)
+  if cmd.returncode == 0:
+    print('Updating rules...')
+    return subprocess.run(update_cmd).returncode
+  else:
+    return cmd.returncode
+
+
 def find_minion_pillar() -> str:
   regex = '^.*_(manager|managersearch|standalone|import|eval)\.sls$'
   
@@ -442,10 +452,7 @@ def main():
         modify.print_help()
     sys.exit(0)
   
-  if isinstance(exit_code, subprocess.CompletedProcess):
-    sys.exit(exit_code.returncode)
-  else:
-    sys.exit(exit_code)
+  sys.exit(exit_code)
 
 
 if __name__ == '__main__':

salt/common/tools/sbin/soup

@@ -19,13 +19,12 @@
 
 UPDATE_DIR=/tmp/sogh/securityonion
 INSTALLEDVERSION=$(cat /etc/soversion)
+POSTVERSION=$INSTALLEDVERSION
 INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk {'print $2'})
 DEFAULT_SALT_DIR=/opt/so/saltstack/default
 BATCHSIZE=5
 SOUP_LOG=/root/soup.log
 
-exec 3>&1 1>${SOUP_LOG} 2>&1
-
 add_common() {
   cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
   cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
@@ -101,19 +100,6 @@ update_registry() {
   salt-call state.apply registry queue=True
 }
 
-check_airgap() {
-  # See if this is an airgap install
-  AIRGAP=$(cat /opt/so/saltstack/local/pillar/global.sls | grep airgap: | awk '{print $2}')
-  if [[ "$AIRGAP" == "True" ]]; then
-      is_airgap=0
-      UPDATE_DIR=/tmp/soagupdate/SecurityOnion
-      AGDOCKER=/tmp/soagupdate/docker
-      AGREPO=/tmp/soagupdate/Packages
-  else 
-      is_airgap=1
-  fi
-}
-
 check_sudoers() {
 	if grep -q "so-setup" /etc/sudoers; then
 		echo "There is an entry for so-setup in the sudoers file, this can be safely deleted using \"visudo\"."
@@ -243,22 +229,10 @@ masterunlock() {
   fi
 }
 
-playbook() {
-  echo "Applying playbook settings"
-  if [[ "$INSTALLEDVERSION" =~ rc.1 ]]; then
-    salt-call state.apply playbook.OLD_db_init
-    rm -f /opt/so/rules/elastalert/playbook/*.yaml
-    so-playbook-ruleupdate >> /root/soup_playbook_rule_update.log 2>&1 &
-  fi
-  if [[ "$INSTALLEDVERSION" != 2.3.30 ]]; then
-    so-playbook-sigma-refresh >> /root/soup_playbook_sigma_refresh.log 2>&1 &
-  fi
-}
-
-pillar_changes() {
+preupgrade_changes() {
     # This function is to add any new pillar items if needed.
-    echo "Checking to see if pillar changes are needed."
-    
+    echo "Checking to see if changes are needed."
+
     [[ "$INSTALLEDVERSION" =~ rc.1 ]] && rc1_to_rc2
     [[ "$INSTALLEDVERSION" =~ rc.2 ]] && rc2_to_rc3
     [[ "$INSTALLEDVERSION" =~ rc.3 ]] && rc3_to_2.3.0
@@ -266,6 +240,34 @@ pillar_changes() {
     [[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30
 }
 
+postupgrade_changes() {
+    # This function is to add any new pillar items if needed.
+    echo "Running post upgrade processes."
+    
+    [[ "$POSTVERSION" =~ rc.1 ]] && post_rc1_to_rc2
+    [[ "$POSTVERSION" == 2.3.20 || "$POSTVERSION" == 2.3.21 ]] && post_2.3.2X_to_2.3.30
+    [[ "$POSTVERSION" == 2.3.30 ]] && post_2.3.30_to_2.3.40
+}
+
+post_rc1_to_2.3.21() {
+  salt-call state.apply playbook.OLD_db_init
+  rm -f /opt/so/rules/elastalert/playbook/*.yaml
+  so-playbook-ruleupdate >> /root/soup_playbook_rule_update.log 2>&1 &
+  POSTVERSION=2.3.21
+}
+
+post_2.3.2X_to_2.3.30() {
+  so-playbook-sigma-refresh >> /root/soup_playbook_sigma_refresh.log 2>&1 &
+  POSTVERSION=2.3.30
+}
+
+post_2.3.30_to_2.3.40() {
+  so-playbook-sigma-refresh >> /root/soup_playbook_sigma_refresh.log 2>&1 &
+  so-kibana-space-defaults
+  POSTVERSION=2.3.40
+}
+
+
 rc1_to_rc2() {
 
   # Move the static file to global.sls
@@ -296,15 +298,14 @@ rc1_to_rc2() {
   done </tmp/nodes.txt
   # Add the nodes back using hostname
   while read p; do
-      local NAME=$(echo $p | awk '{print $1}')
-      local EHOSTNAME=$(echo $p | awk -F"_" '{print $1}')
-      local IP=$(echo $p | awk '{print $2}')
-      echo "Adding the new cross cluster config for $NAME"
-      curl -XPUT http://localhost:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"'$NAME'": {"skip_unavailable": "true", "seeds": ["'$EHOSTNAME':9300"]}}}}}'
+    local NAME=$(echo $p | awk '{print $1}')
+    local EHOSTNAME=$(echo $p | awk -F"_" '{print $1}')
+    local IP=$(echo $p | awk '{print $2}')
+    echo "Adding the new cross cluster config for $NAME"
+    curl -XPUT http://localhost:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"'$NAME'": {"skip_unavailable": "true", "seeds": ["'$EHOSTNAME':9300"]}}}}}'
   done </tmp/nodes.txt
 
   INSTALLEDVERSION=rc.2
-
 }
 
 rc2_to_rc3() {
@@ -334,10 +335,10 @@ rc3_to_2.3.0() {
   fi
 
   {
-      echo "redis_settings:"
-      echo "  redis_maxmemory: 827"
-      echo "playbook:"
-      echo "  api_key: de6639318502476f2fa5aa06f43f51fb389a3d7f" 
+    echo "redis_settings:"
+    echo "  redis_maxmemory: 827"
+    echo "playbook:"
+    echo "  api_key: de6639318502476f2fa5aa06f43f51fb389a3d7f" 
   } >> /opt/so/saltstack/local/pillar/global.sls
 
   sed -i 's/playbook:/playbook_db:/' /opt/so/saltstack/local/pillar/secrets.sls
@@ -385,7 +386,6 @@ up_2.3.0_to_2.3.20(){
   fi
 
   INSTALLEDVERSION=2.3.20
-
 }
 
 up_2.3.2X_to_2.3.30() {
@@ -395,11 +395,11 @@ up_2.3.2X_to_2.3.30() {
     sed -i -r "s/ (\{\{.*}})$/ '\1'/g" "$pillar"
   done
 
-# Change the IMAGEREPO
+  # Change the IMAGEREPO
   sed -i "/  imagerepo: 'securityonion'/c\  imagerepo: 'security-onion-solutions'" /opt/so/saltstack/local/pillar/global.sls
   sed -i "/  imagerepo: securityonion/c\  imagerepo: 'security-onion-solutions'" /opt/so/saltstack/local/pillar/global.sls 
 
-# Strelka rule repo pillar addition
+  # Strelka rule repo pillar addition
   if [ $is_airgap -eq 0 ]; then
       # Add manager as default Strelka YARA rule repo
       sed -i "/^strelka:/a \\  repos: \n    - https://$HOSTNAME/repo/rules/strelka" /opt/so/saltstack/local/pillar/global.sls;
@@ -410,16 +410,26 @@ up_2.3.2X_to_2.3.30() {
   check_log_size_limit
 }
 
-space_check() {
-  # Check to see if there is enough space
+verify_upgradespace() {
   CURRENTSPACE=$(df -BG / | grep -v Avail | awk '{print $4}' | sed 's/.$//')
   if [ "$CURRENTSPACE" -lt "10" ]; then
-      echo "You are low on disk space. Upgrade will try and clean up space.";
-      clean_dockers
+      echo "You are low on disk space."
+      return 1
   else
-      echo "Plenty of space for upgrading"
+      return 0
   fi
-  
+}
+
+upgrade_space() {
+  if ! verify_upgradespace; then
+    clean_dockers
+    if ! verify_upgradespace; then
+      echo "There is not enough space to perform the upgrade. Please free up space and try again"
+      exit 1
+    fi
+  else
+      echo "You have enough space for upgrade. Proceeding with soup."
+  fi  
 }
 
 thehive_maint() {
@@ -427,16 +437,16 @@ thehive_maint() {
   COUNT=0
   THEHIVE_CONNECTED="no"
   while [[ "$COUNT" -le 240 ]]; do
-      curl --output /dev/null --silent --head --fail -k "https://localhost/thehive/api/alert"
-          if [ $? -eq 0 ]; then
-              THEHIVE_CONNECTED="yes"
-              echo "connected!"
-              break
-          else
-              ((COUNT+=1))
-              sleep 1
-              echo -n "."
-          fi
+    curl --output /dev/null --silent --head --fail -k "https://localhost/thehive/api/alert"
+      if [ $? -eq 0 ]; then
+        THEHIVE_CONNECTED="yes"
+        echo "connected!"
+        break
+      else
+        ((COUNT+=1))
+        sleep 1
+        echo -n "."
+      fi
   done
   if [ "$THEHIVE_CONNECTED" == "yes" ]; then
     echo "Migrating thehive databases if needed."
@@ -471,83 +481,84 @@ update_version() {
 }
 
 upgrade_check() {
-    # Let's make sure we actually need to update.
-    NEWVERSION=$(cat $UPDATE_DIR/VERSION)
-    if [ "$INSTALLEDVERSION" == "$NEWVERSION" ]; then
-      echo "You are already running the latest version of Security Onion."
-      exit 0
-    fi 
+  # Let's make sure we actually need to update.
+  NEWVERSION=$(cat $UPDATE_DIR/VERSION)
+  if [ "$INSTALLEDVERSION" == "$NEWVERSION" ]; then
+    echo "You are already running the latest version of Security Onion."
+    exit 0
+  fi 
 }
 
 upgrade_check_salt() {
-    NEWSALTVERSION=$(grep version: $UPDATE_DIR/salt/salt/master.defaults.yaml | awk {'print $2'})
-    if [ "$INSTALLEDSALTVERSION" == "$NEWSALTVERSION" ]; then
-      echo "You are already running the correct version of Salt for Security Onion."
-    else
-      UPGRADESALT=1
-    fi
+  NEWSALTVERSION=$(grep version: $UPDATE_DIR/salt/salt/master.defaults.yaml | awk {'print $2'})
+  if [ "$INSTALLEDSALTVERSION" == "$NEWSALTVERSION" ]; then
+    echo "You are already running the correct version of Salt for Security Onion."
+  else
+    UPGRADESALT=1
+  fi
 }   
 upgrade_salt() {
-      SALTUPGRADED=True
-      echo "Performing upgrade of Salt from $INSTALLEDSALTVERSION to $NEWSALTVERSION."
-      echo ""
-      # If CentOS
-      if [ "$OS" == "centos" ]; then
-        echo "Removing yum versionlock for Salt."
-        echo ""
-        yum versionlock delete "salt-*"
-        echo "Updating Salt packages and restarting services."
-        echo ""
-	if [ $is_airgap -eq 0 ]; then
-          sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable "$NEWSALTVERSION"
-	else 
-          sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable "$NEWSALTVERSION"
-	fi
-        echo "Applying yum versionlock for Salt."
-        echo ""
-        yum versionlock add "salt-*"
-      # Else do Ubuntu things
-      elif [ "$OS" == "ubuntu" ]; then
-        echo "Removing apt hold for Salt."
-        echo ""
-        apt-mark unhold "salt-common"
-        apt-mark unhold "salt-master"
-        apt-mark unhold "salt-minion"
-        echo "Updating Salt packages and restarting services."
-        echo ""
-        sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable "$NEWSALTVERSION"
-        echo "Applying apt hold for Salt."
-        echo ""
-        apt-mark hold "salt-common"
-        apt-mark hold "salt-master"
-        apt-mark hold "salt-minion"
-      fi
+  SALTUPGRADED=True
+  echo "Performing upgrade of Salt from $INSTALLEDSALTVERSION to $NEWSALTVERSION."
+  echo ""
+  # If CentOS
+  if [ "$OS" == "centos" ]; then
+    echo "Removing yum versionlock for Salt."
+    echo ""
+    yum versionlock delete "salt-*"
+    echo "Updating Salt packages and restarting services."
+    echo ""
+    if [ $is_airgap -eq 0 ]; then
+      sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable "$NEWSALTVERSION"
+    else 
+      sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable "$NEWSALTVERSION"
+    fi
+    echo "Applying yum versionlock for Salt."
+    echo ""
+    yum versionlock add "salt-*"
+  # Else do Ubuntu things
+  elif [ "$OS" == "ubuntu" ]; then
+    echo "Removing apt hold for Salt."
+    echo ""
+    apt-mark unhold "salt-common"
+    apt-mark unhold "salt-master"
+    apt-mark unhold "salt-minion"
+    echo "Updating Salt packages and restarting services."
+    echo ""
+    sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable "$NEWSALTVERSION"
+    echo "Applying apt hold for Salt."
+    echo ""
+    apt-mark hold "salt-common"
+    apt-mark hold "salt-master"
+    apt-mark hold "salt-minion"
+  fi
 }
 
 verify_latest_update_script() {
-    # Check to see if the update scripts match. If not run the new one.
-    CURRENTSOUP=$(md5sum /opt/so/saltstack/default/salt/common/tools/sbin/soup | awk '{print $1}')
-    GITSOUP=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/soup | awk '{print $1}')
-    CURRENTCMN=$(md5sum /opt/so/saltstack/default/salt/common/tools/sbin/so-common | awk '{print $1}')
-    GITCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-common | awk '{print $1}')
-    CURRENTIMGCMN=$(md5sum /opt/so/saltstack/default/salt/common/tools/sbin/so-image-common | awk '{print $1}')
-    GITIMGCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-image-common | awk '{print $1}')
+  # Check to see if the update scripts match. If not run the new one.
+  CURRENTSOUP=$(md5sum /opt/so/saltstack/default/salt/common/tools/sbin/soup | awk '{print $1}')
+  GITSOUP=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/soup | awk '{print $1}')
+  CURRENTCMN=$(md5sum /opt/so/saltstack/default/salt/common/tools/sbin/so-common | awk '{print $1}')
+  GITCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-common | awk '{print $1}')
+  CURRENTIMGCMN=$(md5sum /opt/so/saltstack/default/salt/common/tools/sbin/so-image-common | awk '{print $1}')
+  GITIMGCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-image-common | awk '{print $1}')
 
-    if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" ]]; then
-      echo "This version of the soup script is up to date. Proceeding."
-    else
-      echo "You are not running the latest soup version. Updating soup and its components. Might take multiple runs to complete"
-      cp $UPDATE_DIR/salt/common/tools/sbin/soup $DEFAULT_SALT_DIR/salt/common/tools/sbin/
-      cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
-      cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
-      salt-call state.apply common queue=True
-      echo ""
-      echo "soup has been updated. Please run soup again."
-      exit 0
-    fi
+  if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" ]]; then
+    echo "This version of the soup script is up to date. Proceeding."
+  else
+    echo "You are not running the latest soup version. Updating soup and its components. Might take multiple runs to complete"
+    cp $UPDATE_DIR/salt/common/tools/sbin/soup $DEFAULT_SALT_DIR/salt/common/tools/sbin/
+    cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
+    cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
+    salt-call state.apply common queue=True
+    echo ""
+    echo "soup has been updated. Please run soup again."
+    exit 0
+  fi
 }
 
 main () {
+echo "### Preparing soup at `date` ###"
 while getopts ":b" opt; do
   case "$opt" in
     b ) # process option b
@@ -557,9 +568,10 @@ while getopts ":b" opt; do
          echo "Batch size must be a number greater than 0."
          exit 1
        fi
-      ;;
-    \? ) echo "Usage: cmd [-b]"
-      ;;
+    ;;
+    \? ) 
+      echo "Usage: cmd [-b]"
+    ;;
   esac
 done
 
@@ -573,6 +585,8 @@ check_airgap
 echo "Found that Security Onion $INSTALLEDVERSION is currently installed."
 echo ""
 set_os
+set_palette
+check_elastic_license
 echo ""
 if [ $is_airgap -eq 0 ]; then
   # Let's mount the ISO since this is airgap
@@ -599,7 +613,7 @@ fi
 
 echo "Let's see if we need to update Security Onion."
 upgrade_check
-space_check
+upgrade_space
 
 echo "Checking for Salt Master and Minion updates."
 upgrade_check_salt
@@ -649,8 +663,7 @@ else
   echo ""
 fi
 
-echo "Making pillar changes."
-pillar_changes
+preupgrade_changes
 echo ""
 
 if [ $is_airgap -eq 0 ]; then
@@ -704,7 +717,7 @@ echo "Starting Salt Master service."
 systemctl start salt-master
 echo "Running a highstate. This could take several minutes."
 salt-call state.highstate -l info queue=True
-playbook
+postupgrade_changes
 unmount_update
 thehive_maint
 
@@ -736,6 +749,39 @@ if [[ -n $lsl_msg ]]; then
   esac
 fi
 
+NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l)
+
+if [ $NUM_MINIONS -gt 1 ]; then
+
+	cat << EOF
+	
+This appears to be a distributed deployment. Other nodes should update themselves at the next Salt highstate (typically within 15 minutes). Do not manually restart anything until you know that all the search/heavy nodes in your deployment are updated. This is especially important if you are using true clustering for Elasticsearch.
+
+Each minion is on a random 15 minute check-in period and things like network bandwidth can be a factor in how long the actual upgrade takes. If you have a heavy node on a slow link, it is going to take a while to get the containers to it. Depending on what changes happened between the versions, Elasticsearch might not be able to talk to said heavy node until the update is complete.
+
+If it looks like you’re missing data after the upgrade, please avoid restarting services and instead make sure at least one search node has completed its upgrade. The best way to do this is to run 'sudo salt-call state.highstate' from a search node and make sure there are no errors. Typically if it works on one node it will work on the rest. Forward nodes are less complex and will update as they check in so you can monitor those from the Grid section of SOC.
+
+For more information, please see https://docs.securityonion.net/en/2.3/soup.html#distributed-deployments.
+EOF
+
+fi
+echo "### soup has been served at `date` ###"
 }
 
-main "$@" | tee /dev/fd/3
+cat << EOF
+
+SOUP - Security Onion UPdater
+
+Please review the following for more information about the update process and recent updates:
+https://docs.securityonion.net/soup
+https://blog.securityonion.net
+
+Please note that soup only updates Security Onion components and does NOT update the underlying operating system (OS). When you installed Security Onion, there was an option to automatically update the OS packages. If you did not enable this option, then you will want to ensure that the OS is fully updated before running soup.
+
+Press Enter to continue or Ctrl-C to cancel.
+
+EOF
+
+read input
+
+main "$@" | tee -a $SOUP_LOG