highstate , merge with dev fix conflict

salt/common/tools/sbin/so-common

@@ -17,8 +17,8 @@
 
 # Check for prerequisites
 if [ "$(id -u)" -ne 0 ]; then
-    echo "This script must be run using sudo!"
+  echo "This script must be run using sudo!"
-    exit 1
+  exit 1
 fi
 
 # Define a banner to separate sections
@@ -29,19 +29,43 @@ header() {
     printf '%s\n' "$banner" "$*" "$banner"
 }
 
+lookup_salt_value() {
+  key=$1
+  group=$2
+  kind=$3
+
+  if [ -z "$kind" ]; then
+    kind=pillar
+  fi
+
+  if [ -n "$group" ]; then
+    group=${group}:
+  fi
+
+  salt-call --no-color  ${kind}.get ${group}${key} --out=newline_values_only
+}
+
 lookup_pillar() {
-    key=$1
+  key=$1
-    salt-call --no-color  pillar.get global:${key} --out=newline_values_only
+  pillar=$2
+  if [ -z "$pillar" ]; then
+    pillar=global
+  fi
+  lookup_salt_value "$key" "$pillar" "pillar"
 }
 
 lookup_pillar_secret() {
-    key=$1
+  lookup_pillar "$1" "secrets"
-    salt-call --no-color  pillar.get secrets:${key} --out=newline_values_only
 }
 
 lookup_grain() {
-    key=$1
+  lookup_salt_value "$1" "" "grains"
-    salt-call --no-color  grains.get ${key} --out=newline_values_only
+}
+
+lookup_role() {
+  id=$(lookup_grain id)
+  pieces=($(echo $id | tr '_' ' '))
+  echo ${pieces[1]}
 }
 
 check_container() {
@@ -50,9 +74,9 @@ check_container() {
 }
 
 check_password() {
-    local password=$1
+  local password=$1
-    echo "$password" | egrep -v "'|\"|\\$|\\\\" > /dev/null 2>&1
+  echo "$password" | egrep -v "'|\"|\\$|\\\\" > /dev/null 2>&1
-    return $?
+  return $?
 }
 
 set_os() {
@@ -96,3 +120,18 @@ require_manager() {
     exit 1
   fi
 }
+
+is_single_node_grid() {
+  role=$(lookup_role)
+  if [ "$role" != "eval" ] && [ "$role" != "standalone" ] && [ "$role" != "import" ]; then
+    return 1
+  fi
+  return 0
+}
+
+fail() {
+  msg=$1
+  echo "ERROR: $msg"
+  echo "Exiting."
+  exit 1
+}

salt/common/tools/sbin/so-ip-update

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+. $(dirname $0)/so-common
+
+if [ "$FORCE_IP_UPDATE" != "1" ]; then
+	is_single_node_grid || fail "Cannot update the IP on a distributed grid"
+fi
+
+echo "This tool will update a manager's IP address to the new IP assigned to the management network interface."
+
+echo
+echo "WARNING: This tool is still undergoing testing, use at your own risk!"
+echo
+
+if [ -z "$OLD_IP" ]; then
+	OLD_IP=$(lookup_pillar "managerip")
+	
+	if [ -z "$OLD_IP" ]; then
+		fail "Unable to find old IP; possible salt system failure"
+	fi
+
+	echo "Found old IP $OLD_IP."
+fi
+
+if [ -z "$NEW_IP" ]; then
+	iface=$(lookup_pillar "mainint" "host")
+	NEW_IP=$(ip -4 addr list $iface | grep inet | cut -d' ' -f6 | cut -d/ -f1)
+
+	if [ -z "$NEW_IP" ]; then
+		fail "Unable to detect new IP on interface $iface.    "
+	fi
+
+	echo "Detected new IP $NEW_IP on interface $iface."
+fi
+
+if [ "$OLD_IP" == "$NEW_IP" ]; then
+	fail "IP address has not changed"
+fi
+
+echo "About to change old IP $OLD_IP to new IP $NEW_IP."
+
+read -n 1 -p "Would you like to continue? (y/N) " CONTINUE
+echo
+
+if [ "$CONTINUE" == "y" ]; then
+  for file in $(grep -rlI $OLD_IP /opt/so/saltstack /etc); do
+  	echo "Updating file: $file"
+    sed -i "s|$OLD_IP|$NEW_IP|g" $file
+  done
+
+	echo "The IP has been changed from $OLD_IP to $NEW_IP."
+
+	if [ -z "$SKIP_STATE_APPLY" ]; then
+		echo "Re-applying salt states."
+		salt-call state.highstate queue=True
+	fi
+else
+	echo "Exiting without changes."
+fi

salt/common/tools/sbin/soup

@@ -114,6 +114,12 @@ check_airgap() {
   fi
 }
 
+check_sudoers() {
+	if grep -q "so-setup" /etc/sudoers; then
+		echo "There is an entry for so-setup in the sudoers file, this can be safely deleted using \"visudo\"."
+	fi
+}
+
 clean_dockers() {
   # Place Holder for cleaning up old docker images
   echo "Trying to clean up old dockers."
@@ -191,7 +197,6 @@ pillar_changes() {
     [[ "$INSTALLEDVERSION" =~ rc.1 ]] && rc1_to_rc2
     [[ "$INSTALLEDVERSION" =~ rc.2 ]] && rc2_to_rc3
     [[ "$INSTALLEDVERSION" =~ rc.3 ]] && rc3_to_2.3.0
-
 }
 
 rc1_to_rc2() {
@@ -292,6 +297,7 @@ unmount_update() {
   umount /tmp/soagupdate
 }
 
+
 update_centos_repo() {
   # Update the files in the repo
   echo "Syncing new updates to /nsm/repo"
@@ -525,6 +531,8 @@ if [ "$UPGRADESALT" == "1" ]; then
   echo ""
 fi
 
+check_sudoers
+
 }
 
 main "$@" | tee /dev/fd/3

salt/filebeat/etc/filebeat.yml

@@ -115,7 +115,7 @@ filebeat.inputs:
       fields: ["source", "prospector", "input", "offset", "beat"]
  
   fields_under_root: true
-  clean_removed: false
+  clean_removed: true
   close_removed: false
 
 - type: log

salt/salt/map.jinja

@@ -12,12 +12,12 @@
 {% if grains.saltversion|string != SALTVERSION|string %}
   {% if grains.os|lower in ['centos', 'redhat'] %}
     {% if ISAIRGAP is sameas true %}
-      {% set UPGRADECOMMAND = 'yum clean all && yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -r -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && salt-call state.highstate && sleep 300 && salt-call state.apply salt.minion queue=True'%}
+      {% set UPGRADECOMMAND = 'yum clean all && yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -r -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && salt-call state.highstate' %}
     {% else %}
-      {% set UPGRADECOMMAND = 'yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && salt-call state.highstate && sleep 300 && salt-call state.apply salt.minion queue=True' %}
+      {% set UPGRADECOMMAND = 'yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*" && salt-call state.highstate' %}
     {% endif %}
   {% elif grains.os|lower == 'ubuntu' %}
-    {% set UPGRADECOMMAND = 'apt-mark unhold salt-common && apt-mark unhold salt-minion && /usr/sbin/bootstrap-salt.sh -F -x python3 stable ' ~ SALTVERSION ~ ' && apt-mark hold salt-common && apt-mark hold salt-minion && salt-call state.highstate && sleep 300 && salt-call state.apply salt.minion queue=True' %}
+    {% set UPGRADECOMMAND = 'apt-mark unhold salt-common && apt-mark unhold salt-minion && /usr/sbin/bootstrap-salt.sh -F -x python3 stable ' ~ SALTVERSION ~ ' && apt-mark hold salt-common && apt-mark hold salt-minion && salt-call state.highstate' %}
   {% endif %}
 {% else %}
   {% set UPGRADECOMMAND = 'echo Already running Salt Minon version ' ~ SALTVERSION %}

setup/so-functions

@@ -1019,6 +1019,10 @@ install_cleanup() {
 	# If Mysql is running stop it
 	/usr/sbin/so-mysql-stop
 
+	if [[ $setup_type == 'iso' ]]; then
+		info "Removing so-setup permission entry from sudoers file"
+		sed -i '/so-setup/d' /etc/sudoers
+	fi
 }
 
 import_registry_docker() {
@@ -1378,20 +1382,11 @@ reinstall_init() {
 	info "Putting system in state to run setup again"
 
 	{
-		local minion_config=/etc/salt/minion
+		# Kill any salt processes
-		
+		pkill -9 -ef /usr/bin/salt
-		# Remove startup_states from minion config so we don't immediately highstate when salt starts back up
-		if [[ -f $minion_config ]] && grep -q "startup_states" $minion_config; then
-			sed -i '/startup_states/d' $minion_config
-		fi
 
-		if command -v salt-call &> /dev/null; then
+		# Remove all salt configs
-			# Disable schedule so highstate doesn't start running during the install
+		rm -rf /etc/salt/global /etc/salt/minion /etc/salt/master /etc/salt/pki/*
-			salt-call -l info schedule.disable
-
-			# Kill any currently running salt jobs, also to prevent issues with highstate.
-			salt-call -l info saltutil.kill_all_jobs
-		fi
 
 		if command -v docker &> /dev/null; then
 			# Stop and remove all so-* containers so files can be changed with more safety
@@ -1410,7 +1405,7 @@ reinstall_init() {
 		# Backup /nsm for the same reason
 		while IFS= read -r -d '' dir; do
 			mv "$dir" "${dir}_old_${date_string}"
-		done < <(find /nsm -maxdepth 1 -mindepth 1 -type d -print0)
+		done < <(find /nsm -maxdepth 1 -mindepth 1 -type d -not -path "/nsm/docker-registry" -print0)
 
 		# Remove the old launcher package in case the config changes
 		remove_package launcher-final

setup/so-setup

@@ -54,11 +54,12 @@ while [[ $# -gt 0 ]]; do
 	esac
 done
 
-if [[ -f $setup_log ]]; then
+is_reinstall=false
+if [[ -f /root/accept_changes ]]; then
 	is_reinstall=true
 
 	# Move last setup log to backup
-	mv $setup_log $setup_log.bak
+	mv "$setup_log" "$setup_log.bak"
 fi
 
 # Begin Installation pre-processing
@@ -317,7 +318,6 @@ if [[ $is_import ]]; then
 	PLAYBOOK=0
 fi
 
-
 # Start user prompts
 
 if [[ $is_helix || $is_sensor ]]; then
@@ -427,6 +427,7 @@ if [[ $is_manager || $is_import ]]; then whiptail_so_allow; fi
 whiptail_make_changes
 
 # From here on changes will be made.
+echo "1" > /root/accept_changes
 
 if [[ $is_reinstall ]]; then
 	reinstall_init

setup/so-whiptail

@@ -168,8 +168,10 @@ whiptail_cancel() {
 			echo "/root/installtmp removed";
 		} >> $setup_log 2>&1
 	fi
-	exit
 
+	title "User cancelled setup, no changes made."
+	
+	exit
 }
 
 whiptail_check_exitstatus() {