From b14670030349a2747a00ace665568ab5f51ac47b Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 17 Nov 2020 15:36:25 -0500 Subject: [PATCH 01/11] [feat] Remove so-setup permission from sudoers file after iso setup Closes #1701 --- salt/common/tools/sbin/soup | 7 +++++++ setup/so-functions | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 0453ea29d..db806a443 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -191,6 +191,7 @@ pillar_changes() { [[ "$INSTALLEDVERSION" =~ rc.1 ]] && rc1_to_rc2 [[ "$INSTALLEDVERSION" =~ rc.2 ]] && rc2_to_rc3 [[ "$INSTALLEDVERSION" =~ rc.3 ]] && rc3_to_2.3.0 + [[ "$INSTALLEDVERSION" =~ 2.3.2 ]] && up_2.3.2_to_2.3.10 } @@ -292,6 +293,12 @@ unmount_update() { umount /tmp/soagupdate } +up_2.3.2_to_2.3.10() { + if grep -q "so-setup" /etc/sudoers; then + echo "[ INFO ] There is an entry for so-setup in the sudoers file, this can be safely deleted using \"visudo\"." + fi +} + update_centos_repo() { # Update the files in the repo echo "Syncing new updates to /nsm/repo" diff --git a/setup/so-functions b/setup/so-functions index bd2c05179..c21f8407a 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1019,6 +1019,10 @@ install_cleanup() { # If Mysql is running stop it /usr/sbin/so-mysql-stop + if [[ $install_type == 'iso' ]]; then + info "Removing so-setup permission entry from sudoers file" + sed -i '/so-setup/d' /etc/sudoers + fi } import_registry_docker() { From ee3708a428a9561ac12fa302addd045b32f19c70 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 17 Nov 2020 15:44:20 -0500 Subject: [PATCH 02/11] [fix] Move sudoers check in soup to correct place + fix styling issue --- salt/common/tools/sbin/soup | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index db806a443..42e6c2637 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -114,6 +114,12 @@ check_airgap() { fi } +check_sudoers() { + if grep -q "so-setup" /etc/sudoers; then + echo "There is an entry for so-setup in the sudoers file, this can be safely deleted using \"visudo\"." + fi +} + clean_dockers() { # Place Holder for cleaning up old docker images echo "Trying to clean up old dockers." @@ -191,8 +197,6 @@ pillar_changes() { [[ "$INSTALLEDVERSION" =~ rc.1 ]] && rc1_to_rc2 [[ "$INSTALLEDVERSION" =~ rc.2 ]] && rc2_to_rc3 [[ "$INSTALLEDVERSION" =~ rc.3 ]] && rc3_to_2.3.0 - [[ "$INSTALLEDVERSION" =~ 2.3.2 ]] && up_2.3.2_to_2.3.10 - } rc1_to_rc2() { @@ -293,11 +297,6 @@ unmount_update() { umount /tmp/soagupdate } -up_2.3.2_to_2.3.10() { - if grep -q "so-setup" /etc/sudoers; then - echo "[ INFO ] There is an entry for so-setup in the sudoers file, this can be safely deleted using \"visudo\"." - fi -} update_centos_repo() { # Update the files in the repo @@ -468,7 +467,7 @@ fi echo "Checking if Salt was upgraded." echo "" # Check that Salt was upgraded, should be 3 'salt' packages on a manager node. salt-minion, salt-master and salt or salt-common depending on Ubuntu or CentOS. we could add salt-syndic in the future so checking that there are at least 3 packages -if [[ `rpm -qa | grep salt | grep $NEWSALTVERSION | wc -l` < 3 ]]; then +if [[ $(rpm -qa | grep salt | grep -c $NEWSALTVERSION) -lt 3 ]]; then echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG." echo "Once the issue is resolved, run soup again." echo "Exiting." @@ -531,6 +530,8 @@ if [ "$UPGRADESALT" == "1" ]; then echo "" fi +check_sudoers + } main "$@" | tee /dev/fd/3 From 8a4defcffa3c1f6038674492a0b52e5af271beed Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 18 Nov 2020 11:16:31 -0500 Subject: [PATCH 03/11] [refactor] Check for setup log earlier * Check for sosetuo.log before any scripts besides so-variables are sourced to make sure the log hasn't been created yet. --- setup/so-setup | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index e2728a571..8c6378150 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -24,10 +24,19 @@ fi cd "$(dirname "$0")" || exit 255 +source ./so-variables + +is_reinstall=false +if [[ -f $setup_log ]]; then + is_reinstall=true + + # Move last setup log to backup + mv $setup_log $setup_log.bak +fi + source ./so-functions source ./so-common-functions source ./so-whiptail -source ./so-variables # Parse command line arguments setup_type=$1 @@ -54,12 +63,6 @@ while [[ $# -gt 0 ]]; do esac done -if [[ -f $setup_log ]]; then - is_reinstall=true - - # Move last setup log to backup - mv $setup_log $setup_log.bak -fi # Begin Installation pre-processing parse_install_username From ad74b4b3e06a9e25677659b1fc6cb761735aef8e Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 18 Nov 2020 12:29:54 -0500 Subject: [PATCH 04/11] [refactor][fix] Update reinstall logic * Only set reinstall flag if new accept_changes file exists * Instead of stopping highstate from running, kill all salt processes and remove their configs * Make end of non-reinstall logs clear in cases where user cancels (and log not rotated) --- setup/so-functions | 17 +++++------------ setup/so-setup | 20 +++++++++----------- setup/so-variables | 3 +++ setup/so-whiptail | 6 +++++- 4 files changed, 22 insertions(+), 24 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index c21f8407a..1d6ac642c 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1382,20 +1382,13 @@ reinstall_init() { info "Putting system in state to run setup again" { - local minion_config=/etc/salt/minion + rm -f "$change_file" - # Remove startup_states from minion config so we don't immediately highstate when salt starts back up - if [[ -f $minion_config ]] && grep -q "startup_states" $minion_config; then - sed -i '/startup_states/d' $minion_config - fi + # Kill any salt processes + pkill -9 -ef /usr/bin/salt - if command -v salt-call &> /dev/null; then - # Disable schedule so highstate doesn't start running during the install - salt-call -l info schedule.disable - - # Kill any currently running salt jobs, also to prevent issues with highstate. - salt-call -l info saltutil.kill_all_jobs - fi + # Remove all salt configs + rm -rf /etc/salt/global /etc/salt/minion /etc/salt/master /etc/salt/pki/* if command -v docker &> /dev/null; then # Stop and remove all so-* containers so files can be changed with more safety diff --git a/setup/so-setup b/setup/so-setup index 8c6378150..24089dffc 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -24,19 +24,10 @@ fi cd "$(dirname "$0")" || exit 255 -source ./so-variables - -is_reinstall=false -if [[ -f $setup_log ]]; then - is_reinstall=true - - # Move last setup log to backup - mv $setup_log $setup_log.bak -fi - source ./so-functions source ./so-common-functions source ./so-whiptail +source ./so-variables # Parse command line arguments setup_type=$1 @@ -63,6 +54,13 @@ while [[ $# -gt 0 ]]; do esac done +is_reinstall=false +if [[ -f $change_file ]]; then + is_reinstall=true + + # Move last setup log to backup + mv "$setup_log" "$setup_log.bak" +fi # Begin Installation pre-processing parse_install_username @@ -320,7 +318,6 @@ if [[ $is_import ]]; then PLAYBOOK=0 fi - # Start user prompts if [[ $is_helix || $is_sensor ]]; then @@ -428,6 +425,7 @@ fi if [[ $is_manager || $is_import ]]; then whiptail_so_allow; fi whiptail_make_changes +touch $change_file # From here on changes will be made. diff --git a/setup/so-variables b/setup/so-variables index 83b9b4325..8c85954c2 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -21,6 +21,9 @@ export node_es_port setup_log="/root/sosetup.log" export setup_log +change_file="/root/accept_changes" +export change_file + filesystem_root=$(df / | awk '$3 ~ /[0-9]+/ { print $2 * 1000 }') export filesystem_root diff --git a/setup/so-whiptail b/setup/so-whiptail index a1f07868c..2c47b69e1 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -168,8 +168,12 @@ whiptail_cancel() { echo "/root/installtmp removed"; } >> $setup_log 2>&1 fi - exit + echo "----------" >> "$setup_log" 2>&1 + info "User cancelled setup, no changes made." + echo "----------" >> "$setup_log" 2>&1 + + exit } whiptail_check_exitstatus() { From 0542e0aa047ea2d29fd26f9aab23209b25e71b09 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 18 Nov 2020 12:35:16 -0500 Subject: [PATCH 05/11] [fix] info -> title --- setup/so-whiptail | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 2c47b69e1..07e534c0f 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -169,9 +169,7 @@ whiptail_cancel() { } >> $setup_log 2>&1 fi - echo "----------" >> "$setup_log" 2>&1 - info "User cancelled setup, no changes made." - echo "----------" >> "$setup_log" 2>&1 + title "User cancelled setup, no changes made." exit } From 57e9f69c9701fe989f816da7e707e6812c52eccb Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 18 Nov 2020 12:35:33 -0500 Subject: [PATCH 06/11] Add new so-ip-update script (Work in progress) --- salt/common/tools/sbin/so-common | 61 +++++++++++++++++++++++------ salt/common/tools/sbin/so-ip-update | 59 ++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 11 deletions(-) create mode 100644 salt/common/tools/sbin/so-ip-update diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index ab54d634e..1dfa22a5f 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -17,8 +17,8 @@ # Check for prerequisites if [ "$(id -u)" -ne 0 ]; then - echo "This script must be run using sudo!" - exit 1 + echo "This script must be run using sudo!" + exit 1 fi # Define a banner to separate sections @@ -29,19 +29,43 @@ header() { printf '%s\n' "$banner" "$*" "$banner" } +lookup_salt_value() { + key=$1 + group=$2 + kind=$3 + + if [ -z "$kind" ]; then + kind=pillar + fi + + if [ -n "$group" ]; then + group=${group}: + fi + + salt-call --no-color ${kind}.get ${group}${key} --out=newline_values_only +} + lookup_pillar() { - key=$1 - salt-call --no-color pillar.get global:${key} --out=newline_values_only + key=$1 + pillar=$2 + if [ -z "$pillar" ]; then + pillar=global + fi + lookup_salt_value "$key" "$pillar" "pillar" } lookup_pillar_secret() { - key=$1 - salt-call --no-color pillar.get secrets:${key} --out=newline_values_only + lookup_pillar "$1" "secrets" } lookup_grain() { - key=$1 - salt-call --no-color grains.get ${key} --out=newline_values_only + lookup_salt_value "$1" "" "grains" +} + +lookup_role() { + id=$(lookup_grain id) + pieces=($(echo $id | tr '_' ' ')) + echo ${pieces[1]} } check_container() { @@ -50,9 +74,9 @@ check_container() { } check_password() { - local password=$1 - echo "$password" | egrep -v "'|\"|\\$|\\\\" > /dev/null 2>&1 - return $? + local password=$1 + echo "$password" | egrep -v "'|\"|\\$|\\\\" > /dev/null 2>&1 + return $? } set_os() { @@ -96,3 +120,18 @@ require_manager() { exit 1 fi } + +is_single_node_grid() { + role=$(lookup_role) + if [ "$role" != "eval" ] && [ "$role" != "standalone" ] && [ "$role" != "import" ]; then + return 1 + fi + return 0 +} + +fail() { + msg=$1 + echo "ERROR: $msg" + echo "Exiting." + exit 1 +} diff --git a/salt/common/tools/sbin/so-ip-update b/salt/common/tools/sbin/so-ip-update new file mode 100644 index 000000000..7321a5587 --- /dev/null +++ b/salt/common/tools/sbin/so-ip-update @@ -0,0 +1,59 @@ +#!/bin/bash + +. $(dirname $0)/so-common + +if [ "$FORCE_IP_UPDATE" != "1" ]; then + is_single_node_grid || fail "Cannot update the IP on a distributed grid" +fi + +echo "This tool will update a manager's IP address to the new IP assigned to the management network interface." + +echo +echo "WARNING: This tool is still undergoing testing, use at your own risk!" +echo + +if [ -z "$OLD_IP" ]; then + OLD_IP=$(lookup_pillar "managerip") + + if [ -z "$OLD_IP" ]; then + fail "Unable to find old IP; possible salt system failure" + fi + + echo "Found old IP $OLD_IP." +fi + +if [ -z "$NEW_IP" ]; then + iface=$(lookup_pillar "mainint" "host") + NEW_IP=$(ip -4 addr list $iface | grep inet | cut -d' ' -f6 | cut -d/ -f1) + + if [ -z "$NEW_IP" ]; then + fail "Unable to detect new IP on interface $iface. " + fi + + echo "Detected new IP $NEW_IP on interface $iface." +fi + +if [ "$OLD_IP" == "$NEW_IP" ]; then + fail "IP address has not changed" +fi + +echo "About to change old IP $OLD_IP to new IP $NEW_IP." + +read -n 1 -p "Would you like to continue? (y/N) " CONTINUE +echo + +if [ "$CONTINUE" == "y" ]; then + for file in $(grep -rlI $OLD_IP /opt/so/saltstack /etc); do + echo "Updating file: $file" + sed -i "s|$OLD_IP|$NEW_IP|g" $file + done + + echo "The IP has been changed from $OLD_IP to $NEW_IP." + + if [ -z "$SKIP_STATE_APPLY" ]; then + echo "Re-applying salt states." + salt-call state.highstate queue=True + fi +else + echo "Exiting without changes." +fi \ No newline at end of file From 34fd80182e3fa2124dcd235dac0bde637e7ccf9a Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 18 Nov 2020 12:54:29 -0500 Subject: [PATCH 07/11] [fix][wip] Don't use variable for accept_changes file --- setup/so-setup | 4 ++-- setup/so-variables | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 24089dffc..e8993c1da 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -55,7 +55,7 @@ while [[ $# -gt 0 ]]; do done is_reinstall=false -if [[ -f $change_file ]]; then +if [[ -f /root/accept_changes ]]; then is_reinstall=true # Move last setup log to backup @@ -425,9 +425,9 @@ fi if [[ $is_manager || $is_import ]]; then whiptail_so_allow; fi whiptail_make_changes -touch $change_file # From here on changes will be made. +echo "1" > /root/accept_changes if [[ $is_reinstall ]]; then reinstall_init diff --git a/setup/so-variables b/setup/so-variables index 8c85954c2..83b9b4325 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -21,9 +21,6 @@ export node_es_port setup_log="/root/sosetup.log" export setup_log -change_file="/root/accept_changes" -export change_file - filesystem_root=$(df / | awk '$3 ~ /[0-9]+/ { print $2 * 1000 }') export filesystem_root From 6b4af30fc1ff11b5736b45e5f72e99a1541456ff Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 18 Nov 2020 13:47:32 -0500 Subject: [PATCH 08/11] Change clean_removed to true cleanup tracking of Zeek logs removed from current --- salt/filebeat/etc/filebeat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 3587b6ffd..799a37337 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -115,7 +115,7 @@ filebeat.inputs: fields: ["source", "prospector", "input", "offset", "beat"] fields_under_root: true - clean_removed: false + clean_removed: true close_removed: false - type: log From 81b9658499e378d00fe31bf5e678a60252349fb0 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 18 Nov 2020 13:51:18 -0500 Subject: [PATCH 09/11] [fix] Don't remove accept_changes file --- setup/so-functions | 2 -- 1 file changed, 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 1d6ac642c..5875fb8a4 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1382,8 +1382,6 @@ reinstall_init() { info "Putting system in state to run setup again" { - rm -f "$change_file" - # Kill any salt processes pkill -9 -ef /usr/bin/salt From 280cde43ff57df00e9d30902fce2794a67bbd519 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 18 Nov 2020 13:51:39 -0500 Subject: [PATCH 10/11] [fix] install_type -> setup_type --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 5875fb8a4..03ebf53be 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1019,7 +1019,7 @@ install_cleanup() { # If Mysql is running stop it /usr/sbin/so-mysql-stop - if [[ $install_type == 'iso' ]]; then + if [[ $setup_type == 'iso' ]]; then info "Removing so-setup permission entry from sudoers file" sed -i '/so-setup/d' /etc/sudoers fi From e65c53dbb1a29979359ccb0453850ecb7a7cf07d Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 18 Nov 2020 14:01:33 -0500 Subject: [PATCH 11/11] [fix] Don't rename /nsm/docker-registry --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 03ebf53be..f1a1ec1b5 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1405,7 +1405,7 @@ reinstall_init() { # Backup /nsm for the same reason while IFS= read -r -d '' dir; do mv "$dir" "${dir}_old_${date_string}" - done < <(find /nsm -maxdepth 1 -mindepth 1 -type d -print0) + done < <(find /nsm -maxdepth 1 -mindepth 1 -type d -not -path "/nsm/docker-registry" -print0) # Remove the old launcher package in case the config changes remove_package launcher-final