diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 64084dbd0..eca96da5c 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -406,12 +406,17 @@ function update_logstash_outputs() { curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_logstash" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" } +function checkMine() { + local func=$1 + # make sure the minion sees itself in the mine since it needs to see itself for states as opposed to using salt-run + retry 20 1 "salt '$MINION_ID' mine.get '\*' '$func'" "$MINION_ID" + +} + function updateMine() { - salt "$MINION_ID" mine.send network.ip_addrs interface="$MNIC" -} -function apply_ES_state() { - salt-call state.apply elasticsearch concurrent=True + retry 20 1 "salt '$MINION_ID' mine.update" True } + function createEVAL() { is_pcaplimit=true add_elasticsearch_to_minion @@ -547,8 +552,6 @@ function createSEARCHNODE() { add_elasticsearch_to_minion add_logstash_to_minion add_telegraf_to_minion - updateMine - apply_ES_state } function createRECEIVER() { @@ -563,6 +566,19 @@ function createDESKTOP() { } function testConnection() { + # the minion should be trying to auth every 10 seconds so 15 seconds should be more than enough time to see this in the log + # this retry was put in because it is possible that a minion is attempted to be pinged before it has authenticated and connected to the Salt master + # causing the first ping to fail and typically wouldn't be successful until the second ping + # this check may pass without the minion being authenticated if it was previously connected and the line exists in the log + retry 15 1 "grep 'Authentication accepted from $MINION_ID' /opt/so/log/salt/master" + local retauth=$? + if [[ $retauth != 0 ]]; then + echo "The Minion did not authenticate with the Salt master in the allotted time" + echo "Deleting the key" + deleteminion + exit 1 + fi + retry 15 3 "salt '$MINION_ID' test.ping" True local ret=$? if [[ $ret != 0 ]]; then @@ -582,9 +598,9 @@ if [[ "$OPERATION" = 'delete' ]]; then deleteminion fi -if [[ "$OPERATION" = 'add' || "$OPERATION" = 'setup' ]]; then +if [[ "$OPERATION" == 'add' || "$OPERATION" == 'setup' ]]; then # Skip this if its setup - if [ $OPERATION != 'setup' ]; then + if [[ $OPERATION == 'add' ]]; then # Accept the salt key acceptminion # Test to see if the minion was accepted @@ -605,8 +621,19 @@ if [[ "$OPERATION" = 'add' || "$OPERATION" = 'setup' ]]; then else add_sensoroni_to_minion fi + create$NODETYPE echo "Minion file created for $MINION_ID" + + if [[ "$OPERATION" == 'add' ]]; then + # tell the minion to populate the mine with data from mine_functions which is populated during setup + # this only needs to happen on non managers since they handle this during setup + # and they need to wait for ca creation to update the mine + updateMine + checkMine "network.ip_addrs" + # run this async so the cli doesn't wait for a return + salt "$MINION_ID" state.highstate --async + fi fi if [[ "$OPERATION" = 'test' ]]; then diff --git a/salt/salt/engines/master/checkmine.py b/salt/salt/engines/master/checkmine.py index afeb92536..f33392575 100644 --- a/salt/salt/engines/master/checkmine.py +++ b/salt/salt/engines/master/checkmine.py @@ -71,6 +71,7 @@ def start(interval=60): log.error('checkmine engine: found minion %s is not in the mine' % (minion)) mine_flush(minion) mine_update(minion) + continue # Update the mine if the ip in the mine doesn't match returned from manage.alived network_ip_addrs = __salt__['saltutil.runner']('mine.get', tgt=minion, fun='network.ip_addrs') @@ -85,4 +86,5 @@ def start(interval=60): log.error('checkmine engine: found minion %s is not in the mine' % (minion)) mine_flush(minion) mine_update(minion) + sleep(interval) diff --git a/salt/setup/highstate_cron.sls b/salt/setup/highstate_cron.sls index 862968d97..f8f76e737 100644 --- a/salt/setup/highstate_cron.sls +++ b/salt/setup/highstate_cron.sls @@ -3,5 +3,5 @@ post_setup_cron: - name: 'PATH=$PATH:/usr/sbin salt-call state.highstate' - identifier: post_setup_cron - user: root - - minute: '*/1' + - minute: '*/5' - identifier: post_setup_cron diff --git a/salt/soc/files/bin/salt-relay.sh b/salt/soc/files/bin/salt-relay.sh index ac56ec6e3..fea81728d 100755 --- a/salt/soc/files/bin/salt-relay.sh +++ b/salt/soc/files/bin/salt-relay.sh @@ -67,10 +67,10 @@ function manage_minion() { response=$(so-minion "-o=$op" "-m=$minion_id") exit_code=$? if [[ exit_code -eq 0 ]]; then - log "Successful command execution" + log "Successful '$op' command execution on $minion_id" respond "$id" "true" else - log "Unsuccessful command execution: $response ($exit_code)" + log "Unsuccessful '$op' command execution on $minion_id: $response ($exit_code)" respond "$id" "false" fi } diff --git a/setup/so-functions b/setup/so-functions index 7ead07ca7..1c9b0f43d 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2500,7 +2500,7 @@ wait_for_salt_minion() { local maxAttempts=20 until check_salt_minion_status; do attempt=$((attempt+1)) - if [[ $attempt -gt $maxAttempts ]]; then + if [[ $attempt -eq $maxAttempts ]]; then fail_setup fi sleep 10