#!/bin/bash
#
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.

{% from 'salt/schedule.map.jinja' import SCHEDULEMERGED %}

# this script checks the time the file /opt/so/log/salt/state-apply-test was last modified and restarts the salt-minion service if it is outside a threshold date/time
# the file is modified via file.touch using a scheduled job healthcheck.salt-minion.state-apply-test that runs a state.apply. 
# by default the file should be updated every 5-8 minutes. 
# this allows us to test that the minion is able apply states and communicate with the master
# if the file is unable to be touched via the state.apply, then we assume there is a possibilty that the minion is hung (though it could be possible the master is down as well)
# we then stop the service, pkill salt-minion, the start the salt-minion service back up

. /usr/sbin/so-common

QUIET=false
UPTIME_REQ=1800 #in seconds, how long the box has to be up before considering restarting salt-minion due to /opt/so/log/salt/state-apply-test not being touched
HIGHSTATE_UPTIME_REQ=900 #in seconds; if the box has been up this long and no highstate has completed since boot, force one
CURRENT_TIME=$(date +%s)
SYSTEM_START_TIME=$(date -d "$(</proc/uptime awk '{print $1}') seconds ago" +%s)
LAST_HIGHSTATE_END=$([ -e "/opt/so/log/salt/lasthighstate" ] && date -r /opt/so/log/salt/lasthighstate +%s || echo 0)
LAST_HEALTHCHECK_STATE_APPLY=$([ -e "/opt/so/log/salt/state-apply-test" ] && date -r /opt/so/log/salt/state-apply-test +%s || echo 0)
# SETTING THRESHOLD TO ANYTHING UNDER 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default
# THRESHOLD is derived from the salt schedule highstate interval + 1 hour, so the minion-check grace period tracks the schedule automatically.
THRESHOLD=$(( ({{ SCHEDULEMERGED.highstate_interval_hours }} + 1) * 3600 )) #within how many seconds the file /opt/so/log/salt/state-apply-test must have been touched/modified before the salt minion is restarted
THRESHOLD_DATE=$((LAST_HEALTHCHECK_STATE_APPLY+THRESHOLD))

logCmd() {
  cmd=$1
  info "Executing command: $cmd"
  $cmd >> "/opt/so/log/salt/so-salt-minion-check"
}

log() {
    msg=$1
    level=${2:-I}
    now=$(TZ=GMT date +"%Y-%m-%dT%H:%M:%SZ")
    if ! $QUIET; then
      echo $msg
    fi
    echo -e "$now | $level | $msg" >> "/opt/so/log/salt/so-salt-minion-check" 2>&1
}

error() {
    log "$1" "E"
}

info() {
    log "$1" "I"
}

usage()
{
cat <<EOF

Check health of salt-minion and restart it if needed
  Options:
  -h                This message
  -q                Don't output to terminal

EOF
}

while getopts ":q" opt; do
  case "$opt" in
    q )
       QUIET=true
      ;;
    * ) usage
        exit 0
      ;;
  esac
done

log "running so-salt-minion-check"

RESTARTED=false

# Check 1 (minion-restart-check): if the minion has stopped applying states (the
# state-apply-test healthcheck file has gone stale), restart the salt-minion service.
if [ $CURRENT_TIME -ge $((SYSTEM_START_TIME+$UPTIME_REQ))  ]; then
  if [ $THRESHOLD_DATE -le $CURRENT_TIME ]; then
    log "[minion-restart-check] salt-minion is unable to apply states; restarting salt-minion" E
    log "[minion-restart-check] state-apply-test not touched by required date `date -d @$THRESHOLD_DATE`, last touched `date -d @$LAST_HEALTHCHECK_STATE_APPLY`" I
    log "[minion-restart-check] last highstate completed at `date -d @$LAST_HIGHSTATE_END`" I
    log "[minion-restart-check] checking if any jobs are running" I
    logCmd "salt-call --local saltutil.running" I
    log "[minion-restart-check] ensure salt.minion-state-apply-test is enabled" I
    logCmd "salt-call state.enable salt.minion-state-apply-test" I
    log "[minion-restart-check] ensure highstate is enabled" I
    logCmd "salt-call state.enable highstate" I
    log "[minion-restart-check] killing all salt-minion processes" I
    logCmd "pkill -9 -ef /usr/bin/salt-minion" I
    log "[minion-restart-check] starting salt-minion service" I
    logCmd "systemctl start salt-minion" I
    log "[minion-restart-check] waiting for salt-minion to become ready, then applying highstate in the background (queued)" I
    nohup bash -c '/usr/sbin/so-salt-minion-wait; salt-call state.highstate queue=True' >> "/opt/so/log/salt/so-salt-minion-check" 2>&1 &
    RESTARTED=true
  else
    log "[minion-restart-check] healthy: state-apply-test last touched `date -d @$LAST_HEALTHCHECK_STATE_APPLY`, must go stale past `date -d @$THRESHOLD_DATE` to trigger a salt-minion restart" I
  fi
else
  log "[minion-restart-check] skipped: system uptime $((CURRENT_TIME-SYSTEM_START_TIME))s is below the ${UPTIME_REQ}s minimum required before a salt-minion restart" I
fi

# Check 2 (boot-highstate-check): if the host has been up long enough but no highstate
# has completed since this boot, force one. This recovers a host whose boot highstate
# (so-boot-highstate.service) failed or was skipped, even while the minion is otherwise
# healthy (touching state-apply-test). We deliberately do NOT enable highstate here: if
# soup has disabled it during an upgrade, Salt will refuse the highstate and we avoid
# forcing one mid-upgrade.
if $RESTARTED; then
  log "[boot-highstate-check] skipped: minion-restart-check already queued a highstate this run" I
elif [ $CURRENT_TIME -lt $((SYSTEM_START_TIME+HIGHSTATE_UPTIME_REQ)) ]; then
  log "[boot-highstate-check] skipped: system uptime $((CURRENT_TIME-SYSTEM_START_TIME))s is below the ${HIGHSTATE_UPTIME_REQ}s minimum required before forcing a highstate" I
elif [ $LAST_HIGHSTATE_END -ge $SYSTEM_START_TIME ]; then
  log "[boot-highstate-check] healthy: a highstate completed at `date -d @$LAST_HIGHSTATE_END`, after this boot at `date -d @$SYSTEM_START_TIME`" I
elif salt-call --local saltutil.running 2>/dev/null | grep -q 'state.highstate'; then
  log "[boot-highstate-check] no highstate has completed since boot, but one is already running; skipping" I
else
  log "[boot-highstate-check] no highstate has completed since boot after $((CURRENT_TIME-SYSTEM_START_TIME))s uptime; applying highstate" E
  nohup bash -c 'salt-call state.highstate -l info queue=True' >> "/opt/so/log/salt/so-salt-minion-check" 2>&1 &
fi
