From 8da070d5118d4862f2e78db8e48f8944aef53dbb Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Thu, 5 Nov 2020 17:46:23 -0500
Subject: [PATCH] 
 https://github.com/Security-Onion-Solutions/securityonion/issues/1764

---
 salt/common/tools/sbin/so-status | 53 ++++++++++++++++++++++----------
 salt/salt/lasthighstate.sls      |  0
 2 files changed, 37 insertions(+), 16 deletions(-)
 create mode 100644 salt/salt/lasthighstate.sls

diff --git a/salt/common/tools/sbin/so-status b/salt/common/tools/sbin/so-status
index 519d9f39d..714e71bc9 100755
--- a/salt/common/tools/sbin/so-status
+++ b/salt/common/tools/sbin/so-status
@@ -23,11 +23,16 @@ if ! [ "$(id -u)" = 0 ]; then
 fi
 
 # Constants
+SYSTEM_START_TIME=$(date -d "$(</proc/uptime awk '{print $1}') seconds ago" +%s)
+LAST_HIGHSTATE_END=$(date -r /opt/so/log/salt/lasthighstate +%s) # file populated by salt.lasthighstate state at end of successful highstate run
+HIGHSTATE_RUNNING=$(salt-call --local saltutil.running --out=json | jq -r '.local[].fun' | grep -q 'state.highstate' && echo $?)
 ERROR_STRING="ERROR"
 SUCCESS_STRING="OK"
 PENDING_STRING="PENDING"
 MISSING_STRING='MISSING'
 DISABLED_STRING='DISABLED'
+WAIT_START_STRING='WAIT_START'
+STARTING_STRING='STARTING'
 CALLER=$(ps -o comm= $PPID)
 declare -a BAD_STATUSES=("removing" "paused" "exited" "dead")
 declare -a PENDING_STATUSES=("paused" "created" "restarting")
@@ -111,28 +116,44 @@ parse_status() {
     local container_state=${1}
     local service_name=${2}
 
-    [[ $container_state = "missing" ]] && printf $MISSING_STRING && return 1
-
     for state in "${GOOD_STATUSES[@]}"; do
         [[ $container_state = "$state" ]] && printf $SUCCESS_STRING && return 0
     done
 
-    for state in "${PENDING_STATUSES[@]}"; do
-        [[ $container_state = "$state" ]] && printf $PENDING_STRING && return 0
-    done
+    # if a highstate has finished running since the system has started
+    # then the containers should be running so let's check the status
+    if [ $LAST_HIGHSTATE_END -ge $SYSTEM_START_TIME ]; then
 
-    # This is technically not needed since the default is error state
-    for state in "${BAD_STATUSES[@]}"; do
-        if [[ " ${DISABLED_CONTAINERS[@]} " =~ " ${service_name} " ]]; then
-            printf $DISABLED_STRING
-            return 0
-        elif [[ $container_state = "$state" ]]; then
-            printf $ERROR_STRING
-            return 1
-        fi
-    done
+        [[ $container_state = "missing" ]] && printf $MISSING_STRING && return 1
 
-    printf $ERROR_STRING && return 1
+        for state in "${PENDING_STATUSES[@]}"; do
+            [[ $container_state = "$state" ]] && printf $PENDING_STRING && return 0
+        done
+
+        # This is technically not needed since the default is error state
+        for state in "${BAD_STATUSES[@]}"; do
+            if [[ " ${DISABLED_CONTAINERS[@]} " =~ " ${service_name} " ]]; then
+                printf $DISABLED_STRING
+                return 0
+            elif [[ $container_state = "$state" ]]; then
+                printf $ERROR_STRING
+                return 1
+            fi
+        done
+
+        printf $ERROR_STRING && return 1
+    
+    # if a highstate has not run since system start time, but a highstate is currently running
+    # then show that the containers are STARTING
+    elif [ $HIGHSTATE_RUNNING -eq 0 ]; then
+        printf $STARTING_STRING && return 0
+
+    # if a highstate has not finished running since system startup and isn't currently running
+    # then just show that the containers are WAIT_START; waiting to be started
+    else
+        printf $WAIT_START_STRING && return 1
+
+    fi
 }
 
 # {% raw %}
diff --git a/salt/salt/lasthighstate.sls b/salt/salt/lasthighstate.sls
new file mode 100644
index 000000000..e69de29bb