From 8da070d5118d4862f2e78db8e48f8944aef53dbb Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Thu, 5 Nov 2020 17:46:23 -0500
Subject: [PATCH 1/6] 
 https://github.com/Security-Onion-Solutions/securityonion/issues/1764

---
 salt/common/tools/sbin/so-status | 53 ++++++++++++++++++++++----------
 salt/salt/lasthighstate.sls      |  0
 2 files changed, 37 insertions(+), 16 deletions(-)
 create mode 100644 salt/salt/lasthighstate.sls

diff --git a/salt/common/tools/sbin/so-status b/salt/common/tools/sbin/so-status
index 519d9f39d..714e71bc9 100755
--- a/salt/common/tools/sbin/so-status
+++ b/salt/common/tools/sbin/so-status
@@ -23,11 +23,16 @@ if ! [ "$(id -u)" = 0 ]; then
 fi
 
 # Constants
+SYSTEM_START_TIME=$(date -d "$(</proc/uptime awk '{print $1}') seconds ago" +%s)
+LAST_HIGHSTATE_END=$(date -r /opt/so/log/salt/lasthighstate +%s) # file populated by salt.lasthighstate state at end of successful highstate run
+HIGHSTATE_RUNNING=$(salt-call --local saltutil.running --out=json | jq -r '.local[].fun' | grep -q 'state.highstate' && echo $?)
 ERROR_STRING="ERROR"
 SUCCESS_STRING="OK"
 PENDING_STRING="PENDING"
 MISSING_STRING='MISSING'
 DISABLED_STRING='DISABLED'
+WAIT_START_STRING='WAIT_START'
+STARTING_STRING='STARTING'
 CALLER=$(ps -o comm= $PPID)
 declare -a BAD_STATUSES=("removing" "paused" "exited" "dead")
 declare -a PENDING_STATUSES=("paused" "created" "restarting")
@@ -111,28 +116,44 @@ parse_status() {
     local container_state=${1}
     local service_name=${2}
 
-    [[ $container_state = "missing" ]] && printf $MISSING_STRING && return 1
-
     for state in "${GOOD_STATUSES[@]}"; do
         [[ $container_state = "$state" ]] && printf $SUCCESS_STRING && return 0
     done
 
-    for state in "${PENDING_STATUSES[@]}"; do
-        [[ $container_state = "$state" ]] && printf $PENDING_STRING && return 0
-    done
+    # if a highstate has finished running since the system has started
+    # then the containers should be running so let's check the status
+    if [ $LAST_HIGHSTATE_END -ge $SYSTEM_START_TIME ]; then
 
-    # This is technically not needed since the default is error state
-    for state in "${BAD_STATUSES[@]}"; do
-        if [[ " ${DISABLED_CONTAINERS[@]} " =~ " ${service_name} " ]]; then
-            printf $DISABLED_STRING
-            return 0
-        elif [[ $container_state = "$state" ]]; then
-            printf $ERROR_STRING
-            return 1
-        fi
-    done
+        [[ $container_state = "missing" ]] && printf $MISSING_STRING && return 1
 
-    printf $ERROR_STRING && return 1
+        for state in "${PENDING_STATUSES[@]}"; do
+            [[ $container_state = "$state" ]] && printf $PENDING_STRING && return 0
+        done
+
+        # This is technically not needed since the default is error state
+        for state in "${BAD_STATUSES[@]}"; do
+            if [[ " ${DISABLED_CONTAINERS[@]} " =~ " ${service_name} " ]]; then
+                printf $DISABLED_STRING
+                return 0
+            elif [[ $container_state = "$state" ]]; then
+                printf $ERROR_STRING
+                return 1
+            fi
+        done
+
+        printf $ERROR_STRING && return 1
+    
+    # if a highstate has not run since system start time, but a highstate is currently running
+    # then show that the containers are STARTING
+    elif [ $HIGHSTATE_RUNNING -eq 0 ]; then
+        printf $STARTING_STRING && return 0
+
+    # if a highstate has not finished running since system startup and isn't currently running
+    # then just show that the containers are WAIT_START; waiting to be started
+    else
+        printf $WAIT_START_STRING && return 1
+
+    fi
 }
 
 # {% raw %}
diff --git a/salt/salt/lasthighstate.sls b/salt/salt/lasthighstate.sls
new file mode 100644
index 000000000..e69de29bb

From 1a75ebdca312369e7d75ba5947f7930d5c798249 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Thu, 5 Nov 2020 17:51:51 -0500
Subject: [PATCH 2/6] 
 https://github.com/Security-Onion-Solutions/securityonion/issues/1764

---
 salt/common/tools/sbin/so-status | 2 +-
 salt/salt/lasthighstate.sls      | 4 ++++
 salt/top.sls                     | 1 +
 3 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/salt/common/tools/sbin/so-status b/salt/common/tools/sbin/so-status
index 714e71bc9..05f32d7ce 100755
--- a/salt/common/tools/sbin/so-status
+++ b/salt/common/tools/sbin/so-status
@@ -24,7 +24,7 @@ fi
 
 # Constants
 SYSTEM_START_TIME=$(date -d "$(</proc/uptime awk '{print $1}') seconds ago" +%s)
-LAST_HIGHSTATE_END=$(date -r /opt/so/log/salt/lasthighstate +%s) # file populated by salt.lasthighstate state at end of successful highstate run
+LAST_HIGHSTATE_END=$([ -f "/opt/so/log/salt/lasthighstate" ] && date -r /opt/so/log/salt/lasthighstate +%s || echo 0) # file populated by salt.lasthighstate state at end of successful highstate run
 HIGHSTATE_RUNNING=$(salt-call --local saltutil.running --out=json | jq -r '.local[].fun' | grep -q 'state.highstate' && echo $?)
 ERROR_STRING="ERROR"
 SUCCESS_STRING="OK"
diff --git a/salt/salt/lasthighstate.sls b/salt/salt/lasthighstate.sls
index e69de29bb..606bd1082 100644
--- a/salt/salt/lasthighstate.sls
+++ b/salt/salt/lasthighstate.sls
@@ -0,0 +1,4 @@
+lasthighstate:
+  file.touch:
+    - name: /opt/so/log/salt/lasthighstate
+    - order: last
\ No newline at end of file
diff --git a/salt/top.sls b/salt/top.sls
index 2b55c4724..36fd171e1 100644
--- a/salt/top.sls
+++ b/salt/top.sls
@@ -35,6 +35,7 @@ base:
     - common
     - patch.os.schedule
     - motd
+    - salt.lasthighstate
   
   '*_helix and G@saltversion:{{saltversion}}':
     - match: compound

From 47cac59adb20d5d74696a67470aa25409b3fbf42 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Thu, 5 Nov 2020 17:52:28 -0500
Subject: [PATCH 3/6] 
 https://github.com/Security-Onion-Solutions/securityonion/issues/1764

---
 salt/common/tools/sbin/so-status | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/salt/common/tools/sbin/so-status b/salt/common/tools/sbin/so-status
index 05f32d7ce..26d390454 100755
--- a/salt/common/tools/sbin/so-status
+++ b/salt/common/tools/sbin/so-status
@@ -24,7 +24,8 @@ fi
 
 # Constants
 SYSTEM_START_TIME=$(date -d "$(</proc/uptime awk '{print $1}') seconds ago" +%s)
-LAST_HIGHSTATE_END=$([ -f "/opt/so/log/salt/lasthighstate" ] && date -r /opt/so/log/salt/lasthighstate +%s || echo 0) # file populated by salt.lasthighstate state at end of successful highstate run
+# file populated by salt.lasthighstate state at end of successful highstate run
+LAST_HIGHSTATE_END=$([ -f "/opt/so/log/salt/lasthighstate" ] && date -r /opt/so/log/salt/lasthighstate +%s || echo 0)
 HIGHSTATE_RUNNING=$(salt-call --local saltutil.running --out=json | jq -r '.local[].fun' | grep -q 'state.highstate' && echo $?)
 ERROR_STRING="ERROR"
 SUCCESS_STRING="OK"

From 695bace3e8fae387b37f2c57f66db5508ab3360f Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Thu, 5 Nov 2020 17:54:10 -0500
Subject: [PATCH 4/6] 
 https://github.com/Security-Onion-Solutions/securityonion/issues/1764

---
 salt/common/tools/sbin/so-status | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/common/tools/sbin/so-status b/salt/common/tools/sbin/so-status
index 26d390454..8d50bc2fd 100755
--- a/salt/common/tools/sbin/so-status
+++ b/salt/common/tools/sbin/so-status
@@ -25,7 +25,7 @@ fi
 # Constants
 SYSTEM_START_TIME=$(date -d "$(</proc/uptime awk '{print $1}') seconds ago" +%s)
 # file populated by salt.lasthighstate state at end of successful highstate run
-LAST_HIGHSTATE_END=$([ -f "/opt/so/log/salt/lasthighstate" ] && date -r /opt/so/log/salt/lasthighstate +%s || echo 0)
+LAST_HIGHSTATE_END=$([ -e "/opt/so/log/salt/lasthighstate" ] && date -r /opt/so/log/salt/lasthighstate +%s || echo 0)
 HIGHSTATE_RUNNING=$(salt-call --local saltutil.running --out=json | jq -r '.local[].fun' | grep -q 'state.highstate' && echo $?)
 ERROR_STRING="ERROR"
 SUCCESS_STRING="OK"

From a95129b8c29e981fe7544c8f2c3938fc1ca136cc Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Thu, 5 Nov 2020 18:03:42 -0500
Subject: [PATCH 5/6] add color -
 https://github.com/Security-Onion-Solutions/securityonion/issues/1764

---
 salt/common/tools/sbin/so-status | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/salt/common/tools/sbin/so-status b/salt/common/tools/sbin/so-status
index 8d50bc2fd..17bf50523 100755
--- a/salt/common/tools/sbin/so-status
+++ b/salt/common/tools/sbin/so-status
@@ -167,11 +167,11 @@ print_line() {
 
     local PADDING_CONSTANT=14
 
-    if [[ $service_state = "$ERROR_STRING" ]] || [[ $service_state = "$MISSING_STRING" ]]; then
+    if [[ $service_state = "$ERROR_STRING" ]] || [[ $service_state = "$MISSING_STRING" ]] || [[ $service_state = "$WAIT_START_STRING" ]]; then
         state_color="\e[1;31m"
     elif [[ $service_state = "$SUCCESS_STRING" ]]; then
         state_color="\e[1;32m"
-    elif [[ $service_state = "$PENDING_STRING" ]] || [[ $service_state = "$DISABLED_STRING" ]]; then
+    elif [[ $service_state = "$PENDING_STRING" ]] || [[ $service_state = "$DISABLED_STRING" ]] || [[ $service_state = "$STARTING_STRING" ]]; then
         state_color="\e[1;33m"
     fi
 

From c7b4a5351cb99325a412bfd09301bfd2332d7792 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Thu, 5 Nov 2020 18:17:11 -0500
Subject: [PATCH 6/6] fix logic -
 https://github.com/Security-Onion-Solutions/securityonion/issues/1764

---
 salt/common/tools/sbin/so-status | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/common/tools/sbin/so-status b/salt/common/tools/sbin/so-status
index 17bf50523..0aff7a5a2 100755
--- a/salt/common/tools/sbin/so-status
+++ b/salt/common/tools/sbin/so-status
@@ -146,7 +146,7 @@ parse_status() {
     
     # if a highstate has not run since system start time, but a highstate is currently running
     # then show that the containers are STARTING
-    elif [ $HIGHSTATE_RUNNING -eq 0 ]; then
+    elif [[ "$HIGHSTATE_RUNNING" == 0 ]]; then
         printf $STARTING_STRING && return 0
 
     # if a highstate has not finished running since system startup and isn't currently running