Refactor hive and cortex init to use wait loops instead of hardcoded sleeps

salt/common/tools/sbin/so-common

@@ -140,3 +140,32 @@ get_random_value() {
   length=${1:-20}
   head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1
 }
+
+wait_for_web_response() {
+  url=$1
+  expected=$2
+  maxAttempts=${3:-300}
+  logfile=/root/wait_for_web_response.log
+  attempt=0
+  while [[ $attempt -lt $maxAttempts ]]; do
+    attempt=$((attempt+1))
+    echo "Waiting for value '$expected' at '$url' ($attempt/$maxAttempts)"
+    result=$(curl -ks -L $url)
+    exitcode=$?
+
+    echo "--------------------------------------------------" >> $logfile
+    echo "$(date) - Checking web URL: $url ($attempt/$maxAttempts)" >> $logfile
+    echo "$result" >> $logfile
+    echo "exit code=$exitcode" >> $logfile
+    echo "" >> $logfile
+
+    if [[ $exitcode -eq 0 && "$result" =~ $expected ]]; then
+      echo "Received expected response; proceeding."
+      return 0
+    fi
+    echo "Server is not ready"
+    sleep 1
+  done
+  echo "Server still not ready after $maxAttempts attempts; giving up."
+  return 1
+}

salt/thehive/scripts/cortex_init

@@ -8,6 +8,8 @@
 # {%- set CORTEXORGUSER = salt['pillar.get']('global:cortexorguser', 'soadmin') %}
 # {%- set CORTEXORGUSERKEY = salt['pillar.get']('global:cortexorguserkey', '') %}
 
+. /usr/sbin/so-common
+
 default_salt_dir=/opt/so/saltstack/default
 
 cortex_clean(){
@@ -17,7 +19,8 @@ cortex_clean(){
 }
 
 cortex_init(){
-    CORTEX_API_URL="{{URLBASE}}/cortex/api"
+    CORTEX_URL="{{URLBASE}}/cortex"
+    CORTEX_API_URL="{{CORTEX_URL}}/api"
     CORTEX_USER="{{CORTEXUSER}}"
     CORTEX_PASSWORD="{{CORTEXPASSWORD}}"
     CORTEX_KEY="{{CORTEXKEY}}"    
@@ -27,7 +30,7 @@ cortex_init(){
     CORTEX_ORG_USER_KEY="{{CORTEXORGUSERKEY}}"
     SOCTOPUS_CONFIG="$default_salt_dir/salt/soctopus/files/SOCtopus.conf"
 
-
+    if wait_for_web_response https://$CORTEX_URL "Cortex"; then
         # Migrate DB
         curl -v -k -XPOST -L "https://$CORTEX_API_URL/maintenance/migrate"
 
@@ -53,21 +56,19 @@ cortex_init(){
         #sed -i "s/cortex_key = .*/cortex_key = $CORTEX_KEY/" $SOCTOPUS_CONFIG
 
         touch /opt/so/state/cortex.txt
+    else
+        echo "We experienced an issue connecting to Cortex!"
+    fi
 }
 
 if [ -f /opt/so/state/cortex.txt ]; then
     cortex_clean
     exit 0
 else
-    rm -f garbage_file
-    while ! wget -O garbage_file {{MANAGERIP}}:9500 2>/dev/null
-    do
-      echo "Waiting for Elasticsearch..."
-      rm -f garbage_file
-      sleep 1
-    done
-    rm -f garbage_file
-    sleep 5
+    if wait_for_web_response http://{{MANAGERIP}}:9400 '"status":"green"'; then
         cortex_init
         cortex_clean
+    else
+        echo "TheHive Elasticsearch server is not ready; unable to proceed with cortex init."
+    fi
 fi

salt/thehive/scripts/hive_init

@@ -5,6 +5,8 @@
 # {%- set THEHIVEPASSWORD = salt['pillar.get']('global:hivepassword', 'hivechangeme') %}
 # {%- set THEHIVEKEY = salt['pillar.get']('global:hivekey', '') %}
 
+. /usr/sbin/so-common
+
 thehive_clean(){
     sed -i '/^  hiveuser:/d' /opt/so/saltstack/local/pillar/global.sls
     sed -i '/^  hivepassword:/d' /opt/so/saltstack/local/pillar/global.sls 
@@ -19,23 +21,7 @@ thehive_init(){
     SOCTOPUS_CONFIG="/opt/so/saltstack/salt/soctopus/files/SOCtopus.conf"
 
     echo -n "Waiting for TheHive..."
-    COUNT=0
-    THEHIVE_CONNECTED="no"
-    while [[ "$COUNT" -le 240 ]]; do
-        curl --output /dev/null --silent --head --fail -k "https://$THEHIVE_URL"
-            if [ $? -eq 0 ]; then
-                THEHIVE_CONNECTED="yes"
-                echo "connected!"
-                break
-            else
-                ((COUNT+=1))
-                sleep 1
-                echo -n "."
-            fi
-    done
-    
-    if [ "$THEHIVE_CONNECTED" == "yes" ]; then
-    
+    if wait_for_web_response https://$THEHIVE_URL "TheHive"; then
         # Migrate DB
         curl -v -k -XPOST -L "https://$THEHIVE_API_URL/maintenance/migrate"
 
@@ -47,7 +33,6 @@ thehive_init(){
         # reputation
         curl -v -k -L "https://$THEHIVE_API_URL/list/custom_fields" -H "Authorization: Bearer $THEHIVE_KEY" -H "Content-Type: application/json" -d "{\"value\":{\"name\": \"reputation\", \"reference\": \"reputation\", \"description\": \"This field provides an overall reputation status for an address/domain.\", \"type\": \"string\", \"options\": []}}"
    
-   
         touch /opt/so/state/thehive.txt
     else
         echo "We experienced an issue connecting to TheHive!"
@@ -58,15 +43,10 @@ if [ -f /opt/so/state/thehive.txt ]; then
     thehive_clean
     exit 0
 else
-    rm -f garbage_file
-    while ! wget -O garbage_file {{MANAGERIP}}:9400 2>/dev/null
-    do
-      echo "Waiting for Elasticsearch..."
-      rm -f garbage_file
-      sleep 1
-    done
-    rm -f garbage_file
-    sleep 5
+    if wait_for_web_response http://{{MANAGERIP}}:9400 '"status":"green"'; then
         thehive_init
         thehive_clean
+    else
+        echo "TheHive Elasticsearch server is not ready; unable to proceed with hive init."
+    fi
 fi