Use the retry function so-elasticsearch-query

salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines

@@ -15,52 +15,16 @@ ELASTICSEARCH_INGEST_PIPELINES="/opt/so/conf/elasticsearch/ingest/"
 
 # Wait for ElasticSearch to initialize
 if [ ! -f /opt/so/state/espipelines.txt ]; then
-echo "State file /opt/so/state/espipelines.txt not found. Running so-elasticsearch-pipelines."
+  echo "State file /opt/so/state/espipelines.txt not found. Running so-elasticsearch-pipelines."
-
   echo -n "Waiting for ElasticSearch..."
-  COUNT=0
+  retry 240 1 "so-elasticsearch-query / -k --output /dev/null --silent --head --fail" || fail "Connection attempt timed out.  Unable to connect to ElasticSearch.  \nPlease try: \n  -checking log(s) in /var/log/elasticsearch/\n  -running 'sudo docker ps' \n  -running 'sudo so-elastic-restart'"
-  ELASTICSEARCH_CONNECTED="no"
-  while [[ "$COUNT" -le 240 ]]; do
-      curl -K /opt/so/conf/elasticsearch/curl.config -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
-          if [ $? -eq 0 ]; then
-                ELASTICSEARCH_CONNECTED="yes"
-                echo "connected!"
-                break
-          else
-                ((COUNT+=1))
-                sleep 1
-                echo -n "."
-          fi
-    done
-  if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
-        echo
-        echo -e "Connection attempt timed out.  Unable to connect to ElasticSearch.  \nPlease try: \n  -checking log(s) in /var/log/elasticsearch/\n  -running 'sudo docker ps' \n  -running 'sudo so-elastic-restart'"
-        echo
-	exit 1
-  fi
 
   cd ${ELASTICSEARCH_INGEST_PIPELINES}
   echo "Loading pipelines..."
   for i in .[a-z]* *; 
   do 
     echo $i;
-    SUCCESSFUL="no"
+    retry 5 5 "so-elasticsearch-query _ingest/pipeline/$i -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load pipeline: $i"
-    while [[ "$TRYCOUNT" -le 4 ]]; do
-      RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -k -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null);
-      if [ "$RESPONSE" == '{"acknowledged":true}' ]; then
-        SUCCESSFUL="yes"
-	break
-      else
-        ((TRYCOUNT+=1))
-        sleep 5
-	echo -n "Attempt $TRYCOUNT/5 unsuccessful..."
-      fi
-    done
-    if ! [ "$SUCCESSFUL" == "yes" ];then
-      echo -n "Could not load pipeline."
-      echo -n "$RESPONSE"
-      exit 1
-    fi
   done
   echo
 

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load

@@ -32,155 +32,71 @@ if [ ! -f /opt/so/state/estemplates.txt ]; then
   {% endif %}
 
   if [ -f "$file" ]; then
-	# Wait for ElasticSearch to initialize
+    # Wait for ElasticSearch to initialize
-	echo -n "Waiting for ElasticSearch..."
+    echo -n "Waiting for ElasticSearch..."
-	COUNT=0
+    retry 240 1 "so-elasticsearch-query / -k --output /dev/null --silent --head --fail" || fail "Connection attempt timed out.  Unable to connect to ElasticSearch.  \nPlease try: \n  -checking log(s) in /var/log/elasticsearch/\n  -running 'sudo docker ps' \n  -running 'sudo so-elastic-restart'"
-	ELASTICSEARCH_CONNECTED="no"
+    {% if GLOBALS.role != 'so-heavynode' %}
-	while [[ "$COUNT" -le 240 ]]; do
+    SESSIONCOOKIE=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
-      		so-elasticsearch-query / -k --output /dev/null --silent --head --fail
+    INSTALLED=$(elastic_fleet_package_is_installed {{ SUPPORTED_PACKAGES[0] }} )
-		if [ $? -eq 0 ]; then
+    if [ "$INSTALLED" != "installed" ]; then
-			ELASTICSEARCH_CONNECTED="yes"
+      echo
-			echo "connected!"
+      echo "Packages not yet installed."
-			break
+      echo
-		else
+      exit 0
-			((COUNT+=1))
+    fi
-			sleep 1
+    {% endif %}
-			echo -n "."
-		fi
-	done
-	if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
-		echo
-		echo -e "Connection attempt timed out.  Unable to connect to ElasticSearch.  \nPlease try: \n  -checking log(s) in /var/log/elasticsearch/\n  -running 'sudo docker ps' \n  -running 'sudo so-elastic-restart'"
-		echo
-		exit 1
-	fi
 
-	{% if GLOBALS.role != 'so-heavynode' %}
+    cd ${ELASTICSEARCH_TEMPLATES}/component/ecs
-        SESSIONCOOKIE=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
-	INSTALLED=$(elastic_fleet_package_is_installed {{ SUPPORTED_PACKAGES[0] }} )
-	if [ "$INSTALLED" != "installed" ]; then
-        	echo
-        	echo "Packages not yet installed."
-        	echo
-        	exit 0
-	fi
-        {% endif %}
 
-	cd ${ELASTICSEARCH_TEMPLATES}/component/ecs
+    echo "Loading ECS component templates..."
+    for i in *; do 
+      TEMPLATE=$(echo $i | cut -d '.' -f1) 
+      echo "$TEMPLATE-mappings" 
+      retry 5 5 "so-elasticsearch-query _component_template/${TEMPLATE}-mappings -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load template: $TEMPLATE-mappings"
+    done
+    echo
 
-	echo "Loading ECS component templates..."
+    cd ${ELASTICSEARCH_TEMPLATES}/component/elastic-agent
-	for i in *; do 
-	  TEMPLATE=$(echo $i | cut -d '.' -f1) 
-	  echo "$TEMPLATE-mappings" 
-          SUCCESSFUL="no"
-          while [[ "$TRYCOUNT" -le 4 ]]; do
-	    RESPONSE=$(so-elasticsearch-query _component_template/${TEMPLATE}-mappings -d@$i -XPUT 2>/dev/null); 
-            if [ "$RESPONSE" == '{"acknowledged":true}' ]; then
-              SUCCESSFUL="yes"
-              break
-            else
-              ((TRYCOUNT+=1))
-              sleep 5
-              echo -n "Attempt $TRYCOUNT/5 unsuccessful..."
-            fi
-          done
-            if ! [ "$SUCCESSFUL" == "yes" ];then
-              echo -n "Could not load template."
-              echo -n "$RESPONSE"
-              exit 1
-            fi
-        done
-	echo
 
-	cd ${ELASTICSEARCH_TEMPLATES}/component/elastic-agent
+    echo "Loading Elastic Agent component templates..."
+    {% if GLOBALS.role == 'so-heavynode' %}
+    component_pattern="so-*"
+    {% else %}
+    component_pattern="*"
+    {% endif %}
+    for i in $component_pattern; do 
+      TEMPLATE=${i::-5} 
+      echo "$TEMPLATE"
+      retry 5 5 "so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load template: $TEMPLATE"
+    done
+    echo
 	    
-	echo "Loading Elastic Agent component templates..."
+    # Load SO-specific component templates
-	{% if GLOBALS.role == 'so-heavynode' %}
+    cd ${ELASTICSEARCH_TEMPLATES}/component/so
-        component_pattern="so-*"
-        {% else %}
-        component_pattern="*"
-        {% endif %}
-        for i in $component_pattern; do 
-	  TEMPLATE=${i::-5} 
-	  echo "$TEMPLATE"
-	  SUCCESSFUL="no"
-          while [[ "$TRYCOUNT" -le 4 ]]; do
-	    RESPONSE=$(so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null); 
-	    if [ "$RESPONSE" == '{"acknowledged":true}' ]; then
-              SUCCESSFUL="yes"
-              break
-            else
-              ((TRYCOUNT+=1))
-              sleep 5
-              echo -n "Attempt $TRYCOUNT/5 unsuccessful..."
-            fi
-          done
-            if ! [ "$SUCCESSFUL" == "yes" ];then
-              echo -n "Could not load template."
-              echo -n "$RESPONSE"
-              exit 1
-            fi
-        done
-	echo
 
-	# Load SO-specific component templates
+    echo "Loading Security Onion component templates..."
-	cd ${ELASTICSEARCH_TEMPLATES}/component/so
+    for i in *; do 
+      TEMPLATE=$(echo $i | cut -d '.' -f1); 
+      echo "$TEMPLATE"
+      retry 5 5 "so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load template: $TEMPLATE"  
+    done
+    echo
 
-	echo "Loading Security Onion component templates..."
+    # Load SO index templates
-	for i in *; do 
+    cd ${ELASTICSEARCH_TEMPLATES}/index
-	  TEMPLATE=$(echo $i | cut -d '.' -f1); 
-	  echo "$TEMPLATE"
-	  SUCCESSFUL="no"
-          while [[ "$TRYCOUNT" -le 4 ]]; do 
-	    RESPONSE=$(so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null);
-            if [ "$RESPONSE" == '{"acknowledged":true}' ]; then
-              SUCCESSFUL="yes"
-              break
-            else
-              ((TRYCOUNT+=1))
-              sleep 5
-              echo -n "Attempt $TRYCOUNT/5 unsuccessful..."
-            fi
-          done
-            if ! [ "$SUCCESSFUL" == "yes" ];then
-              echo -n "Could not load template."
-              echo -n "$RESPONSE"
-              exit 1
-            fi
-        done
-	echo
 
-	# Load SO index templates
+    echo "Loading Security Onion index templates..."
-	cd ${ELASTICSEARCH_TEMPLATES}/index
+    shopt -s extglob
-
+    {% if GLOBALS.role == 'so-heavynode' %}
-	echo "Loading Security Onion index templates..."
+    pattern="!(*1password*|*aws*|*azure*|*cloudflare*|*elastic_agent*|*fim*|*github*|*google*|*osquery*|*system*|*windows*)"
-	shopt -s extglob
+    {% else %}
-        {% if GLOBALS.role == 'so-heavynode' %}
+    pattern="*"
-        pattern="!(*1password*|*aws*|*azure*|*cloudflare*|*elastic_agent*|*fim*|*github*|*google*|*osquery*|*system*|*windows*)"
+    {% endif %}
-        {% else %}
+    for i in $pattern; do
-        pattern="*"
+      TEMPLATE=${i::-14}
-        {% endif %}
+      echo "$TEMPLATE"
-        for i in $pattern; do
+      retry 5 5 "so-elasticsearch-query _index_template/$TEMPLATE -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load template: $TEMPLATE"     
-          TEMPLATE=${i::-14}
+      done
-          echo "$TEMPLATE"
-	  SUCCESSFUL="no"
-          while [[ "$TRYCOUNT" -le 4 ]]; do
-	    RESPONSE=$(so-elasticsearch-query _index_template/$TEMPLATE -d@$i -XPUT 2>/dev/null);
-            if [ "$RESPONSE" == '{"acknowledged":true}' ]; then
-              SUCCESSFUL="yes"
-              break
-            else
-              ((TRYCOUNT+=1))
-              sleep 5
-              echo -n "Attempt $TRYCOUNT/5 unsuccessful..."
-            fi
-          done
-            if ! [ "$SUCCESSFUL" == "yes" ];then
-              echo -n "Could not load template."
-              echo -n "$RESPONSE"
-              exit 1
-            fi
-        done
   else
     {% if GLOBALS.role == 'so-heavynode' %}
     echo "Common template does not exist. Exiting..."