allow template loads to partially succeed only on the initial attempt

salt/common/tools/sbin/so-common

@@ -397,6 +397,10 @@ retry() {
 								echo "<Start of output>"
 								echo "$output"
 								echo "<End of output>"
+								if [[ $exitcode -eq 0 ]]; then
+									echo "Forcing exit code to 1"
+									exitcode=1
+								fi
                         fi
                 elif [ -n "$failedOutput" ]; then
                         if [[ "$output" =~ "$failedOutput" ]]; then
@@ -405,7 +409,7 @@ retry() {
 								echo "$output"
 								echo "<End of output>"
 								if [[ $exitcode -eq 0 ]]; then
-									echo "The exitcode was 0, but we are setting to 1 since we found $failedOutput in the output."
+									echo "Forcing exit code to 1"
 									exitcode=1
 								fi
                         else

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load

@@ -7,8 +7,42 @@
 {%  from 'vars/globals.map.jinja' import GLOBALS %}
 {%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %}
 
-if [ ! -f /opt/so/state/estemplates.txt ]; then
+STATE_FILE_INITIAL=/opt/so/state/estemplates_initial_load_attempt.txt
-  echo "State file /opt/so/state/estemplates.txt not found. Running so-elasticsearch-templates-load."
+STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt
+
+if [[ -f $STATE_FILE_INITIAL ]]; then
+  # The initial template load has already run. As this is a subsequent load, all dependencies should 
+  # already be satisified. Therefore, immediately exit/abort this script upon any template load failure
+  # since this is an unrecoverable failure.
+  should_exit_on_failure=1
+else
+  # This is the initial template load, and there likely are some components not yet setup in Elasticsearch.
+  # Therefore load as many templates as possible at this time and if an error occurs proceed to the next 
+  # template. But if at least one template fails to load do not mark the templates as having been loaded.
+  # This will allow the next load to resume the load of the templates that failed to load initially.
+  should_exit_on_failure=0
+  echo "This is the initial template load"
+fi
+
+load_failures=0
+
+load_template() {
+  uri=$1
+  file=$2
+
+  echo "Loading template file $i" 
+  if ! retry 3 5 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}"; then
+    if [[ $should_exit_on_failure -eq 1 ]]; then
+      fail "Could not load template file: $file"
+    else
+      load_failures=$((load_failures+1))
+      echo "Incremented load failure counter: $load_failures"
+    fi
+  fi
+}
+
+if [ ! -f $STATE_FILE_SUCCESS ]; then
+  echo "State file $STATE_FILE_SUCCESS not found. Running so-elasticsearch-templates-load."
 
   . /usr/sbin/so-common
 
@@ -44,13 +78,14 @@ if [ ! -f /opt/so/state/estemplates.txt ]; then
     fi
     {% endif %}
 
+    touch $STATE_FILE_INITIAL
+
     cd ${ELASTICSEARCH_TEMPLATES}/component/ecs
 
     echo "Loading ECS component templates..."
     for i in *; do 
       TEMPLATE=$(echo $i | cut -d '.' -f1) 
-      echo "$TEMPLATE-mappings" 
+      load_template "_component_template/${TEMPLATE}-mappings" "$i"
-      retry 24 5 "so-elasticsearch-query _component_template/${TEMPLATE}-mappings -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load template: $TEMPLATE-mappings"
     done
     echo
 
@@ -64,8 +99,7 @@ if [ ! -f /opt/so/state/estemplates.txt ]; then
     {% endif %}
     for i in $component_pattern; do 
       TEMPLATE=${i::-5} 
-      echo "$TEMPLATE"
+      load_template "_component_template/$TEMPLATE" "$i"
-      retry 24 5 "so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load template: $TEMPLATE"
     done
     echo
 	    
@@ -75,8 +109,7 @@ if [ ! -f /opt/so/state/estemplates.txt ]; then
     echo "Loading Security Onion component templates..."
     for i in *; do 
       TEMPLATE=$(echo $i | cut -d '.' -f1); 
-      echo "$TEMPLATE"
+      load_template "_component_template/$TEMPLATE" "$i"
-      retry 24 5 "so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load template: $TEMPLATE"  
     done
     echo
 
@@ -92,9 +125,8 @@ if [ ! -f /opt/so/state/estemplates.txt ]; then
     {% endif %}
     for i in $pattern; do
       TEMPLATE=${i::-14}
-      echo "$TEMPLATE"
+      load_template "_index_template/$TEMPLATE" "$i"
-      retry 60 5 "so-elasticsearch-query _index_template/$TEMPLATE -d@$i -XPUT" "{\"acknowledged\":true}" || fail "Could not load template: $TEMPLATE"     
+    done
-      done
   else
     {% if GLOBALS.role == 'so-heavynode' %}
     echo "Common template does not exist. Exiting..."
@@ -105,5 +137,13 @@ if [ ! -f /opt/so/state/estemplates.txt ]; then
   fi
   
   cd - >/dev/null
-  touch /opt/so/state/estemplates.txt
+
+  if [[ $load_failures -eq 0 ]]; then
+    echo "All template loaded successfully"
+    touch $STATE_FILE_SUCCESS
+  else
+    echo "Encountered $load_failures templates that were unable to load, likely due to missing dependencies that will be available later; will retry on next highstate"
+  fi
+else
+  echo "Templates already loaded"
 fi