State file changes and retry logic

salt/elasticsearch/enabled.sls

@@ -110,7 +110,7 @@ escomponenttemplates:
     - group: 939
     - clean: True
     - onchanges_in:
-      - cmd: so-elasticsearch-templates
+      - file: so-elasticsearch-templates-reload
       
 # Auto-generate templates from defaults file
 {%     for index, settings in ES_INDEX_SETTINGS.items() %}
@@ -123,7 +123,7 @@ es_index_template_{{index}}:
       TEMPLATE_CONFIG: {{ settings.index_template }}
     - template: jinja
     - onchanges_in:
-      - cmd: so-elasticsearch-templates
+      - file: so-elasticsearch-templates-reload
 {%       endif %}
 {%     endfor %}
 
@@ -142,7 +142,7 @@ es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}:
     - user: 930
     - group: 939
     - onchanges_in:
-      - cmd: so-elasticsearch-templates
+      - file: so-elasticsearch-templates-reload
 {%       endfor %}
 {%     endif %}
 
@@ -167,6 +167,10 @@ so-elasticsearch-ilm-policy-load:
     - onchanges:
       - file: so-elasticsearch-ilm-policy-load-script
 
+so-elasticsearch-templates-reload:
+  file.absent:
+    - name: /opt/so/state/estemplates.txt
+
 so-elasticsearch-templates:
   cmd.run:
     - name: /usr/sbin/so-elasticsearch-templates-load

salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines

@@ -42,13 +42,25 @@ echo "State file /opt/so/state/espipelines.txt not found. Running so-elasticsear
   cd ${ELASTICSEARCH_INGEST_PIPELINES}
   echo "Loading pipelines..."
   for i in .[a-z]* *; 
-    do echo $i;
+  do 
-    RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -k -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null);
+    echo $i;
-    ERRORS=$(echo $RESPONSE | grep -E "Connection attempt timed out|error");
+    SUCCESSFUL="no"
-    if ! [ -z "$ERRORS" ]; then
+    while [[ "$TRYCOUNT" -le 4 ]]; do
-      echo $ERRORS;
+      RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -k -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null);
-      RETURN_CODE=1;
+      if [ "$RESPONSE" == '{"acknowledged":true}' ]; then
-    fi;
+        SUCCESSFUL="yes"
+	break
+      else
+        ((TRYCOUNT+=1))
+        sleep 5
+	echo -n "Attempt $TRYCOUNT/5 unsuccessful..."
+      fi
+    done
+    if ! [ "$SUCCESSFUL" == "yes" ];then
+      echo -n "Could not load pipeline."
+      echo -n "$RESPONSE"
+      exit 1
+    fi
   done
   echo
 

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load

@@ -7,25 +7,31 @@
 {%  from 'vars/globals.map.jinja' import GLOBALS %}
 {%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %}
 
-. /usr/sbin/so-common
+RETURN_CODE=0
-{% if GLOBALS.role != 'so-heavynode' %}
-if [ -f /usr/sbin/so-elastic-fleet-common ]; then
-  . /usr/sbin/so-elastic-fleet-common
-fi
-{% endif %}
 
-default_conf_dir=/opt/so/conf
+if [ ! -f /opt/so/state/estemplates.txt ]; then
+  echo "State file /opt/so/state/estemplates.txt not found. Running so-elasticsearch-templates-load."
 
-# Define a default directory to load pipelines from
+  . /usr/sbin/so-common
-ELASTICSEARCH_TEMPLATES="$default_conf_dir/elasticsearch/templates/"
 
-{% if GLOBALS.role == 'so-heavynode' %}
+  {% if GLOBALS.role != 'so-heavynode' %}
-file="/opt/so/conf/elasticsearch/templates/index/so-common-template.json"
+  if [ -f /usr/sbin/so-elastic-fleet-common ]; then
-{% else %}
+    . /usr/sbin/so-elastic-fleet-common
-file="/usr/sbin/so-elastic-fleet-common"
+  fi
-{% endif %}
+  {% endif %}
 
-if [ -f "$file" ]; then
+  default_conf_dir=/opt/so/conf
+
+  # Define a default directory to load pipelines from
+  ELASTICSEARCH_TEMPLATES="$default_conf_dir/elasticsearch/templates/"
+
+  {% if GLOBALS.role == 'so-heavynode' %}
+  file="/opt/so/conf/elasticsearch/templates/index/so-common-template.json"
+  {% else %}
+  file="/usr/sbin/so-elastic-fleet-common"
+  {% endif %}
+
+  if [ -f "$file" ]; then
 	# Wait for ElasticSearch to initialize
 	echo -n "Waiting for ElasticSearch..."
 	COUNT=0
@@ -59,12 +65,32 @@ if [ -f "$file" ]; then
         	exit 0
 	fi
         {% endif %}
-	set -e
 
 	cd ${ELASTICSEARCH_TEMPLATES}/component/ecs
 
 	echo "Loading ECS component templates..."
-	for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); echo "$TEMPLATE-mappings"; so-elasticsearch-query _component_template/$TEMPLATE-mappings -d@$i -XPUT 2>/dev/null; echo; done
+	for i in *; do 
+	  TEMPLATE=$(echo $i | cut -d '.' -f1) 
+	  echo "$TEMPLATE-mappings" 
+          SUCCESSFUL="no"
+          while [[ "$TRYCOUNT" -le 4 ]]; do
+	    RESPONSE=$(so-elasticsearch-query _component_template/${TEMPLATE}-mappings -d@$i -XPUT 2>/dev/null); 
+            if [ "$RESPONSE" == '{"acknowledged":true}' ]; then
+              SUCCESSFUL="yes"
+              break
+            else
+              ((TRYCOUNT+=1))
+              sleep 5
+              echo -n "Attempt $TRYCOUNT/5 unsuccessful..."
+            fi
+          done
+            if ! [ "$SUCCESSFUL" == "yes" ];then
+              echo -n "Could not load template."
+              echo -n "$RESPONSE"
+              exit 1
+            fi
+        done
+	echo
 
 	cd ${ELASTICSEARCH_TEMPLATES}/component/elastic-agent
 
@@ -74,13 +100,54 @@ if [ -f "$file" ]; then
         {% else %}
         component_pattern="*"
         {% endif %}
-        for i in $component_pattern; do TEMPLATE=${i::-5}; echo "$TEMPLATE"; so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done
+        for i in $component_pattern; do 
+	  TEMPLATE=${i::-5} 
+	  echo "$TEMPLATE"
+	  SUCCESSFUL="no"
+          while [[ "$TRYCOUNT" -le 4 ]]; do
+	    RESPONSE=$(so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null); 
+	    if [ "$RESPONSE" == '{"acknowledged":true}' ]; then
+              SUCCESSFUL="yes"
+              break
+            else
+              ((TRYCOUNT+=1))
+              sleep 5
+              echo -n "Attempt $TRYCOUNT/5 unsuccessful..."
+            fi
+          done
+            if ! [ "$SUCCESSFUL" == "yes" ];then
+              echo -n "Could not load template."
+              echo -n "$RESPONSE"
+              exit 1
+            fi
+        done
+	echo
 	    
 	# Load SO-specific component templates
 	cd ${ELASTICSEARCH_TEMPLATES}/component/so
 
 	echo "Loading Security Onion component templates..."
-	for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); echo "$TEMPLATE"; so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done
+	for i in *; do 
+	  TEMPLATE=$(echo $i | cut -d '.' -f1); 
+	  echo "$TEMPLATE"
+	  SUCCESSFUL="no"
+          while [[ "$TRYCOUNT" -le 4 ]]; do 
+	    RESPONSE=$(so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null);
+            if [ "$RESPONSE" == '{"acknowledged":true}' ]; then
+              SUCCESSFUL="yes"
+              break
+            else
+              ((TRYCOUNT+=1))
+              sleep 5
+              echo -n "Attempt $TRYCOUNT/5 unsuccessful..."
+            fi
+          done
+            if ! [ "$SUCCESSFUL" == "yes" ];then
+              echo -n "Could not load template."
+              echo -n "$RESPONSE"
+              exit 1
+            fi
+        done
 	echo
 
 	# Load SO index templates
@@ -94,18 +161,41 @@ if [ -f "$file" ]; then
         pattern="*"
         {% endif %}
         for i in $pattern; do
-            TEMPLATE=${i::-14};
+          TEMPLATE=${i::-14}
-            echo "$TEMPLATE";
+          echo "$TEMPLATE"
-            so-elasticsearch-query _index_template/$TEMPLATE -d@$i -XPUT 2>/dev/null;
+	  SUCCESSFUL="no"
-            echo;
+          while [[ "$TRYCOUNT" -le 4 ]]; do
+	    RESPONSE=$(so-elasticsearch-query _index_template/$TEMPLATE -d@$i -XPUT 2>/dev/null);
+            if [ "$RESPONSE" == '{"acknowledged":true}' ]; then
+              SUCCESSFUL="yes"
+              break
+            else
+              ((TRYCOUNT+=1))
+              sleep 5
+              echo -n "Attempt $TRYCOUNT/5 unsuccessful..."
+            fi
+          done
+            if ! [ "$SUCCESSFUL" == "yes" ];then
+              echo -n "Could not load template."
+              echo -n "$RESPONSE"
+              exit 1
+            fi
         done
-	echo
+  else
+    {% if GLOBALS.role == 'so-heavynode' %}
+    echo "Common template does not exist. Exiting..."
+    {% else %}
+    echo "Elastic Fleet not configured. Exiting..."
+    {% endif %}
+    RETURN_CODE=1
+    exit 0
+  fi
+  cd - >/dev/null
+  if [[ "$RETURN_CODE" != "1" ]]; then
+    touch /opt/so/state/estemplates.txt
+  else
+    echo "Errors were detected. This script will run again during the next application of the state."
+  fi
 else
-	{% if GLOBALS.role == 'so-heavynode' %}
+  exit $RETURN_CODE
-        echo "Common template does not exist. Exiting..."
-        {% else %}
-        echo "Elastic Fleet not configured. Exiting..."
-	{% endif %}
-        exit 0
 fi
-	cd - >/dev/null