Merge pull request #7446 from Security-Onion-Solutions/fixpipelineload

Only load pipelines on change

salt/elasticsearch/init.sls

@@ -268,6 +268,15 @@ es_repo_dir:
     - require:
       - file: nsmesdir
 
+so-pipelines-reload:
+  file.absent:
+    - name: /opt/so/state/espipelines.txt
+    - onchanges:
+      - file: esingestconf
+      - file: esingestdynamicconf
+      - file: esyml
+      - file: so-elasticsearch-pipelines-script
+
 auth_users:
   file.managed:
     - name: /opt/so/conf/elasticsearch/users.tmp
@@ -358,9 +367,6 @@ so-elasticsearch:
     - watch:
       - file: cacertz
       - file: esyml
-      - file: esingestconf
-      - file: esingestdynamicconf
-      - file: so-elasticsearch-pipelines-script
     - require:
       - file: esyml
       - file: eslog4jfile
@@ -397,11 +403,6 @@ so-elasticsearch-templates:
 so-elasticsearch-pipelines:
   cmd.run:
     - name: /usr/sbin/so-elasticsearch-pipelines {{ grains.host }}
-    - onchanges:
-      - file: esingestconf
-      - file: esingestdynamicconf
-      - file: esyml
-      - file: so-elasticsearch-pipelines-script
     - require:
       - docker_container: so-elasticsearch
       - file: so-elasticsearch-pipelines-script

salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines

@@ -23,33 +23,40 @@ ELASTICSEARCH_PORT=9200
 ELASTICSEARCH_INGEST_PIPELINES="/opt/so/conf/elasticsearch/ingest/"
 
 # Wait for ElasticSearch to initialize
-echo -n "Waiting for ElasticSearch..."
-COUNT=0
-ELASTICSEARCH_CONNECTED="no"
-while [[ "$COUNT" -le 240 ]]; do
-    {{ ELASTICCURL }} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
-	if [ $? -eq 0 ]; then
-		ELASTICSEARCH_CONNECTED="yes"
+
+if [ ! -f /opt/so/state/espipelines.txt ]; then
+
+  echo -n "Waiting for ElasticSearch..."
+  COUNT=0
+  ELASTICSEARCH_CONNECTED="no"
+  while [[ "$COUNT" -le 240 ]]; do
+      {{ ELASTICCURL }} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
+	  if [ $? -eq 0 ]; then
+	  	ELASTICSEARCH_CONNECTED="yes"
 		echo "connected!"
 		break
-	else
+	  else
 		((COUNT+=1))
 		sleep 1
 		echo -n "."
-	fi
-done
-if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
+	  fi
+    done
+  if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
 	echo
 	echo -e "Connection attempt timed out.  Unable to connect to ElasticSearch.  \nPlease try: \n  -checking log(s) in /var/log/elasticsearch/\n  -running 'sudo docker ps' \n  -running 'sudo so-elastic-restart'"
 	echo
-fi
+  fi
 
-cd ${ELASTICSEARCH_INGEST_PIPELINES}
+  cd ${ELASTICSEARCH_INGEST_PIPELINES}
 
-echo "Loading pipelines..."
-for i in *; do echo $i; RESPONSE=$({{ ELASTICCURL }} -k -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done
-echo
+  echo "Loading pipelines..."
+  for i in *; do echo $i; RESPONSE=$({{ ELASTICCURL }} -k -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done
+  echo
 
-cd - >/dev/null
-
-exit $RETURN_CODE
+  cd - >/dev/null
+  if [[ "$RETURN_CODE" != "1" ]]; then
+    touch /opt/so/state/espipelines.txt
+  fi  
+else 
+  exit $RETURN_CODE
+fi