From 74eda68d8424472460de791d0ec5321ba4a31293 Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 6 Nov 2023 13:16:35 +0000 Subject: [PATCH 01/13] Exit if unable to communicate with Elasticsearch --- salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines index 350ac97c5..f6f56eaf1 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines @@ -37,6 +37,7 @@ if [ ! -f /opt/so/state/espipelines.txt ]; then echo echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" echo + exit 1 fi cd ${ELASTICSEARCH_INGEST_PIPELINES} From c30a0d5b5b0bc43df6a0a4df12c38a5c6c583383 Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 6 Nov 2023 14:29:01 +0000 Subject: [PATCH 02/13] Better error handling and state file management --- .../tools/sbin/so-elasticsearch-pipelines | 45 +++++++++++-------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines index f6f56eaf1..ff826d2c9 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines @@ -6,7 +6,6 @@ . /usr/sbin/so-common - RETURN_CODE=0 ELASTICSEARCH_HOST=$1 ELASTICSEARCH_PORT=9200 @@ -15,41 +14,51 @@ ELASTICSEARCH_PORT=9200 ELASTICSEARCH_INGEST_PIPELINES="/opt/so/conf/elasticsearch/ingest/" # Wait for ElasticSearch to initialize - if [ ! -f /opt/so/state/espipelines.txt ]; then +echo "State file /opt/so/state/espipelines.txt not found. Running so-elasticsearch-pipelines." echo -n "Waiting for ElasticSearch..." COUNT=0 ELASTICSEARCH_CONNECTED="no" while [[ "$COUNT" -le 240 ]]; do curl -K /opt/so/conf/elasticsearch/curl.config -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" - if [ $? -eq 0 ]; then - ELASTICSEARCH_CONNECTED="yes" - echo "connected!" - break - else - ((COUNT+=1)) - sleep 1 - echo -n "." - fi + if [ $? -eq 0 ]; then + ELASTICSEARCH_CONNECTED="yes" + echo "connected!" + break + else + ((COUNT+=1)) + sleep 1 + echo -n "." + fi done if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then - echo - echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" - echo + echo + echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" + echo exit 1 fi cd ${ELASTICSEARCH_INGEST_PIPELINES} - echo "Loading pipelines..." - for i in .[a-z]* *; do echo $i; RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -k -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done + for i in .[a-z]* *; + do echo $i; + RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -k -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); + ERRORS=$(echo $RESPONSE | grep -E "Connection attempt timed out|error"); + if ! [ -z "$ERRORS" ]; then + echo $ERRORS; + RETURN_CODE=1; + fi; + done echo cd - >/dev/null + if [[ "$RETURN_CODE" != "1" ]]; then touch /opt/so/state/espipelines.txt - fi -else + else + echo "Errors were detected. This script will run again during the next application of the state." + fi +else exit $RETURN_CODE fi From 0b4a246ddbaef67effb5e33affdee35cd11deeac Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 7 Nov 2023 16:44:42 +0000 Subject: [PATCH 03/13] State file changes and retry logic --- salt/elasticsearch/enabled.sls | 10 +- .../tools/sbin/so-elasticsearch-pipelines | 30 +++- .../so-elasticsearch-templates-load | 154 ++++++++++++++---- 3 files changed, 150 insertions(+), 44 deletions(-) diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index fa0f824b4..f7ab7749f 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -110,7 +110,7 @@ escomponenttemplates: - group: 939 - clean: True - onchanges_in: - - cmd: so-elasticsearch-templates + - file: so-elasticsearch-templates-reload # Auto-generate templates from defaults file {% for index, settings in ES_INDEX_SETTINGS.items() %} @@ -123,7 +123,7 @@ es_index_template_{{index}}: TEMPLATE_CONFIG: {{ settings.index_template }} - template: jinja - onchanges_in: - - cmd: so-elasticsearch-templates + - file: so-elasticsearch-templates-reload {% endif %} {% endfor %} @@ -142,7 +142,7 @@ es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}: - user: 930 - group: 939 - onchanges_in: - - cmd: so-elasticsearch-templates + - file: so-elasticsearch-templates-reload {% endfor %} {% endif %} @@ -167,6 +167,10 @@ so-elasticsearch-ilm-policy-load: - onchanges: - file: so-elasticsearch-ilm-policy-load-script +so-elasticsearch-templates-reload: + file.absent: + - name: /opt/so/state/estemplates.txt + so-elasticsearch-templates: cmd.run: - name: /usr/sbin/so-elasticsearch-templates-load diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines index ff826d2c9..2ddc5fa52 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines @@ -41,19 +41,31 @@ echo "State file /opt/so/state/espipelines.txt not found. Running so-elasticsear cd ${ELASTICSEARCH_INGEST_PIPELINES} echo "Loading pipelines..." - for i in .[a-z]* *; - do echo $i; - RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -k -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); - ERRORS=$(echo $RESPONSE | grep -E "Connection attempt timed out|error"); - if ! [ -z "$ERRORS" ]; then - echo $ERRORS; - RETURN_CODE=1; - fi; + for i in .[a-z]* *; + do + echo $i; + SUCCESSFUL="no" + while [[ "$TRYCOUNT" -le 4 ]]; do + RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -k -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); + if [ "$RESPONSE" == '{"acknowledged":true}' ]; then + SUCCESSFUL="yes" + break + else + ((TRYCOUNT+=1)) + sleep 5 + echo -n "Attempt $TRYCOUNT/5 unsuccessful..." + fi + done + if ! [ "$SUCCESSFUL" == "yes" ];then + echo -n "Could not load pipeline." + echo -n "$RESPONSE" + exit 1 + fi done echo cd - >/dev/null - + if [[ "$RETURN_CODE" != "1" ]]; then touch /opt/so/state/espipelines.txt else diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load index 857da5434..d1e5dc41a 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load @@ -7,25 +7,31 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %} -. /usr/sbin/so-common -{% if GLOBALS.role != 'so-heavynode' %} -if [ -f /usr/sbin/so-elastic-fleet-common ]; then - . /usr/sbin/so-elastic-fleet-common -fi -{% endif %} +RETURN_CODE=0 -default_conf_dir=/opt/so/conf +if [ ! -f /opt/so/state/estemplates.txt ]; then + echo "State file /opt/so/state/estemplates.txt not found. Running so-elasticsearch-templates-load." -# Define a default directory to load pipelines from -ELASTICSEARCH_TEMPLATES="$default_conf_dir/elasticsearch/templates/" + . /usr/sbin/so-common -{% if GLOBALS.role == 'so-heavynode' %} -file="/opt/so/conf/elasticsearch/templates/index/so-common-template.json" -{% else %} -file="/usr/sbin/so-elastic-fleet-common" -{% endif %} + {% if GLOBALS.role != 'so-heavynode' %} + if [ -f /usr/sbin/so-elastic-fleet-common ]; then + . /usr/sbin/so-elastic-fleet-common + fi + {% endif %} -if [ -f "$file" ]; then + default_conf_dir=/opt/so/conf + + # Define a default directory to load pipelines from + ELASTICSEARCH_TEMPLATES="$default_conf_dir/elasticsearch/templates/" + + {% if GLOBALS.role == 'so-heavynode' %} + file="/opt/so/conf/elasticsearch/templates/index/so-common-template.json" + {% else %} + file="/usr/sbin/so-elastic-fleet-common" + {% endif %} + + if [ -f "$file" ]; then # Wait for ElasticSearch to initialize echo -n "Waiting for ElasticSearch..." COUNT=0 @@ -59,12 +65,32 @@ if [ -f "$file" ]; then exit 0 fi {% endif %} - set -e cd ${ELASTICSEARCH_TEMPLATES}/component/ecs echo "Loading ECS component templates..." - for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); echo "$TEMPLATE-mappings"; so-elasticsearch-query _component_template/$TEMPLATE-mappings -d@$i -XPUT 2>/dev/null; echo; done + for i in *; do + TEMPLATE=$(echo $i | cut -d '.' -f1) + echo "$TEMPLATE-mappings" + SUCCESSFUL="no" + while [[ "$TRYCOUNT" -le 4 ]]; do + RESPONSE=$(so-elasticsearch-query _component_template/${TEMPLATE}-mappings -d@$i -XPUT 2>/dev/null); + if [ "$RESPONSE" == '{"acknowledged":true}' ]; then + SUCCESSFUL="yes" + break + else + ((TRYCOUNT+=1)) + sleep 5 + echo -n "Attempt $TRYCOUNT/5 unsuccessful..." + fi + done + if ! [ "$SUCCESSFUL" == "yes" ];then + echo -n "Could not load template." + echo -n "$RESPONSE" + exit 1 + fi + done + echo cd ${ELASTICSEARCH_TEMPLATES}/component/elastic-agent @@ -74,13 +100,54 @@ if [ -f "$file" ]; then {% else %} component_pattern="*" {% endif %} - for i in $component_pattern; do TEMPLATE=${i::-5}; echo "$TEMPLATE"; so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done - + for i in $component_pattern; do + TEMPLATE=${i::-5} + echo "$TEMPLATE" + SUCCESSFUL="no" + while [[ "$TRYCOUNT" -le 4 ]]; do + RESPONSE=$(so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null); + if [ "$RESPONSE" == '{"acknowledged":true}' ]; then + SUCCESSFUL="yes" + break + else + ((TRYCOUNT+=1)) + sleep 5 + echo -n "Attempt $TRYCOUNT/5 unsuccessful..." + fi + done + if ! [ "$SUCCESSFUL" == "yes" ];then + echo -n "Could not load template." + echo -n "$RESPONSE" + exit 1 + fi + done + echo + # Load SO-specific component templates cd ${ELASTICSEARCH_TEMPLATES}/component/so echo "Loading Security Onion component templates..." - for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); echo "$TEMPLATE"; so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done + for i in *; do + TEMPLATE=$(echo $i | cut -d '.' -f1); + echo "$TEMPLATE" + SUCCESSFUL="no" + while [[ "$TRYCOUNT" -le 4 ]]; do + RESPONSE=$(so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null); + if [ "$RESPONSE" == '{"acknowledged":true}' ]; then + SUCCESSFUL="yes" + break + else + ((TRYCOUNT+=1)) + sleep 5 + echo -n "Attempt $TRYCOUNT/5 unsuccessful..." + fi + done + if ! [ "$SUCCESSFUL" == "yes" ];then + echo -n "Could not load template." + echo -n "$RESPONSE" + exit 1 + fi + done echo # Load SO index templates @@ -94,18 +161,41 @@ if [ -f "$file" ]; then pattern="*" {% endif %} for i in $pattern; do - TEMPLATE=${i::-14}; - echo "$TEMPLATE"; - so-elasticsearch-query _index_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; - echo; + TEMPLATE=${i::-14} + echo "$TEMPLATE" + SUCCESSFUL="no" + while [[ "$TRYCOUNT" -le 4 ]]; do + RESPONSE=$(so-elasticsearch-query _index_template/$TEMPLATE -d@$i -XPUT 2>/dev/null); + if [ "$RESPONSE" == '{"acknowledged":true}' ]; then + SUCCESSFUL="yes" + break + else + ((TRYCOUNT+=1)) + sleep 5 + echo -n "Attempt $TRYCOUNT/5 unsuccessful..." + fi + done + if ! [ "$SUCCESSFUL" == "yes" ];then + echo -n "Could not load template." + echo -n "$RESPONSE" + exit 1 + fi done - echo + else + {% if GLOBALS.role == 'so-heavynode' %} + echo "Common template does not exist. Exiting..." + {% else %} + echo "Elastic Fleet not configured. Exiting..." + {% endif %} + RETURN_CODE=1 + exit 0 + fi + cd - >/dev/null + if [[ "$RETURN_CODE" != "1" ]]; then + touch /opt/so/state/estemplates.txt + else + echo "Errors were detected. This script will run again during the next application of the state." + fi else - {% if GLOBALS.role == 'so-heavynode' %} - echo "Common template does not exist. Exiting..." - {% else %} - echo "Elastic Fleet not configured. Exiting..." - {% endif %} - exit 0 + exit $RETURN_CODE fi - cd - >/dev/null From 1676c84f9c6ef6b2332c7668dfbe8d1d8f58cac0 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 7 Nov 2023 19:56:50 +0000 Subject: [PATCH 04/13] Use the retry function so-elasticsearch-query --- .../tools/sbin/so-elasticsearch-pipelines | 42 +--- .../so-elasticsearch-templates-load | 198 +++++------------- 2 files changed, 60 insertions(+), 180 deletions(-) diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines index 2ddc5fa52..4afc9bd4d 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines @@ -15,52 +15,16 @@ ELASTICSEARCH_INGEST_PIPELINES="/opt/so/conf/elasticsearch/ingest/" # Wait for ElasticSearch to initialize if [ ! -f /opt/so/state/espipelines.txt ]; then -echo "State file /opt/so/state/espipelines.txt not found. Running so-elasticsearch-pipelines." - + echo "State file /opt/so/state/espipelines.txt not found. Running so-elasticsearch-pipelines." echo -n "Waiting for ElasticSearch..." - COUNT=0 - ELASTICSEARCH_CONNECTED="no" - while [[ "$COUNT" -le 240 ]]; do - curl -K /opt/so/conf/elasticsearch/curl.config -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" - if [ $? -eq 0 ]; then - ELASTICSEARCH_CONNECTED="yes" - echo "connected!" - break - else - ((COUNT+=1)) - sleep 1 - echo -n "." - fi - done - if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then - echo - echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" - echo - exit 1 - fi + retry 240 1 "so-elasticsearch-query / -k --output /dev/null --silent --head --fail" || fail "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" cd ${ELASTICSEARCH_INGEST_PIPELINES} echo "Loading pipelines..." for i in .[a-z]* *; do echo $i; - SUCCESSFUL="no" - while [[ "$TRYCOUNT" -le 4 ]]; do - RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -k -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); - if [ "$RESPONSE" == '{"acknowledged":true}' ]; then - SUCCESSFUL="yes" - break - else - ((TRYCOUNT+=1)) - sleep 5 - echo -n "Attempt $TRYCOUNT/5 unsuccessful..." - fi - done - if ! [ "$SUCCESSFUL" == "yes" ];then - echo -n "Could not load pipeline." - echo -n "$RESPONSE" - exit 1 - fi + retry 5 5 "so-elasticsearch-query _ingest/pipeline/$i -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load pipeline: $i" done echo diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load index d1e5dc41a..c0d4f9cba 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load @@ -32,155 +32,71 @@ if [ ! -f /opt/so/state/estemplates.txt ]; then {% endif %} if [ -f "$file" ]; then - # Wait for ElasticSearch to initialize - echo -n "Waiting for ElasticSearch..." - COUNT=0 - ELASTICSEARCH_CONNECTED="no" - while [[ "$COUNT" -le 240 ]]; do - so-elasticsearch-query / -k --output /dev/null --silent --head --fail - if [ $? -eq 0 ]; then - ELASTICSEARCH_CONNECTED="yes" - echo "connected!" - break - else - ((COUNT+=1)) - sleep 1 - echo -n "." - fi - done - if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then - echo - echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" - echo - exit 1 - fi + # Wait for ElasticSearch to initialize + echo -n "Waiting for ElasticSearch..." + retry 240 1 "so-elasticsearch-query / -k --output /dev/null --silent --head --fail" || fail "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" + {% if GLOBALS.role != 'so-heavynode' %} + SESSIONCOOKIE=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') + INSTALLED=$(elastic_fleet_package_is_installed {{ SUPPORTED_PACKAGES[0] }} ) + if [ "$INSTALLED" != "installed" ]; then + echo + echo "Packages not yet installed." + echo + exit 0 + fi + {% endif %} - {% if GLOBALS.role != 'so-heavynode' %} - SESSIONCOOKIE=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') - INSTALLED=$(elastic_fleet_package_is_installed {{ SUPPORTED_PACKAGES[0] }} ) - if [ "$INSTALLED" != "installed" ]; then - echo - echo "Packages not yet installed." - echo - exit 0 - fi - {% endif %} + cd ${ELASTICSEARCH_TEMPLATES}/component/ecs - cd ${ELASTICSEARCH_TEMPLATES}/component/ecs + echo "Loading ECS component templates..." + for i in *; do + TEMPLATE=$(echo $i | cut -d '.' -f1) + echo "$TEMPLATE-mappings" + retry 5 5 "so-elasticsearch-query _component_template/${TEMPLATE}-mappings -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load template: $TEMPLATE-mappings" + done + echo - echo "Loading ECS component templates..." - for i in *; do - TEMPLATE=$(echo $i | cut -d '.' -f1) - echo "$TEMPLATE-mappings" - SUCCESSFUL="no" - while [[ "$TRYCOUNT" -le 4 ]]; do - RESPONSE=$(so-elasticsearch-query _component_template/${TEMPLATE}-mappings -d@$i -XPUT 2>/dev/null); - if [ "$RESPONSE" == '{"acknowledged":true}' ]; then - SUCCESSFUL="yes" - break - else - ((TRYCOUNT+=1)) - sleep 5 - echo -n "Attempt $TRYCOUNT/5 unsuccessful..." - fi - done - if ! [ "$SUCCESSFUL" == "yes" ];then - echo -n "Could not load template." - echo -n "$RESPONSE" - exit 1 - fi - done - echo + cd ${ELASTICSEARCH_TEMPLATES}/component/elastic-agent - cd ${ELASTICSEARCH_TEMPLATES}/component/elastic-agent - - echo "Loading Elastic Agent component templates..." - {% if GLOBALS.role == 'so-heavynode' %} - component_pattern="so-*" - {% else %} - component_pattern="*" - {% endif %} - for i in $component_pattern; do - TEMPLATE=${i::-5} - echo "$TEMPLATE" - SUCCESSFUL="no" - while [[ "$TRYCOUNT" -le 4 ]]; do - RESPONSE=$(so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null); - if [ "$RESPONSE" == '{"acknowledged":true}' ]; then - SUCCESSFUL="yes" - break - else - ((TRYCOUNT+=1)) - sleep 5 - echo -n "Attempt $TRYCOUNT/5 unsuccessful..." - fi - done - if ! [ "$SUCCESSFUL" == "yes" ];then - echo -n "Could not load template." - echo -n "$RESPONSE" - exit 1 - fi - done - echo + echo "Loading Elastic Agent component templates..." + {% if GLOBALS.role == 'so-heavynode' %} + component_pattern="so-*" + {% else %} + component_pattern="*" + {% endif %} + for i in $component_pattern; do + TEMPLATE=${i::-5} + echo "$TEMPLATE" + retry 5 5 "so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load template: $TEMPLATE" + done + echo - # Load SO-specific component templates - cd ${ELASTICSEARCH_TEMPLATES}/component/so + # Load SO-specific component templates + cd ${ELASTICSEARCH_TEMPLATES}/component/so - echo "Loading Security Onion component templates..." - for i in *; do - TEMPLATE=$(echo $i | cut -d '.' -f1); - echo "$TEMPLATE" - SUCCESSFUL="no" - while [[ "$TRYCOUNT" -le 4 ]]; do - RESPONSE=$(so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null); - if [ "$RESPONSE" == '{"acknowledged":true}' ]; then - SUCCESSFUL="yes" - break - else - ((TRYCOUNT+=1)) - sleep 5 - echo -n "Attempt $TRYCOUNT/5 unsuccessful..." - fi - done - if ! [ "$SUCCESSFUL" == "yes" ];then - echo -n "Could not load template." - echo -n "$RESPONSE" - exit 1 - fi - done - echo + echo "Loading Security Onion component templates..." + for i in *; do + TEMPLATE=$(echo $i | cut -d '.' -f1); + echo "$TEMPLATE" + retry 5 5 "so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load template: $TEMPLATE" + done + echo - # Load SO index templates - cd ${ELASTICSEARCH_TEMPLATES}/index + # Load SO index templates + cd ${ELASTICSEARCH_TEMPLATES}/index - echo "Loading Security Onion index templates..." - shopt -s extglob - {% if GLOBALS.role == 'so-heavynode' %} - pattern="!(*1password*|*aws*|*azure*|*cloudflare*|*elastic_agent*|*fim*|*github*|*google*|*osquery*|*system*|*windows*)" - {% else %} - pattern="*" - {% endif %} - for i in $pattern; do - TEMPLATE=${i::-14} - echo "$TEMPLATE" - SUCCESSFUL="no" - while [[ "$TRYCOUNT" -le 4 ]]; do - RESPONSE=$(so-elasticsearch-query _index_template/$TEMPLATE -d@$i -XPUT 2>/dev/null); - if [ "$RESPONSE" == '{"acknowledged":true}' ]; then - SUCCESSFUL="yes" - break - else - ((TRYCOUNT+=1)) - sleep 5 - echo -n "Attempt $TRYCOUNT/5 unsuccessful..." - fi - done - if ! [ "$SUCCESSFUL" == "yes" ];then - echo -n "Could not load template." - echo -n "$RESPONSE" - exit 1 - fi - done + echo "Loading Security Onion index templates..." + shopt -s extglob + {% if GLOBALS.role == 'so-heavynode' %} + pattern="!(*1password*|*aws*|*azure*|*cloudflare*|*elastic_agent*|*fim*|*github*|*google*|*osquery*|*system*|*windows*)" + {% else %} + pattern="*" + {% endif %} + for i in $pattern; do + TEMPLATE=${i::-14} + echo "$TEMPLATE" + retry 5 5 "so-elasticsearch-query _index_template/$TEMPLATE -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load template: $TEMPLATE" + done else {% if GLOBALS.role == 'so-heavynode' %} echo "Common template does not exist. Exiting..." From 7772657b4bc9342a2800560f4919fbff0e4d03d2 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 7 Nov 2023 21:06:35 +0000 Subject: [PATCH 05/13] Remove RETURN_CODE --- .../tools/sbin/so-elasticsearch-pipelines | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines index 4afc9bd4d..fa0e57fc4 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines @@ -6,7 +6,6 @@ . /usr/sbin/so-common -RETURN_CODE=0 ELASTICSEARCH_HOST=$1 ELASTICSEARCH_PORT=9200 @@ -30,11 +29,4 @@ if [ ! -f /opt/so/state/espipelines.txt ]; then cd - >/dev/null - if [[ "$RETURN_CODE" != "1" ]]; then - touch /opt/so/state/espipelines.txt - else - echo "Errors were detected. This script will run again during the next application of the state." - fi -else - exit $RETURN_CODE -fi + touch /opt/so/state/espipelines.txt From 570624da7eb806bc01a1f7b4aec6cebeaa262be0 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 7 Nov 2023 21:09:29 +0000 Subject: [PATCH 06/13] Remove RETURN_CODE --- .../sbin_jinja/so-elasticsearch-templates-load | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load index c0d4f9cba..d870ada90 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load @@ -7,8 +7,6 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %} -RETURN_CODE=0 - if [ ! -f /opt/so/state/estemplates.txt ]; then echo "State file /opt/so/state/estemplates.txt not found. Running so-elasticsearch-templates-load." @@ -103,15 +101,9 @@ if [ ! -f /opt/so/state/estemplates.txt ]; then {% else %} echo "Elastic Fleet not configured. Exiting..." {% endif %} - RETURN_CODE=1 exit 0 fi + cd - >/dev/null - if [[ "$RETURN_CODE" != "1" ]]; then - touch /opt/so/state/estemplates.txt - else - echo "Errors were detected. This script will run again during the next application of the state." - fi -else - exit $RETURN_CODE -fi + + touch /opt/so/state/estemplates.txt From 69ec1987af24a2e36a24e649e667f7ad8dc0f3f6 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 7 Nov 2023 17:28:37 -0500 Subject: [PATCH 07/13] Fix if statement --- salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines index fa0e57fc4..71c40c1ca 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines @@ -28,5 +28,5 @@ if [ ! -f /opt/so/state/espipelines.txt ]; then echo cd - >/dev/null - touch /opt/so/state/espipelines.txt +fi From 749e22e4b9502205bbbc9a43b3577ab186c76c3f Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 7 Nov 2023 17:29:38 -0500 Subject: [PATCH 08/13] Fix if statement --- .../tools/sbin_jinja/so-elasticsearch-templates-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load index d870ada90..985630402 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load @@ -105,5 +105,5 @@ if [ ! -f /opt/so/state/estemplates.txt ]; then fi cd - >/dev/null - touch /opt/so/state/estemplates.txt +fi From de9f9549afa6ef300982690a9beadfd01a48ea4e Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 7 Nov 2023 23:55:03 +0000 Subject: [PATCH 09/13] Extend template loading to 24 attempts and a total of ~2 minutes --- .../tools/sbin_jinja/so-elasticsearch-templates-load | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load index 985630402..a25d49390 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load @@ -50,7 +50,7 @@ if [ ! -f /opt/so/state/estemplates.txt ]; then for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1) echo "$TEMPLATE-mappings" - retry 5 5 "so-elasticsearch-query _component_template/${TEMPLATE}-mappings -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load template: $TEMPLATE-mappings" + retry 24 5 "so-elasticsearch-query _component_template/${TEMPLATE}-mappings -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load template: $TEMPLATE-mappings" done echo @@ -65,7 +65,7 @@ if [ ! -f /opt/so/state/estemplates.txt ]; then for i in $component_pattern; do TEMPLATE=${i::-5} echo "$TEMPLATE" - retry 5 5 "so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load template: $TEMPLATE" + retry 24 5 "so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load template: $TEMPLATE" done echo @@ -76,7 +76,7 @@ if [ ! -f /opt/so/state/estemplates.txt ]; then for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); echo "$TEMPLATE" - retry 5 5 "so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load template: $TEMPLATE" + retry 24 5 "so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load template: $TEMPLATE" done echo @@ -93,7 +93,7 @@ if [ ! -f /opt/so/state/estemplates.txt ]; then for i in $pattern; do TEMPLATE=${i::-14} echo "$TEMPLATE" - retry 5 5 "so-elasticsearch-query _index_template/$TEMPLATE -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load template: $TEMPLATE" + retry 24 5 "so-elasticsearch-query _index_template/$TEMPLATE -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load template: $TEMPLATE" done else {% if GLOBALS.role == 'so-heavynode' %} From b46e86c39b0c1730c5051d86e39156915f7ba31f Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 8 Nov 2023 02:29:09 +0000 Subject: [PATCH 10/13] Extend index template loading to 60 attempts and a total of ~5 minutes --- .../tools/sbin_jinja/so-elasticsearch-templates-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load index a25d49390..c3c5ff69f 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load @@ -93,7 +93,7 @@ if [ ! -f /opt/so/state/estemplates.txt ]; then for i in $pattern; do TEMPLATE=${i::-14} echo "$TEMPLATE" - retry 24 5 "so-elasticsearch-query _index_template/$TEMPLATE -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load template: $TEMPLATE" + retry 60 5 "so-elasticsearch-query _index_template/$TEMPLATE -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load template: $TEMPLATE" done else {% if GLOBALS.role == 'so-heavynode' %} From 653fda124f13303a0111c8c126ea2d7e8b6a42f9 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 8 Nov 2023 13:02:17 +0000 Subject: [PATCH 11/13] Check expected with retry --- .../tools/sbin_jinja/so-elasticsearch-templates-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load index c3c5ff69f..268b6138d 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load @@ -93,7 +93,7 @@ if [ ! -f /opt/so/state/estemplates.txt ]; then for i in $pattern; do TEMPLATE=${i::-14} echo "$TEMPLATE" - retry 60 5 "so-elasticsearch-query _index_template/$TEMPLATE -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load template: $TEMPLATE" + retry 60 5 "so-elasticsearch-query _index_template/$TEMPLATE -d@$i -XPUT" "{\"acknowledged\":true}" || fail "Could not load template: $TEMPLATE" done else {% if GLOBALS.role == 'so-heavynode' %} From d256be3eb3ba768872c83670995e50f4a8191a8e Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 8 Nov 2023 10:32:11 -0500 Subject: [PATCH 12/13] allow template loads to partially succeed only on the initial attempt --- salt/common/tools/sbin/so-common | 6 +- .../so-elasticsearch-templates-load | 64 +++++++++++++++---- 2 files changed, 57 insertions(+), 13 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 8089db28b..e09d2c8ae 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -397,6 +397,10 @@ retry() { echo "" echo "$output" echo "" + if [[ $exitcode -eq 0 ]]; then + echo "Forcing exit code to 1" + exitcode=1 + fi fi elif [ -n "$failedOutput" ]; then if [[ "$output" =~ "$failedOutput" ]]; then @@ -405,7 +409,7 @@ retry() { echo "$output" echo "" if [[ $exitcode -eq 0 ]]; then - echo "The exitcode was 0, but we are setting to 1 since we found $failedOutput in the output." + echo "Forcing exit code to 1" exitcode=1 fi else diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load index 268b6138d..33caff435 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load @@ -7,8 +7,42 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %} -if [ ! -f /opt/so/state/estemplates.txt ]; then - echo "State file /opt/so/state/estemplates.txt not found. Running so-elasticsearch-templates-load." +STATE_FILE_INITIAL=/opt/so/state/estemplates_initial_load_attempt.txt +STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt + +if [[ -f $STATE_FILE_INITIAL ]]; then + # The initial template load has already run. As this is a subsequent load, all dependencies should + # already be satisified. Therefore, immediately exit/abort this script upon any template load failure + # since this is an unrecoverable failure. + should_exit_on_failure=1 +else + # This is the initial template load, and there likely are some components not yet setup in Elasticsearch. + # Therefore load as many templates as possible at this time and if an error occurs proceed to the next + # template. But if at least one template fails to load do not mark the templates as having been loaded. + # This will allow the next load to resume the load of the templates that failed to load initially. + should_exit_on_failure=0 + echo "This is the initial template load" +fi + +load_failures=0 + +load_template() { + uri=$1 + file=$2 + + echo "Loading template file $i" + if ! retry 3 5 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}"; then + if [[ $should_exit_on_failure -eq 1 ]]; then + fail "Could not load template file: $file" + else + load_failures=$((load_failures+1)) + echo "Incremented load failure counter: $load_failures" + fi + fi +} + +if [ ! -f $STATE_FILE_SUCCESS ]; then + echo "State file $STATE_FILE_SUCCESS not found. Running so-elasticsearch-templates-load." . /usr/sbin/so-common @@ -44,13 +78,14 @@ if [ ! -f /opt/so/state/estemplates.txt ]; then fi {% endif %} + touch $STATE_FILE_INITIAL + cd ${ELASTICSEARCH_TEMPLATES}/component/ecs echo "Loading ECS component templates..." for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1) - echo "$TEMPLATE-mappings" - retry 24 5 "so-elasticsearch-query _component_template/${TEMPLATE}-mappings -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load template: $TEMPLATE-mappings" + load_template "_component_template/${TEMPLATE}-mappings" "$i" done echo @@ -64,8 +99,7 @@ if [ ! -f /opt/so/state/estemplates.txt ]; then {% endif %} for i in $component_pattern; do TEMPLATE=${i::-5} - echo "$TEMPLATE" - retry 24 5 "so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load template: $TEMPLATE" + load_template "_component_template/$TEMPLATE" "$i" done echo @@ -75,8 +109,7 @@ if [ ! -f /opt/so/state/estemplates.txt ]; then echo "Loading Security Onion component templates..." for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); - echo "$TEMPLATE" - retry 24 5 "so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load template: $TEMPLATE" + load_template "_component_template/$TEMPLATE" "$i" done echo @@ -92,9 +125,8 @@ if [ ! -f /opt/so/state/estemplates.txt ]; then {% endif %} for i in $pattern; do TEMPLATE=${i::-14} - echo "$TEMPLATE" - retry 60 5 "so-elasticsearch-query _index_template/$TEMPLATE -d@$i -XPUT" "{\"acknowledged\":true}" || fail "Could not load template: $TEMPLATE" - done + load_template "_index_template/$TEMPLATE" "$i" + done else {% if GLOBALS.role == 'so-heavynode' %} echo "Common template does not exist. Exiting..." @@ -105,5 +137,13 @@ if [ ! -f /opt/so/state/estemplates.txt ]; then fi cd - >/dev/null - touch /opt/so/state/estemplates.txt + + if [[ $load_failures -eq 0 ]]; then + echo "All template loaded successfully" + touch $STATE_FILE_SUCCESS + else + echo "Encountered $load_failures templates that were unable to load, likely due to missing dependencies that will be available later; will retry on next highstate" + fi +else + echo "Templates already loaded" fi From 3701c1d847eb6b6e7f0b5cb22d4c7b9441641cf3 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 8 Nov 2023 11:50:56 -0500 Subject: [PATCH 13/13] ignore retry logging --- setup/so-verify | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-verify b/setup/so-verify index e4d90b937..dc84b1e88 100755 --- a/setup/so-verify +++ b/setup/so-verify @@ -37,7 +37,7 @@ log_has_errors() { # Failed to restart snapd.mounts-pre.target: Operation refused, unit snapd.mounts-pre.target # may be requested by dependency only (it is configured to refuse manual start/stop). - # Exit code 100 failure is likely apt-get running in the background, we wait for it to unlock. + # Command failed with exit code is output during retry loops. grep -E "FAILED|Failed|failed|ERROR|Result: False|Error is not recoverable" "$setup_log" | \ grep -vE "The Salt Master has cached the public key for this node" | \ @@ -57,7 +57,7 @@ log_has_errors() { grep -vE "Login Failed Details" | \ grep -vE "response from daemon: unauthorized" | \ grep -vE "Reading first line of patchfile" | \ - grep -vE "Command failed with exit code 100; will retry" | \ + grep -vE "Command failed with exit code" | \ grep -vE "Running scope as unit" &> "$error_log" if [[ $? -eq 0 ]]; then