From d551faeb163d536c5722f5549fa92ea36bc0a798 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 6 Jul 2023 17:19:28 +0000 Subject: [PATCH 1/3] Heavy node template considerations --- .../so-elasticsearch-templates-load | 62 ++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load index 991f54819..9232c9c05 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load @@ -4,17 +4,20 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. {%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} +{% from 'vars/globals.map.jinja' import GLOBALS %} {%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %} . /usr/sbin/so-common +{% if GLOBALS.role != 'so-heavynode' %} . /usr/sbin/so-elastic-fleet-common - +{% endif %} default_conf_dir=/opt/so/conf # Define a default directory to load pipelines from ELASTICSEARCH_TEMPLATES="$default_conf_dir/elasticsearch/templates/" +{% if GLOBALS.role != 'so-heavynode' %} if [ -f /usr/sbin/so-elastic-fleet-common ]; then # Wait for ElasticSearch to initialize echo -n "Waiting for ElasticSearch..." @@ -77,4 +80,61 @@ else echo "Elastic Fleet not configured. Exiting..." exit 0 fi +{% else %} + # Wait for ElasticSearch to initialize + echo -n "Waiting for ElasticSearch..." + COUNT=0 + ELASTICSEARCH_CONNECTED="no" + while [[ "$COUNT" -le 240 ]]; do + so-elasticsearch-query / -k --output /dev/null --silent --head --fail + if [ $? -eq 0 ]; then + ELASTICSEARCH_CONNECTED="yes" + echo "connected!" + break + else + ((COUNT+=1)) + sleep 1 + echo -n "." + fi + done + if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then + echo + echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" + echo + exit 1 + fi + + set -e + + cd ${ELASTICSEARCH_TEMPLATES}/component/ecs + + echo "Loading ECS component templates..." + for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); echo "$TEMPLATE-mappings"; so-elasticsearch-query _component_template/$TEMPLATE-mappings -d@$i -XPUT 2>/dev/null; echo; done + + cd ${ELASTICSEARCH_TEMPLATES}/component/elastic-agent + + echo "Loading Elastic Agent component templates..." + for i in so-*; do TEMPLATE=${i::-5}; echo "$TEMPLATE"; so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done + + # Load SO-specific component templates + cd ${ELASTICSEARCH_TEMPLATES}/component/so + + echo "Loading Security Onion component templates..." + for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); echo "$TEMPLATE"; so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done + echo + + # Load SO index templates + cd ${ELASTICSEARCH_TEMPLATES}/index + + echo "Loading Security Onion index templates..." + shopt -s extglob + exclude_pattern="!(*1password*|*aws*|*azure*|*cloudflare*|*elastic_agent*|*fim*|*github*|*google*|*osquery*|*system*|*windows*)" + for i in $exclude_pattern; do + TEMPLATE=${i::-14}; + echo "$TEMPLATE"; + so-elasticsearch-query _index_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; + echo; + done + echo +{% endif %} cd - >/dev/null From 910125f13a219cb509d2abf45aed4955f50e1c40 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 6 Jul 2023 17:49:06 +0000 Subject: [PATCH 2/3] Restructure logic --- .../so-elasticsearch-templates-load | 99 +++++++------------ 1 file changed, 34 insertions(+), 65 deletions(-) diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load index 9232c9c05..426cebee4 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load @@ -17,8 +17,13 @@ default_conf_dir=/opt/so/conf # Define a default directory to load pipelines from ELASTICSEARCH_TEMPLATES="$default_conf_dir/elasticsearch/templates/" -{% if GLOBALS.role != 'so-heavynode' %} -if [ -f /usr/sbin/so-elastic-fleet-common ]; then +{% if GLOBALS.role == 'so-heavynode' %} +file="/opt/so/conf/elasticsearch/templates/index/so-common-template.json" +{% else %} +file="/usr/sbin/so-elastic-fleet-common" +{% endif %} + +if [ -f "$file" ]; then # Wait for ElasticSearch to initialize echo -n "Waiting for ElasticSearch..." COUNT=0 @@ -42,7 +47,8 @@ if [ -f /usr/sbin/so-elastic-fleet-common ]; then exit 1 fi - SESSIONCOOKIE=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') + {% if GLOBALS.role != 'so-heavynode' %} + SESSIONCOOKIE=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') INSTALLED=$(elastic_fleet_package_is_installed {{ SUPPORTED_PACKAGES[0] }} ) if [ "$INSTALLED" != "installed" ]; then echo @@ -50,7 +56,7 @@ if [ -f /usr/sbin/so-elastic-fleet-common ]; then echo exit 0 fi - + {% endif %} set -e cd ${ELASTICSEARCH_TEMPLATES}/component/ecs @@ -61,7 +67,12 @@ if [ -f /usr/sbin/so-elastic-fleet-common ]; then cd ${ELASTICSEARCH_TEMPLATES}/component/elastic-agent echo "Loading Elastic Agent component templates..." - for i in *; do TEMPLATE=${i::-5}; echo "$TEMPLATE"; so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done + {% if GLOBALS.role != 'so-heavynode' %} + component_pattern="so-*" + {% else %} + component_pattern="*" + {% endif %} + for i in $component_pattern; do TEMPLATE=${i::-5}; echo "$TEMPLATE"; so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done # Load SO-specific component templates cd ${ELASTICSEARCH_TEMPLATES}/component/so @@ -74,67 +85,25 @@ if [ -f /usr/sbin/so-elastic-fleet-common ]; then cd ${ELASTICSEARCH_TEMPLATES}/index echo "Loading Security Onion index templates..." - for i in *; do TEMPLATE=${i::-14}; echo "$TEMPLATE"; so-elasticsearch-query _index_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done + shopt -s extglob + {% if GLOBALS.role == 'so-heavynode' %} + pattern="!(*1password*|*aws*|*azure*|*cloudflare*|*elastic_agent*|*fim*|*github*|*google*|*osquery*|*system*|*windows*)" + {% else %} + pattern="*" + {% endif %} + for i in $pattern; do + TEMPLATE=${i::-14}; + echo "$TEMPLATE"; + so-elasticsearch-query _index_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; + echo; + done echo else - echo "Elastic Fleet not configured. Exiting..." - exit 0 + {% if GLOBALS.role == 'so-heavynode' %} + echo "Common template does not exist. Exiting..." + {% else %} + echo "Elastic Fleet not configured. Exiting..." + {% endif %} + exit 0 fi -{% else %} - # Wait for ElasticSearch to initialize - echo -n "Waiting for ElasticSearch..." - COUNT=0 - ELASTICSEARCH_CONNECTED="no" - while [[ "$COUNT" -le 240 ]]; do - so-elasticsearch-query / -k --output /dev/null --silent --head --fail - if [ $? -eq 0 ]; then - ELASTICSEARCH_CONNECTED="yes" - echo "connected!" - break - else - ((COUNT+=1)) - sleep 1 - echo -n "." - fi - done - if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then - echo - echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" - echo - exit 1 - fi - - set -e - - cd ${ELASTICSEARCH_TEMPLATES}/component/ecs - - echo "Loading ECS component templates..." - for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); echo "$TEMPLATE-mappings"; so-elasticsearch-query _component_template/$TEMPLATE-mappings -d@$i -XPUT 2>/dev/null; echo; done - - cd ${ELASTICSEARCH_TEMPLATES}/component/elastic-agent - - echo "Loading Elastic Agent component templates..." - for i in so-*; do TEMPLATE=${i::-5}; echo "$TEMPLATE"; so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done - - # Load SO-specific component templates - cd ${ELASTICSEARCH_TEMPLATES}/component/so - - echo "Loading Security Onion component templates..." - for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); echo "$TEMPLATE"; so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done - echo - - # Load SO index templates - cd ${ELASTICSEARCH_TEMPLATES}/index - - echo "Loading Security Onion index templates..." - shopt -s extglob - exclude_pattern="!(*1password*|*aws*|*azure*|*cloudflare*|*elastic_agent*|*fim*|*github*|*google*|*osquery*|*system*|*windows*)" - for i in $exclude_pattern; do - TEMPLATE=${i::-14}; - echo "$TEMPLATE"; - so-elasticsearch-query _index_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; - echo; - done - echo -{% endif %} cd - >/dev/null From 0b5ee49873d04729c2f60a6304d8341cf136d741 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 6 Jul 2023 20:46:35 +0000 Subject: [PATCH 3/3] Fix inverted logic for component template --- .../tools/sbin_jinja/so-elasticsearch-templates-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load index 426cebee4..aac6279fc 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load @@ -67,7 +67,7 @@ if [ -f "$file" ]; then cd ${ELASTICSEARCH_TEMPLATES}/component/elastic-agent echo "Loading Elastic Agent component templates..." - {% if GLOBALS.role != 'so-heavynode' %} + {% if GLOBALS.role == 'so-heavynode' %} component_pattern="so-*" {% else %} component_pattern="*"