From 6298397534c1c76fa54ed5b54f7853d797066a26 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Sat, 11 Apr 2026 04:40:47 -0500 Subject: [PATCH] rework elasticsearch template load script -- for core templates --- salt/elasticsearch/enabled.sls | 20 +- .../so-elasticsearch-component-templates-list | 19 +- .../sbin/so-elasticsearch-templates-load | 190 ++++++++++++++++++ .../so-elasticsearch-templates-load | 165 --------------- 4 files changed, 219 insertions(+), 175 deletions(-) create mode 100755 salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load delete mode 100755 salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index 61b9bad01..d95ec2f98 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -10,7 +10,10 @@ {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %} {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %} {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %} -{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, ALL_ADDON_SETTINGS, SO_MANAGED_INDICES %} +{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, SO_MANAGED_INDICES %} +{% if GLOBALS.role != 'so-heavynode' %} +{% from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %} +{% endif %} include: - ca @@ -140,17 +143,17 @@ so_index_template_{{index}}: - defaults: TEMPLATE_CONFIG: {{ settings.index_template }} - template: jinja - - show_changes: False - onchanges_in: - file: so-elasticsearch-templates-reload {% endif %} {% endfor %} +{% if GLOBALS.role != "so-heavynode" %} # Auto-generate optional index templates for integration | input | content packages # These index templates are not used by default (until user adds package to an agent policy). # Pre-configured with standard defaults, and incorporated into SOC configuration for user customization. -{% for index,settings in ALL_ADDON_SETTINGS.items() %} -{% if settings.index_template is defined %} +{% for index,settings in ALL_ADDON_SETTINGS.items() %} +{% if settings.index_template is defined %} addon_index_template_{{index}}: file.managed: - name: /opt/so/conf/elasticsearch/templates/addon-index/{{ index }}-template.json @@ -161,8 +164,9 @@ addon_index_template_{{index}}: - show_changes: False - onchanges_in: - file: addon-elasticsearch-templates-reload -{% endif %} -{% endfor %} +{% endif %} +{% endfor %} +{% endif %} {% if GLOBALS.role in GLOBALS.manager_roles %} so-es-cluster-settings: @@ -195,7 +199,11 @@ addon-elasticsearch-templates-reload: so-elasticsearch-templates: cmd.run: +{%- if GLOBALS.role == "so-heavynode" %} + - name: /usr/sbin/so-elasticsearch-templates-load --heavynode +{%- else %} - name: /usr/sbin/so-elasticsearch-templates-load +{%- endif %} - cwd: /opt/so - template: jinja - require: diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-component-templates-list b/salt/elasticsearch/tools/sbin/so-elasticsearch-component-templates-list index 2fccce9cb..6946e30da 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-component-templates-list +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-component-templates-list @@ -6,8 +6,19 @@ # Elastic License 2.0. . /usr/sbin/so-common -if [ "$1" == "" ]; then - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_component_template | jq '.component_templates[] |.name'| sort + +if [[ -z "$1" ]]; then + if output=$(so-elasticsearch-query "_component_template" --retry 3 --retry-delay 1 --fail); then + jq '[.component_templates[] | .name] | sort' <<< "$output" + else + echo "Failed to retrieve component templates from Elasticsearch." + exit 1 + fi else - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_component_template/$1 | jq -fi + if output=$(so-elasticsearch-query "_component_template/$1" --retry 3 --retry-delay 1 --fail); then + jq <<< "$output" + else + echo "Failed to retrieve component template '$1' from Elasticsearch." + exit 1 + fi +fi \ No newline at end of file diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load new file mode 100755 index 000000000..f44225ac6 --- /dev/null +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load @@ -0,0 +1,190 @@ +#!/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-common + +SO_STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt +ADDON_STATE_FILE_SUCCESS=/opt/so/state/addon-estemplates.txt +ELASTICSEARCH_TEMPLATES_DIR="/opt/so/conf/elasticsearch/templates" +SO_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/index" +ADDON_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/addon-index" +LOAD_FAILURES=0 +LOAD_FAILURES_NAMES=() +IS_HEAVYNODE="false" +FORCE="false" +VERBOSE="false" + +# If soup is running, ignore errors +pgrep soup >/dev/null && should_exit_on_failure=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --heavynode) + IS_HEAVYNODE="true" + ;; + --force) + FORCE="true" + ;; + --verbose) + VERBOSE="true" + ;; + *) + echo "Usage: $0 [options]" + echo "Options:" + echo " --heavynode Only loads index templates specific to heavynodes" + echo " --force Force reload all templates regardless of state file (default: false)" + echo " --verbose Enable verbose output" + exit 1 + ;; + esac + shift +done + +load_template() { + local uri="$1" + local file="$2" + + echo "Loading template file $file" + if ! output=$(retry 3 3 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}"); then + echo "$output" + + return 1 + + elif [[ "$VERBOSE" == "true" ]]; then + echo "$output" + fi + +} + +check_required_component_template_exists() { + local required + local missing + local file=$1 + + required=$(jq '[((.composed_of //[]) - (.ignore_missing_component_templates // []))[]]' "$file") + missing=$(jq -n --argjson required "$required" --argjson component_templates "$component_templates" '(($required) - ($component_templates))') + + if [[ $(jq length <<<"$missing") -gt 0 ]]; then + + return 1 + fi +} + +check_heavynode_compatiable_index_template() { + # The only templates that are relevant to heavynodes are from datasets defined in elasticagent/files/elastic-agent.yml.jinja. + # Heavynodes do not have fleet server packages installed and do not support elastic agents reporting directly to them. + local -A heavynode_index_templates=( + ["so-import"]=1 + ["so-syslog"]=1 + ["so-logs-soc"]=1 + ["so-suricata"]=1 + ["so-suricata.alerts"]=1 + ["so-zeek"]=1 + ["so-strelka"]=1 + ) + + local template_name="$1" + + if [[ ! -v heavynode_index_templates["$template_name"] ]]; then + + return 1 + fi + +} + +load_component_templates() { + local printed_name="$1" + local pattern="${ELASTICSEARCH_TEMPLATES_DIR}/component/$2" + + # current state of nullglob shell option + shopt -q nullglob && nullglob_set=1 || nullglob_set=0 + + shopt -s nullglob + echo -e "\nLoading $printed_name component templates...\n" + for component in "$pattern"/*.json; do + tmpl_name=$(basename "${component%.json}") + if ! load_template "_component_template/${tmpl_name}-mappings" "$component"; then + LOAD_FAILURES=$((LOAD_FAILURES + 1)) + LOAD_FAILURES_NAMES+=("$component") + fi + done + + # restore nullglob shell option if needed + if [[ $nullglob_set -eq 1 ]]; then + shopt -u nullglob + fi +} + +if [[ "$FORCE" == "true" || ! -f "$SO_STATE_FILE_SUCCESS" ]]; then + # Cannot load templates if Elasticsearch is not responding. + # NOTE: Slightly faster exit w/ failure than previous "retry 240 1" if there is a problem with Elasticsearch the + # script should exit sooner rather than hang at the 'so-elasticsearch-templates' salt state. + retry 3 15 "so-elasticsearch-query / --output /dev/null --fail" || + fail "Elasticsearch is not responding. Please review Elasticsearch logs /opt/so/log/elasticsearch/securityonion.log for more details. Additionally, consider running so-elasticsearch-troubleshoot." + + if [[ "$IS_HEAVYNODE" == "false" ]]; then + # TODO: Better way to check if fleet server is installed vs checking for Elastic Defend component template. + fleet_check="logs-endpoint.alerts@package" + if ! so-elasticsearch-query "_component_template/$fleet_check" --output /dev/null --retry 5 --retry-delay 3 --fail; then + echo -e "\nPackage $fleet_check not yet installed. Fleet Server may not be fully configured yet." + # Fleet Server is required because some SO index templates depend on components installed via + # specific integrations eg Elastic Defend. These are components that we do not manually create / manage + # via /opt/so/saltstack/salt/elasticsearch/templates/component/ + + exit 0 + fi + fi + + load_component_templates "ECS" "ecs" + load_component_templates "Elastic Agent" "elastic-agent" + load_component_templates "Security Onion" "securityonion" + + component_templates=$(so-elasticsearch-component-templates-list) + echo -e "Loading Security Onion index templates...\n" + for so_idx_tmpl in "${SO_TEMPLATES_DIR}"/*.json; do + tmpl_name=$(basename "${so_idx_tmpl%-template.json}") + + if [[ "$IS_HEAVYNODE" == "true" ]]; then + # TODO: Better way to load only heavynode specific templates + if ! check_heavynode_compatiable_index_template "$tmpl_name"; then + if [[ "$VERBOSE" == "true" ]]; then + echo "Skipping over $so_idx_tmpl, template is not a heavynode specific index template." + fi + + continue + fi + fi + + if check_required_component_template_exists "$so_idx_tmpl"; then + if ! load_template "_index_template/$tmpl_name" "$so_idx_tmpl"; then + LOAD_FAILURES=$((LOAD_FAILURES + 1)) + LOAD_FAILURES_NAMES+=("$so_idx_tmpl") + fi + else + echo "Skipping over $so_idx_tmpl due to missing required component template(s)." + LOAD_FAILURES=$((LOAD_FAILURES + 1)) + LOAD_FAILURES_NAMES+=("$so_idx_tmpl") + + continue + fi + done + + if [[ $LOAD_FAILURES -eq 0 ]]; then + echo "All templates loaded successfully." + + touch "$SO_STATE_FILE_SUCCESS" + else + echo "Encountered $LOAD_FAILURES failure(s) loading templates:" + for failed_template in "${LOAD_FAILURES_NAMES[@]}"; do + echo " - $failed_template" + done + fi + +else + + echo "Templates already loaded" + +fi \ No newline at end of file diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load deleted file mode 100755 index ad3fe1344..000000000 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load +++ /dev/null @@ -1,165 +0,0 @@ -#!/bin/bash -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. -{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} -{% from 'vars/globals.map.jinja' import GLOBALS %} - -STATE_FILE_INITIAL=/opt/so/state/estemplates_initial_load_attempt.txt -STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt - -if [[ -f $STATE_FILE_INITIAL ]]; then - # The initial template load has already run. As this is a subsequent load, all dependencies should - # already be satisified. Therefore, immediately exit/abort this script upon any template load failure - # since this is an unrecoverable failure. - should_exit_on_failure=1 -else - # This is the initial template load, and there likely are some components not yet setup in Elasticsearch. - # Therefore load as many templates as possible at this time and if an error occurs proceed to the next - # template. But if at least one template fails to load do not mark the templates as having been loaded. - # This will allow the next load to resume the load of the templates that failed to load initially. - should_exit_on_failure=0 - echo "This is the initial template load" -fi - -# If soup is running, ignore errors -pgrep soup > /dev/null && should_exit_on_failure=0 - -load_failures=0 - -load_template() { - uri=$1 - file=$2 - - echo "Loading template file $i" - if ! retry 3 1 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}"; then - if [[ $should_exit_on_failure -eq 1 ]]; then - fail "Could not load template file: $file" - else - load_failures=$((load_failures+1)) - echo "Incremented load failure counter: $load_failures" - fi - fi -} - -if [ ! -f $STATE_FILE_SUCCESS ]; then - echo "State file $STATE_FILE_SUCCESS not found. Running so-elasticsearch-templates-load." - - . /usr/sbin/so-common - - {% if GLOBALS.role != 'so-heavynode' %} - if [ -f /usr/sbin/so-elastic-fleet-common ]; then - . /usr/sbin/so-elastic-fleet-common - fi - {% endif %} - - default_conf_dir=/opt/so/conf - - # Define a default directory to load pipelines from - ELASTICSEARCH_TEMPLATES="$default_conf_dir/elasticsearch/templates/" - - {% if GLOBALS.role == 'so-heavynode' %} - file="/opt/so/conf/elasticsearch/templates/index/so-common-template.json" - {% else %} - file="/usr/sbin/so-elastic-fleet-common" - {% endif %} - - if [ -f "$file" ]; then - # Wait for ElasticSearch to initialize - echo -n "Waiting for ElasticSearch..." - retry 240 1 "so-elasticsearch-query / -k --output /dev/null --silent --head --fail" || fail "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" - {% if GLOBALS.role != 'so-heavynode' %} - TEMPLATE="logs-endpoint.alerts@package" - INSTALLED=$(so-elasticsearch-query _component_template/$TEMPLATE | jq -r .component_templates[0].name) - if [ "$INSTALLED" != "$TEMPLATE" ]; then - echo - echo "Packages not yet installed." - echo - exit 0 - fi - {% endif %} - - touch $STATE_FILE_INITIAL - - cd ${ELASTICSEARCH_TEMPLATES}/component/ecs - - echo "Loading ECS component templates..." - for i in *; do - TEMPLATE=$(echo $i | cut -d '.' -f1) - load_template "_component_template/${TEMPLATE}-mappings" "$i" - done - echo - - cd ${ELASTICSEARCH_TEMPLATES}/component/elastic-agent - - echo "Loading Elastic Agent component templates..." - {% if GLOBALS.role == 'so-heavynode' %} - component_pattern="so-*" - {% else %} - component_pattern="*" - {% endif %} - for i in $component_pattern; do - TEMPLATE=${i::-5} - load_template "_component_template/$TEMPLATE" "$i" - done - echo - - # Load SO-specific component templates - cd ${ELASTICSEARCH_TEMPLATES}/component/so - - echo "Loading Security Onion component templates..." - for i in *; do - TEMPLATE=$(echo $i | cut -d '.' -f1); - load_template "_component_template/$TEMPLATE" "$i" - done - echo - - # Load SO index templates - cd ${ELASTICSEARCH_TEMPLATES}/index - - echo "Loading Security Onion index templates..." - shopt -s extglob - {% if GLOBALS.role == 'so-heavynode' %} - pattern="!(*1password*|*aws*|*azure*|*cloudflare*|*elastic_agent*|*fim*|*github*|*google*|*osquery*|*system*|*windows*|*endpoint*|*elasticsearch*|*generic*|*fleet_server*|*soc*)" - {% else %} - pattern="*" - {% endif %} - # Index templates will be skipped if the following conditions are met: - # 1. The template is part of the "so-logs-" template group - # 2. The template name does not correlate to at least one existing component template - # In this situation, the script will treat the skipped template as a temporary failure - # and allow the templates to be loaded again on the next run or highstate, whichever - # comes first. - COMPONENT_LIST=$(so-elasticsearch-component-templates-list) - for i in $pattern; do - TEMPLATE=${i::-14} - COMPONENT_PATTERN=${TEMPLATE:3} - MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -vE "detections|osquery") - if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ \.generic|logs-winlog\.winlog ]]; then - load_failures=$((load_failures+1)) - echo "Component template does not exist for $COMPONENT_PATTERN. The index template will not be loaded. Load failures: $load_failures" - else - load_template "_index_template/$TEMPLATE" "$i" - fi - done - else - {% if GLOBALS.role == 'so-heavynode' %} - echo "Common template does not exist. Exiting..." - {% else %} - echo "Elastic Fleet not configured. Exiting..." - {% endif %} - exit 0 - fi - - cd - >/dev/null - - if [[ $load_failures -eq 0 ]]; then - echo "All templates loaded successfully" - touch $STATE_FILE_SUCCESS - else - echo "Encountered $load_failures templates that were unable to load, likely due to missing dependencies that will be available later; will retry on next highstate" - fi -else - echo "Templates already loaded" -fi