CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2026-04-13 00:02:03 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #15761 from Security-Onion-Solutions/reyesj2-es932

Browse Source 
rework elasticsearch template load script -- for core templates
This commit is contained in:
Jorge Reyes
2026-04-11 04:42:45 -05:00
committed by GitHub
parent 9272afa9e5 6298397534
commit 08f34d408f
4 changed files with 219 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
salt/elasticsearch/enabled.sls
View File

@@ -10,7 +10,10 @@
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %}
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, ALL_ADDON_SETTINGS, SO_MANAGED_INDICES %}
{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, SO_MANAGED_INDICES %}
{% if GLOBALS.role != 'so-heavynode' %}
{% from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %}
{% endif %}
include:
- ca
@@ -140,17 +143,17 @@ so_index_template_{{index}}:
- defaults:
TEMPLATE_CONFIG: {{ settings.index_template }}
- template: jinja
- show_changes: False
- onchanges_in:
- file: so-elasticsearch-templates-reload
{% endif %}
{% endfor %}
{% if GLOBALS.role != "so-heavynode" %}
# Auto-generate optional index templates for integration | input | content packages
# These index templates are not used by default (until user adds package to an agent policy).
# Pre-configured with standard defaults, and incorporated into SOC configuration for user customization.
{% for index,settings in ALL_ADDON_SETTINGS.items() %}
{% if settings.index_template is defined %}
{% for index,settings in ALL_ADDON_SETTINGS.items() %}
{% if settings.index_template is defined %}
addon_index_template_{{index}}:
file.managed:
- name: /opt/so/conf/elasticsearch/templates/addon-index/{{ index }}-template.json
@@ -161,8 +164,9 @@ addon_index_template_{{index}}:
- show_changes: False
- onchanges_in:
- file: addon-elasticsearch-templates-reload
{% endif %}
{% endfor %}
{% endif %}
{% endfor %}
{% endif %}
{% if GLOBALS.role in GLOBALS.manager_roles %}
so-es-cluster-settings:
@@ -195,7 +199,11 @@ addon-elasticsearch-templates-reload:
so-elasticsearch-templates:
cmd.run:
{%- if GLOBALS.role == "so-heavynode" %}
- name: /usr/sbin/so-elasticsearch-templates-load --heavynode
{%- else %}
- name: /usr/sbin/so-elasticsearch-templates-load
{%- endif %}
- cwd: /opt/so
- template: jinja
- require:

19
salt/elasticsearch/tools/sbin/so-elasticsearch-component-templates-list
View File

@@ -6,8 +6,19 @@
# Elastic License 2.0.
. /usr/sbin/so-common
if [ "$1" == "" ]; then
curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_component_template | jq '.component_templates[] |.name'| sort
if [[ -z "$1" ]]; then
if output=$(so-elasticsearch-query "_component_template" --retry 3 --retry-delay 1 --fail); then
jq '[.component_templates[] | .name] | sort' <<< "$output"
else
echo "Failed to retrieve component templates from Elasticsearch."
exit 1
fi
else
curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_component_template/$1 | jq
fi
if output=$(so-elasticsearch-query "_component_template/$1" --retry 3 --retry-delay 1 --fail); then
jq <<< "$output"
else
echo "Failed to retrieve component template '$1' from Elasticsearch."
exit 1
fi
fi

190
salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load Executable file
View File

@@ -0,0 +1,190 @@
#!/bin/bash
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
. /usr/sbin/so-common
SO_STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt
ADDON_STATE_FILE_SUCCESS=/opt/so/state/addon-estemplates.txt
ELASTICSEARCH_TEMPLATES_DIR="/opt/so/conf/elasticsearch/templates"
SO_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/index"
ADDON_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/addon-index"
LOAD_FAILURES=0
LOAD_FAILURES_NAMES=()
IS_HEAVYNODE="false"
FORCE="false"
VERBOSE="false"
# If soup is running, ignore errors
pgrep soup >/dev/null && should_exit_on_failure=0
while [[ $# -gt 0 ]]; do
case "$1" in
--heavynode)
IS_HEAVYNODE="true"
;;
--force)
FORCE="true"
;;
--verbose)
VERBOSE="true"
;;
*)
echo "Usage: $0 [options]"
echo "Options:"
echo " --heavynode Only loads index templates specific to heavynodes"
echo " --force Force reload all templates regardless of state file (default: false)"
echo " --verbose Enable verbose output"
exit 1
;;
esac
shift
done
load_template() {
local uri="$1"
local file="$2"
echo "Loading template file $file"
if ! output=$(retry 3 3 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}"); then
echo "$output"
return 1
elif [[ "$VERBOSE" == "true" ]]; then
echo "$output"
fi
}
check_required_component_template_exists() {
local required
local missing
local file=$1
required=$(jq '[((.composed_of //[]) - (.ignore_missing_component_templates // []))[]]' "$file")
missing=$(jq -n --argjson required "$required" --argjson component_templates "$component_templates" '(($required) - ($component_templates))')
if [[ $(jq length <<<"$missing") -gt 0 ]]; then
return 1
fi
}
check_heavynode_compatiable_index_template() {
# The only templates that are relevant to heavynodes are from datasets defined in elasticagent/files/elastic-agent.yml.jinja.
# Heavynodes do not have fleet server packages installed and do not support elastic agents reporting directly to them.
local -A heavynode_index_templates=(
["so-import"]=1
["so-syslog"]=1
["so-logs-soc"]=1
["so-suricata"]=1
["so-suricata.alerts"]=1
["so-zeek"]=1
["so-strelka"]=1
)
local template_name="$1"
if [[ ! -v heavynode_index_templates["$template_name"] ]]; then
return 1
fi
}
load_component_templates() {
local printed_name="$1"
local pattern="${ELASTICSEARCH_TEMPLATES_DIR}/component/$2"
# current state of nullglob shell option
shopt -q nullglob && nullglob_set=1 || nullglob_set=0
shopt -s nullglob
echo -e "\nLoading $printed_name component templates...\n"
for component in "$pattern"/*.json; do
tmpl_name=$(basename "${component%.json}")
if ! load_template "_component_template/${tmpl_name}-mappings" "$component"; then
LOAD_FAILURES=$((LOAD_FAILURES + 1))
LOAD_FAILURES_NAMES+=("$component")
fi
done
# restore nullglob shell option if needed
if [[ $nullglob_set -eq 1 ]]; then
shopt -u nullglob
fi
}
if [[ "$FORCE" == "true" || ! -f "$SO_STATE_FILE_SUCCESS" ]]; then
# Cannot load templates if Elasticsearch is not responding.
# NOTE: Slightly faster exit w/ failure than previous "retry 240 1" if there is a problem with Elasticsearch the
# script should exit sooner rather than hang at the 'so-elasticsearch-templates' salt state.
retry 3 15 "so-elasticsearch-query / --output /dev/null --fail" ||
fail "Elasticsearch is not responding. Please review Elasticsearch logs /opt/so/log/elasticsearch/securityonion.log for more details. Additionally, consider running so-elasticsearch-troubleshoot."
if [[ "$IS_HEAVYNODE" == "false" ]]; then
# TODO: Better way to check if fleet server is installed vs checking for Elastic Defend component template.
fleet_check="logs-endpoint.alerts@package"
if ! so-elasticsearch-query "_component_template/$fleet_check" --output /dev/null --retry 5 --retry-delay 3 --fail; then
echo -e "\nPackage $fleet_check not yet installed. Fleet Server may not be fully configured yet."
# Fleet Server is required because some SO index templates depend on components installed via
# specific integrations eg Elastic Defend. These are components that we do not manually create / manage
# via /opt/so/saltstack/salt/elasticsearch/templates/component/
exit 0
fi
fi
load_component_templates "ECS" "ecs"
load_component_templates "Elastic Agent" "elastic-agent"
load_component_templates "Security Onion" "securityonion"
component_templates=$(so-elasticsearch-component-templates-list)
echo -e "Loading Security Onion index templates...\n"
for so_idx_tmpl in "${SO_TEMPLATES_DIR}"/*.json; do
tmpl_name=$(basename "${so_idx_tmpl%-template.json}")
if [[ "$IS_HEAVYNODE" == "true" ]]; then
# TODO: Better way to load only heavynode specific templates
if ! check_heavynode_compatiable_index_template "$tmpl_name"; then
if [[ "$VERBOSE" == "true" ]]; then
echo "Skipping over $so_idx_tmpl, template is not a heavynode specific index template."
fi
continue
fi
fi
if check_required_component_template_exists "$so_idx_tmpl"; then
if ! load_template "_index_template/$tmpl_name" "$so_idx_tmpl"; then
LOAD_FAILURES=$((LOAD_FAILURES + 1))
LOAD_FAILURES_NAMES+=("$so_idx_tmpl")
fi
else
echo "Skipping over $so_idx_tmpl due to missing required component template(s)."
LOAD_FAILURES=$((LOAD_FAILURES + 1))
LOAD_FAILURES_NAMES+=("$so_idx_tmpl")
continue
fi
done
if [[ $LOAD_FAILURES -eq 0 ]]; then
echo "All templates loaded successfully."
touch "$SO_STATE_FILE_SUCCESS"
else
echo "Encountered $LOAD_FAILURES failure(s) loading templates:"
for failed_template in "${LOAD_FAILURES_NAMES[@]}"; do
echo " - $failed_template"
done
fi
else
echo "Templates already loaded"
fi

165
salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load
View File

@@ -1,165 +0,0 @@
#!/bin/bash
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
STATE_FILE_INITIAL=/opt/so/state/estemplates_initial_load_attempt.txt
STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt
if [[ -f $STATE_FILE_INITIAL ]]; then
# The initial template load has already run. As this is a subsequent load, all dependencies should
# already be satisified. Therefore, immediately exit/abort this script upon any template load failure
# since this is an unrecoverable failure.
should_exit_on_failure=1
else
# This is the initial template load, and there likely are some components not yet setup in Elasticsearch.
# Therefore load as many templates as possible at this time and if an error occurs proceed to the next
# template. But if at least one template fails to load do not mark the templates as having been loaded.
# This will allow the next load to resume the load of the templates that failed to load initially.
should_exit_on_failure=0
echo "This is the initial template load"
fi
# If soup is running, ignore errors
pgrep soup > /dev/null && should_exit_on_failure=0
load_failures=0
load_template() {
uri=$1
file=$2
echo "Loading template file $i"
if ! retry 3 1 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}"; then
if [[ $should_exit_on_failure -eq 1 ]]; then
fail "Could not load template file: $file"
else
load_failures=$((load_failures+1))
echo "Incremented load failure counter: $load_failures"
fi
fi
}
if [ ! -f $STATE_FILE_SUCCESS ]; then
echo "State file $STATE_FILE_SUCCESS not found. Running so-elasticsearch-templates-load."
. /usr/sbin/so-common
{% if GLOBALS.role != 'so-heavynode' %}
if [ -f /usr/sbin/so-elastic-fleet-common ]; then
. /usr/sbin/so-elastic-fleet-common
fi
{% endif %}
default_conf_dir=/opt/so/conf
# Define a default directory to load pipelines from
ELASTICSEARCH_TEMPLATES="$default_conf_dir/elasticsearch/templates/"
{% if GLOBALS.role == 'so-heavynode' %}
file="/opt/so/conf/elasticsearch/templates/index/so-common-template.json"
{% else %}
file="/usr/sbin/so-elastic-fleet-common"
{% endif %}
if [ -f "$file" ]; then
# Wait for ElasticSearch to initialize
echo -n "Waiting for ElasticSearch..."
retry 240 1 "so-elasticsearch-query / -k --output /dev/null --silent --head --fail" || fail "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'"
{% if GLOBALS.role != 'so-heavynode' %}
TEMPLATE="logs-endpoint.alerts@package"
INSTALLED=$(so-elasticsearch-query _component_template/$TEMPLATE | jq -r .component_templates[0].name)
if [ "$INSTALLED" != "$TEMPLATE" ]; then
echo
echo "Packages not yet installed."
echo
exit 0
fi
{% endif %}
touch $STATE_FILE_INITIAL
cd ${ELASTICSEARCH_TEMPLATES}/component/ecs
echo "Loading ECS component templates..."
for i in *; do
TEMPLATE=$(echo $i | cut -d '.' -f1)
load_template "_component_template/${TEMPLATE}-mappings" "$i"
done
echo
cd ${ELASTICSEARCH_TEMPLATES}/component/elastic-agent
echo "Loading Elastic Agent component templates..."
{% if GLOBALS.role == 'so-heavynode' %}
component_pattern="so-*"
{% else %}
component_pattern="*"
{% endif %}
for i in $component_pattern; do
TEMPLATE=${i::-5}
load_template "_component_template/$TEMPLATE" "$i"
done
echo
# Load SO-specific component templates
cd ${ELASTICSEARCH_TEMPLATES}/component/so
echo "Loading Security Onion component templates..."
for i in *; do
TEMPLATE=$(echo $i | cut -d '.' -f1);
load_template "_component_template/$TEMPLATE" "$i"
done
echo
# Load SO index templates
cd ${ELASTICSEARCH_TEMPLATES}/index
echo "Loading Security Onion index templates..."
shopt -s extglob
{% if GLOBALS.role == 'so-heavynode' %}
pattern="!(*1password*|*aws*|*azure*|*cloudflare*|*elastic_agent*|*fim*|*github*|*google*|*osquery*|*system*|*windows*|*endpoint*|*elasticsearch*|*generic*|*fleet_server*|*soc*)"
{% else %}
pattern="*"
{% endif %}
# Index templates will be skipped if the following conditions are met:
# 1. The template is part of the "so-logs-" template group
# 2. The template name does not correlate to at least one existing component template
# In this situation, the script will treat the skipped template as a temporary failure
# and allow the templates to be loaded again on the next run or highstate, whichever
# comes first.
COMPONENT_LIST=$(so-elasticsearch-component-templates-list)
for i in $pattern; do
TEMPLATE=${i::-14}
COMPONENT_PATTERN=${TEMPLATE:3}
MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -vE "detections|osquery")
if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ \.generic|logs-winlog\.winlog ]]; then
load_failures=$((load_failures+1))
echo "Component template does not exist for $COMPONENT_PATTERN. The index template will not be loaded. Load failures: $load_failures"
else
load_template "_index_template/$TEMPLATE" "$i"
fi
done
else
{% if GLOBALS.role == 'so-heavynode' %}
echo "Common template does not exist. Exiting..."
{% else %}
echo "Elastic Fleet not configured. Exiting..."
{% endif %}
exit 0
fi
cd - >/dev/null
if [[ $load_failures -eq 0 ]]; then
echo "All templates loaded successfully"
touch $STATE_FILE_SUCCESS
else
echo "Encountered $load_failures templates that were unable to load, likely due to missing dependencies that will be available later; will retry on next highstate"
fi
else
echo "Templates already loaded"
fi