CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2026-04-24 21:47:48 +02:00
Code Issues Packages Projects Releases Wiki Activity

rework elasticsearch index template generation

Browse Source
This commit is contained in:
reyesj2
2026-04-09 16:44:51 -05:00
parent f0b67a415a
commit 89e49d0bf3
8 changed files with 392 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
salt/elasticfleet/content-defaults.map.jinja
+123
View File
@@ -0,0 +1,123 @@
{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
this file except in compliance with the Elastic License 2.0. #}
{% import_json '/opt/so/state/esfleet_content_package_components.json' as ADDON_CONTENT_PACKAGE_COMPONENTS %}
{% import_json '/opt/so/state/esfleet_component_templates.json' as INSTALLED_COMPONENT_TEMPLATES %}
{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
{% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
{% set ADDON_CONTENT_INTEGRATION_DEFAULTS = {} %}
{% set DEBUG_STUFF = {} %}
{% for pkg in ADDON_CONTENT_PACKAGE_COMPONENTS %}
{% if pkg.name in CORE_ESFLEET_PACKAGES %}
{# skip core content packages #}
{% elif pkg.name not in CORE_ESFLEET_PACKAGES %}
{# generate defaults for each content package #}
{% if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0%}
{% for pattern in pkg.dataStreams %}
{# in ES 9.3.2 'input' type integrations no longer create default component templates and instead they wait for user input during 'integration' setup (fleet ui config)
title: generic is an artifact of that and is not in use #}
{% if pattern.title == "generic" %}
{% continue %}
{% endif %}
{% if "metrics-" in pattern.name %}
{% set integration_type = "metrics-" %}
{% elif "logs-" in pattern.name %}
{% set integration_type = "logs-" %}
{% else %}
{% set integration_type = "" %}
{% endif %}
{# on content integrations the component name is user defined at the time it is added to an agent policy #}
{% set component_name = pattern.title %}
{% set index_pattern = pattern.name %}
{# component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
{% set component_name_x = component_name.replace(".","_x_") %}
{# pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
{% set integration_key = "so-" ~ integration_type ~ pkg.name + '_x_' ~ component_name_x %}
{# Default integration settings #}
{% set integration_defaults = {
"index_sorting": false,
"index_template": {
"composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
"data_stream": {
"allow_custom_routing": false,
"hidden": false
},
"ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
"index_patterns": [index_pattern],
"priority": 501,
"template": {
"settings": {
"index": {
"lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
"number_of_replicas": 0
}
}
}
},
"policy": {
"phases": {
"cold": {
"actions": {
"allocate":{
"number_of_replicas": ""
},
"set_priority": {"priority": 0}
},
"min_age": "60d"
},
"delete": {
"actions": {
"delete": {}
},
"min_age": "365d"
},
"hot": {
"actions": {
"rollover": {
"max_age": "30d",
"max_primary_shard_size": "50gb"
},
"forcemerge":{
"max_num_segments": ""
},
"shrink":{
"max_primary_shard_size": "",
"method": "COUNT",
"number_of_shards": ""
},
"set_priority": {"priority": 100}
},
"min_age": "0ms"
},
"warm": {
"actions": {
"allocate": {
"number_of_replicas": ""
},
"forcemerge": {
"max_num_segments": ""
},
"shrink":{
"max_primary_shard_size": "",
"method": "COUNT",
"number_of_shards": ""
},
"set_priority": {"priority": 50}
},
"min_age": "30d"
}
}
}
} %}
{% do ADDON_CONTENT_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %}
{% endfor %}
{% else %}
{% endif %}
{% endif %}
{% endfor %}
salt/elasticfleet/input-defaults.map.jinja
+123
View File
@@ -0,0 +1,123 @@
{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
this file except in compliance with the Elastic License 2.0. #}
{% import_json '/opt/so/state/esfleet_input_package_components.json' as ADDON_INPUT_PACKAGE_COMPONENTS %}
{% import_json '/opt/so/state/esfleet_component_templates.json' as INSTALLED_COMPONENT_TEMPLATES %}
{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
{% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
{% set ADDON_INPUT_INTEGRATION_DEFAULTS = {} %}
{% set DEBUG_STUFF = {} %}
{% for pkg in ADDON_INPUT_PACKAGE_COMPONENTS %}
{% if pkg.name in CORE_ESFLEET_PACKAGES %}
{# skip core input packages #}
{% elif pkg.name not in CORE_ESFLEET_PACKAGES %}
{# generate defaults for each input package #}
{% if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0 %}
{% for pattern in pkg.dataStreams %}
{# in ES 9.3.2 'input' type integrations no longer create default component templates and instead they wait for user input during 'integration' setup (fleet ui config)
title: generic is an artifact of that and is not in use #}
{% if pattern.title == "generic" %}
{% continue %}
{% endif %}
{% if "metrics-" in pattern.name %}
{% set integration_type = "metrics-" %}
{% elif "logs-" in pattern.name %}
{% set integration_type = "logs-" %}
{% else %}
{% set integration_type = "" %}
{% endif %}
{# on input integrations the component name is user defined at the time it is added to an agent policy #}
{% set component_name = pattern.title %}
{% set index_pattern = pattern.name %}
{# component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
{% set component_name_x = component_name.replace(".","_x_") %}
{# pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
{% set integration_key = "so-" ~ integration_type ~ pkg.name + '_x_' ~ component_name_x %}
{# Default integration settings #}
{% set integration_defaults = {
"index_sorting": false,
"index_template": {
"composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
"data_stream": {
"allow_custom_routing": false,
"hidden": false
},
"ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
"index_patterns": [index_pattern],
"priority": 501,
"template": {
"settings": {
"index": {
"lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
"number_of_replicas": 0
}
}
}
},
"policy": {
"phases": {
"cold": {
"actions": {
"allocate":{
"number_of_replicas": ""
},
"set_priority": {"priority": 0}
},
"min_age": "60d"
},
"delete": {
"actions": {
"delete": {}
},
"min_age": "365d"
},
"hot": {
"actions": {
"rollover": {
"max_age": "30d",
"max_primary_shard_size": "50gb"
},
"forcemerge":{
"max_num_segments": ""
},
"shrink":{
"max_primary_shard_size": "",
"method": "COUNT",
"number_of_shards": ""
},
"set_priority": {"priority": 100}
},
"min_age": "0ms"
},
"warm": {
"actions": {
"allocate": {
"number_of_replicas": ""
},
"forcemerge": {
"max_num_segments": ""
},
"shrink":{
"max_primary_shard_size": "",
"method": "COUNT",
"number_of_shards": ""
},
"set_priority": {"priority": 50}
},
"min_age": "30d"
}
}
}
} %}
{% do ADDON_INPUT_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %}
{% do DEBUG_STUFF.update({integration_key: "Generating defaults for "+ pkg.name })%}
{% endfor %}
{% endif %}
{% endif %}
{% endfor %}
salt/elasticfleet/integration-defaults.map.jinja
+5 -22
View File
@@ -59,8 +59,8 @@
{# skip core integrations #}
{% elif pkg.name not in CORE_ESFLEET_PACKAGES %}
{# generate defaults for each integration #}
{% if pkg.es_index_patterns is defined and pkg.es_index_patterns is not none %}
{% for pattern in pkg.es_index_patterns %}
{% if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0 %}
{% for pattern in pkg.dataStreams %}
{% if "metrics-" in pattern.name %}
{% set integration_type = "metrics-" %}
{% elif "logs-" in pattern.name %}
@@ -75,44 +75,27 @@
{% if component_name in WEIRD_INTEGRATIONS %}
{% set component_name = WEIRD_INTEGRATIONS[component_name] %}
{% endif %}
{# create duplicate of component_name, so we can split generics from @custom component templates in the index template below and overwrite the default @package when needed
eg. having to replace unifiedlogs.generic@package with filestream.generic@package, but keep the ability to customize unifiedlogs.generic@custom and its ILM policy #}
{% set custom_component_name = component_name %}
{# duplicate integration_type to assist with sometimes needing to overwrite component templates with 'logs-filestream.generic@package' (there is no metrics-filestream.generic@package) #}
{% set generic_integration_type = integration_type %}
{# component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
{% set component_name_x = component_name.replace(".","_x_") %}
{# pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
{% set integration_key = "so-" ~ integration_type ~ component_name_x %}
{# if its a .generic template make sure that a .generic@package for the integration exists. Else default to logs-filestream.generic@package #}
{% if ".generic" in component_name and integration_type ~ component_name ~ "@package" not in INSTALLED_COMPONENT_TEMPLATES %}
{# these generic templates by default are directed to index_pattern of 'logs-generic-*', overwrite that here to point to eg gcp_pubsub.generic-* #}
{% set index_pattern = integration_type ~ component_name ~ "-*" %}
{# includes use of .generic component template, but it doesn't exist in installed component templates. Redirect it to filestream.generic@package #}
{% set component_name = "filestream.generic" %}
{% set generic_integration_type = "logs-" %}
{% endif %}
{# Default integration settings #}
{% set integration_defaults = {
"index_sorting": false,
"index_template": {
"composed_of": [generic_integration_type ~ component_name ~ "@package", integration_type ~ custom_component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
"composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
"data_stream": {
"allow_custom_routing": false,
"hidden": false
},
"ignore_missing_component_templates": [integration_type ~ custom_component_name ~ "@custom"],
"ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
"index_patterns": [index_pattern],
"priority": 501,
"template": {
"settings": {
"index": {
"lifecycle": {"name": "so-" ~ integration_type ~ custom_component_name ~ "-logs"},
"lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
"number_of_replicas": 0
}
}
salt/elasticfleet/tools/sbin/so-elastic-fleet-common
+26 -2
View File
@@ -135,9 +135,33 @@ elastic_fleet_bulk_package_install() {
fi
}
elastic_fleet_installed_packages() {
if ! fleet_api "epm/packages/installed?perPage=500"; then
elastic_fleet_get_package_list_by_type() {
if ! output=$(fleet_api "epm/packages"); then
return 1
else
is_integration=$(jq '[.items[] | select(.type=="integration") | .name ]' <<< "$output")
is_input=$(jq '[.items[] | select(.type=="input") | .name ]' <<< "$output")
is_content=$(jq '[.items[] | select(.type=="content") | .name ]' <<< "$output")
jq -n --argjson is_integration "${is_integration:-[]}" \
--argjson is_input "${is_input:-[]}" \
--argjson is_content "${is_content:-[]}" \
'{"integration": $is_integration,"input": $is_input, "content": $is_content}'
fi
}
elastic_fleet_installed_packages_components() {
package_type=${1,,}
if [[ "$package_type" != "integration" && "$package_type" != "input" && "$package_type" != "content" ]]; then
echo "Error: Invalid package type ${package_type}. Valid types are 'integration', 'input', or 'content'."
return 1
fi
packages_by_type=$(elastic_fleet_get_package_list_by_type)
packages=$(jq --arg package_type "$package_type" '.[$package_type]' <<< "$packages_by_type")
if ! output=$(fleet_api "epm/packages/installed?perPage=500"); then
return 1
else
jq -c --argjson packages "$packages" '[.items[] | select(.name | IN($packages[])) | {name: .name, dataStreams: .dataStreams}]' <<< "$output"
fi
}
salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
+10 -5
View File
@@ -18,7 +18,9 @@ INSTALLED_PACKAGE_LIST=/tmp/esfleet_installed_packages.json
BULK_INSTALL_PACKAGE_LIST=/tmp/esfleet_bulk_install.json
BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json
BULK_INSTALL_OUTPUT=/opt/so/state/esfleet_bulk_install_results.json
PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json
INTEGRATION_PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json
INPUT_PACKAGE_COMPONENTS=/opt/so/state/esfleet_input_package_components.json
CONTENT_PACKAGE_COMPONENTS=/opt/so/state/esfleet_content_package_components.json
COMPONENT_TEMPLATES=/opt/so/state/esfleet_component_templates.json
PENDING_UPDATE=false
@@ -179,10 +181,13 @@ if [[ -f $STATE_FILE_SUCCESS ]]; then
else
echo "Elastic integrations don't appear to need installation/updating..."
fi
# Write out file for generating index/component/ilm templates
if latest_installed_package_list=$(elastic_fleet_installed_packages); then
echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS
fi
# Write out file for generating index/component/ilm templates, keeping each package type separate
for package_type in "INTEGRATION" "INPUT" "CONTENT"; do
if latest_installed_package_list=$(elastic_fleet_installed_packages_components "$package_type"); then
outfile="${package_type}_PACKAGE_COMPONENTS"
echo $latest_installed_package_list > "${!outfile}"
fi
done
if retry 3 1 "so-elasticsearch-query / --fail --output /dev/null"; then
# Refresh installed component template list
latest_component_templates_list=$(so-elasticsearch-query _component_template | jq '.component_templates[] | .name' | jq -s '.')