From f0b67a415accd492e1e75b233c0dfe7a9f8c2fbe Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 9 Apr 2026 12:40:55 -0500 Subject: [PATCH 1/3] more filestream integration policy updates --- .../grid-nodes_general/import-zeek-logs.json | 11 +++++++++-- .../grid-nodes_general/kratos-logs.json | 14 ++++++++++---- .../grid-nodes_general/zeek-logs.json | 12 +++++++++--- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json index ac03f3c1d..c1fd7f147 100644 --- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json @@ -9,16 +9,22 @@ "namespace": "so", "description": "Zeek Import logs", "policy_id": "so-grid-nodes_general", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/nsm/import/*/zeek/logs/*.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "import", "pipeline": "", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -34,7 +40,8 @@ "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json index 545588521..83d153439 100644 --- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json @@ -15,19 +15,25 @@ "version": "" }, "name": "kratos-logs", + "namespace": "so", "description": "Kratos logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/opt/so/log/kratos/kratos.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "kratos", "pipeline": "kratos", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -48,10 +54,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json index 4af2b2921..9797b9e75 100644 --- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json @@ -9,16 +9,22 @@ "namespace": "so", "description": "Zeek logs", "policy_id": "so-grid-nodes_general", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/nsm/zeek/logs/current/*.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "zeek", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", "exclude_files": ["({%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%})(\\..+)?\\.log$"], @@ -30,10 +36,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } From 89e49d0bf35dcff8202c27462b3c966601737af7 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 9 Apr 2026 16:44:51 -0500 Subject: [PATCH 2/3] rework elasticsearch index template generation --- salt/elasticfleet/content-defaults.map.jinja | 123 ++++++++++++++++++ salt/elasticfleet/input-defaults.map.jinja | 123 ++++++++++++++++++ .../integration-defaults.map.jinja | 27 +--- .../tools/sbin/so-elastic-fleet-common | 28 +++- ...o-elastic-fleet-optional-integrations-load | 15 ++- salt/elasticsearch/config.sls | 7 + salt/elasticsearch/enabled.sls | 58 +++++---- salt/elasticsearch/template.map.jinja | 80 +++++++++--- 8 files changed, 392 insertions(+), 69 deletions(-) create mode 100644 salt/elasticfleet/content-defaults.map.jinja create mode 100644 salt/elasticfleet/input-defaults.map.jinja diff --git a/salt/elasticfleet/content-defaults.map.jinja b/salt/elasticfleet/content-defaults.map.jinja new file mode 100644 index 000000000..f4237d6d1 --- /dev/null +++ b/salt/elasticfleet/content-defaults.map.jinja @@ -0,0 +1,123 @@ +{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one + or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use + this file except in compliance with the Elastic License 2.0. #} + + +{% import_json '/opt/so/state/esfleet_content_package_components.json' as ADDON_CONTENT_PACKAGE_COMPONENTS %} +{% import_json '/opt/so/state/esfleet_component_templates.json' as INSTALLED_COMPONENT_TEMPLATES %} +{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} + +{% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %} +{% set ADDON_CONTENT_INTEGRATION_DEFAULTS = {} %} +{% set DEBUG_STUFF = {} %} + +{% for pkg in ADDON_CONTENT_PACKAGE_COMPONENTS %} +{% if pkg.name in CORE_ESFLEET_PACKAGES %} +{# skip core content packages #} +{% elif pkg.name not in CORE_ESFLEET_PACKAGES %} +{# generate defaults for each content package #} +{% if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0%} +{% for pattern in pkg.dataStreams %} +{# in ES 9.3.2 'input' type integrations no longer create default component templates and instead they wait for user input during 'integration' setup (fleet ui config) + title: generic is an artifact of that and is not in use #} +{% if pattern.title == "generic" %} +{% continue %} +{% endif %} +{% if "metrics-" in pattern.name %} +{% set integration_type = "metrics-" %} +{% elif "logs-" in pattern.name %} +{% set integration_type = "logs-" %} +{% else %} +{% set integration_type = "" %} +{% endif %} +{# on content integrations the component name is user defined at the time it is added to an agent policy #} +{% set component_name = pattern.title %} +{% set index_pattern = pattern.name %} +{# component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #} +{% set component_name_x = component_name.replace(".","_x_") %} +{# pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #} +{% set integration_key = "so-" ~ integration_type ~ pkg.name + '_x_' ~ component_name_x %} +{# Default integration settings #} +{% set integration_defaults = { + "index_sorting": false, + "index_template": { + "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"], + "data_stream": { + "allow_custom_routing": false, + "hidden": false + }, + "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"], + "index_patterns": [index_pattern], + "priority": 501, + "template": { + "settings": { + "index": { + "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"}, + "number_of_replicas": 0 + } + } + } + }, + "policy": { + "phases": { + "cold": { + "actions": { + "allocate":{ + "number_of_replicas": "" + }, + "set_priority": {"priority": 0} + }, + "min_age": "60d" + }, + "delete": { + "actions": { + "delete": {} + }, + "min_age": "365d" + }, + "hot": { + "actions": { + "rollover": { + "max_age": "30d", + "max_primary_shard_size": "50gb" + }, + "forcemerge":{ + "max_num_segments": "" + }, + "shrink":{ + "max_primary_shard_size": "", + "method": "COUNT", + "number_of_shards": "" + }, + "set_priority": {"priority": 100} + }, + "min_age": "0ms" + }, + "warm": { + "actions": { + "allocate": { + "number_of_replicas": "" + }, + "forcemerge": { + "max_num_segments": "" + }, + "shrink":{ + "max_primary_shard_size": "", + "method": "COUNT", + "number_of_shards": "" + }, + "set_priority": {"priority": 50} + }, + "min_age": "30d" + } + } + } + } %} + + +{% do ADDON_CONTENT_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %} +{% endfor %} +{% else %} +{% endif %} +{% endif %} +{% endfor %} diff --git a/salt/elasticfleet/input-defaults.map.jinja b/salt/elasticfleet/input-defaults.map.jinja new file mode 100644 index 000000000..a02844330 --- /dev/null +++ b/salt/elasticfleet/input-defaults.map.jinja @@ -0,0 +1,123 @@ +{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one + or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use + this file except in compliance with the Elastic License 2.0. #} + + +{% import_json '/opt/so/state/esfleet_input_package_components.json' as ADDON_INPUT_PACKAGE_COMPONENTS %} +{% import_json '/opt/so/state/esfleet_component_templates.json' as INSTALLED_COMPONENT_TEMPLATES %} +{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} + +{% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %} +{% set ADDON_INPUT_INTEGRATION_DEFAULTS = {} %} +{% set DEBUG_STUFF = {} %} + +{% for pkg in ADDON_INPUT_PACKAGE_COMPONENTS %} +{% if pkg.name in CORE_ESFLEET_PACKAGES %} +{# skip core input packages #} +{% elif pkg.name not in CORE_ESFLEET_PACKAGES %} +{# generate defaults for each input package #} +{% if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0 %} +{% for pattern in pkg.dataStreams %} +{# in ES 9.3.2 'input' type integrations no longer create default component templates and instead they wait for user input during 'integration' setup (fleet ui config) + title: generic is an artifact of that and is not in use #} +{% if pattern.title == "generic" %} +{% continue %} +{% endif %} +{% if "metrics-" in pattern.name %} +{% set integration_type = "metrics-" %} +{% elif "logs-" in pattern.name %} +{% set integration_type = "logs-" %} +{% else %} +{% set integration_type = "" %} +{% endif %} +{# on input integrations the component name is user defined at the time it is added to an agent policy #} +{% set component_name = pattern.title %} +{% set index_pattern = pattern.name %} +{# component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #} +{% set component_name_x = component_name.replace(".","_x_") %} +{# pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #} +{% set integration_key = "so-" ~ integration_type ~ pkg.name + '_x_' ~ component_name_x %} +{# Default integration settings #} +{% set integration_defaults = { + "index_sorting": false, + "index_template": { + "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"], + "data_stream": { + "allow_custom_routing": false, + "hidden": false + }, + "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"], + "index_patterns": [index_pattern], + "priority": 501, + "template": { + "settings": { + "index": { + "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"}, + "number_of_replicas": 0 + } + } + } + }, + "policy": { + "phases": { + "cold": { + "actions": { + "allocate":{ + "number_of_replicas": "" + }, + "set_priority": {"priority": 0} + }, + "min_age": "60d" + }, + "delete": { + "actions": { + "delete": {} + }, + "min_age": "365d" + }, + "hot": { + "actions": { + "rollover": { + "max_age": "30d", + "max_primary_shard_size": "50gb" + }, + "forcemerge":{ + "max_num_segments": "" + }, + "shrink":{ + "max_primary_shard_size": "", + "method": "COUNT", + "number_of_shards": "" + }, + "set_priority": {"priority": 100} + }, + "min_age": "0ms" + }, + "warm": { + "actions": { + "allocate": { + "number_of_replicas": "" + }, + "forcemerge": { + "max_num_segments": "" + }, + "shrink":{ + "max_primary_shard_size": "", + "method": "COUNT", + "number_of_shards": "" + }, + "set_priority": {"priority": 50} + }, + "min_age": "30d" + } + } + } + } %} + + +{% do ADDON_INPUT_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %} +{% do DEBUG_STUFF.update({integration_key: "Generating defaults for "+ pkg.name })%} +{% endfor %} +{% endif %} +{% endif %} +{% endfor %} diff --git a/salt/elasticfleet/integration-defaults.map.jinja b/salt/elasticfleet/integration-defaults.map.jinja index f85a95ec9..eeb85123a 100644 --- a/salt/elasticfleet/integration-defaults.map.jinja +++ b/salt/elasticfleet/integration-defaults.map.jinja @@ -59,8 +59,8 @@ {# skip core integrations #} {% elif pkg.name not in CORE_ESFLEET_PACKAGES %} {# generate defaults for each integration #} -{% if pkg.es_index_patterns is defined and pkg.es_index_patterns is not none %} -{% for pattern in pkg.es_index_patterns %} +{% if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0 %} +{% for pattern in pkg.dataStreams %} {% if "metrics-" in pattern.name %} {% set integration_type = "metrics-" %} {% elif "logs-" in pattern.name %} @@ -75,44 +75,27 @@ {% if component_name in WEIRD_INTEGRATIONS %} {% set component_name = WEIRD_INTEGRATIONS[component_name] %} {% endif %} - -{# create duplicate of component_name, so we can split generics from @custom component templates in the index template below and overwrite the default @package when needed - eg. having to replace unifiedlogs.generic@package with filestream.generic@package, but keep the ability to customize unifiedlogs.generic@custom and its ILM policy #} -{% set custom_component_name = component_name %} - -{# duplicate integration_type to assist with sometimes needing to overwrite component templates with 'logs-filestream.generic@package' (there is no metrics-filestream.generic@package) #} -{% set generic_integration_type = integration_type %} - {# component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #} {% set component_name_x = component_name.replace(".","_x_") %} {# pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #} {% set integration_key = "so-" ~ integration_type ~ component_name_x %} -{# if its a .generic template make sure that a .generic@package for the integration exists. Else default to logs-filestream.generic@package #} -{% if ".generic" in component_name and integration_type ~ component_name ~ "@package" not in INSTALLED_COMPONENT_TEMPLATES %} -{# these generic templates by default are directed to index_pattern of 'logs-generic-*', overwrite that here to point to eg gcp_pubsub.generic-* #} -{% set index_pattern = integration_type ~ component_name ~ "-*" %} -{# includes use of .generic component template, but it doesn't exist in installed component templates. Redirect it to filestream.generic@package #} -{% set component_name = "filestream.generic" %} -{% set generic_integration_type = "logs-" %} -{% endif %} - {# Default integration settings #} {% set integration_defaults = { "index_sorting": false, "index_template": { - "composed_of": [generic_integration_type ~ component_name ~ "@package", integration_type ~ custom_component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"], + "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"], "data_stream": { "allow_custom_routing": false, "hidden": false }, - "ignore_missing_component_templates": [integration_type ~ custom_component_name ~ "@custom"], + "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"], "index_patterns": [index_pattern], "priority": 501, "template": { "settings": { "index": { - "lifecycle": {"name": "so-" ~ integration_type ~ custom_component_name ~ "-logs"}, + "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"}, "number_of_replicas": 0 } } diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common index 1a597b1db..92532082a 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common @@ -135,9 +135,33 @@ elastic_fleet_bulk_package_install() { fi } -elastic_fleet_installed_packages() { - if ! fleet_api "epm/packages/installed?perPage=500"; then +elastic_fleet_get_package_list_by_type() { + if ! output=$(fleet_api "epm/packages"); then return 1 + else + is_integration=$(jq '[.items[] | select(.type=="integration") | .name ]' <<< "$output") + is_input=$(jq '[.items[] | select(.type=="input") | .name ]' <<< "$output") + is_content=$(jq '[.items[] | select(.type=="content") | .name ]' <<< "$output") + jq -n --argjson is_integration "${is_integration:-[]}" \ + --argjson is_input "${is_input:-[]}" \ + --argjson is_content "${is_content:-[]}" \ + '{"integration": $is_integration,"input": $is_input, "content": $is_content}' + fi +} +elastic_fleet_installed_packages_components() { + package_type=${1,,} + if [[ "$package_type" != "integration" && "$package_type" != "input" && "$package_type" != "content" ]]; then + echo "Error: Invalid package type ${package_type}. Valid types are 'integration', 'input', or 'content'." + return 1 + fi + + packages_by_type=$(elastic_fleet_get_package_list_by_type) + packages=$(jq --arg package_type "$package_type" '.[$package_type]' <<< "$packages_by_type") + + if ! output=$(fleet_api "epm/packages/installed?perPage=500"); then + return 1 + else + jq -c --argjson packages "$packages" '[.items[] | select(.name | IN($packages[])) | {name: .name, dataStreams: .dataStreams}]' <<< "$output" fi } diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load index 8c0f627ef..ab38b7065 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load @@ -18,7 +18,9 @@ INSTALLED_PACKAGE_LIST=/tmp/esfleet_installed_packages.json BULK_INSTALL_PACKAGE_LIST=/tmp/esfleet_bulk_install.json BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json BULK_INSTALL_OUTPUT=/opt/so/state/esfleet_bulk_install_results.json -PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json +INTEGRATION_PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json +INPUT_PACKAGE_COMPONENTS=/opt/so/state/esfleet_input_package_components.json +CONTENT_PACKAGE_COMPONENTS=/opt/so/state/esfleet_content_package_components.json COMPONENT_TEMPLATES=/opt/so/state/esfleet_component_templates.json PENDING_UPDATE=false @@ -179,10 +181,13 @@ if [[ -f $STATE_FILE_SUCCESS ]]; then else echo "Elastic integrations don't appear to need installation/updating..." fi - # Write out file for generating index/component/ilm templates - if latest_installed_package_list=$(elastic_fleet_installed_packages); then - echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS - fi + # Write out file for generating index/component/ilm templates, keeping each package type separate + for package_type in "INTEGRATION" "INPUT" "CONTENT"; do + if latest_installed_package_list=$(elastic_fleet_installed_packages_components "$package_type"); then + outfile="${package_type}_PACKAGE_COMPONENTS" + echo $latest_installed_package_list > "${!outfile}" + fi + done if retry 3 1 "so-elasticsearch-query / --fail --output /dev/null"; then # Refresh installed component template list latest_component_templates_list=$(so-elasticsearch-query _component_template | jq '.component_templates[] | .name' | jq -s '.') diff --git a/salt/elasticsearch/config.sls b/salt/elasticsearch/config.sls index 41ef02164..ac9fa8f72 100644 --- a/salt/elasticsearch/config.sls +++ b/salt/elasticsearch/config.sls @@ -91,6 +91,13 @@ estemplatedir: - group: 939 - makedirs: True +esaddontemplatedir: + file.directory: + - name: /opt/so/conf/elasticsearch/templates/addon-index + - user: 930 + - group: 939 + - makedirs: True + esrolesdir: file.directory: - name: /opt/so/conf/elasticsearch/roles diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index 29ab80329..61b9bad01 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -10,8 +10,7 @@ {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %} {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %} {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %} -{% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %} -{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %} +{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, ALL_ADDON_SETTINGS, SO_MANAGED_INDICES %} include: - ca @@ -117,16 +116,29 @@ escomponenttemplates: - onchanges_in: - file: so-elasticsearch-templates-reload - show_changes: False - -# Auto-generate templates from defaults file + +# Clean up legacy and non-SO managed templates from the elasticsearch/templates/index/ directory +so_index_template_dir: + file.directory: + - name: /opt/so/conf/elasticsearch/templates/index + - clean: True + {%- if SO_MANAGED_INDICES %} + - require: + {%- for index in SO_MANAGED_INDICES %} + - file: so_index_template_{{index}} + {%- endfor %} + {%- endif %} + +# Auto-generate index templates for SO managed indices (directly defined in elasticsearch/defaults.yaml) +# These index templates are for the core SO datasets and are always required {% for index, settings in ES_INDEX_SETTINGS.items() %} - {% if settings.index_template is defined %} -es_index_template_{{index}}: +{% if settings.index_template is defined %} +so_index_template_{{index}}: file.managed: - name: /opt/so/conf/elasticsearch/templates/index/{{ index }}-template.json - source: salt://elasticsearch/base-template.json.jinja - defaults: - TEMPLATE_CONFIG: {{ settings.index_template }} + TEMPLATE_CONFIG: {{ settings.index_template }} - template: jinja - show_changes: False - onchanges_in: @@ -134,25 +146,23 @@ es_index_template_{{index}}: {% endif %} {% endfor %} -{% if TEMPLATES %} -# Sync custom templates to /opt/so/conf/elasticsearch/templates -{% for TEMPLATE in TEMPLATES %} -es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}: +# Auto-generate optional index templates for integration | input | content packages +# These index templates are not used by default (until user adds package to an agent policy). +# Pre-configured with standard defaults, and incorporated into SOC configuration for user customization. +{% for index,settings in ALL_ADDON_SETTINGS.items() %} +{% if settings.index_template is defined %} +addon_index_template_{{index}}: file.managed: - - source: salt://elasticsearch/templates/index/{{TEMPLATE}} -{% if 'jinja' in TEMPLATE.split('.')[-1] %} - - name: /opt/so/conf/elasticsearch/templates/index/{{TEMPLATE.split('/')[1] | replace(".jinja", "")}} + - name: /opt/so/conf/elasticsearch/templates/addon-index/{{ index }}-template.json + - source: salt://elasticsearch/base-template.json.jinja + - defaults: + TEMPLATE_CONFIG: {{ settings.index_template }} - template: jinja -{% else %} - - name: /opt/so/conf/elasticsearch/templates/index/{{TEMPLATE.split('/')[1]}} -{% endif %} - - user: 930 - - group: 939 - show_changes: False - onchanges_in: - - file: so-elasticsearch-templates-reload -{% endfor %} -{% endif %} + - file: addon-elasticsearch-templates-reload +{% endif %} +{% endfor %} {% if GLOBALS.role in GLOBALS.manager_roles %} so-es-cluster-settings: @@ -179,6 +189,10 @@ so-elasticsearch-templates-reload: file.absent: - name: /opt/so/state/estemplates.txt +addon-elasticsearch-templates-reload: + file.absent: + - name: /opt/so/state/addon_estemplates.txt + so-elasticsearch-templates: cmd.run: - name: /usr/sbin/so-elasticsearch-templates-load diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index 2563f8e23..2690fa56f 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -15,14 +15,40 @@ {% set ES_INDEX_SETTINGS_ORIG = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings %} {# start generation of integration default index_settings #} -{% if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') and salt['file.file_exists']('/opt/so/state/esfleet_component_templates.json') %} -{% set check_package_components = salt['file.stats']('/opt/so/state/esfleet_package_components.json') %} -{% if check_package_components.size > 1 %} -{% from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %} -{% for index, settings in ADDON_INTEGRATION_DEFAULTS.items() %} -{% do ES_INDEX_SETTINGS_ORIG.update({index: settings}) %} -{% endfor %} -{% endif%} +{% if salt['file.file_exists']('/opt/so/state/esfleet_component_templates.json') %} +{% set ALL_ADDON_INTEGRATION_DEFAULTS = {} %} +{% set ALL_ADDON_SETTINGS_ORIG = {} %} +{% set ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES = {} %} +{# import integration type defaults #} +{% if salt['file.file_exists']('/opt/so/state/esfleet_integration_package_components.json') %} +{% set check_integration_package_components = salt['file.stats']('/opt/so/state/esfleet_integration_package_components.json') %} +{% if check_integration_package_components.size > 1 %} +{% from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %} +{% do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_INTEGRATION_DEFAULTS) %} +{% endif %} +{% endif %} + +{# import input type defaults #} +{% if salt['file.file_exists']('/opt/so/state/esfleet_input_package_components.json') %} +{% set check_input_package_components = salt['file.stats']('/opt/so/state/esfleet_input_package_components.json') %} +{% if check_input_package_components.size > 1 %} +{% from 'elasticfleet/input-defaults.map.jinja' import ADDON_INPUT_INTEGRATION_DEFAULTS %} +{% do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_INPUT_INTEGRATION_DEFAULTS) %} +{% endif %} +{% endif %} + +{# import content type defaults #} +{% if salt['file.file_exists']('/opt/so/state/esfleet_content_package_components.json') %} +{% set check_content_package_components = salt['file.stats']('/opt/so/state/esfleet_content_package_components.json') %} +{% if check_content_package_components.size > 1 %} +{% from 'elasticfleet/content-defaults.map.jinja' import ADDON_CONTENT_INTEGRATION_DEFAULTS %} +{% do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_CONTENT_INTEGRATION_DEFAULTS) %} +{% endif %} +{% endif %} + +{% for index, settings in ALL_ADDON_INTEGRATION_DEFAULTS.items() %} +{% do ALL_ADDON_SETTINGS_ORIG.update({index: settings}) %} +{% endfor %} {% endif %} {# end generation of integration default index_settings #} @@ -31,25 +57,34 @@ {% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ELASTICSEARCHDEFAULTS.elasticsearch.index_settings[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %} {% endfor %} +{% if ALL_ADDON_SETTINGS_ORIG.keys() | length > 0 %} +{% for index in ALL_ADDON_SETTINGS_ORIG.keys() %} +{% do ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ALL_ADDON_SETTINGS_ORIG[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %} +{% endfor %} +{% endif %} + {% set ES_INDEX_SETTINGS = {} %} -{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update(salt['defaults.merge'](ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %} -{% for index, settings in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.items() %} +{% set ALL_ADDON_SETTINGS = {}%} +{% macro create_final_index_template(DEFINED_SETTINGS, GLOBAL_OVERRIDES, FINAL_INDEX_SETTINGS) %} + +{% do GLOBAL_OVERRIDES.update(salt['defaults.merge'](GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %} +{% for index, settings in GLOBAL_OVERRIDES.items() %} {# prevent this action from being performed on custom defined indices. #} {# the custom defined index is not present in either of the dictionaries and fails to reder. #} -{% if index in ES_INDEX_SETTINGS_ORIG and index in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES %} +{% if index in DEFINED_SETTINGS and index in GLOBAL_OVERRIDES %} {# dont merge policy from the global_overrides if policy isn't defined in the original index settingss #} {# this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #} -{% if not ES_INDEX_SETTINGS_ORIG[index].policy is defined and ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %} -{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %} +{% if not DEFINED_SETTINGS[index].policy is defined and GLOBAL_OVERRIDES[index].policy is defined %} +{% do GLOBAL_OVERRIDES[index].pop('policy') %} {% endif %} {# this prevents and index from inderiting a policy phase from global overrides if it wasnt defined in the defaults. #} -{% if ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %} -{% for phase in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.copy() %} -{% if ES_INDEX_SETTINGS_ORIG[index].policy.phases[phase] is not defined %} -{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.pop(phase) %} +{% if GLOBAL_OVERRIDES[index].policy is defined %} +{% for phase in GLOBAL_OVERRIDES[index].policy.phases.copy() %} +{% if DEFINED_SETTINGS[index].policy.phases[phase] is not defined %} +{% do GLOBAL_OVERRIDES[index].policy.phases.pop(phase) %} {% endif %} {% endfor %} {% endif %} @@ -111,5 +146,14 @@ {% endfor %} {% endif %} -{% do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index]}) %} +{% do FINAL_INDEX_SETTINGS.update({index | replace("_x_", "."): GLOBAL_OVERRIDES[index]}) %} {% endfor %} +{% endmacro %} + +{{ create_final_index_template(ES_INDEX_SETTINGS_ORIG, ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_SETTINGS) }} +{{ create_final_index_template(ALL_ADDON_SETTINGS_ORIG, ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES, ALL_ADDON_SETTINGS) }} + +{% set SO_MANAGED_INDICES = [] %} +{% for index, settings in ES_INDEX_SETTINGS.items() %} +{% do SO_MANAGED_INDICES.append(index) %} +{% endfor %} \ No newline at end of file From 6b8a6267daab3875d3288d2eeadbaf80138c5467 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 9 Apr 2026 16:45:26 -0500 Subject: [PATCH 3/3] remove unused elasticsearch:index_template pillar references --- pillar/elasticsearch/index_templates.sls | 2 -- pillar/top.sls | 3 --- 2 files changed, 5 deletions(-) delete mode 100644 pillar/elasticsearch/index_templates.sls diff --git a/pillar/elasticsearch/index_templates.sls b/pillar/elasticsearch/index_templates.sls deleted file mode 100644 index a02a1818c..000000000 --- a/pillar/elasticsearch/index_templates.sls +++ /dev/null @@ -1,2 +0,0 @@ -elasticsearch: - index_settings: diff --git a/pillar/top.sls b/pillar/top.sls index 6cdc4808a..d3b24677c 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -97,7 +97,6 @@ base: - node_data.ips - secrets - healthcheck.eval - - elasticsearch.index_templates {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} @@ -142,7 +141,6 @@ base: - logstash.nodes - logstash.soc_logstash - logstash.adv_logstash - - elasticsearch.index_templates {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} @@ -256,7 +254,6 @@ base: '*_import': - node_data.ips - secrets - - elasticsearch.index_templates {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %}