securityonion/salt/elasticsearch/template.map.jinja

{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
   https://securityonion.net/license; you may not use this file except in compliance with the
   Elastic License 2.0. #}

{% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
{% set DEFAULT_GLOBAL_OVERRIDES = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings.pop('global_overrides') %}

{% set PILLAR_GLOBAL_OVERRIDES = {} %}
{% set ES_INDEX_PILLAR = salt['pillar.get']('elasticsearch:index_settings', {}) %}
{% if ES_INDEX_PILLAR.global_overrides is defined %}
{%   set PILLAR_GLOBAL_OVERRIDES = ES_INDEX_PILLAR.pop('global_overrides') %}
{% endif %}

{% set ES_INDEX_SETTINGS_ORIG = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings %}

{# start generation of integration default index_settings #}
{% if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') and salt['file.file_exists']('/opt/so/state/esfleet_component_templates.json') %}
{%   set check_package_components = salt['file.stats']('/opt/so/state/esfleet_package_components.json') %}
{%   if check_package_components.size > 1 %}
{%    from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %}
{%    for index, settings in ADDON_INTEGRATION_DEFAULTS.items() %}
{%      do ES_INDEX_SETTINGS_ORIG.update({index: settings}) %}
{%    endfor %}
{%   endif%}
{% endif %}
{# end generation of integration default index_settings #}

{% set ES_INDEX_SETTINGS_GLOBAL_OVERRIDES = {} %}
{% for index in ES_INDEX_SETTINGS_ORIG.keys() %}
{%   do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ELASTICSEARCHDEFAULTS.elasticsearch.index_settings[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %}
{% endfor %}

{% set ES_INDEX_SETTINGS = {} %}
{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update(salt['defaults.merge'](ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %}
{% for index, settings in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.items() %}

{#   prevent this action from being performed on custom defined indices. #}
{#   the custom defined index is not present in either of the dictionaries and fails to reder. #}
{%   if index in ES_INDEX_SETTINGS_ORIG and index in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES %}

{#     dont merge policy from the global_overrides if policy isn't defined in the original index settingss #}
{#     this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #}
{%     if not ES_INDEX_SETTINGS_ORIG[index].policy is defined and ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %}
{%       do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %}
{%     endif %}

{#     this prevents and index from inderiting a policy phase from global overrides if it wasnt defined in the defaults. #}
{%     if ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %}
{%       for phase in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.copy() %}
{%         if ES_INDEX_SETTINGS_ORIG[index].policy.phases[phase] is not defined %}
{%           do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.pop(phase) %}
{%         endif %}
{%       endfor %}
{%     endif %}

{%   endif %}

{%   if settings.index_template is defined %}
{%     if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %}
{%       do settings.index_template.template.settings.index.pop('sort') %}
{%     endif %}
{%   endif %}

{# advanced ilm actions #}
{%   if settings.policy is defined and settings.policy.phases is defined %}
{%     set PHASE_NAMES = ["hot", "warm", "cold"] %}
{%     for P in PHASE_NAMES %}
{%       if settings.policy.phases[P] is defined and settings.policy.phases[P].actions is defined %}
{%         set PHASE = settings.policy.phases[P].actions %}
{#         remove allocate action if number_of_replicas isn't configured #}
{%         if PHASE.allocate is defined %}
{%           if PHASE.allocate.number_of_replicas is not defined or PHASE.allocate.number_of_replicas == "" %}
{%             do PHASE.pop('allocate', none) %}
{%           endif %}
{%         endif %}
{#         start shrink action #}
{%         if PHASE.shrink is defined %}
{%           if PHASE.shrink.method is defined %}
{%             if PHASE.shrink.method == 'COUNT' and PHASE.shrink.number_of_shards is defined and PHASE.shrink.number_of_shards %}
{#               remove max_primary_shard_size value when doing shrink operation by count vs size #}
{%               do PHASE.shrink.pop('max_primary_shard_size', none) %}
{%             elif PHASE.shrink.method == 'SIZE' and PHASE.shrink.max_primary_shard_size is defined and PHASE.shrink.max_primary_shard_size %}
{#               remove number_of_shards value when doing shrink operation by size vs count #}
{%               do PHASE.shrink.pop('number_of_shards', none) %}
{%             else %}
{#               method isn't defined or missing a required config number_of_shards/max_primary_shard_size #}
{%               do PHASE.pop('shrink', none) %}
{%             endif %}
{%           endif %}
{%         endif %}
{#         always remove shrink method since its only used for SOC config, not in the actual ilm policy #}
{%         if PHASE.shrink is defined %}
{%            do PHASE.shrink.pop('method', none) %}
{%         endif %}
{#         end shrink action #}
{#         start force merge #}
{%         if PHASE.forcemerge is defined %}
{%           if PHASE.forcemerge.index_codec is defined and PHASE.forcemerge.index_codec %}
{%             do PHASE.forcemerge.update({'index_codec': 'best_compression'}) %}
{%           else %}
{%             do PHASE.forcemerge.pop('index_codec', none) %}
{%           endif %}
{%           if PHASE.forcemerge.max_num_segments is not defined or not PHASE.forcemerge.max_num_segments %}
{#             max_num_segments is empty, drop it #}
{%             do PHASE.pop('forcemerge', none) %}
{%           endif %}
{%         endif %}
{#         end force merge #}
{%       endif %}
{%     endfor %}
{%   endif %}

{%   do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index]}) %}
{% endfor %}