{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at https://securityonion.net/license; you may not use this file except in compliance with the Elastic License 2.0. #} {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %} {% set DEFAULT_GLOBAL_OVERRIDES = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings.pop('global_overrides') %} {% set PILLAR_GLOBAL_OVERRIDES = {} %} {% set ES_INDEX_PILLAR = salt['pillar.get']('elasticsearch:index_settings', {}) %} {% if ES_INDEX_PILLAR.global_overrides is defined %} {% set PILLAR_GLOBAL_OVERRIDES = ES_INDEX_PILLAR.pop('global_overrides') %} {% endif %} {% set ES_INDEX_SETTINGS_ORIG = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings %} {# start generation of integration default index_settings #} {% if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') and salt['file.file_exists']('/opt/so/state/esfleet_component_templates.json') %} {% set check_package_components = salt['file.stats']('/opt/so/state/esfleet_package_components.json') %} {% if check_package_components.size > 1 %} {% from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %} {% for index, settings in ADDON_INTEGRATION_DEFAULTS.items() %} {% do ES_INDEX_SETTINGS_ORIG.update({index: settings}) %} {% endfor %} {% endif%} {% endif %} {# end generation of integration default index_settings #} {% set ES_INDEX_SETTINGS_GLOBAL_OVERRIDES = {} %} {% for index in ES_INDEX_SETTINGS_ORIG.keys() %} {% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ELASTICSEARCHDEFAULTS.elasticsearch.index_settings[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %} {% endfor %} {% set ES_INDEX_SETTINGS = {} %} {% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update(salt['defaults.merge'](ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %} {% for index, settings in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.items() %} {# prevent this action from being performed on custom defined indices. #} {# the custom defined index is not present in either of the dictionaries and fails to reder. #} {% if index in ES_INDEX_SETTINGS_ORIG and index in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES %} {# dont merge policy from the global_overrides if policy isn't defined in the original index settingss #} {# this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #} {% if not ES_INDEX_SETTINGS_ORIG[index].policy is defined and ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %} {% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %} {% endif %} {# this prevents and index from inderiting a policy phase from global overrides if it wasnt defined in the defaults. #} {% if ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %} {% for phase in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.copy() %} {% if ES_INDEX_SETTINGS_ORIG[index].policy.phases[phase] is not defined %} {% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.pop(phase) %} {% endif %} {% endfor %} {% endif %} {% endif %} {% if settings.index_template is defined %} {% if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %} {% do settings.index_template.template.settings.index.pop('sort') %} {% endif %} {% endif %} {# advanced ilm actions #} {% if settings.policy is defined and settings.policy.phases is defined %} {# start HOT actions #} {# only run if hot action is defined for this index #} {% if settings.policy.phases.hot is defined and settings.policy.phases.hot.actions is defined %} {% set HA = settings.policy.phases.hot.actions %} {% if HA.shrink is defined %} {% if HA.shrink.method is defined %} {% if HA.shrink.method == 'COUNT' and HA.shrink.number_of_shards is defined and HA.shrink.number_of_shards %} {# remove max_primary_shard_size value when doing shrink operation by count vs size #} {% do HA.shrink.pop('max_primary_shard_size', none) %} {% elif HA.shrink.method == 'SIZE' and HA.shrink.max_primary_shard_size is defined and HA.shrink.max_primary_shard_size %} {# remove number_of_shards value when doing shrink operation by size vs count #} {% do HA.shrink.pop('number_of_shards', none) %} {% else %} {# method isn't defined or missing a required config number_of_shards/max_primary_shard_size #} {% do HA.pop('shrink', none) %} {% endif %} {% endif %} {% endif %} {# always remove method since its only used for SOC config, not in the actual ilm policy #} {% if HA.shrink is defined %} {% do HA.shrink.pop('method', none) %} {% endif %} {# end shrink action #} {# start force merge #} {% if HA.forcemerge is defined %} {% if HA.forcemerge.index_codec is defined and HA.forcemerge.index_codec %} {% do HA.forcemerge.update({'index_codec': 'best_compression'}) %} {% else %} {% do HA.forcemerge.pop('index_codec', none) %} {% endif %} {% if HA.forcemerge.max_num_segments is defined and not HA.forcemerge.max_num_segments %} {# max_num_segments is empty, drop it #} {% do HA.pop('forcemerge', none) %} {% endif %} {% endif %} {# end force merge #} {% endif %} {# end HOT actions #} {# Start WARM actions #} {# only run if warm action is defined for this index #} {% if settings.policy.phases.warm is defined and settings.policy.phases.warm.actions is defined %} {% set WA = settings.policy.phases.warm.actions %} {# start warm shrink action #} {% if WA.shrink is defined %} {% if WA.shrink.method is defined %} {% if WA.shrink.method == 'COUNT' and WA.shrink.number_of_shards is defined and WA.shrink.number_of_shards %} {# remove max_primary_shard_size value when doing shrink operation by count vs size #} {% do WA.shrink.pop('max_primary_shard_size', none) %} {% elif WA.shrink.method == 'SIZE' and WA.shrink.max_primary_shard_size is defined and WA.shrink.max_primary_shard_size %} {# remove number_of_shards value when doing shrink operation by size vs count #} {% do WA.shrink.pop('number_of_shards', none) %} {% else %} {# method isn't defined or missing a required config number_of_shards/max_primary_shard_size #} {% do WA.pop('shrink', none) %} {% endif %} {% endif %} {% endif %} {# always remove method since its only used for SOC config, not in the actual ilm policy #} {% if WA.shrink is defined %} {% do WA.shrink.pop('method', none) %} {% endif %} {# end shrink action #} {# start force merge #} {% if WA.forcemerge is defined %} {% if WA.forcemerge.index_codec is defined and WA.forcemerge.index_codec %} {% do WA.forcemerge.update({'index_codec': 'best_compression'}) %} {% else %} {% do WA.forcemerge.pop('index_codec', none) %} {% endif %} {% if WA.forcemerge.max_num_segments is defined and not WA.forcemerge.max_num_segments %} {# max_num_segments is empty, drop it #} {% do WA.pop('forcemerge', none) %} {% endif %} {% endif %} {# end force merge #} {% if WA.allocate is defined %} {% if WA.allocate.number_of_replicas is defined and not WA.allocate.number_of_replicas %} {% do WA.pop('allocate', none) %} {% endif %} {% endif %} {% endif %} {# end WARM actions #} {% endif %} {% do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index]}) %} {% endfor %}