{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at https://securityonion.net/license; you may not use this file except in compliance with the Elastic License 2.0. #} {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %} {% set DEFAULT_GLOBAL_OVERRIDES = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings.pop('global_overrides') %} {% set PILLAR_GLOBAL_OVERRIDES = {} %} {% set ES_INDEX_PILLAR = salt['pillar.get']('elasticsearch:index_settings', {}) %} {% if ES_INDEX_PILLAR.global_overrides is defined %} {% set PILLAR_GLOBAL_OVERRIDES = ES_INDEX_PILLAR.pop('global_overrides') %} {% endif %} {% set ES_INDEX_SETTINGS_ORIG = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings %} {# start generation of integration default index_settings #} {% if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') and salt['file.file_exists']('/opt/so/state/esfleet_component_templates.json') %} {% set check_package_components = salt['file.stats']('/opt/so/state/esfleet_package_components.json') %} {% if check_package_components.size > 1 %} {% from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %} {% for index, settings in ADDON_INTEGRATION_DEFAULTS.items() %} {% do ES_INDEX_SETTINGS_ORIG.update({index: settings}) %} {% endfor %} {% endif%} {% endif %} {# end generation of integration default index_settings #} {% set ES_INDEX_SETTINGS_GLOBAL_OVERRIDES = {} %} {% for index in ES_INDEX_SETTINGS_ORIG.keys() %} {% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ELASTICSEARCHDEFAULTS.elasticsearch.index_settings[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %} {% endfor %} {% set ES_INDEX_SETTINGS = {} %} {% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update(salt['defaults.merge'](ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %} {% for index, settings in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.items() %} {# prevent this action from being performed on custom defined indices. #} {# the custom defined index is not present in either of the dictionaries and fails to reder. #} {% if index in ES_INDEX_SETTINGS_ORIG and index in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES %} {# dont merge policy from the global_overrides if policy isn't defined in the original index settingss #} {# this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #} {% if not ES_INDEX_SETTINGS_ORIG[index].policy is defined and ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %} {% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %} {% endif %} {# this prevents and index from inderiting a policy phase from global overrides if it wasnt defined in the defaults. #} {% if ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %} {% for phase in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.copy() %} {% if ES_INDEX_SETTINGS_ORIG[index].policy.phases[phase] is not defined %} {% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.pop(phase) %} {% endif %} {% endfor %} {% endif %} {% endif %} {% if settings.index_template is defined %} {% if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %} {% do settings.index_template.template.settings.index.pop('sort') %} {% endif %} {% endif %} {# advanced ilm actions #} {% if settings.policy is defined and settings.policy.phases is defined %} {% set PHASE_NAMES = ["hot", "warm", "cold"] %} {% for P in PHASE_NAMES %} {% if settings.policy.phases[P] is defined and settings.policy.phases[P].actions is defined %} {% set PHASE = settings.policy.phases[P].actions %} {# remove allocate action if number_of_replicas isn't configured #} {% if PHASE.allocate is defined %} {% if PHASE.allocate.number_of_replicas is not defined or PHASE.allocate.number_of_replicas == "" %} {% do PHASE.pop('allocate', none) %} {% endif %} {% endif %} {# start shrink action #} {% if PHASE.shrink is defined %} {% if PHASE.shrink.method is defined %} {% if PHASE.shrink.method == 'COUNT' and PHASE.shrink.number_of_shards is defined and PHASE.shrink.number_of_shards %} {# remove max_primary_shard_size value when doing shrink operation by count vs size #} {% do PHASE.shrink.pop('max_primary_shard_size', none) %} {% elif PHASE.shrink.method == 'SIZE' and PHASE.shrink.max_primary_shard_size is defined and PHASE.shrink.max_primary_shard_size %} {# remove number_of_shards value when doing shrink operation by size vs count #} {% do PHASE.shrink.pop('number_of_shards', none) %} {% else %} {# method isn't defined or missing a required config number_of_shards/max_primary_shard_size #} {% do PHASE.pop('shrink', none) %} {% endif %} {% endif %} {% endif %} {# always remove shrink method since its only used for SOC config, not in the actual ilm policy #} {% if PHASE.shrink is defined %} {% do PHASE.shrink.pop('method', none) %} {% endif %} {# end shrink action #} {# start force merge #} {% if PHASE.forcemerge is defined %} {% if PHASE.forcemerge.index_codec is defined and PHASE.forcemerge.index_codec %} {% do PHASE.forcemerge.update({'index_codec': 'best_compression'}) %} {% else %} {% do PHASE.forcemerge.pop('index_codec', none) %} {% endif %} {% if PHASE.forcemerge.max_num_segments is not defined or not PHASE.forcemerge.max_num_segments %} {# max_num_segments is empty, drop it #} {% do PHASE.pop('forcemerge', none) %} {% endif %} {% endif %} {# end force merge #} {% endif %} {% endfor %} {% endif %} {% do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index]}) %} {% endfor %}