diff --git a/salt/elasticfleet/integration-defaults.map.jinja b/salt/elasticfleet/integration-defaults.map.jinja index 500a9e63c..69ce7f3af 100644 --- a/salt/elasticfleet/integration-defaults.map.jinja +++ b/salt/elasticfleet/integration-defaults.map.jinja @@ -121,6 +121,9 @@ "phases": { "cold": { "actions": { + "allocate":{ + "number_of_replicas": "" + }, "set_priority": {"priority": 0} }, "min_age": "60d" @@ -137,12 +140,31 @@ "max_age": "30d", "max_primary_shard_size": "50gb" }, + "forcemerge":{ + "max_num_segments": "" + }, + "shrink":{ + "max_primary_shard_size": "", + "method": "COUNT", + "number_of_shards": "" + }, "set_priority": {"priority": 100} }, "min_age": "0ms" }, "warm": { "actions": { + "allocate": { + "number_of_replicas": "" + }, + "forcemerge": { + "max_num_segments": "" + }, + "shrink":{ + "max_primary_shard_size": "", + "method": "COUNT", + "number_of_shards": "" + }, "set_priority": {"priority": 50} }, "min_age": "30d" diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index bbfaf3244..5cfb9a0e0 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -72,6 +72,8 @@ elasticsearch: actions: set_priority: priority: 0 + allocate: + number_of_replicas: "" min_age: 60d delete: actions: @@ -84,11 +86,25 @@ elasticsearch: max_primary_shard_size: 50gb set_priority: priority: 100 + forcemerge: + max_num_segments: "" + shrink: + max_primary_shard_size: "" + method: COUNT + number_of_shards: "" min_age: 0ms warm: actions: set_priority: priority: 50 + forcemerge: + max_num_segments: "" + shrink: + max_primary_shard_size: "" + method: COUNT + number_of_shards: "" + allocate: + number_of_replicas: "" min_age: 30d so-case: index_sorting: false @@ -1123,6 +1139,8 @@ elasticsearch: actions: set_priority: priority: 0 + allocate: + number_of_replicas: "" min_age: 60d delete: actions: @@ -1135,11 +1153,25 @@ elasticsearch: max_primary_shard_size: 50gb set_priority: priority: 100 + forcemerge: + max_num_segments: "" + shrink: + max_primary_shard_size: "" + method: COUNT + number_of_shards: "" min_age: 0ms warm: actions: set_priority: priority: 50 + allocate: + number_of_replicas: "" + forcemerge: + max_num_segments: "" + shrink: + max_primary_shard_size: "" + method: COUNT + number_of_shards: "" min_age: 30d so-logs-detections_x_alerts: index_sorting: false diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 097a53296..27d5654b5 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -131,6 +131,47 @@ elasticsearch: description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index. global: True helpLink: elasticsearch.html + shrink: + method: + description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size. + options: + - COUNT + - SIZE + global: True + advanced: True + forcedType: string + number_of_shards: + title: shard count + description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'. + global: True + forcedType: int + advanced: True + max_primary_shard_size: + title: max shard size + description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'. + regex: ^(?:[0-9]+(?:gb|tb|pb)|)$ + global: True + forcedType: string + advanced: True + allow_write_after_shrink: + description: Allow writes after shrink. + global: True + forcedType: bool + default: False + advanced: True + forcemerge: + max_num_segments: + description: Reduce the number of segments in each index shard and clean up deleted documents. + global: True + forcedType: int + advanced: True + index_codec: + title: compression + description: Use higher compression for stored fields at the cost of slower performance. + forcedType: bool + global: True + default: False + advanced: True cold: min_age: description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and cold min_age set to 60 then there will be 30 days from index creation to rollover and then an additional 60 days before moving to cold tier. @@ -144,6 +185,12 @@ elasticsearch: description: Used for index recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities. global: True helpLink: elasticsearch.html + allocate: + number_of_replicas: + description: Set the number of replicas. Remains the same as the previous phase by default. + forcedType: int + global: True + advanced: True warm: min_age: description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier. @@ -158,6 +205,52 @@ elasticsearch: forcedType: int global: True helpLink: elasticsearch.html + shrink: + method: + description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size. + options: + - COUNT + - SIZE + global: True + advanced: True + number_of_shards: + title: shard count + description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'. + global: True + forcedType: int + advanced: True + max_primary_shard_size: + title: max shard size + description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'. + regex: ^(?:[0-9]+(?:gb|tb|pb)|)$ + global: True + forcedType: string + advanced: True + allow_write_after_shrink: + description: Allow writes after shrink. + global: True + forcedType: bool + default: False + advanced: True + forcemerge: + max_num_segments: + description: Reduce the number of segments in each index shard and clean up deleted documents. + global: True + forcedType: int + advanced: True + index_codec: + title: compression + description: Use higher compression for stored fields at the cost of slower performance. + forcedType: bool + global: True + default: False + advanced: True + allocate: + number_of_replicas: + description: Set the number of replicas. Remains the same as the previous phase by default. + forcedType: int + global: True + advanced: True delete: min_age: description: Minimum age of index. ex. 90d - This determines when the index should be deleted. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and delete min_age set to 90 then there will be 30 days from index creation to rollover and then an additional 90 days before deletion. @@ -287,6 +380,47 @@ elasticsearch: global: True advanced: True helpLink: elasticsearch.html + shrink: + method: + description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size. + options: + - COUNT + - SIZE + global: True + advanced: True + forcedType: string + number_of_shards: + title: shard count + description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'. + global: True + forcedType: int + advanced: True + max_primary_shard_size: + title: max shard size + description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'. + regex: ^(?:[0-9]+(?:gb|tb|pb)|)$ + global: True + forcedType: string + advanced: True + allow_write_after_shrink: + description: Allow writes after shrink. + global: True + forcedType: bool + default: False + advanced: True + forcemerge: + max_num_segments: + description: Reduce the number of segments in each index shard and clean up deleted documents. + global: True + forcedType: int + advanced: True + index_codec: + title: compression + description: Use higher compression for stored fields at the cost of slower performance. + forcedType: bool + global: True + default: False + advanced: True warm: min_age: description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier. @@ -314,6 +448,52 @@ elasticsearch: global: True advanced: True helpLink: elasticsearch.html + shrink: + method: + description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size. + options: + - COUNT + - SIZE + global: True + advanced: True + number_of_shards: + title: shard count + description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'. + global: True + forcedType: int + advanced: True + max_primary_shard_size: + title: max shard size + description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'. + regex: ^(?:[0-9]+(?:gb|tb|pb)|)$ + global: True + forcedType: string + advanced: True + allow_write_after_shrink: + description: Allow writes after shrink. + global: True + forcedType: bool + default: False + advanced: True + forcemerge: + max_num_segments: + description: Reduce the number of segments in each index shard and clean up deleted documents. + global: True + forcedType: int + advanced: True + index_codec: + title: compression + description: Use higher compression for stored fields at the cost of slower performance. + forcedType: bool + global: True + default: False + advanced: True + allocate: + number_of_replicas: + description: Set the number of replicas. Remains the same as the previous phase by default. + forcedType: int + global: True + advanced: True cold: min_age: description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and cold min_age set to 60 then there will be 30 days from index creation to rollover and then an additional 60 days before moving to cold tier. @@ -330,6 +510,12 @@ elasticsearch: global: True advanced: True helpLink: elasticsearch.html + allocate: + number_of_replicas: + description: Set the number of replicas. Remains the same as the previous phase by default. + forcedType: int + global: True + advanced: True delete: min_age: description: Minimum age of index. ex. 90d - This determines when the index should be deleted. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and delete min_age set to 90 then there will be 30 days from index creation to rollover and then an additional 90 days before deletion. diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index 414d8a6b4..904ab862c 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -61,5 +61,92 @@ {% do settings.index_template.template.settings.index.pop('sort') %} {% endif %} {% endif %} + +{# advanced ilm actions #} +{% if settings.policy is defined and settings.policy.phases is defined %} +{# start HOT actions #} +{# only run if hot action is defined for this index #} +{% if settings.policy.phases.hot is defined and settings.policy.phases.hot.actions is defined %} +{% set HA = settings.policy.phases.hot.actions %} +{% if HA.shrink is defined %} +{% if HA.shrink.method is defined %} +{% if HA.shrink.method == 'COUNT' and HA.shrink.number_of_shards is defined and HA.shrink.number_of_shards %} +{# remove max_primary_shard_size value when doing shrink operation by count vs size #} +{% do HA.shrink.pop('max_primary_shard_size', none) %} +{% elif HA.shrink.method == 'SIZE' and HA.shrink.max_primary_shard_size is defined and HA.shrink.max_primary_shard_size %} +{# remove number_of_shards value when doing shrink operation by size vs count #} +{% do HA.shrink.pop('number_of_shards', none) %} +{% else %} +{# method isn't defined or missing a required config number_of_shards/max_primary_shard_size #} +{% do HA.pop('shrink', none) %} +{% endif %} +{% endif %} +{% endif %} +{# always remove method since its only used for SOC config, not in the actual ilm policy #} +{% if HA.shrink is defined %} +{% do HA.shrink.pop('method', none) %} +{% endif %} +{# end shrink action #} +{# start force merge #} +{% if HA.forcemerge is defined %} +{% if HA.forcemerge.index_codec is defined and HA.forcemerge.index_codec %} +{% do HA.forcemerge.update({'index_codec': 'best_compression'}) %} +{% else %} +{% do HA.forcemerge.pop('index_codec', none) %} +{% endif %} +{% if HA.forcemerge.max_num_segments is defined and not HA.forcemerge.max_num_segments %} +{# max_num_segments is empty, drop it #} +{% do HA.pop('forcemerge', none) %} +{% endif %} +{% endif %} +{# end force merge #} +{% endif %} +{# end HOT actions #} +{# Start WARM actions #} +{# only run if warm action is defined for this index #} +{% if settings.policy.phases.warm is defined and settings.policy.phases.warm.actions is defined %} +{% set WA = settings.policy.phases.warm.actions %} +{# start warm shrink action #} +{% if WA.shrink is defined %} +{% if WA.shrink.method is defined %} +{% if WA.shrink.method == 'COUNT' and WA.shrink.number_of_shards is defined and WA.shrink.number_of_shards %} +{# remove max_primary_shard_size value when doing shrink operation by count vs size #} +{% do WA.shrink.pop('max_primary_shard_size', none) %} +{% elif WA.shrink.method == 'SIZE' and WA.shrink.max_primary_shard_size is defined and WA.shrink.max_primary_shard_size %} +{# remove number_of_shards value when doing shrink operation by size vs count #} +{% do WA.shrink.pop('number_of_shards', none) %} +{% else %} +{# method isn't defined or missing a required config number_of_shards/max_primary_shard_size #} +{% do WA.pop('shrink', none) %} +{% endif %} +{% endif %} +{% endif %} +{# always remove method since its only used for SOC config, not in the actual ilm policy #} +{% if WA.shrink is defined %} +{% do WA.shrink.pop('method', none) %} +{% endif %} +{# end shrink action #} +{# start force merge #} +{% if WA.forcemerge is defined %} +{% if WA.forcemerge.index_codec is defined and WA.forcemerge.index_codec %} +{% do WA.forcemerge.update({'index_codec': 'best_compression'}) %} +{% else %} +{% do WA.forcemerge.pop('index_codec', none) %} +{% endif %} +{% if WA.forcemerge.max_num_segments is defined and not WA.forcemerge.max_num_segments %} +{# max_num_segments is empty, drop it #} +{% do WA.pop('forcemerge', none) %} +{% endif %} +{% endif %} +{# end force merge #} +{% if WA.allocate is defined %} +{% if WA.allocate.number_of_replicas is defined and not WA.allocate.number_of_replicas %} +{% do WA.pop('allocate', none) %} +{% endif %} +{% endif %} +{% endif %} +{# end WARM actions #} +{% endif %} + {% do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index]}) %} {% endfor %}