CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2025-12-06 17:22:49 +01:00
Code Issues Packages Projects Releases Wiki Activity

advanced ilm actions

Browse Source
This commit is contained in:
reyesj2
2025-11-19 13:24:55 -06:00
parent a155f45036
commit b52dd53e29
4 changed files with 327 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
salt/elasticfleet/integration-defaults.map.jinja
View File

@@ -121,6 +121,9 @@
"phases": {
"cold": {
"actions": {
"allocate":{
"number_of_replicas": ""
},
"set_priority": {"priority": 0}
},
"min_age": "60d"
@@ -137,12 +140,31 @@
"max_age": "30d",
"max_primary_shard_size": "50gb"
},
"forcemerge":{
"max_num_segments": ""
},
"shrink":{
"max_primary_shard_size": "",
"method": "COUNT",
"number_of_shards": ""
},
"set_priority": {"priority": 100}
},
"min_age": "0ms"
},
"warm": {
"actions": {
"allocate": {
"number_of_replicas": ""
},
"forcemerge": {
"max_num_segments": ""
},
"shrink":{
"max_primary_shard_size": "",
"method": "COUNT",
"number_of_shards": ""
},
"set_priority": {"priority": 50}
},
"min_age": "30d"

32
salt/elasticsearch/defaults.yaml
View File

@@ -72,6 +72,8 @@ elasticsearch:
actions:
set_priority:
priority: 0
allocate:
number_of_replicas: ""
min_age: 60d
delete:
actions:
@@ -84,11 +86,25 @@ elasticsearch:
max_primary_shard_size: 50gb
set_priority:
priority: 100
forcemerge:
max_num_segments: ""
shrink:
max_primary_shard_size: ""
method: COUNT
number_of_shards: ""
min_age: 0ms
warm:
actions:
set_priority:
priority: 50
forcemerge:
max_num_segments: ""
shrink:
max_primary_shard_size: ""
method: COUNT
number_of_shards: ""
allocate:
number_of_replicas: ""
min_age: 30d
so-case:
index_sorting: false
@@ -1123,6 +1139,8 @@ elasticsearch:
actions:
set_priority:
priority: 0
allocate:
number_of_replicas: ""
min_age: 60d
delete:
actions:
@@ -1135,11 +1153,25 @@ elasticsearch:
max_primary_shard_size: 50gb
set_priority:
priority: 100
forcemerge:
max_num_segments: ""
shrink:
max_primary_shard_size: ""
method: COUNT
number_of_shards: ""
min_age: 0ms
warm:
actions:
set_priority:
priority: 50
allocate:
number_of_replicas: ""
forcemerge:
max_num_segments: ""
shrink:
max_primary_shard_size: ""
method: COUNT
number_of_shards: ""
min_age: 30d
so-logs-detections_x_alerts:
index_sorting: false

186
salt/elasticsearch/soc_elasticsearch.yaml
View File

@@ -131,6 +131,47 @@ elasticsearch:
description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index.
global: True
helpLink: elasticsearch.html
shrink:
method:
description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
options:
- COUNT
- SIZE
global: True
advanced: True
forcedType: string
number_of_shards:
title: shard count
description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
global: True
forcedType: int
advanced: True
max_primary_shard_size:
title: max shard size
description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
regex: ^(?:[0-9]+(?:gb|tb|pb)|)$
global: True
forcedType: string
advanced: True
allow_write_after_shrink:
description: Allow writes after shrink.
global: True
forcedType: bool
default: False
advanced: True
forcemerge:
max_num_segments:
description: Reduce the number of segments in each index shard and clean up deleted documents.
global: True
forcedType: int
advanced: True
index_codec:
title: compression
description: Use higher compression for stored fields at the cost of slower performance.
forcedType: bool
global: True
default: False
advanced: True
cold:
min_age:
description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and cold min_age set to 60 then there will be 30 days from index creation to rollover and then an additional 60 days before moving to cold tier.
@@ -144,6 +185,12 @@ elasticsearch:
description: Used for index recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
global: True
helpLink: elasticsearch.html
allocate:
number_of_replicas:
description: Set the number of replicas. Remains the same as the previous phase by default.
forcedType: int
global: True
advanced: True
warm:
min_age:
description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier.
@@ -158,6 +205,52 @@ elasticsearch:
forcedType: int
global: True
helpLink: elasticsearch.html
shrink:
method:
description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
options:
- COUNT
- SIZE
global: True
advanced: True
number_of_shards:
title: shard count
description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
global: True
forcedType: int
advanced: True
max_primary_shard_size:
title: max shard size
description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
regex: ^(?:[0-9]+(?:gb|tb|pb)|)$
global: True
forcedType: string
advanced: True
allow_write_after_shrink:
description: Allow writes after shrink.
global: True
forcedType: bool
default: False
advanced: True
forcemerge:
max_num_segments:
description: Reduce the number of segments in each index shard and clean up deleted documents.
global: True
forcedType: int
advanced: True
index_codec:
title: compression
description: Use higher compression for stored fields at the cost of slower performance.
forcedType: bool
global: True
default: False
advanced: True
allocate:
number_of_replicas:
description: Set the number of replicas. Remains the same as the previous phase by default.
forcedType: int
global: True
advanced: True
delete:
min_age:
description: Minimum age of index. ex. 90d - This determines when the index should be deleted. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and delete min_age set to 90 then there will be 30 days from index creation to rollover and then an additional 90 days before deletion.
@@ -287,6 +380,47 @@ elasticsearch:
global: True
advanced: True
helpLink: elasticsearch.html
shrink:
method:
description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
options:
- COUNT
- SIZE
global: True
advanced: True
forcedType: string
number_of_shards:
title: shard count
description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
global: True
forcedType: int
advanced: True
max_primary_shard_size:
title: max shard size
description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
regex: ^(?:[0-9]+(?:gb|tb|pb)|)$
global: True
forcedType: string
advanced: True
allow_write_after_shrink:
description: Allow writes after shrink.
global: True
forcedType: bool
default: False
advanced: True
forcemerge:
max_num_segments:
description: Reduce the number of segments in each index shard and clean up deleted documents.
global: True
forcedType: int
advanced: True
index_codec:
title: compression
description: Use higher compression for stored fields at the cost of slower performance.
forcedType: bool
global: True
default: False
advanced: True
warm:
min_age:
description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier.
@@ -314,6 +448,52 @@ elasticsearch:
global: True
advanced: True
helpLink: elasticsearch.html
shrink:
method:
description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
options:
- COUNT
- SIZE
global: True
advanced: True
number_of_shards:
title: shard count
description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
global: True
forcedType: int
advanced: True
max_primary_shard_size:
title: max shard size
description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
regex: ^(?:[0-9]+(?:gb|tb|pb)|)$
global: True
forcedType: string
advanced: True
allow_write_after_shrink:
description: Allow writes after shrink.
global: True
forcedType: bool
default: False
advanced: True
forcemerge:
max_num_segments:
description: Reduce the number of segments in each index shard and clean up deleted documents.
global: True
forcedType: int
advanced: True
index_codec:
title: compression
description: Use higher compression for stored fields at the cost of slower performance.
forcedType: bool
global: True
default: False
advanced: True
allocate:
number_of_replicas:
description: Set the number of replicas. Remains the same as the previous phase by default.
forcedType: int
global: True
advanced: True
cold:
min_age:
description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and cold min_age set to 60 then there will be 30 days from index creation to rollover and then an additional 60 days before moving to cold tier.
@@ -330,6 +510,12 @@ elasticsearch:
global: True
advanced: True
helpLink: elasticsearch.html
allocate:
number_of_replicas:
description: Set the number of replicas. Remains the same as the previous phase by default.
forcedType: int
global: True
advanced: True
delete:
min_age:
description: Minimum age of index. ex. 90d - This determines when the index should be deleted. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and delete min_age set to 90 then there will be 30 days from index creation to rollover and then an additional 90 days before deletion.

87
salt/elasticsearch/template.map.jinja
View File

@@ -61,5 +61,92 @@
{% do settings.index_template.template.settings.index.pop('sort') %}
{% endif %}
{% endif %}
{# advanced ilm actions #}
{% if settings.policy is defined and settings.policy.phases is defined %}
{# start HOT actions #}
{# only run if hot action is defined for this index #}
{% if settings.policy.phases.hot is defined and settings.policy.phases.hot.actions is defined %}
{% set HA = settings.policy.phases.hot.actions %}
{% if HA.shrink is defined %}
{% if HA.shrink.method is defined %}
{% if HA.shrink.method == 'COUNT' and HA.shrink.number_of_shards is defined and HA.shrink.number_of_shards %}
{# remove max_primary_shard_size value when doing shrink operation by count vs size #}
{% do HA.shrink.pop('max_primary_shard_size', none) %}
{% elif HA.shrink.method == 'SIZE' and HA.shrink.max_primary_shard_size is defined and HA.shrink.max_primary_shard_size %}
{# remove number_of_shards value when doing shrink operation by size vs count #}
{% do HA.shrink.pop('number_of_shards', none) %}
{% else %}
{# method isn't defined or missing a required config number_of_shards/max_primary_shard_size #}
{% do HA.pop('shrink', none) %}
{% endif %}
{% endif %}
{% endif %}
{# always remove method since its only used for SOC config, not in the actual ilm policy #}
{% if HA.shrink is defined %}
{% do HA.shrink.pop('method', none) %}
{% endif %}
{# end shrink action #}
{# start force merge #}
{% if HA.forcemerge is defined %}
{% if HA.forcemerge.index_codec is defined and HA.forcemerge.index_codec %}
{% do HA.forcemerge.update({'index_codec': 'best_compression'}) %}
{% else %}
{% do HA.forcemerge.pop('index_codec', none) %}
{% endif %}
{% if HA.forcemerge.max_num_segments is defined and not HA.forcemerge.max_num_segments %}
{# max_num_segments is empty, drop it #}
{% do HA.pop('forcemerge', none) %}
{% endif %}
{% endif %}
{# end force merge #}
{% endif %}
{# end HOT actions #}
{# Start WARM actions #}
{# only run if warm action is defined for this index #}
{% if settings.policy.phases.warm is defined and settings.policy.phases.warm.actions is defined %}
{% set WA = settings.policy.phases.warm.actions %}
{# start warm shrink action #}
{% if WA.shrink is defined %}
{% if WA.shrink.method is defined %}
{% if WA.shrink.method == 'COUNT' and WA.shrink.number_of_shards is defined and WA.shrink.number_of_shards %}
{# remove max_primary_shard_size value when doing shrink operation by count vs size #}
{% do WA.shrink.pop('max_primary_shard_size', none) %}
{% elif WA.shrink.method == 'SIZE' and WA.shrink.max_primary_shard_size is defined and WA.shrink.max_primary_shard_size %}
{# remove number_of_shards value when doing shrink operation by size vs count #}
{% do WA.shrink.pop('number_of_shards', none) %}
{% else %}
{# method isn't defined or missing a required config number_of_shards/max_primary_shard_size #}
{% do WA.pop('shrink', none) %}
{% endif %}
{% endif %}
{% endif %}
{# always remove method since its only used for SOC config, not in the actual ilm policy #}
{% if WA.shrink is defined %}
{% do WA.shrink.pop('method', none) %}
{% endif %}
{# end shrink action #}
{# start force merge #}
{% if WA.forcemerge is defined %}
{% if WA.forcemerge.index_codec is defined and WA.forcemerge.index_codec %}
{% do WA.forcemerge.update({'index_codec': 'best_compression'}) %}
{% else %}
{% do WA.forcemerge.pop('index_codec', none) %}
{% endif %}
{% if WA.forcemerge.max_num_segments is defined and not WA.forcemerge.max_num_segments %}
{# max_num_segments is empty, drop it #}
{% do WA.pop('forcemerge', none) %}
{% endif %}
{% endif %}
{# end force merge #}
{% if WA.allocate is defined %}
{% if WA.allocate.number_of_replicas is defined and not WA.allocate.number_of_replicas %}
{% do WA.pop('allocate', none) %}
{% endif %}
{% endif %}
{% endif %}
{# end WARM actions #}
{% endif %}
{% do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index]}) %}
{% endfor %}