CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2025-12-21 08:23:08 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/2.4/dev' into bravo

Browse Source
This commit is contained in:
Josh Patterson
2025-12-11 17:30:43 -05:00
parent b9ff1704b0 6fe817ca4a
commit 9960db200c
64 changed files with 1241 additions and 782 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
salt/elasticsearch/defaults.yaml
View File

@@ -72,6 +72,8 @@ elasticsearch:
actions:
set_priority:
priority: 0
allocate:
number_of_replicas: ""
min_age: 60d
delete:
actions:
@@ -84,11 +86,25 @@ elasticsearch:
max_primary_shard_size: 50gb
set_priority:
priority: 100
forcemerge:
max_num_segments: ""
shrink:
max_primary_shard_size: ""
method: COUNT
number_of_shards: ""
min_age: 0ms
warm:
actions:
set_priority:
priority: 50
forcemerge:
max_num_segments: ""
shrink:
max_primary_shard_size: ""
method: COUNT
number_of_shards: ""
allocate:
number_of_replicas: ""
min_age: 30d
so-case:
index_sorting: false
@@ -245,7 +261,6 @@ elasticsearch:
set_priority:
priority: 50
min_age: 30d
warm: 7
so-detection:
index_sorting: false
index_template:
@@ -284,6 +299,19 @@ elasticsearch:
hot:
actions: {}
min_age: 0ms
sos-backup:
index_sorting: false
index_template:
composed_of: []
ignore_missing_component_templates: []
index_patterns:
- sos-backup-*
priority: 501
template:
settings:
index:
number_of_replicas: 0
number_of_shards: 1
so-assistant-chat:
index_sorting: false
index_template:
@@ -584,7 +612,6 @@ elasticsearch:
set_priority:
priority: 50
min_age: 30d
warm: 7
so-import:
index_sorting: false
index_template:
@@ -932,7 +959,6 @@ elasticsearch:
set_priority:
priority: 50
min_age: 30d
warm: 7
so-hydra:
close: 30
delete: 365
@@ -1043,7 +1069,6 @@ elasticsearch:
set_priority:
priority: 50
min_age: 30d
warm: 7
so-lists:
index_sorting: false
index_template:
@@ -1127,6 +1152,8 @@ elasticsearch:
actions:
set_priority:
priority: 0
allocate:
number_of_replicas: ""
min_age: 60d
delete:
actions:
@@ -1139,11 +1166,25 @@ elasticsearch:
max_primary_shard_size: 50gb
set_priority:
priority: 100
forcemerge:
max_num_segments: ""
shrink:
max_primary_shard_size: ""
method: COUNT
number_of_shards: ""
min_age: 0ms
warm:
actions:
set_priority:
priority: 50
allocate:
number_of_replicas: ""
forcemerge:
max_num_segments: ""
shrink:
max_primary_shard_size: ""
method: COUNT
number_of_shards: ""
min_age: 30d
so-logs-detections_x_alerts:
index_sorting: false
@@ -3123,7 +3164,6 @@ elasticsearch:
set_priority:
priority: 50
min_age: 30d
warm: 7
so-logs-system_x_application:
index_sorting: false
index_template:

186
salt/elasticsearch/soc_elasticsearch.yaml
View File

@@ -131,6 +131,47 @@ elasticsearch:
description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index.
global: True
helpLink: elasticsearch.html
shrink:
method:
description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
options:
- COUNT
- SIZE
global: True
advanced: True
forcedType: string
number_of_shards:
title: shard count
description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
global: True
forcedType: int
advanced: True
max_primary_shard_size:
title: max shard size
description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
regex: ^[0-9]+(?:gb|tb|pb)$
global: True
forcedType: string
advanced: True
allow_write_after_shrink:
description: Allow writes after shrink.
global: True
forcedType: bool
default: False
advanced: True
forcemerge:
max_num_segments:
description: Reduce the number of segments in each index shard and clean up deleted documents.
global: True
forcedType: int
advanced: True
index_codec:
title: compression
description: Use higher compression for stored fields at the cost of slower performance.
forcedType: bool
global: True
default: False
advanced: True
cold:
min_age:
description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and cold min_age set to 60 then there will be 30 days from index creation to rollover and then an additional 60 days before moving to cold tier.
@@ -144,6 +185,12 @@ elasticsearch:
description: Used for index recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
global: True
helpLink: elasticsearch.html
allocate:
number_of_replicas:
description: Set the number of replicas. Remains the same as the previous phase by default.
forcedType: int
global: True
advanced: True
warm:
min_age:
description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier.
@@ -158,6 +205,52 @@ elasticsearch:
forcedType: int
global: True
helpLink: elasticsearch.html
shrink:
method:
description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
options:
- COUNT
- SIZE
global: True
advanced: True
number_of_shards:
title: shard count
description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
global: True
forcedType: int
advanced: True
max_primary_shard_size:
title: max shard size
description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
regex: ^[0-9]+(?:gb|tb|pb)$
global: True
forcedType: string
advanced: True
allow_write_after_shrink:
description: Allow writes after shrink.
global: True
forcedType: bool
default: False
advanced: True
forcemerge:
max_num_segments:
description: Reduce the number of segments in each index shard and clean up deleted documents.
global: True
forcedType: int
advanced: True
index_codec:
title: compression
description: Use higher compression for stored fields at the cost of slower performance.
forcedType: bool
global: True
default: False
advanced: True
allocate:
number_of_replicas:
description: Set the number of replicas. Remains the same as the previous phase by default.
forcedType: int
global: True
advanced: True
delete:
min_age:
description: Minimum age of index. ex. 90d - This determines when the index should be deleted. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and delete min_age set to 90 then there will be 30 days from index creation to rollover and then an additional 90 days before deletion.
@@ -287,6 +380,47 @@ elasticsearch:
global: True
advanced: True
helpLink: elasticsearch.html
shrink:
method:
description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
options:
- COUNT
- SIZE
global: True
advanced: True
forcedType: string
number_of_shards:
title: shard count
description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
global: True
forcedType: int
advanced: True
max_primary_shard_size:
title: max shard size
description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
regex: ^[0-9]+(?:gb|tb|pb)$
global: True
forcedType: string
advanced: True
allow_write_after_shrink:
description: Allow writes after shrink.
global: True
forcedType: bool
default: False
advanced: True
forcemerge:
max_num_segments:
description: Reduce the number of segments in each index shard and clean up deleted documents.
global: True
forcedType: int
advanced: True
index_codec:
title: compression
description: Use higher compression for stored fields at the cost of slower performance.
forcedType: bool
global: True
default: False
advanced: True
warm:
min_age:
description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier.
@@ -314,6 +448,52 @@ elasticsearch:
global: True
advanced: True
helpLink: elasticsearch.html
shrink:
method:
description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
options:
- COUNT
- SIZE
global: True
advanced: True
number_of_shards:
title: shard count
description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
global: True
forcedType: int
advanced: True
max_primary_shard_size:
title: max shard size
description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
regex: ^[0-9]+(?:gb|tb|pb)$
global: True
forcedType: string
advanced: True
allow_write_after_shrink:
description: Allow writes after shrink.
global: True
forcedType: bool
default: False
advanced: True
forcemerge:
max_num_segments:
description: Reduce the number of segments in each index shard and clean up deleted documents.
global: True
forcedType: int
advanced: True
index_codec:
title: compression
description: Use higher compression for stored fields at the cost of slower performance.
forcedType: bool
global: True
default: False
advanced: True
allocate:
number_of_replicas:
description: Set the number of replicas. Remains the same as the previous phase by default.
forcedType: int
global: True
advanced: True
cold:
min_age:
description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and cold min_age set to 60 then there will be 30 days from index creation to rollover and then an additional 60 days before moving to cold tier.
@@ -330,6 +510,12 @@ elasticsearch:
global: True
advanced: True
helpLink: elasticsearch.html
allocate:
number_of_replicas:
description: Set the number of replicas. Remains the same as the previous phase by default.
forcedType: int
global: True
advanced: True
delete:
min_age:
description: Minimum age of index. ex. 90d - This determines when the index should be deleted. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and delete min_age set to 90 then there will be 30 days from index creation to rollover and then an additional 90 days before deletion.

50
salt/elasticsearch/template.map.jinja
View File

@@ -61,5 +61,55 @@
{% do settings.index_template.template.settings.index.pop('sort') %}
{% endif %}
{% endif %}
{# advanced ilm actions #}
{% if settings.policy is defined and settings.policy.phases is defined %}
{% set PHASE_NAMES = ["hot", "warm", "cold"] %}
{% for P in PHASE_NAMES %}
{% if settings.policy.phases[P] is defined and settings.policy.phases[P].actions is defined %}
{% set PHASE = settings.policy.phases[P].actions %}
{# remove allocate action if number_of_replicas isn't configured #}
{% if PHASE.allocate is defined %}
{% if PHASE.allocate.number_of_replicas is not defined or PHASE.allocate.number_of_replicas == "" %}
{% do PHASE.pop('allocate', none) %}
{% endif %}
{% endif %}
{# start shrink action #}
{% if PHASE.shrink is defined %}
{% if PHASE.shrink.method is defined %}
{% if PHASE.shrink.method == 'COUNT' and PHASE.shrink.number_of_shards is defined and PHASE.shrink.number_of_shards %}
{# remove max_primary_shard_size value when doing shrink operation by count vs size #}
{% do PHASE.shrink.pop('max_primary_shard_size', none) %}
{% elif PHASE.shrink.method == 'SIZE' and PHASE.shrink.max_primary_shard_size is defined and PHASE.shrink.max_primary_shard_size %}
{# remove number_of_shards value when doing shrink operation by size vs count #}
{% do PHASE.shrink.pop('number_of_shards', none) %}
{% else %}
{# method isn't defined or missing a required config number_of_shards/max_primary_shard_size #}
{% do PHASE.pop('shrink', none) %}
{% endif %}
{% endif %}
{% endif %}
{# always remove shrink method since its only used for SOC config, not in the actual ilm policy #}
{% if PHASE.shrink is defined %}
{% do PHASE.shrink.pop('method', none) %}
{% endif %}
{# end shrink action #}
{# start force merge #}
{% if PHASE.forcemerge is defined %}
{% if PHASE.forcemerge.index_codec is defined and PHASE.forcemerge.index_codec %}
{% do PHASE.forcemerge.update({'index_codec': 'best_compression'}) %}
{% else %}
{% do PHASE.forcemerge.pop('index_codec', none) %}
{% endif %}
{% if PHASE.forcemerge.max_num_segments is not defined or not PHASE.forcemerge.max_num_segments %}
{# max_num_segments is empty, drop it #}
{% do PHASE.pop('forcemerge', none) %}
{% endif %}
{% endif %}
{# end force merge #}
{% endif %}
{% endfor %}
{% endif %}
{% do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index]}) %}
{% endfor %}