mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2025-12-07 17:52:46 +01:00
Merge pull request #9764 from Security-Onion-Solutions/fix/elasticsearch_ilm_dynamic_policy_loadiing
ILM Policy Changes
This commit is contained in:
weslambert
GitHub
@@ -6,68 +6,16 @@
|
||||
|
||||
. /usr/sbin/so-common
|
||||
|
||||
{% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %}
|
||||
{%- set ES_INDEX_SETTINGS = salt['pillar.get']('elasticsearch:index_settings', default=ESCONFIG.elasticsearch.index_settings, merge=True) %}
|
||||
{%- set NODEIP = salt['pillar.get']('host:mainip', '') %}
|
||||
|
||||
# Set up ILM policies
|
||||
{%- for index, settings in ES_INDEX_SETTINGS.items() %}
|
||||
{%- if settings.policy is defined %}
|
||||
echo
|
||||
echo "Setting up default Security Onion index lifecycle management policies..."
|
||||
|
||||
# Elasticsearch logs
|
||||
echo "Setting up {{ index }}-logs policy..."
|
||||
curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
|
||||
echo
|
||||
echo "Setting up Elasticsearch ILM policy..."
|
||||
curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/elasticsearch-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }'
|
||||
echo
|
||||
|
||||
# Import logs
|
||||
echo
|
||||
echo "Setting up Import ILM policy..."
|
||||
curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-import-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }'
|
||||
echo
|
||||
|
||||
# Kibana logs
|
||||
echo
|
||||
echo "Setting up Kibana ILM policy..."
|
||||
curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-kibana-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }'
|
||||
echo
|
||||
|
||||
# Kratos logs
|
||||
echo
|
||||
echo "Setting up Kratos ILM policy..."
|
||||
curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-kratos-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }'
|
||||
echo
|
||||
|
||||
# Logstash logs
|
||||
echo
|
||||
echo "Setting up Logstash ILM policy..."
|
||||
curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-logstash-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }'
|
||||
echo
|
||||
|
||||
# Redis logs
|
||||
echo
|
||||
echo "Setting up Redis ILM policy..."
|
||||
curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-redis-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }'
|
||||
echo
|
||||
|
||||
# Strelka logs
|
||||
echo
|
||||
echo "Setting up Strelka ILM policy..."
|
||||
curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-strelka-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }'
|
||||
echo
|
||||
|
||||
# Suricata logs
|
||||
echo
|
||||
echo "Setting up Suricata ILM policy..."
|
||||
curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-suricata-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }'
|
||||
echo
|
||||
|
||||
# Syslog logs
|
||||
echo
|
||||
echo "Setting up Syslog ILM policy..."
|
||||
curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-syslog-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }'
|
||||
echo
|
||||
|
||||
# Zeek logs
|
||||
echo
|
||||
echo "Setting up Zeek ILM policy..."
|
||||
curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-zeek-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }'
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
echo
|
||||
|
||||
@@ -1430,6 +1430,8 @@ elasticsearch:
|
||||
date_detection: false
|
||||
settings:
|
||||
index:
|
||||
lifecycle:
|
||||
name: so-elasticsearch-logs
|
||||
mapping:
|
||||
total_fields:
|
||||
limit: 5000
|
||||
@@ -1498,6 +1500,25 @@ elasticsearch:
|
||||
- common-settings
|
||||
- common-dynamic-mappings
|
||||
priority: 500
|
||||
policy:
|
||||
phases:
|
||||
hot:
|
||||
min_age: 0ms
|
||||
actions:
|
||||
set_priority:
|
||||
priority: 100
|
||||
rollover:
|
||||
max_age: 30d
|
||||
max_primary_shard_size: 50gb
|
||||
cold:
|
||||
min_age: 30d
|
||||
actions:
|
||||
set_priority:
|
||||
priority: 0
|
||||
delete:
|
||||
min_age: 365d
|
||||
actions:
|
||||
delete: {}
|
||||
so-endgame:
|
||||
index_sorting: False
|
||||
index_template:
|
||||
@@ -2183,6 +2204,8 @@ elasticsearch:
|
||||
date_detection: false
|
||||
settings:
|
||||
index:
|
||||
lifecycle:
|
||||
name: so-suricata-logs
|
||||
mapping:
|
||||
total_fields:
|
||||
limit: 5000
|
||||
@@ -2251,6 +2274,25 @@ elasticsearch:
|
||||
- common-settings
|
||||
- common-dynamic-mappings
|
||||
priority: 500
|
||||
policy:
|
||||
phases:
|
||||
hot:
|
||||
min_age: 0ms
|
||||
actions:
|
||||
set_priority:
|
||||
priority: 100
|
||||
rollover:
|
||||
max_age: 30d
|
||||
max_primary_shard_size: 50gb
|
||||
cold:
|
||||
min_age: 30d
|
||||
actions:
|
||||
set_priority:
|
||||
priority: 0
|
||||
delete:
|
||||
min_age: 365d
|
||||
actions:
|
||||
delete: {}
|
||||
so-imperva:
|
||||
index_sorting: False
|
||||
index_template:
|
||||
@@ -2351,6 +2393,8 @@ elasticsearch:
|
||||
date_detection: false
|
||||
settings:
|
||||
index:
|
||||
lifecycle:
|
||||
name: so-import-logs
|
||||
mapping:
|
||||
total_fields:
|
||||
limit: 5000
|
||||
@@ -2419,6 +2463,25 @@ elasticsearch:
|
||||
- common-dynamic-mappings
|
||||
- winlog-mappings
|
||||
priority: 500
|
||||
policy:
|
||||
phases:
|
||||
hot:
|
||||
min_age: 0ms
|
||||
actions:
|
||||
set_priority:
|
||||
priority: 100
|
||||
rollover:
|
||||
max_age: 30d
|
||||
max_primary_shard_size: 50gb
|
||||
cold:
|
||||
min_age: 30d
|
||||
actions:
|
||||
set_priority:
|
||||
priority: 0
|
||||
delete:
|
||||
min_age: 365d
|
||||
actions:
|
||||
delete: {}
|
||||
so-infoblox:
|
||||
index_sorting: False
|
||||
index_template:
|
||||
@@ -2671,6 +2734,25 @@ elasticsearch:
|
||||
- common-settings
|
||||
- common-dynamic-mappings
|
||||
priority: 500
|
||||
policy:
|
||||
phases:
|
||||
hot:
|
||||
min_age: 0ms
|
||||
actions:
|
||||
set_priority:
|
||||
priority: 100
|
||||
rollover:
|
||||
max_age: 30d
|
||||
max_primary_shard_size: 50gb
|
||||
cold:
|
||||
min_age: 30d
|
||||
actions:
|
||||
set_priority:
|
||||
priority: 0
|
||||
delete:
|
||||
min_age: 365d
|
||||
actions:
|
||||
delete: {}
|
||||
so-kratos:
|
||||
warm: 7
|
||||
close: 30
|
||||
@@ -2754,6 +2836,25 @@ elasticsearch:
|
||||
- common-settings
|
||||
- common-dynamic-mappings
|
||||
priority: 500
|
||||
policy:
|
||||
phases:
|
||||
hot:
|
||||
min_age: 0ms
|
||||
actions:
|
||||
set_priority:
|
||||
priority: 100
|
||||
rollover:
|
||||
max_age: 30d
|
||||
max_primary_shard_size: 50gb
|
||||
cold:
|
||||
min_age: 30d
|
||||
actions:
|
||||
set_priority:
|
||||
priority: 0
|
||||
delete:
|
||||
min_age: 365d
|
||||
actions:
|
||||
delete: {}
|
||||
so-logstash:
|
||||
index_sorting: False
|
||||
index_template:
|
||||
@@ -2770,6 +2871,8 @@ elasticsearch:
|
||||
date_detection: false
|
||||
settings:
|
||||
index:
|
||||
lifecycle:
|
||||
name: so-logstash-logs
|
||||
mapping:
|
||||
total_fields:
|
||||
limit: 5000
|
||||
@@ -2838,6 +2941,25 @@ elasticsearch:
|
||||
- common-settings
|
||||
- common-dynamic-mappings
|
||||
priority: 500
|
||||
policy:
|
||||
phases:
|
||||
hot:
|
||||
min_age: 0ms
|
||||
actions:
|
||||
set_priority:
|
||||
priority: 100
|
||||
rollover:
|
||||
max_age: 30d
|
||||
max_primary_shard_size: 50gb
|
||||
cold:
|
||||
min_age: 30d
|
||||
actions:
|
||||
set_priority:
|
||||
priority: 0
|
||||
delete:
|
||||
min_age: 365d
|
||||
actions:
|
||||
delete: {}
|
||||
so-microsoft:
|
||||
index_sorting: False
|
||||
index_template:
|
||||
@@ -3691,6 +3813,8 @@ elasticsearch:
|
||||
date_detection: false
|
||||
settings:
|
||||
index:
|
||||
lifecycle:
|
||||
name: so-redis-logs
|
||||
mapping:
|
||||
total_fields:
|
||||
limit: 5000
|
||||
@@ -3759,6 +3883,25 @@ elasticsearch:
|
||||
- common-settings
|
||||
- common-dynamic-mappings
|
||||
priority: 500
|
||||
policy:
|
||||
phases:
|
||||
hot:
|
||||
min_age: 0ms
|
||||
actions:
|
||||
set_priority:
|
||||
priority: 100
|
||||
rollover:
|
||||
max_age: 30d
|
||||
max_primary_shard_size: 50gb
|
||||
cold:
|
||||
min_age: 30d
|
||||
actions:
|
||||
set_priority:
|
||||
priority: 0
|
||||
delete:
|
||||
min_age: 365d
|
||||
actions:
|
||||
delete: {}
|
||||
so-snort:
|
||||
index_sorting: False
|
||||
index_template:
|
||||
@@ -4262,6 +4405,25 @@ elasticsearch:
|
||||
- common-settings
|
||||
- common-dynamic-mappings
|
||||
priority: 500
|
||||
policy:
|
||||
phases:
|
||||
hot:
|
||||
min_age: 0ms
|
||||
actions:
|
||||
set_priority:
|
||||
priority: 100
|
||||
rollover:
|
||||
max_age: 30d
|
||||
max_primary_shard_size: 50gb
|
||||
cold:
|
||||
min_age: 30d
|
||||
actions:
|
||||
set_priority:
|
||||
priority: 0
|
||||
delete:
|
||||
min_age: 365d
|
||||
actions:
|
||||
delete: {}
|
||||
so-syslog:
|
||||
index_sorting: False
|
||||
index_template:
|
||||
@@ -4347,6 +4509,25 @@ elasticsearch:
|
||||
- common-settings
|
||||
- common-dynamic-mappings
|
||||
priority: 500
|
||||
policy:
|
||||
phases:
|
||||
hot:
|
||||
min_age: 0ms
|
||||
actions:
|
||||
set_priority:
|
||||
priority: 100
|
||||
rollover:
|
||||
max_age: 30d
|
||||
max_primary_shard_size: 50gb
|
||||
cold:
|
||||
min_age: 30d
|
||||
actions:
|
||||
set_priority:
|
||||
priority: 0
|
||||
delete:
|
||||
min_age: 365d
|
||||
actions:
|
||||
delete: {}
|
||||
so-tomcat:
|
||||
index_sorting: False
|
||||
index_template:
|
||||
@@ -4447,6 +4628,8 @@ elasticsearch:
|
||||
date_detection: false
|
||||
settings:
|
||||
index:
|
||||
lifecycle:
|
||||
name: so-zeek-logs
|
||||
mapping:
|
||||
total_fields:
|
||||
limit: 5000
|
||||
@@ -4517,6 +4700,25 @@ elasticsearch:
|
||||
- common-settings
|
||||
- common-dynamic-mappings
|
||||
priority: 500
|
||||
policy:
|
||||
phases:
|
||||
hot:
|
||||
min_age: 0ms
|
||||
actions:
|
||||
set_priority:
|
||||
priority: 100
|
||||
rollover:
|
||||
max_age: 30d
|
||||
max_primary_shard_size: 50gb
|
||||
cold:
|
||||
min_age: 30d
|
||||
actions:
|
||||
set_priority:
|
||||
priority: 0
|
||||
delete:
|
||||
min_age: 365d
|
||||
actions:
|
||||
delete: {}
|
||||
so-zscaler:
|
||||
index_sorting: False
|
||||
index_template:
|
||||
|
||||
Reference in New Issue
Block a user