diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-policy-load b/salt/common/tools/sbin/so-elasticsearch-ilm-policy-load index 09cc6dc08..dda521736 100644 --- a/salt/common/tools/sbin/so-elasticsearch-ilm-policy-load +++ b/salt/common/tools/sbin/so-elasticsearch-ilm-policy-load @@ -6,68 +6,16 @@ . /usr/sbin/so-common +{% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %} +{%- set ES_INDEX_SETTINGS = salt['pillar.get']('elasticsearch:index_settings', default=ESCONFIG.elasticsearch.index_settings, merge=True) %} {%- set NODEIP = salt['pillar.get']('host:mainip', '') %} -# Set up ILM policies +{%- for index, settings in ES_INDEX_SETTINGS.items() %} + {%- if settings.policy is defined %} echo -echo "Setting up default Security Onion index lifecycle management policies..." - -# Elasticsearch logs +echo "Setting up {{ index }}-logs policy..." +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' echo -echo "Setting up Elasticsearch ILM policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/elasticsearch-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }' -echo - -# Import logs -echo -echo "Setting up Import ILM policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-import-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }' -echo - -# Kibana logs -echo -echo "Setting up Kibana ILM policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-kibana-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }' -echo - -# Kratos logs -echo -echo "Setting up Kratos ILM policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-kratos-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }' -echo - -# Logstash logs -echo -echo "Setting up Logstash ILM policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-logstash-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }' -echo - -# Redis logs -echo -echo "Setting up Redis ILM policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-redis-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }' -echo - -# Strelka logs -echo -echo "Setting up Strelka ILM policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-strelka-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }' -echo - -# Suricata logs -echo -echo "Setting up Suricata ILM policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-suricata-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }' -echo - -# Syslog logs -echo -echo "Setting up Syslog ILM policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-syslog-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }' -echo - -# Zeek logs -echo -echo "Setting up Zeek ILM policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-zeek-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }' + {%- endif %} +{%- endfor %} echo diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 3d1182255..a0c431881 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1430,6 +1430,8 @@ elasticsearch: date_detection: false settings: index: + lifecycle: + name: so-elasticsearch-logs mapping: total_fields: limit: 5000 @@ -1498,6 +1500,25 @@ elasticsearch: - common-settings - common-dynamic-mappings priority: 500 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} so-endgame: index_sorting: False index_template: @@ -2183,6 +2204,8 @@ elasticsearch: date_detection: false settings: index: + lifecycle: + name: so-suricata-logs mapping: total_fields: limit: 5000 @@ -2251,6 +2274,25 @@ elasticsearch: - common-settings - common-dynamic-mappings priority: 500 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} so-imperva: index_sorting: False index_template: @@ -2351,6 +2393,8 @@ elasticsearch: date_detection: false settings: index: + lifecycle: + name: so-import-logs mapping: total_fields: limit: 5000 @@ -2419,6 +2463,25 @@ elasticsearch: - common-dynamic-mappings - winlog-mappings priority: 500 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} so-infoblox: index_sorting: False index_template: @@ -2671,6 +2734,25 @@ elasticsearch: - common-settings - common-dynamic-mappings priority: 500 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} so-kratos: warm: 7 close: 30 @@ -2754,6 +2836,25 @@ elasticsearch: - common-settings - common-dynamic-mappings priority: 500 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} so-logstash: index_sorting: False index_template: @@ -2770,6 +2871,8 @@ elasticsearch: date_detection: false settings: index: + lifecycle: + name: so-logstash-logs mapping: total_fields: limit: 5000 @@ -2838,6 +2941,25 @@ elasticsearch: - common-settings - common-dynamic-mappings priority: 500 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} so-microsoft: index_sorting: False index_template: @@ -3691,6 +3813,8 @@ elasticsearch: date_detection: false settings: index: + lifecycle: + name: so-redis-logs mapping: total_fields: limit: 5000 @@ -3759,6 +3883,25 @@ elasticsearch: - common-settings - common-dynamic-mappings priority: 500 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} so-snort: index_sorting: False index_template: @@ -4262,6 +4405,25 @@ elasticsearch: - common-settings - common-dynamic-mappings priority: 500 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} so-syslog: index_sorting: False index_template: @@ -4347,6 +4509,25 @@ elasticsearch: - common-settings - common-dynamic-mappings priority: 500 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} so-tomcat: index_sorting: False index_template: @@ -4447,6 +4628,8 @@ elasticsearch: date_detection: false settings: index: + lifecycle: + name: so-zeek-logs mapping: total_fields: limit: 5000 @@ -4517,6 +4700,25 @@ elasticsearch: - common-settings - common-dynamic-mappings priority: 500 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} so-zscaler: index_sorting: False index_template: