Merge pull request #9764 from Security-Onion-Solutions/fix/elasticsearch_ilm_dynamic_policy_loadiing

ILM Policy Changes

salt/common/tools/sbin/so-elasticsearch-ilm-policy-load

@@ -6,68 +6,16 @@
 
 . /usr/sbin/so-common
 
+{% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %}
+{%- set ES_INDEX_SETTINGS = salt['pillar.get']('elasticsearch:index_settings', default=ESCONFIG.elasticsearch.index_settings, merge=True) %}
 {%- set NODEIP = salt['pillar.get']('host:mainip', '') %}
 
-# Set up ILM policies
+{%- for index, settings in ES_INDEX_SETTINGS.items() %}
+  {%- if settings.policy is defined %}
 echo
-echo "Setting up default Security Onion index lifecycle management policies..."
+echo "Setting up {{ index }}-logs policy..."
-
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
-# Elasticsearch logs
 echo
-echo "Setting up Elasticsearch ILM policy..."
+  {%- endif %}
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/elasticsearch-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }'
+{%- endfor %}
-echo
-
-# Import logs
-echo
-echo "Setting up Import ILM policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-import-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }'
-echo
-
-# Kibana logs
-echo
-echo "Setting up Kibana ILM policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-kibana-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }'
-echo
-
-# Kratos logs
-echo
-echo "Setting up Kratos ILM policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-kratos-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }'
-echo
-
-# Logstash logs
-echo
-echo "Setting up Logstash ILM policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-logstash-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }'
-echo
-
-# Redis logs
-echo
-echo "Setting up Redis ILM policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-redis-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }'
-echo
-
-# Strelka logs
-echo
-echo "Setting up Strelka ILM policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-strelka-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }'
-echo
-
-# Suricata logs
-echo
-echo "Setting up Suricata ILM policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-suricata-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }'
-echo
-
-# Syslog logs
-echo
-echo "Setting up Syslog ILM policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-syslog-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }'
-echo
-
-# Zeek logs
-echo
-echo "Setting up Zeek ILM policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-zeek-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }'
 echo

salt/elasticsearch/defaults.yaml

@@ -1430,6 +1430,8 @@ elasticsearch:
             date_detection: false
           settings:
             index:
+              lifecycle:
+                name: so-elasticsearch-logs
               mapping:
                 total_fields:
                   limit: 5000
@@ -1498,6 +1500,25 @@ elasticsearch:
           - common-settings
           - common-dynamic-mappings
         priority: 500
+      policy:
+        phases:
+          hot:
+            min_age: 0ms
+            actions:
+              set_priority:
+                priority: 100
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+          cold:
+            min_age: 30d
+            actions:
+              set_priority:
+                priority: 0
+          delete:
+            min_age: 365d
+            actions:
+              delete: {}
     so-endgame:
       index_sorting: False
       index_template:
@@ -2183,6 +2204,8 @@ elasticsearch:
             date_detection: false
           settings:
             index:
+              lifecycle:
+                name: so-suricata-logs
               mapping:
                 total_fields:
                   limit: 5000
@@ -2251,6 +2274,25 @@ elasticsearch:
           - common-settings
           - common-dynamic-mappings
         priority: 500
+      policy:
+        phases:
+          hot:
+            min_age: 0ms
+            actions:
+              set_priority:
+                priority: 100
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+          cold:
+            min_age: 30d
+            actions:
+              set_priority:
+                priority: 0
+          delete:
+            min_age: 365d
+            actions:
+              delete: {}
     so-imperva:
       index_sorting: False
       index_template:
@@ -2351,6 +2393,8 @@ elasticsearch:
             date_detection: false
           settings:
             index:
+              lifecycle:
+                name: so-import-logs
               mapping:
                 total_fields:
                   limit: 5000
@@ -2419,6 +2463,25 @@ elasticsearch:
           - common-dynamic-mappings
           - winlog-mappings
         priority: 500
+      policy:
+        phases:
+          hot:
+            min_age: 0ms
+            actions:
+              set_priority:
+                priority: 100
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+          cold:
+            min_age: 30d
+            actions:
+              set_priority:
+                priority: 0
+          delete:
+            min_age: 365d
+            actions:
+              delete: {}
     so-infoblox:
       index_sorting: False
       index_template:
@@ -2671,6 +2734,25 @@ elasticsearch:
           - common-settings
           - common-dynamic-mappings
         priority: 500
+      policy:
+        phases:
+          hot:
+            min_age: 0ms
+            actions:
+              set_priority:
+                priority: 100
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+          cold:
+            min_age: 30d
+            actions:
+              set_priority:
+                priority: 0
+          delete:
+            min_age: 365d
+            actions:
+              delete: {}
     so-kratos:
       warm: 7
       close: 30
@@ -2754,6 +2836,25 @@ elasticsearch:
           - common-settings
           - common-dynamic-mappings
         priority: 500
+      policy:
+        phases:
+          hot:
+            min_age: 0ms
+            actions:
+              set_priority:
+                priority: 100
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+          cold:
+            min_age: 30d
+            actions:
+              set_priority:
+                priority: 0
+          delete:
+            min_age: 365d
+            actions:
+              delete: {}
     so-logstash:
       index_sorting: False
       index_template:
@@ -2770,6 +2871,8 @@ elasticsearch:
             date_detection: false
           settings:
             index:
+              lifecycle:
+                name: so-logstash-logs
               mapping:
                 total_fields:
                   limit: 5000
@@ -2838,6 +2941,25 @@ elasticsearch:
           - common-settings
           - common-dynamic-mappings
         priority: 500
+      policy:
+        phases:
+          hot:
+            min_age: 0ms
+            actions:
+              set_priority:
+                priority: 100
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+          cold:
+            min_age: 30d
+            actions:
+              set_priority:
+                priority: 0
+          delete:
+            min_age: 365d
+            actions:
+              delete: {}
     so-microsoft:
       index_sorting: False
       index_template:
@@ -3691,6 +3813,8 @@ elasticsearch:
             date_detection: false
           settings:
             index:
+              lifecycle:
+                name: so-redis-logs
               mapping:
                 total_fields:
                   limit: 5000
@@ -3759,6 +3883,25 @@ elasticsearch:
           - common-settings
           - common-dynamic-mappings
         priority: 500
+      policy:
+        phases:
+          hot:
+            min_age: 0ms
+            actions:
+              set_priority:
+                priority: 100
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+          cold:
+            min_age: 30d
+            actions:
+              set_priority:
+                priority: 0
+          delete:
+            min_age: 365d
+            actions:
+              delete: {}
     so-snort:
       index_sorting: False
       index_template:
@@ -4262,6 +4405,25 @@ elasticsearch:
           - common-settings
           - common-dynamic-mappings
         priority: 500
+      policy:
+        phases:
+          hot:
+            min_age: 0ms
+            actions:
+              set_priority:
+                priority: 100
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+          cold:
+            min_age: 30d
+            actions:
+              set_priority:
+                priority: 0
+          delete:
+            min_age: 365d
+            actions:
+              delete: {}
     so-syslog:
       index_sorting: False
       index_template:
@@ -4347,6 +4509,25 @@ elasticsearch:
           - common-settings
           - common-dynamic-mappings
         priority: 500
+      policy:
+        phases:
+          hot:
+            min_age: 0ms
+            actions:
+              set_priority:
+                priority: 100
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+          cold:
+            min_age: 30d
+            actions:
+              set_priority:
+                priority: 0
+          delete:
+            min_age: 365d
+            actions:
+              delete: {}
     so-tomcat:
       index_sorting: False
       index_template:
@@ -4447,6 +4628,8 @@ elasticsearch:
             date_detection: false
           settings:
             index:
+              lifecycle:
+                name: so-zeek-logs
               mapping:
                 total_fields:
                   limit: 5000
@@ -4517,6 +4700,25 @@ elasticsearch:
           - common-settings
           - common-dynamic-mappings
         priority: 500
+      policy:
+        phases:
+          hot:
+            min_age: 0ms
+            actions:
+              set_priority:
+                priority: 100
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+          cold:
+            min_age: 30d
+            actions:
+              set_priority:
+                priority: 0
+          delete:
+            min_age: 365d
+            actions:
+              delete: {}
     so-zscaler:
       index_sorting: False
       index_template: