merge

salt/common/tools/sbin/so-elasticsearch-ilm-policy-load

@@ -6,68 +6,16 @@
 
 . /usr/sbin/so-common
 
+{% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %}
+{%- set ES_INDEX_SETTINGS = salt['pillar.get']('elasticsearch:index_settings', default=ESCONFIG.elasticsearch.index_settings, merge=True) %}
 {%- set NODEIP = salt['pillar.get']('host:mainip', '') %}
 
-# Set up ILM policies
+{%- for index, settings in ES_INDEX_SETTINGS.items() %}
+  {%- if settings.policy is defined %}
 echo
-echo "Setting up default Security Onion index lifecycle management policies..."
-
-# Elasticsearch logs
+echo "Setting up {{ index }}-logs policy..."
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
 echo
-echo "Setting up Elasticsearch ILM policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/elasticsearch-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }'
-echo
-
-# Import logs
-echo
-echo "Setting up Import ILM policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-import-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }'
-echo
-
-# Kibana logs
-echo
-echo "Setting up Kibana ILM policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-kibana-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }'
-echo
-
-# Kratos logs
-echo
-echo "Setting up Kratos ILM policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-kratos-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }'
-echo
-
-# Logstash logs
-echo
-echo "Setting up Logstash ILM policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-logstash-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }'
-echo
-
-# Redis logs
-echo
-echo "Setting up Redis ILM policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-redis-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }'
-echo
-
-# Strelka logs
-echo
-echo "Setting up Strelka ILM policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-strelka-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }'
-echo
-
-# Suricata logs
-echo
-echo "Setting up Suricata ILM policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-suricata-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }'
-echo
-
-# Syslog logs
-echo
-echo "Setting up Syslog ILM policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-syslog-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }'
-echo
-
-# Zeek logs
-echo
-echo "Setting up Zeek ILM policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-zeek-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }'
+  {%- endif %}
+{%- endfor %}
 echo

salt/common/tools/sbin/so-image-common

@@ -25,7 +25,6 @@ container_list() {
   if [ $MANAGERCHECK == 'so-import' ]; then
     TRUSTED_CONTAINERS=(
       "so-elasticsearch"
-      "so-filebeat"
       "so-idstools"
       "so-influxdb"
       "so-kibana"
@@ -47,7 +46,6 @@ container_list() {
       "so-elastic-agent"
       "so-elastic-agent-builder"
       "so-elasticsearch"
-      "so-filebeat"
       "so-idh"
       "so-idstools"
       "so-influxdb"
@@ -72,7 +70,6 @@ container_list() {
     )
   else
     TRUSTED_CONTAINERS=(
-      "so-filebeat"
       "so-idstools"
       "so-elasticsearch"
       "so-logstash"

salt/elasticsearch/defaults.yaml

@@ -1430,6 +1430,8 @@ elasticsearch:
             date_detection: false
           settings:
             index:
+              lifecycle:
+                name: so-elasticsearch-logs
               mapping:
                 total_fields:
                   limit: 5000
@@ -1498,6 +1500,25 @@ elasticsearch:
           - common-settings
           - common-dynamic-mappings
         priority: 500
+      policy:
+        phases:
+          hot:
+            min_age: 0ms
+            actions:
+              set_priority:
+                priority: 100
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+          cold:
+            min_age: 30d
+            actions:
+              set_priority:
+                priority: 0
+          delete:
+            min_age: 365d
+            actions:
+              delete: {}
     so-endgame:
       index_sorting: False
       index_template:
@@ -2183,6 +2204,8 @@ elasticsearch:
             date_detection: false
           settings:
             index:
+              lifecycle:
+                name: so-suricata-logs
               mapping:
                 total_fields:
                   limit: 5000
@@ -2251,6 +2274,25 @@ elasticsearch:
           - common-settings
           - common-dynamic-mappings
         priority: 500
+      policy:
+        phases:
+          hot:
+            min_age: 0ms
+            actions:
+              set_priority:
+                priority: 100
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+          cold:
+            min_age: 30d
+            actions:
+              set_priority:
+                priority: 0
+          delete:
+            min_age: 365d
+            actions:
+              delete: {}
     so-imperva:
       index_sorting: False
       index_template:
@@ -2351,6 +2393,8 @@ elasticsearch:
             date_detection: false
           settings:
             index:
+              lifecycle:
+                name: so-import-logs
               mapping:
                 total_fields:
                   limit: 5000
@@ -2419,6 +2463,25 @@ elasticsearch:
           - common-dynamic-mappings
           - winlog-mappings
         priority: 500
+      policy:
+        phases:
+          hot:
+            min_age: 0ms
+            actions:
+              set_priority:
+                priority: 100
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+          cold:
+            min_age: 30d
+            actions:
+              set_priority:
+                priority: 0
+          delete:
+            min_age: 365d
+            actions:
+              delete: {}
     so-infoblox:
       index_sorting: False
       index_template:
@@ -2671,6 +2734,25 @@ elasticsearch:
           - common-settings
           - common-dynamic-mappings
         priority: 500
+      policy:
+        phases:
+          hot:
+            min_age: 0ms
+            actions:
+              set_priority:
+                priority: 100
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+          cold:
+            min_age: 30d
+            actions:
+              set_priority:
+                priority: 0
+          delete:
+            min_age: 365d
+            actions:
+              delete: {}
     so-kratos:
       warm: 7
       close: 30
@@ -2754,6 +2836,25 @@ elasticsearch:
           - common-settings
           - common-dynamic-mappings
         priority: 500
+      policy:
+        phases:
+          hot:
+            min_age: 0ms
+            actions:
+              set_priority:
+                priority: 100
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+          cold:
+            min_age: 30d
+            actions:
+              set_priority:
+                priority: 0
+          delete:
+            min_age: 365d
+            actions:
+              delete: {}
     so-logstash:
       index_sorting: False
       index_template:
@@ -2770,6 +2871,8 @@ elasticsearch:
             date_detection: false
           settings:
             index:
+              lifecycle:
+                name: so-logstash-logs
               mapping:
                 total_fields:
                   limit: 5000
@@ -2838,6 +2941,25 @@ elasticsearch:
           - common-settings
           - common-dynamic-mappings
         priority: 500
+      policy:
+        phases:
+          hot:
+            min_age: 0ms
+            actions:
+              set_priority:
+                priority: 100
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+          cold:
+            min_age: 30d
+            actions:
+              set_priority:
+                priority: 0
+          delete:
+            min_age: 365d
+            actions:
+              delete: {}
     so-microsoft:
       index_sorting: False
       index_template:
@@ -3691,6 +3813,8 @@ elasticsearch:
             date_detection: false
           settings:
             index:
+              lifecycle:
+                name: so-redis-logs
               mapping:
                 total_fields:
                   limit: 5000
@@ -3759,6 +3883,25 @@ elasticsearch:
           - common-settings
           - common-dynamic-mappings
         priority: 500
+      policy:
+        phases:
+          hot:
+            min_age: 0ms
+            actions:
+              set_priority:
+                priority: 100
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+          cold:
+            min_age: 30d
+            actions:
+              set_priority:
+                priority: 0
+          delete:
+            min_age: 365d
+            actions:
+              delete: {}
     so-snort:
       index_sorting: False
       index_template:
@@ -4262,6 +4405,25 @@ elasticsearch:
           - common-settings
           - common-dynamic-mappings
         priority: 500
+      policy:
+        phases:
+          hot:
+            min_age: 0ms
+            actions:
+              set_priority:
+                priority: 100
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+          cold:
+            min_age: 30d
+            actions:
+              set_priority:
+                priority: 0
+          delete:
+            min_age: 365d
+            actions:
+              delete: {}
     so-syslog:
       index_sorting: False
       index_template:
@@ -4347,6 +4509,25 @@ elasticsearch:
           - common-settings
           - common-dynamic-mappings
         priority: 500
+      policy:
+        phases:
+          hot:
+            min_age: 0ms
+            actions:
+              set_priority:
+                priority: 100
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+          cold:
+            min_age: 30d
+            actions:
+              set_priority:
+                priority: 0
+          delete:
+            min_age: 365d
+            actions:
+              delete: {}
     so-tomcat:
       index_sorting: False
       index_template:
@@ -4447,6 +4628,8 @@ elasticsearch:
             date_detection: false
           settings:
             index:
+              lifecycle:
+                name: so-zeek-logs
               mapping:
                 total_fields:
                   limit: 5000
@@ -4517,6 +4700,25 @@ elasticsearch:
           - common-settings
           - common-dynamic-mappings
         priority: 500
+      policy:
+        phases:
+          hot:
+            min_age: 0ms
+            actions:
+              set_priority:
+                priority: 100
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+          cold:
+            min_age: 30d
+            actions:
+              set_priority:
+                priority: 0
+          delete:
+            min_age: 365d
+            actions:
+              delete: {}
     so-zscaler:
       index_sorting: False
       index_template:

salt/elasticsearch/init.sls

@@ -53,9 +53,19 @@ es_sync_scripts:
     - source: salt://elasticsearch/tools/sbin
     - exclude_pat:
         - so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state
+        - so-elasticsearch-ilm-policy-load 
     - defaults:
         GLOBALS: {{ GLOBALS }}
 
+so-elasticsearch-ilm-policy-load-script:
+  file.managed:
+    - name: /usr/sbin/so-elasticsearch-ilm-policy-load
+    - source: salt://elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-load
+    - user: 930
+    - group: 939
+    - mode: 754
+    - template: jinja
+
 so-elasticsearch-pipelines-script:
   file.managed:
     - name: /usr/sbin/so-elasticsearch-pipelines
@@ -362,6 +372,16 @@ so-es-cluster-settings:
       - docker_container: so-elasticsearch
       - file: es_sync_scripts
 
+so-elasticsearch-ilm-policy-load:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-ilm-policy-load
+    - cwd: /opt/so
+    - require:
+      - docker_container: so-elasticsearch
+      - file: so-elasticsearch-ilm-policy-load-script
+    - onchanges:
+      - file: so-elasticsearch-ilm-policy-load-script
+
 so-elasticsearch-templates:
   cmd.run:
     - name: /usr/sbin/so-elasticsearch-templates-load

salt/elasticsearch/soc_elasticsearch.yaml

@@ -36,7 +36,7 @@ elasticsearch:
             global: True
             helpLink: elasticsearch.html
   index_settings: 
-    so-aws: &indexSettings
+    so-elasticsearch: &indexSettings
       warm: 
         description: Age (in days) of this index before it will move to warm storage, if warm nodes are present. Once moved, events on this index can take longer to fetch.
         global: True
@@ -75,45 +75,51 @@ elasticsearch:
                   description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs.
                   global: True
                   helpLink: elasticsearch.html
-    so-azure: *indexSettings
-    so-barracuda: *indexSettings
-    so-beats: *indexSettings
-    so-bluecoat: *indexSettings
-    so-cef: *indexSettings
-    so-checkpoint: *indexSettings
-    so-cisco: *indexSettings
-    so-cyberark: *indexSettings
-    so-cylance: *indexSettings
-    so-elasticsearch: *indexSettings
+      policy:
+        phases:
+          hot:
+            min_age:
+              description: Minimum age
+              global: True
+              helpLink: elasticsearch.html
+            actions:
+              set_priority:
+                priority:
+                  description: Priority of index, used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
+                  global: True
+                  helpLink: elasticsearch.html
+              rollover:
+                max_age:
+                  description: Maximum age of index.  Once an index reaches this limit, it will be rolled over into a new index.
+                  global: True
+                  helpLink: elasticsearch.html
+                max_primary_shard_size:
+                  description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index.
+                  global: True
+                  helpLink: elasticsearch.html
+          cold:
+            min_age:
+              description: Minimum age of index, determining when it should be sent to the cold tier.  While still searchable, this tier is typically optimized for lower storage costs rather than search speed.
+              global: True
+              helpLink: elasticsearch.html
+            actions:
+              set_priority:
+                priority:
+                  description: Used for index recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
+                  global: True
+                  helpLink: elasticsearch.html
+          delete:
+            min_age:
+              description: Minimum age of index, determining when it should be deleted.
+              global: True
+              helpLink: elastic
     so-endgame: *indexSettings
-    so-f5: *indexSettings
     so-firewall: *indexSettings
-    so-fortinet: *indexSettings
-    so-gcp: *indexSettings
-    so-google_workspace: *indexSettings
-    so-ids: *indexSettings
-    so-imperva: *indexSettings
     so-import: *indexSettings
-    so-infoblox: *indexSettings
-    so-juniper: *indexSettings
     so-kibana: *indexSettings
     so-logstash: *indexSettings
-    so-microsoft: *indexSettings
-    so-misp: *indexSettings
-    so-netflow: *indexSettings
-    so-netscout: *indexSettings
-    so-o365: *indexSettings
-    so-okta: *indexSettings
     so-osquery: *indexSettings
-    so-proofpoint: *indexSettings
-    so-radware: *indexSettings
     so-redis: *indexSettings
-    so-snort: *indexSettings
-    so-snyk: *indexSettings
-    so-sonicwall: *indexSettings
-    so-sophos: *indexSettings
     so-strelka: *indexSettings
     so-syslog: *indexSettings
-    so-tomcat: *indexSettings
     so-zeek: *indexSettings
-    so-zscaler: *indexSettings

salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-load

@@ -0,0 +1,21 @@
+#/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-common
+
+{% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %}
+{%- set ES_INDEX_SETTINGS = salt['pillar.get']('elasticsearch:index_settings', default=ESCONFIG.elasticsearch.index_settings, merge=True) %}
+{%- set NODEIP = salt['pillar.get']('host:mainip', '') %}
+
+{%- for index, settings in ES_INDEX_SETTINGS.items() %}
+  {%- if settings.policy is defined %}
+echo
+echo "Setting up {{ index }}-logs policy..."
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
+echo
+  {%- endif %}
+{%- endfor %}
+echo