update index template priorities + explicity add datastream config options

salt/common/tools/sbin/so-common

@@ -550,6 +550,22 @@ retry() {
         return $exitcode
 }
 
+rollover_index() {
+  idx=$1
+  exists=$(so-elasticsearch-query $idx -o /dev/null -w "%{http_code}")
+  if [[ $exists -eq 200 ]]; then
+    rollover=$(so-elasticsearch-query $idx/_rollover -o /dev/null -w "%{http_code}" -XPOST)
+
+    if [[ $rollover -eq 200 ]]; then
+      echo "Successfully triggered rollover for $idx..."
+    else
+      echo "Could not trigger rollover for $idx..."
+    fi
+  else
+    echo "Could not find index $idx..."
+  fi
+}
+
 run_check_net_err() {
 	local cmd=$1
 	local err_msg=${2:-"Unknown error occured, please check /root/$WHATWOULDYOUSAYYAHDOHERE.log for details."} # Really need to rename that variable

salt/elasticsearch/defaults.yaml

@@ -117,7 +117,7 @@ elasticsearch:
         ignore_missing_component_templates: []
         index_patterns:
         - so-case*
-        priority: 500
+        priority: 501
         template:
           mappings:
             date_detection: false
@@ -129,8 +129,6 @@ elasticsearch:
                 match_mapping_type: string
           settings:
             index:
-              lifecycle:
-                name: so-case-logs
               mapping:
                 total_fields:
                   limit: 1500
@@ -141,14 +139,7 @@ elasticsearch:
               sort:
                 field: '@timestamp'
                 order: desc
-      policy:
-        phases:
-          hot:
-            actions: {}
-            min_age: 0ms
     so-common:
-      close: 30
-      delete: 365
       index_sorting: false
       index_template:
         composed_of:
@@ -212,7 +203,9 @@ elasticsearch:
         - common-settings
         - common-dynamic-mappings
         - winlog-mappings
-        data_stream: {}
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
         ignore_missing_component_templates: []
         index_patterns:
         - logs-*-so*
@@ -272,7 +265,7 @@ elasticsearch:
         ignore_missing_component_templates: []
         index_patterns:
         - so-detection*
-        priority: 500
+        priority: 501
         template:
           mappings:
             date_detection: false
@@ -284,8 +277,6 @@ elasticsearch:
                 match_mapping_type: string
           settings:
             index:
-              lifecycle:
-                name: so-detection-logs
               mapping:
                 total_fields:
                   limit: 1500
@@ -296,11 +287,6 @@ elasticsearch:
               sort:
                 field: '@timestamp'
                 order: desc
-      policy:
-        phases:
-          hot:
-            actions: {}
-            min_age: 0ms
     sos-backup:
       index_sorting: false
       index_template:
@@ -460,7 +446,7 @@ elasticsearch:
         ignore_missing_component_templates: []
         index_patterns:
         - endgame*
-        priority: 500
+        priority: 501
         template:
           mappings:
             date_detection: false
@@ -508,8 +494,6 @@ elasticsearch:
                 priority: 50
             min_age: 30d
     so-idh:
-      close: 30
-      delete: 365
       index_sorting: false
       index_template:
         composed_of:
@@ -566,8 +550,8 @@ elasticsearch:
         - common-dynamic-mappings
         ignore_missing_component_templates: []
         index_patterns:
-        - so-idh-*
-        priority: 500
+        - logs-idh-so*
+        priority: 501
         template:
           mappings:
             date_detection: false
@@ -677,11 +661,13 @@ elasticsearch:
         - common-dynamic-mappings
         - winlog-mappings
         - hash-mappings
-        data_stream: {}
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
         ignore_missing_component_templates: []
         index_patterns:
         - logs-import-so*
-        priority: 500
+        priority: 501
         template:
           mappings:
             date_detection: false
@@ -736,7 +722,7 @@ elasticsearch:
         ignore_missing_component_templates: []
         index_patterns:
         - so-ip*
-        priority: 500
+        priority: 501
         template:
           mappings:
             date_detection: false
@@ -751,19 +737,12 @@ elasticsearch:
               mapping:
                 total_fields:
                   limit: 1500
-              lifecycle:
-                name: so-ip-mappings-logs
               number_of_replicas: 0
               number_of_shards: 1
               refresh_interval: 30s
               sort:
                 field: '@timestamp'
                 order: desc
-      policy:
-        phases:
-          hot:
-            actions: {}
-            min_age: 0ms
     so-items:
       index_sorting: false
       index_template:
@@ -772,7 +751,7 @@ elasticsearch:
         ignore_missing_component_templates: []
         index_patterns:
         - .items-default-**
-        priority: 500
+        priority: 501
         template:
           mappings:
             date_detection: false
@@ -851,8 +830,6 @@ elasticsearch:
                 priority: 50
             min_age: 30d
     so-kratos:
-      close: 30
-      delete: 365
       index_sorting: false
       index_template:
         composed_of:
@@ -873,7 +850,7 @@ elasticsearch:
         ignore_missing_component_templates: []
         index_patterns:
         - logs-kratos-so*
-        priority: 500
+        priority: 501
         template:
           mappings:
             date_detection: false
@@ -921,8 +898,6 @@ elasticsearch:
                 priority: 50
             min_age: 30d
     so-hydra:
-      close: 30
-      delete: 365
       index_sorting: false
       index_template:
         composed_of:
@@ -983,7 +958,7 @@ elasticsearch:
         ignore_missing_component_templates: []
         index_patterns:
         - logs-hydra-so*
-        priority: 500
+        priority: 501
         template:
           mappings:
             date_detection: false
@@ -1038,7 +1013,7 @@ elasticsearch:
         ignore_missing_component_templates: []
         index_patterns:
         - .lists-default-**
-        priority: 500
+        priority: 501
         template:
           mappings:
             date_detection: false
@@ -1524,6 +1499,9 @@ elasticsearch:
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
         - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
         ignore_missing_component_templates:
         - logs-elastic_agent.cloudbeat@custom
         index_patterns:
@@ -1759,6 +1737,9 @@ elasticsearch:
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
         - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
         ignore_missing_component_templates:
         - logs-elastic_agent.heartbeat@custom
         index_patterns:
@@ -3018,8 +2999,6 @@ elasticsearch:
                 priority: 50
             min_age: 30d
     so-logs-soc:
-      close: 30
-      delete: 365
       index_sorting: false
       index_template:
         composed_of:
@@ -3074,11 +3053,13 @@ elasticsearch:
         - dtc-user_agent-mappings
         - common-settings
         - common-dynamic-mappings
-        data_stream: {}
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
         ignore_missing_component_templates: []
         index_patterns:
         - logs-soc-so*
-        priority: 500
+        priority: 501
         template:
           mappings:
             date_detection: false
@@ -3668,10 +3649,13 @@ elasticsearch:
         - vulnerability-mappings
         - common-settings
         - common-dynamic-mappings
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
         ignore_missing_component_templates: []
         index_patterns:
         - logs-logstash-default*
-        priority: 500
+        priority: 501
         template:
           mappings:
             date_detection: false
@@ -3971,8 +3955,8 @@ elasticsearch:
         - common-dynamic-mappings
         ignore_missing_component_templates: []
         index_patterns:
-        - logs-redis-default*
-        priority: 500
+        - logs-redis.log*
+        priority: 501
         template:
           mappings:
             date_detection: false
@@ -4083,11 +4067,13 @@ elasticsearch:
         - common-settings
         - common-dynamic-mappings
         - hash-mappings
-        data_stream: {}
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
         ignore_missing_component_templates: []
         index_patterns:
         - logs-strelka-so*
-        priority: 500
+        priority: 501
         template:
           mappings:
             date_detection: false
@@ -4197,11 +4183,13 @@ elasticsearch:
         - common-settings
         - common-dynamic-mappings
         - hash-mappings
-        data_stream: {}
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
         ignore_missing_component_templates: []
         index_patterns:
         - logs-suricata-so*
-        priority: 500
+        priority: 501
         template:
           mappings:
             date_detection: false
@@ -4311,11 +4299,13 @@ elasticsearch:
         - common-settings
         - common-dynamic-mappings
         - hash-mappings
-        data_stream: {}
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
         ignore_missing_component_templates: []
         index_patterns:
         - logs-suricata.alerts-*
-        priority: 500
+        priority: 501
         template:
           mappings:
             date_detection: false
@@ -4425,11 +4415,13 @@ elasticsearch:
         - vulnerability-mappings
         - common-settings
         - common-dynamic-mappings
-        data_stream: {}
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
         ignore_missing_component_templates: []
         index_patterns:
         - logs-syslog-so*
-        priority: 500
+        priority: 501
         template:
           mappings:
             date_detection: false
@@ -4541,11 +4533,13 @@ elasticsearch:
         - common-settings
         - common-dynamic-mappings
         - hash-mappings
-        data_stream: {}
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
         ignore_missing_component_templates: []
         index_patterns:
         - logs-zeek-so*
-        priority: 500
+        priority: 501
         template:
           mappings:
             date_detection: false

salt/manager/tools/sbin/soup

@@ -403,7 +403,15 @@ migrate_pcap_to_suricata() {
 }
 
 post_to_3.0.0() {
-  echo "Nothing to apply"
+  for idx in "logs-idh-so" "logs-redis.log-default"; do
+    rollover_index "$idx"
+  done
+
+  # Remove ILM for so-case and so-detection indices
+  for idx in "so-case" "so-casehistory" "so-detection" "so-detectionhistory"; do
+    so-elasticsearch-query $idx/_ilm/remove -XPOST
+  done
+
   POSTVERSION=3.0.0
 }