From 88841c981427d39b2b949fb089f97a96a8801c4c Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 16 Mar 2026 16:52:42 -0500 Subject: [PATCH 1/5] remove ilm configs from non-datastream indices --- salt/elasticsearch/defaults.yaml | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index c9f771a91..c9c87fcde 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -129,8 +129,6 @@ elasticsearch: match_mapping_type: string settings: index: - lifecycle: - name: so-case-logs mapping: total_fields: limit: 1500 @@ -141,11 +139,6 @@ elasticsearch: sort: field: '@timestamp' order: desc - policy: - phases: - hot: - actions: {} - min_age: 0ms so-common: close: 30 delete: 365 @@ -284,8 +277,6 @@ elasticsearch: match_mapping_type: string settings: index: - lifecycle: - name: so-detection-logs mapping: total_fields: limit: 1500 @@ -296,11 +287,6 @@ elasticsearch: sort: field: '@timestamp' order: desc - policy: - phases: - hot: - actions: {} - min_age: 0ms sos-backup: index_sorting: false index_template: @@ -751,19 +737,12 @@ elasticsearch: mapping: total_fields: limit: 1500 - lifecycle: - name: so-ip-mappings-logs number_of_replicas: 0 number_of_shards: 1 refresh_interval: 30s sort: field: '@timestamp' order: desc - policy: - phases: - hot: - actions: {} - min_age: 0ms so-items: index_sorting: false index_template: From 84f4e460f67b36c70c6f2cbf3942760cfffbb047 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 16 Mar 2026 16:53:22 -0500 Subject: [PATCH 2/5] update index patterns --- salt/elasticsearch/defaults.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index c9c87fcde..42ff8a658 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -552,7 +552,7 @@ elasticsearch: - common-dynamic-mappings ignore_missing_component_templates: [] index_patterns: - - so-idh-* + - logs-idh-so* priority: 500 template: mappings: @@ -3950,7 +3950,7 @@ elasticsearch: - common-dynamic-mappings ignore_missing_component_templates: [] index_patterns: - - logs-redis-default* + - logs-redis.log* priority: 500 template: mappings: From eaf3f10adc24d3dd344bddd4e2d914b7eed0d06a Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 16 Mar 2026 17:26:45 -0500 Subject: [PATCH 3/5] remove unused close/delete configs on datastream index templates --- salt/elasticsearch/defaults.yaml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 42ff8a658..7efaea0e5 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -140,8 +140,6 @@ elasticsearch: field: '@timestamp' order: desc so-common: - close: 30 - delete: 365 index_sorting: false index_template: composed_of: @@ -494,8 +492,6 @@ elasticsearch: priority: 50 min_age: 30d so-idh: - close: 30 - delete: 365 index_sorting: false index_template: composed_of: @@ -830,8 +826,6 @@ elasticsearch: priority: 50 min_age: 30d so-kratos: - close: 30 - delete: 365 index_sorting: false index_template: composed_of: @@ -900,8 +894,6 @@ elasticsearch: priority: 50 min_age: 30d so-hydra: - close: 30 - delete: 365 index_sorting: false index_template: composed_of: @@ -2997,8 +2989,6 @@ elasticsearch: priority: 50 min_age: 30d so-logs-soc: - close: 30 - delete: 365 index_sorting: false index_template: composed_of: From 1a943aefc517bd65cffed44149c1f82c876c77d1 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 17 Mar 2026 13:49:20 -0500 Subject: [PATCH 4/5] rollover datastreams to get latest index templates + remove existing ilm policies from so-case / so-detection indices --- salt/common/tools/sbin/so-common | 16 ++++++++++++++++ salt/manager/tools/sbin/soup | 10 +++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 4bb3e21d5..122a85a0d 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -550,6 +550,22 @@ retry() { return $exitcode } +rollover_index() { + idx=$1 + exists=$(so-elasticsearch-query $idx -o /dev/null -w "%{http_code}") + if [[ $exists -eq 200 ]]; then + rollover=$(so-elasticsearch-query $idx/_rollover -o /dev/null -w "%{http_code}" -XPOST) + + if [[ $rollover -eq 200 ]]; then + echo "Successfully triggered rollover for $idx..." + else + echo "Could not trigger rollover for $idx..." + fi + else + echo "Could not find index $idx..." + fi +} + run_check_net_err() { local cmd=$1 local err_msg=${2:-"Unknown error occured, please check /root/$WHATWOULDYOUSAYYAHDOHERE.log for details."} # Really need to rename that variable diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index b0e2632a0..48661afc7 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -403,7 +403,15 @@ migrate_pcap_to_suricata() { } post_to_3.0.0() { - echo "Nothing to apply" + for idx in "logs-idh-so" "logs-redis.log-default"; do + rollover_index "$idx" + done + + # Remove ILM for so-case and so-detection indices + for idx in "so-case" "so-casehistory" "so-detection" "so-detectionhistory"; do + so-elasticsearch-query $idx/_ilm/remove -XPOST + done + POSTVERSION=3.0.0 } From a52e5d04745ce5b0406baf3d64a6571fe1a3bd6d Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 17 Mar 2026 13:50:15 -0500 Subject: [PATCH 5/5] update index template priorities + explicity add datastream config options --- salt/elasticsearch/defaults.yaml | 77 +++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 26 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 7efaea0e5..a4952933d 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -117,7 +117,7 @@ elasticsearch: ignore_missing_component_templates: [] index_patterns: - so-case* - priority: 500 + priority: 501 template: mappings: date_detection: false @@ -203,7 +203,9 @@ elasticsearch: - common-settings - common-dynamic-mappings - winlog-mappings - data_stream: {} + data_stream: + allow_custom_routing: false + hidden: false ignore_missing_component_templates: [] index_patterns: - logs-*-so* @@ -263,7 +265,7 @@ elasticsearch: ignore_missing_component_templates: [] index_patterns: - so-detection* - priority: 500 + priority: 501 template: mappings: date_detection: false @@ -444,7 +446,7 @@ elasticsearch: ignore_missing_component_templates: [] index_patterns: - endgame* - priority: 500 + priority: 501 template: mappings: date_detection: false @@ -549,7 +551,7 @@ elasticsearch: ignore_missing_component_templates: [] index_patterns: - logs-idh-so* - priority: 500 + priority: 501 template: mappings: date_detection: false @@ -659,11 +661,13 @@ elasticsearch: - common-dynamic-mappings - winlog-mappings - hash-mappings - data_stream: {} + data_stream: + allow_custom_routing: false + hidden: false ignore_missing_component_templates: [] index_patterns: - logs-import-so* - priority: 500 + priority: 501 template: mappings: date_detection: false @@ -718,7 +722,7 @@ elasticsearch: ignore_missing_component_templates: [] index_patterns: - so-ip* - priority: 500 + priority: 501 template: mappings: date_detection: false @@ -747,7 +751,7 @@ elasticsearch: ignore_missing_component_templates: [] index_patterns: - .items-default-** - priority: 500 + priority: 501 template: mappings: date_detection: false @@ -846,7 +850,7 @@ elasticsearch: ignore_missing_component_templates: [] index_patterns: - logs-kratos-so* - priority: 500 + priority: 501 template: mappings: date_detection: false @@ -954,7 +958,7 @@ elasticsearch: ignore_missing_component_templates: [] index_patterns: - logs-hydra-so* - priority: 500 + priority: 501 template: mappings: date_detection: false @@ -1009,7 +1013,7 @@ elasticsearch: ignore_missing_component_templates: [] index_patterns: - .lists-default-** - priority: 500 + priority: 501 template: mappings: date_detection: false @@ -1495,6 +1499,9 @@ elasticsearch: - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false ignore_missing_component_templates: - logs-elastic_agent.cloudbeat@custom index_patterns: @@ -1730,6 +1737,9 @@ elasticsearch: - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false ignore_missing_component_templates: - logs-elastic_agent.heartbeat@custom index_patterns: @@ -3043,11 +3053,13 @@ elasticsearch: - dtc-user_agent-mappings - common-settings - common-dynamic-mappings - data_stream: {} + data_stream: + allow_custom_routing: false + hidden: false ignore_missing_component_templates: [] index_patterns: - logs-soc-so* - priority: 500 + priority: 501 template: mappings: date_detection: false @@ -3637,10 +3649,13 @@ elasticsearch: - vulnerability-mappings - common-settings - common-dynamic-mappings + data_stream: + allow_custom_routing: false + hidden: false ignore_missing_component_templates: [] index_patterns: - logs-logstash-default* - priority: 500 + priority: 501 template: mappings: date_detection: false @@ -3941,7 +3956,7 @@ elasticsearch: ignore_missing_component_templates: [] index_patterns: - logs-redis.log* - priority: 500 + priority: 501 template: mappings: date_detection: false @@ -4052,11 +4067,13 @@ elasticsearch: - common-settings - common-dynamic-mappings - hash-mappings - data_stream: {} + data_stream: + allow_custom_routing: false + hidden: false ignore_missing_component_templates: [] index_patterns: - logs-strelka-so* - priority: 500 + priority: 501 template: mappings: date_detection: false @@ -4166,11 +4183,13 @@ elasticsearch: - common-settings - common-dynamic-mappings - hash-mappings - data_stream: {} + data_stream: + allow_custom_routing: false + hidden: false ignore_missing_component_templates: [] index_patterns: - logs-suricata-so* - priority: 500 + priority: 501 template: mappings: date_detection: false @@ -4280,11 +4299,13 @@ elasticsearch: - common-settings - common-dynamic-mappings - hash-mappings - data_stream: {} + data_stream: + allow_custom_routing: false + hidden: false ignore_missing_component_templates: [] index_patterns: - logs-suricata.alerts-* - priority: 500 + priority: 501 template: mappings: date_detection: false @@ -4394,11 +4415,13 @@ elasticsearch: - vulnerability-mappings - common-settings - common-dynamic-mappings - data_stream: {} + data_stream: + allow_custom_routing: false + hidden: false ignore_missing_component_templates: [] index_patterns: - logs-syslog-so* - priority: 500 + priority: 501 template: mappings: date_detection: false @@ -4510,11 +4533,13 @@ elasticsearch: - common-settings - common-dynamic-mappings - hash-mappings - data_stream: {} + data_stream: + allow_custom_routing: false + hidden: false ignore_missing_component_templates: [] index_patterns: - logs-zeek-so* - priority: 500 + priority: 501 template: mappings: date_detection: false