From 6dbe0645e5007d23b21747eefad91354096eee8c Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Mon, 11 Nov 2024 13:51:48 -0600
Subject: [PATCH 1/6] use auto_expand_replica, configure ilm for so-case* &
 so-detection*

Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com>
---
 salt/elasticsearch/defaults.yaml | 16 ++++++++++++++++
 salt/manager/tools/sbin/soup     | 11 ++++++++++-
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml
index 133c333e1..84e1731d5 100644
--- a/salt/elasticsearch/defaults.yaml
+++ b/salt/elasticsearch/defaults.yaml
@@ -111,15 +111,23 @@ elasticsearch:
                 match_mapping_type: string
           settings:
             index:
+              lifecycle:
+                name: so-case-logs
               mapping:
                 total_fields:
                   limit: 1500
               number_of_replicas: 0
+              auto_expand_replicas: 0-2
               number_of_shards: 1
               refresh_interval: 30s
               sort:
                 field: '@timestamp'
                 order: desc
+      policy:
+        phases:
+          hot:
+            actions: {}
+            min_age: 0ms
     so-common:
       close: 30
       delete: 365
@@ -258,15 +266,23 @@ elasticsearch:
                 match_mapping_type: string
           settings:
             index:
+              lifecycle:
+                name: so-detection-logs
               mapping:
                 total_fields:
                   limit: 1500
               number_of_replicas: 0
+              auto_expand_replicas: 0-2
               number_of_shards: 1
               refresh_interval: 30s
               sort:
                 field: '@timestamp'
                 order: desc
+      policy:
+        phases:
+          hot:
+            actions: {}
+            min_age: 0ms
     so-endgame:
       index_sorting: false
       index_template:
diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index bd2db98d7..6fa4c44ab 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -520,7 +520,16 @@ post_to_2.4.110() {
 }
 
 post_to_2.4.120() {
-  echo "Nothing to apply"
+  for idx in "so-detection" "so-detectionhistory" "so-case" "so-casehistory"; do
+    JSON_STRING=$( jq -n \
+                  --arg INDEX_NAME "$idx" \
+                  '{"settings": {"index.auto_expand_replicas":"0-2","index.lifecycle.name":($INDEX_NAME) + "-logs"}}'
+                )
+    echo "Updating $idx index settings"
+    retry 5 15 "so-elasticsearch-query $idx/_settings -d "$JSON_STRING" -XPUT| grep '{\"acknowledged\":true}'"
+    echo ""
+  done
+
   POSTVERSION=2.4.120
 }
 

From 43f7989d731dcb0e542a6c68eb6baeb4b3fb037b Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Mon, 11 Nov 2024 14:47:17 -0600
Subject: [PATCH 2/6] ()

Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com>
---
 salt/manager/tools/sbin/soup | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 6fa4c44ab..df2e83f5d 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -523,7 +523,7 @@ post_to_2.4.120() {
   for idx in "so-detection" "so-detectionhistory" "so-case" "so-casehistory"; do
     JSON_STRING=$( jq -n \
                   --arg INDEX_NAME "$idx" \
-                  '{"settings": {"index.auto_expand_replicas":"0-2","index.lifecycle.name":($INDEX_NAME) + "-logs"}}'
+                  '{"settings": {"index.auto_expand_replicas":"0-2","index.lifecycle.name":($INDEX_NAME + "-logs")}}'
                 )
     echo "Updating $idx index settings"
     retry 5 15 "so-elasticsearch-query $idx/_settings -d "$JSON_STRING" -XPUT| grep '{\"acknowledged\":true}'"

From ce9bd18947745b337da3898dd70dc4e2022c3a3a Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Mon, 11 Nov 2024 14:59:42 -0600
Subject: [PATCH 3/6] no error when versionlock dir exists after re-running
 soup

Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com>
---
 salt/manager/tools/sbin/soup | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index df2e83f5d..d5aa26bae 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -725,7 +725,7 @@ up_to_2.4.110() {
 
 up_to_2.4.120() {
   # this is needed for the new versionlock state
-  mkdir /opt/so/saltstack/local/pillar/versionlock
+  mkdir -p /opt/so/saltstack/local/pillar/versionlock
   touch /opt/so/saltstack/local/pillar/versionlock/adv_versionlock.sls /opt/so/saltstack/local/pillar/versionlock/soc_versionlock.sls
 
   # New Grid Integration added this release

From aaf9f536956b1206a4f51ebb0cac06bf19f77588 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Mon, 11 Nov 2024 22:40:06 -0600
Subject: [PATCH 4/6] update soup; check for index before applying new index
 setting

Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com>
---
 salt/manager/tools/sbin/soup | 28 ++++++++++++++++++----------
 1 file changed, 18 insertions(+), 10 deletions(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index d5aa26bae..98a7f9402 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -520,16 +520,7 @@ post_to_2.4.110() {
 }
 
 post_to_2.4.120() {
-  for idx in "so-detection" "so-detectionhistory" "so-case" "so-casehistory"; do
-    JSON_STRING=$( jq -n \
-                  --arg INDEX_NAME "$idx" \
-                  '{"settings": {"index.auto_expand_replicas":"0-2","index.lifecycle.name":($INDEX_NAME + "-logs")}}'
-                )
-    echo "Updating $idx index settings"
-    retry 5 15 "so-elasticsearch-query $idx/_settings -d "$JSON_STRING" -XPUT| grep '{\"acknowledged\":true}'"
-    echo ""
-  done
-
+  update_elasticsearch_index_settings
   POSTVERSION=2.4.120
 }
 
@@ -947,6 +938,23 @@ update_airgap_repo() {
   createrepo /nsm/repo
 }
 
+update_elasticsearch_index_settings() {
+  # Update managed indices to reflect latest index template
+  for idx in "so-detection" "so-detectionhistory" "so-case" "so-casehistory"; do
+    JSON_STRING=$( jq -n --arg INDEX_NAME "$idx" '{"settings": {"index.auto_expand_replicas":"0-2","index.lifecycle.name":($INDEX_NAME + "-logs")}}')
+    echo "Checking if index \"$idx\" exists"
+    exists=$(curl -K /opt/so/conf/elasticsearch/curl.config -s -o /dev/null -w "%{http_code}" -k -L -H "Content-Type: application/json" "https://localhost:9200/$idx")
+    if [ $exists -eq 200 ]; then
+      echo "$idx index found..."
+      echo "Updating $idx index settings"
+      so-elasticsearch-query $idx/_settings -d "$JSON_STRING" -XPUT
+      echo -e "\n"
+    else
+      echo -e "Skipping $idx... index does not exist\n"
+    fi
+  done
+}
+
 update_salt_mine() {
     echo "Populating the mine with mine_functions for each host."
     set +e

From a4d763c1e571152d82b0e4c48a05d7f7a316932a Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Tue, 12 Nov 2024 14:50:04 -0600
Subject: [PATCH 5/6] use curl vs es query to force PUT request

Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com>
---
 salt/manager/tools/sbin/soup | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 98a7f9402..3a1e60fd2 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -947,7 +947,7 @@ update_elasticsearch_index_settings() {
     if [ $exists -eq 200 ]; then
       echo "$idx index found..."
       echo "Updating $idx index settings"
-      so-elasticsearch-query $idx/_settings -d "$JSON_STRING" -XPUT
+      curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -H "Content-Type: application/json" "https://localhost:9200/$idx/_settings" -d "$JSON_STRING" -XPUT
       echo -e "\n"
     else
       echo -e "Skipping $idx... index does not exist\n"

From f3a88de0c393568aed54ef361ae3de9cac685805 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Tue, 12 Nov 2024 16:28:01 -0600
Subject: [PATCH 6/6] so-(case/detection)history uses same ilm policy as
 so-(case/detection)

Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com>
---
 salt/manager/tools/sbin/soup | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 3a1e60fd2..96d955422 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -941,7 +941,14 @@ update_airgap_repo() {
 update_elasticsearch_index_settings() {
   # Update managed indices to reflect latest index template
   for idx in "so-detection" "so-detectionhistory" "so-case" "so-casehistory"; do
-    JSON_STRING=$( jq -n --arg INDEX_NAME "$idx" '{"settings": {"index.auto_expand_replicas":"0-2","index.lifecycle.name":($INDEX_NAME + "-logs")}}')
+    ilm_name=$idx
+    if [ "$idx" = "so-detectionhistory" ]; then
+      ilm_name="so-detection"
+    elif [ "$idx" = "so-casehistory" ]; then
+      ilm_name="so-case"
+    fi
+    JSON_STRING=$( jq -n --arg ILM_NAME "$ilm_name" '{"settings": {"index.auto_expand_replicas":"0-2","index.lifecycle.name":($ILM_NAME + "-logs")}}')
+
     echo "Checking if index \"$idx\" exists"
     exists=$(curl -K /opt/so/conf/elasticsearch/curl.config -s -o /dev/null -w "%{http_code}" -k -L -H "Content-Type: application/json" "https://localhost:9200/$idx")
     if [ $exists -eq 200 ]; then