Merge remote-tracking branch 'origin/2.4/dev' into reyesj2/kafka

Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com>

salt/manager/tools/sbin/so-minion

@@ -201,11 +201,7 @@ function add_idh_to_minion() {
 	"idh:"\
 	"  enabled: True"\
 	"  restrict_management_ip: $IDH_MGTRESTRICT"\
-	"  services:" >> "$PILLARFILE"
-	IFS=',' read -ra IDH_SERVICES_ARRAY <<< "$IDH_SERVICES"
-	for service in ${IDH_SERVICES_ARRAY[@]}; do
-	  echo "    - $service" | tr '[:upper:]' '[:lower:]' | tr -d '"' >> "$PILLARFILE"
-	done
+	"  " >> $PILLARFILE
 }
 
 function add_logstash_to_minion() {

salt/manager/tools/sbin/soup

@@ -438,7 +438,13 @@ post_to_2.4.60() {
 }
 
 post_to_2.4.70() {
-  echo "Nothing to apply"
+  printf "\nRemoving idh.services from any existing IDH node pillar files\n"
+  for file in /opt/so/saltstack/local/pillar/minions/*.sls; do
+    if [[ $file =~ "_idh.sls" && ! $file =~ "/opt/so/saltstack/local/pillar/minions/adv_" ]]; then
+      echo "Removing idh.services from: $file"
+      so-yaml.py remove "$file" idh.services
+    fi
+  done
   POSTVERSION=2.4.70
 }
 
@@ -583,7 +589,9 @@ up_to_2.4.60() {
 
 up_to_2.4.70() {
   playbook_migration
+  suricata_idstools_migration
   toggle_telemetry
+  add_detection_test_pillars
 
   # Kafka configuration changes
 
@@ -603,6 +611,18 @@ up_to_2.4.70() {
   INSTALLEDVERSION=2.4.70
 }
 
+add_detection_test_pillars() {
+  if [[ -n "$SOUP_INTERNAL_TESTING" ]]; then
+    echo "Adding detection pillar values for automated testing"
+    so-yaml.py add /opt/so/saltstack/local/pillar/soc/soc_soc.sls soc.config.server.modules.elastalertengine.allowRegex SecurityOnion
+    so-yaml.py add /opt/so/saltstack/local/pillar/soc/soc_soc.sls soc.config.server.modules.elastalertengine.failAfterConsecutiveErrorCount 1
+    so-yaml.py add /opt/so/saltstack/local/pillar/soc/soc_soc.sls soc.config.server.modules.strelkaengine.allowRegex "EquationGroup_Toolset_Apr17__ELV_.*"
+    so-yaml.py add /opt/so/saltstack/local/pillar/soc/soc_soc.sls soc.config.server.modules.strelkaengine.failAfterConsecutiveErrorCount 1
+    so-yaml.py add /opt/so/saltstack/local/pillar/soc/soc_soc.sls soc.config.server.modules.suricataengine.allowRegex "(200033\\d|2100538|2102466)"
+    so-yaml.py add /opt/so/saltstack/local/pillar/soc/soc_soc.sls soc.config.server.modules.suricataengine.failAfterConsecutiveErrorCount 1
+  fi
+}
+
 toggle_telemetry() {
   if [[ -z $UNATTENDED && $is_airgap -ne 0 ]]; then
     cat << ASSIST_EOF
@@ -637,6 +657,38 @@ ASSIST_EOF
   fi
 }
 
+suricata_idstools_migration() {
+  #Backup the pillars for idstools
+  mkdir -p /nsm/backup/detections-migration/idstools
+  rsync -av /opt/so/saltstack/local/pillar/idstools/* /nsm/backup/detections-migration/idstools
+  if [[ $? -eq 0 ]]; then
+  	echo "IDStools configuration has been backed up."
+  else
+  	fail "Error: rsync failed to copy the files. IDStools configuration has not been backed up."
+  fi
+
+  #Backup Thresholds
+  mkdir -p /nsm/backup/detections-migration/suricata
+  rsync -av /opt/so/saltstack/local/salt/suricata/thresholding /nsm/backup/detections-migration/suricata
+  if [[ $? -eq 0 ]]; then
+  	echo "Suricata thresholds have been backed up."
+  else
+  	fail "Error: rsync failed to copy the files. Thresholds have not been backed up."
+  fi
+
+  #Backup local rules
+  mkdir -p /nsm/backup/detections-migration/suricata/local-rules
+  rsync -av /opt/so/rules/nids/suri/local.rules /nsm/backup/detections-migration/suricata/local-rules
+  if [[ -f /opt/so/saltstack/local/salt/idstools/rules/local.rules ]]; then
+      rsync -av /opt/so/saltstack/local/salt/idstools/rules/local.rules /nsm/backup/detections-migration/suricata/local-rules/local.rules.bak
+  fi 
+
+  #Tell SOC to migrate
+  mkdir -p /opt/so/conf/soc/migrations
+  echo "0" > /opt/so/conf/soc/migrations/suricata-migration-2.4.70
+  chown -R socore:socore /opt/so/conf/soc/migrations
+}
+
 playbook_migration() {
   # Start SOC Detections migration
   mkdir -p /nsm/backup/detections-migration/{suricata,sigma/rules,elastalert}
@@ -648,22 +700,21 @@ playbook_migration() {
   if grep -A 1 'playbook:'  /opt/so/saltstack/local/pillar/minions/* | grep -q 'enabled: True'; then
   
     # Check for active Elastalert rules
-    active_rules_count=$(find /opt/so/rules/elastalert/playbook/ -type f -name "*.yaml" | wc -l)
+    active_rules_count=$(find /opt/so/rules/elastalert/playbook/ -type f \( -name "*.yaml" -o -name "*.yml" \) | wc -l)
 
     if [[ "$active_rules_count" -gt 0 ]]; then
-        # Prompt the user to AGREE if active Elastalert rules found
+        # Prompt the user to press ENTER if active Elastalert rules found
         echo
         echo "$active_rules_count Active Elastalert/Playbook rules found."
         echo "In preparation for the new Detections module, they will be backed up and then disabled."
         echo
-        echo "If you would like to proceed, then type AGREE and press ENTER."
+        echo "Press ENTER to proceed."
         echo
         # Read user input
-        read INPUT
-        if [ "${INPUT^^}" != 'AGREE' ]; then fail "SOUP canceled."; fi
+        read -r
 
         echo "Backing up the Elastalert rules..."
-        rsync -av --stats /opt/so/rules/elastalert/playbook/*.yaml /nsm/backup/detections-migration/elastalert/
+        rsync -av --ignore-missing-args --stats /opt/so/rules/elastalert/playbook/*.{yaml,yml} /nsm/backup/detections-migration/elastalert/
 
         # Verify that rsync completed successfully
         if [[ $? -eq 0 ]]; then
@@ -1029,6 +1080,7 @@ main() {
       backup_old_states_pillars
     fi
     copy_new_files
+    create_local_directories "/opt/so/saltstack/default"
     apply_hotfix
     echo "Hotfix applied"
     update_version
@@ -1095,6 +1147,7 @@ main() {
     echo "Copying new Security Onion code from $UPDATE_DIR to $DEFAULT_SALT_DIR."
     copy_new_files
     echo ""
+    create_local_directories "/opt/so/saltstack/default"
     update_version
 
     echo ""