Merge remote-tracking branch 'origin/2.4/dev' into reyesj2/kafka

Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com>

salt/manager/init.sls

@@ -73,17 +73,6 @@ manager_sbin:
     - exclude_pat:
       - "*_test.py"
 
-yara_update_scripts:
-  file.recurse:
-    - name: /usr/sbin/
-    - source: salt://manager/tools/sbin_jinja/
-    - user: socore
-    - group: socore
-    - file_mode: 755
-    - template: jinja
-    - defaults:
-        EXCLUDEDRULES: {{ STRELKAMERGED.rules.excluded }}
-
 so-repo-file:
   file.managed:
     - name: /opt/so/conf/reposync/repodownload.conf

salt/manager/tools/sbin/so-minion

@@ -201,11 +201,7 @@ function add_idh_to_minion() {
 	"idh:"\
 	"  enabled: True"\
 	"  restrict_management_ip: $IDH_MGTRESTRICT"\
-	"  services:" >> "$PILLARFILE"
-	IFS=',' read -ra IDH_SERVICES_ARRAY <<< "$IDH_SERVICES"
-	for service in ${IDH_SERVICES_ARRAY[@]}; do
-	  echo "    - $service" | tr '[:upper:]' '[:lower:]' | tr -d '"' >> "$PILLARFILE"
-	done
+	"  " >> $PILLARFILE
 }
 
 function add_logstash_to_minion() {

salt/manager/tools/sbin/soup

@@ -438,7 +438,13 @@ post_to_2.4.60() {
 }
 
 post_to_2.4.70() {
-  echo "Nothing to apply"
+  printf "\nRemoving idh.services from any existing IDH node pillar files\n"
+  for file in /opt/so/saltstack/local/pillar/minions/*.sls; do
+    if [[ $file =~ "_idh.sls" && ! $file =~ "/opt/so/saltstack/local/pillar/minions/adv_" ]]; then
+      echo "Removing idh.services from: $file"
+      so-yaml.py remove "$file" idh.services
+    fi
+  done
   POSTVERSION=2.4.70
 }
 
@@ -583,7 +589,9 @@ up_to_2.4.60() {
 
 up_to_2.4.70() {
   playbook_migration
+  suricata_idstools_migration
   toggle_telemetry
+  add_detection_test_pillars
 
   # Kafka configuration changes
 
@@ -603,6 +611,18 @@ up_to_2.4.70() {
   INSTALLEDVERSION=2.4.70
 }
 
+add_detection_test_pillars() {
+  if [[ -n "$SOUP_INTERNAL_TESTING" ]]; then
+    echo "Adding detection pillar values for automated testing"
+    so-yaml.py add /opt/so/saltstack/local/pillar/soc/soc_soc.sls soc.config.server.modules.elastalertengine.allowRegex SecurityOnion
+    so-yaml.py add /opt/so/saltstack/local/pillar/soc/soc_soc.sls soc.config.server.modules.elastalertengine.failAfterConsecutiveErrorCount 1
+    so-yaml.py add /opt/so/saltstack/local/pillar/soc/soc_soc.sls soc.config.server.modules.strelkaengine.allowRegex "EquationGroup_Toolset_Apr17__ELV_.*"
+    so-yaml.py add /opt/so/saltstack/local/pillar/soc/soc_soc.sls soc.config.server.modules.strelkaengine.failAfterConsecutiveErrorCount 1
+    so-yaml.py add /opt/so/saltstack/local/pillar/soc/soc_soc.sls soc.config.server.modules.suricataengine.allowRegex "(200033\\d|2100538|2102466)"
+    so-yaml.py add /opt/so/saltstack/local/pillar/soc/soc_soc.sls soc.config.server.modules.suricataengine.failAfterConsecutiveErrorCount 1
+  fi
+}
+
 toggle_telemetry() {
   if [[ -z $UNATTENDED && $is_airgap -ne 0 ]]; then
     cat << ASSIST_EOF
@@ -637,6 +657,38 @@ ASSIST_EOF
   fi
 }
 
+suricata_idstools_migration() {
+  #Backup the pillars for idstools
+  mkdir -p /nsm/backup/detections-migration/idstools
+  rsync -av /opt/so/saltstack/local/pillar/idstools/* /nsm/backup/detections-migration/idstools
+  if [[ $? -eq 0 ]]; then
+  	echo "IDStools configuration has been backed up."
+  else
+  	fail "Error: rsync failed to copy the files. IDStools configuration has not been backed up."
+  fi
+
+  #Backup Thresholds
+  mkdir -p /nsm/backup/detections-migration/suricata
+  rsync -av /opt/so/saltstack/local/salt/suricata/thresholding /nsm/backup/detections-migration/suricata
+  if [[ $? -eq 0 ]]; then
+  	echo "Suricata thresholds have been backed up."
+  else
+  	fail "Error: rsync failed to copy the files. Thresholds have not been backed up."
+  fi
+
+  #Backup local rules
+  mkdir -p /nsm/backup/detections-migration/suricata/local-rules
+  rsync -av /opt/so/rules/nids/suri/local.rules /nsm/backup/detections-migration/suricata/local-rules
+  if [[ -f /opt/so/saltstack/local/salt/idstools/rules/local.rules ]]; then
+      rsync -av /opt/so/saltstack/local/salt/idstools/rules/local.rules /nsm/backup/detections-migration/suricata/local-rules/local.rules.bak
+  fi 
+
+  #Tell SOC to migrate
+  mkdir -p /opt/so/conf/soc/migrations
+  echo "0" > /opt/so/conf/soc/migrations/suricata-migration-2.4.70
+  chown -R socore:socore /opt/so/conf/soc/migrations
+}
+
 playbook_migration() {
   # Start SOC Detections migration
   mkdir -p /nsm/backup/detections-migration/{suricata,sigma/rules,elastalert}
@@ -648,22 +700,21 @@ playbook_migration() {
   if grep -A 1 'playbook:'  /opt/so/saltstack/local/pillar/minions/* | grep -q 'enabled: True'; then
   
     # Check for active Elastalert rules
-    active_rules_count=$(find /opt/so/rules/elastalert/playbook/ -type f -name "*.yaml" | wc -l)
+    active_rules_count=$(find /opt/so/rules/elastalert/playbook/ -type f \( -name "*.yaml" -o -name "*.yml" \) | wc -l)
 
     if [[ "$active_rules_count" -gt 0 ]]; then
-        # Prompt the user to AGREE if active Elastalert rules found
+        # Prompt the user to press ENTER if active Elastalert rules found
         echo
         echo "$active_rules_count Active Elastalert/Playbook rules found."
         echo "In preparation for the new Detections module, they will be backed up and then disabled."
         echo
-        echo "If you would like to proceed, then type AGREE and press ENTER."
+        echo "Press ENTER to proceed."
         echo
         # Read user input
-        read INPUT
-        if [ "${INPUT^^}" != 'AGREE' ]; then fail "SOUP canceled."; fi
+        read -r
 
         echo "Backing up the Elastalert rules..."
-        rsync -av --stats /opt/so/rules/elastalert/playbook/*.yaml /nsm/backup/detections-migration/elastalert/
+        rsync -av --ignore-missing-args --stats /opt/so/rules/elastalert/playbook/*.{yaml,yml} /nsm/backup/detections-migration/elastalert/
 
         # Verify that rsync completed successfully
         if [[ $? -eq 0 ]]; then
@@ -1029,6 +1080,7 @@ main() {
       backup_old_states_pillars
     fi
     copy_new_files
+    create_local_directories "/opt/so/saltstack/default"
     apply_hotfix
     echo "Hotfix applied"
     update_version
@@ -1095,6 +1147,7 @@ main() {
     echo "Copying new Security Onion code from $UPDATE_DIR to $DEFAULT_SALT_DIR."
     copy_new_files
     echo ""
+    create_local_directories "/opt/so/saltstack/default"
     update_version
 
     echo ""

salt/manager/tools/sbin_jinja/so-yara-download

@@ -1,51 +0,0 @@
-#!/bin/bash
-NOROOT=1
-. /usr/sbin/so-common
-
-{%- set proxy = salt['pillar.get']('manager:proxy') %}
-{%- set noproxy = salt['pillar.get']('manager:no_proxy', '') %}
-
-# Download the rules from the internet
-{%- if proxy %}
-export http_proxy={{ proxy }} 
-export https_proxy={{ proxy }} 
-export no_proxy="{{ noproxy }}" 
-{%- endif %}
-
-repos="/opt/so/conf/strelka/repos.txt"
-output_dir=/nsm/rules/yara
-gh_status=$(curl -s -o /dev/null -w "%{http_code}" https://github.com)
-clone_dir="/tmp"
-if [ "$gh_status" == "200" ]  || [ "$gh_status" == "301" ]; then
-
-  while IFS= read -r repo; do
-    if ! $(echo "$repo" | grep -qE '^#'); then
-      # Remove old repo if existing bc of previous error condition or unexpected disruption
-      repo_name=`echo $repo | awk -F '/' '{print $NF}'`
-      [ -d $output_dir/$repo_name ] && rm -rf $output_dir/$repo_name
-
-      # Clone repo and make appropriate directories for rules
-      git clone $repo $clone_dir/$repo_name
-      echo "Analyzing rules from $clone_dir/$repo_name..."
-      mkdir -p $output_dir/$repo_name
-      # Ensure a copy of the license is available for the rules
-      [ -f $clone_dir/$repo_name/LICENSE ] && cp $clone_dir/$repo_name/LICENSE $output_dir/$repo_name
-
-      # Copy over rules
-      for i in $(find $clone_dir/$repo_name -name "*.yar*"); do
-        rule_name=$(echo $i | awk -F '/' '{print $NF}')
-        cp $i $output_dir/$repo_name
-     done
-     rm -rf $clone_dir/$repo_name
-    fi
-    done < $repos
-
-  echo "Done!"
-
-/usr/sbin/so-yara-update 
-  
-else
-  echo "Server returned $gh_status status code."
-  echo "No connectivity to Github...exiting..."
-  exit 1
-fi

salt/manager/tools/sbin_jinja/so-yara-update

@@ -1,41 +0,0 @@
-#!/bin/bash
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-NOROOT=1
-. /usr/sbin/so-common
-
-echo "Starting to check for yara rule updates at $(date)..."
-
-newcounter=0
-excludedcounter=0
-excluded_rules=({{ EXCLUDEDRULES | join(' ') }})
-
-# Pull down the SO Rules
-SORULEDIR=/nsm/rules/yara
-OUTPUTDIR=/opt/so/saltstack/local/salt/strelka/rules
-
-mkdir -p $OUTPUTDIR
-# remove all rules prior to copy so we can clear out old rules
-rm -f $OUTPUTDIR/*
-
-for i in $(find $SORULEDIR -name "*.yar" -o -name "*.yara"); do
-  rule_name=$(echo $i | awk -F '/' '{print $NF}')
-  if [[ ! "${excluded_rules[*]}" =~ ${rule_name} ]]; then
-    echo "Adding rule: $rule_name..."
-    cp $i $OUTPUTDIR/$rule_name
-    ((newcounter++))
-  else
-    echo "Excluding rule: $rule_name..."
-    ((excludedcounter++))
-  fi
-done
-
-if [ "$newcounter" -gt 0 ] || [ "$excludedcounter" -gt 0 ];then
-  echo "$newcounter rules added."
-  echo "$excludedcounter rule(s) excluded."
-fi
-
-echo "Finished rule updates at $(date)..."