Merge branch 'dev' into hotfix/soup_salt

2026-06-14 06:11:32 +02:00 · 2021-06-10 15:48:00 -04:00
parent ff807c9a6f a2e1b1de3a
commit c2339c84e7
65 changed files with 9335 additions and 533 deletions
@@ -17,4 +17,4 @@

 . /usr/sbin/so-common

-salt-call state.highstate
+salt-call state.highstate -linfo 
@@ -153,16 +153,16 @@ Do you agree to the terms of the Elastic License?
 If so, type AGREE to accept the Elastic License and continue.  Otherwise, press Enter to exit this program without making any changes.
 EOM

-AGREED=$(whiptail --title "Security Onion Setup" --inputbox \
-"$message" 20 75 3>&1 1>&2 2>&3)
+	AGREED=$(whiptail --title "$whiptail_title" --inputbox \
+	"$message" 20 75 3>&1 1>&2 2>&3)

-if [ "${AGREED^^}" = 'AGREE' ]; then
-  mkdir -p /opt/so/state
-  touch /opt/so/state/yeselastic.txt 
-else
-  echo "Starting in 2.3.40 you must accept the Elastic license if you want to run Security Onion."
-  exit 1
-fi
+	if [ "${AGREED^^}" = 'AGREE' ]; then
+		mkdir -p /opt/so/state
+		touch /opt/so/state/yeselastic.txt 
+	else
+		echo "Starting in 2.3.40 you must accept the Elastic license if you want to run Security Onion."
+		exit 1
+	fi

 }

@@ -302,6 +302,7 @@ retry() {
 	cmd=$3
 	expectedOutput=$4
 	attempt=0
+	local exitcode=0
 	while [[ $attempt -lt $maxAttempts ]]; do			
 		attempt=$((attempt+1))
 		echo "Executing command with retry support: $cmd"
@@ -321,7 +322,29 @@ retry() {
 		sleep $sleepDelay
 	done
 	echo "Command continues to fail; giving up."
-	return 1
+	return $exitcode
+}
+
+run_check_net_err() {
+	local cmd=$1
+	local err_msg=${2:-"Unknown error occured, please check /root/$WHATWOULDYOUSAYYAHDOHERE.log for details."} # Really need to rename that variable
+	local no_retry=$3
+
+	local exit_code
+	if [[ -z $no_retry ]]; then
+		retry 5 60 "$cmd"
+		exit_code=$?
+	else
+		eval "$cmd"
+		exit_code=$?
+	fi
+	
+	if [[ $exit_code -ne 0 ]]; then
+		ERR_HANDLED=true
+		[[ -z $no_retry ]] || echo "Command failed with error $exit_code"
+		echo "$err_msg"
+		exit $exit_code
+	fi
 }

 set_os() {
@@ -35,6 +35,7 @@ if [ ! -f $BACKUPFILE ]; then
  {%- endfor %}
  tar -rf $BACKUPFILE /etc/pki
  tar -rf $BACKUPFILE /etc/salt
+  tar -rf $BACKUPFILE /opt/so/conf/kratos

 fi

@@ -32,13 +32,15 @@ def get_image_version(string) -> str:
  ver = string.split(':')[-1]
  if ver == 'latest':
    # Version doesn't like "latest", so use a high semver
-    return '999999.9.9'
+    return '99999.9.9'
  else:
    try:
      Version(ver)
    except InvalidVersion:
-      # Strip the last substring following a hyphen for automated branches
-      ver = '-'.join(ver.split('-')[:-1])  
+      # Also return a very high semver for any version 
+      # with a dash in it since it will likely be a dev version of some kind
+      if '-' in ver:
+        return '999999.9.9'
    return ver


@@ -73,8 +75,12 @@ def main(quiet):
          for tag in group:
            if not quiet: print(f'Removing image {tag}')
            client.images.remove(tag)
-    except InvalidVersion as e:
-      print(f'so-{get_so_image_basename(t_list[0])}: {e.args[0]}', file=sys.stderr)
+    except (docker.errors.APIError, InvalidVersion) as e:
+      print(f'so-{get_so_image_basename(t_list[0])}: {e}', file=sys.stderr)
+      exit(1)
+    except Exception as e:
+      print('Unhandled exception occurred:')
+      print(f'so-{get_so_image_basename(t_list[0])}: {e}', file=sys.stderr)
      exit(1)

  if no_prunable and not quiet:
@@ -86,4 +92,4 @@ if __name__ == "__main__":
  main_parser.add_argument('-q', '--quiet', action='store_const', const=True, required=False)
  args = main_parser.parse_args(sys.argv[1:])
 
-  main(args.quiet)
+  main(args.quiet)
@@ -0,0 +1,67 @@
+{%- set mainint = salt['pillar.get']('host:mainint') %}
+{%- set MYIP = salt['grains.get']('ip_interfaces:' ~ mainint)[0] %}
+
+#!/bin/bash
+# Copyright 2014,2015,2016,2017,2018,2019 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+default_conf_dir=/opt/so/conf
+ELASTICSEARCH_HOST="{{ MYIP }}"
+ELASTICSEARCH_PORT=9200
+#ELASTICSEARCH_AUTH=""
+
+# Define a default directory to load pipelines from
+FB_MODULE_YML="/usr/share/filebeat/module-setup.yml"
+
+
+# Wait for ElasticSearch to initialize
+echo -n "Waiting for ElasticSearch..."
+COUNT=0
+ELASTICSEARCH_CONNECTED="no"
+while [[ "$COUNT" -le 240 ]]; do
+        curl -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
+        if [ $? -eq 0 ]; then
+                ELASTICSEARCH_CONNECTED="yes"
+                echo "connected!"
+                break
+        else
+                ((COUNT+=1))
+                sleep 1
+                echo -n "."
+        fi
+done
+if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
+        echo
+        echo -e "Connection attempt timed out.  Unable to connect to ElasticSearch.  \nPlease try: \n  -checking log(s) in /var/log/elasticsearch/\n  -running 'sudo docker ps' \n  -running 'sudo so-elastic-restart'"
+        echo
+fi
+echo "Testing to see if the pipelines are already applied"
+ESVER=$(curl -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" |jq .version.number |tr -d \")
+PIPELINES=$(curl -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"/_ingest/pipeline/filebeat-$ESVER-suricata-eve-pipeline | jq . | wc -c)
+
+if [[ "$PIPELINES" -lt 5 ]]; then
+  echo "Setting up ingest pipeline(s)"
+
+  for MODULE in activemq apache auditd aws azure barracuda bluecoat cef checkpoint cisco coredns crowdstrike cyberark cylance elasticsearch envoyproxy f5 fortinet gcp google_workspace googlecloud gsuite haproxy ibmmq icinga iis imperva infoblox iptables juniper kafka kibana logstash microsoft misp mongodb mssql mysql nats netscout nginx o365 okta osquery panw postgresql rabbitmq radware redis santa snort snyk sonicwall sophos squid suricata system tomcat traefik zeek zscaler
+  do
+    echo "Loading $MODULE"
+    docker exec -i so-filebeat filebeat setup modules -pipelines -modules $MODULE -c $FB_MODULE_YML
+    sleep 2
+  done
+else
+  exit 0
+fi
+
+
@@ -18,6 +18,7 @@
 # NOTE: This script depends on so-common
 IMAGEREPO=security-onion-solutions

+# shellcheck disable=SC2120
 container_list() {
  MANAGERCHECK=$1
  
@@ -128,13 +129,13 @@ update_docker_containers() {
  mkdir -p $SIGNPATH >> "$LOG_FILE" 2>&1 

  # Let's make sure we have the public key
-  retry 50 10 "curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" >> "$LOG_FILE" 2>&1
+  run_check_net_err \
+  "curl --retry 5 --retry-delay 60 -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" \
+  "Could not pull signature key file, please ensure connectivity to https://raw.gihubusercontent.com" \
+  noretry >> "$LOG_FILE" 2>&1
  result=$?
  if [[ $result -eq 0 ]]; then
    cat $SIGNPATH/KEYS | gpg --import - >> "$LOG_FILE" 2>&1
-  else
-    echo "Failed to pull signature key file: $result"
-    exit 1
  fi
  
  # Download the containers from the interwebs
@@ -148,14 +149,15 @@ update_docker_containers() {

    # Pull down the trusted docker image
    local image=$i:$VERSION$IMAGE_TAG_SUFFIX
-    retry 50 10 "docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image" >> "$LOG_FILE" 2>&1 
+    run_check_net_err \
+    "docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image" \
+    "Could not pull $image, please ensure connectivity to $CONTAINER_REGISTRY" >> "$LOG_FILE" 2>&1 
    
    # Get signature
-    retry 50 10 "curl -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" >> "$LOG_FILE" 2>&1
-    if [[ $? -ne 0 ]]; then
-      echo "Unable to pull signature file for $image" >> "$LOG_FILE" 2>&1 
-      exit 1
-    fi
+    run_check_net_err \
+    "curl --retry 5 --retry-delay 60 -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" \
+    "Could not pull signature file for $image, please ensure connectivity to https://sigs.securityonion.net " \
+    noretry >> "$LOG_FILE" 2>&1
    # Dump our hash values
    DOCKERINSPECT=$(docker inspect $CONTAINER_REGISTRY/$IMAGEREPO/$image)
       
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+wdurregex="^[0-9]+w$"
+ddurregex="^[0-9]+d$"
+
+echo -e "\nThis script is used to reduce the size of InfluxDB by removing old data and retaining only the duration specified."
+echo "The duration will need to be specified as an integer followed by the duration unit without a space."
+echo -e "\nFor example, to purge all data but retain the past 12 weeks, specify 12w for the duration."
+echo "The duration units are as follows:"
+echo "  w  - week(s)"
+echo "  d  - day(s)"
+
+while true; do
+  echo ""
+  read -p 'Enter the duration of past data that you would like to retain: ' duration
+  duration=$(echo $duration | tr '[:upper:]' '[:lower:]')
+
+  if [[ "$duration" =~ $wdurregex ]] || [[ "$duration" =~ $ddurregex ]]; then
+    break
+  fi
+
+  echo -e "\nInvalid duration."
+done
+
+echo -e "\nInfluxDB will now be cleaned and leave only the past $duration worth of data."
+read -r -p "Are you sure you want to continue? [y/N] " yorn
+if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then
+  echo -e "\nCleaning InfluxDb and saving only the past $duration. This may could take several minutes depending on how much data needs to be cleaned."
+    if docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"DELETE FROM /.*/ WHERE \"time\" >= '2020-01-01T00:00:00.0000000Z' AND \"time\" <= now() - $duration\""; then
+      echo -e "\nInfluxDb clean complete."
+    else
+      echo -e "\nSomething went wrong with cleaning InfluxDB. Please verify that the so-influxdb Docker container is running, and check the log at /opt/so/log/influxdb/influxdb.log for any details."
+    fi
+else
+  echo -e "\nExiting as requested."
+fi
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+echo -e "\nThis script is used to reduce the size of InfluxDB by downsampling old data into the so_long_term retention policy."
+
+echo -e "\nInfluxDB will now be migrated. This could take a few hours depending on how large the database is and hardware resources available."
+read -r -p "Are you sure you want to continue? [y/N] " yorn
+if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then
+  echo -e "\nMigrating InfluxDb started at `date`. This may take several hours depending on how much data needs to be moved."
+  day=0
+  startdate=`date`
+  
+  while docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"SELECT mean(*) INTO \"so_long_term\".:MEASUREMENT FROM \"autogen\"./.*/ WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d GROUP BY time(5m),*\""; do
+    # why 2020-07-21? 
+    migrationdate=`date -d "2020-07-21 + ${day} days" +"%y-%m-%d"`
+    echo "Migration of $migrationdate started at $startdate and completed at `date`."
+
+    newdaytomigrate=$(date -d "$migrationdate + 1 days" +"%s")
+    today=$(date +"%s")
+    if [ $newdaytomigrate -ge $today ]; then
+      break
+    else
+      ((day=day+1))
+      startdate=`date`
+      echo -e "\nMigrating the next day's worth of data."
+    fi
+  done
+
+  echo -e "\nInfluxDb data migration complete."
+
+else
+  echo -e "\nExiting as requested."
+fi
@@ -0,0 +1,26 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+if [ $# -lt 2 ]; then
+  echo "Usage: $0 <steno-query> Output-Filename"
+  exit 1
+fi
+
+docker exec -it so-sensoroni scripts/stenoquery.sh "$1" -w /nsm/pcapout/$2.pcap
+
+echo ""
+echo "If successful, the output was written to: /nsm/pcapout/$2.pcap"
@@ -22,5 +22,5 @@ salt-call state.apply playbook.db_init,playbook,playbook.automation_user_create
 /usr/sbin/so-soctopus-restart

 echo "Importing Plays - this will take some time...."
-wait 5
-/usr/sbin/so-playbook-ruleupdate
+sleep 5
+/usr/sbin/so-playbook-ruleupdate
@@ -17,18 +17,6 @@

 . /usr/sbin/so-common

-#check_boss_raid() {
-#    BOSSBIN=/opt/boss/mvcli
-#    BOSSRC=$($BOSSBIN info -o vd | grep functional)
-#
-#    if [[ $BOSSRC ]]; then
-#        # Raid is good
-#        BOSSRAID=0
-#    else
-#        BOSSRAID=1
-#    fi
-#}
-
 check_lsi_raid() {
    # For use for LSI on Ubuntu
    #MEGA=/opt/MegaRAID/MegeCli/MegaCli64
@@ -66,13 +54,11 @@ mkdir -p /opt/so/log/raid
  {%- if grains['sosmodel'] in ['SOSMN', 'SOSSNNV'] %}
 #check_boss_raid
 check_software_raid
-#echo "osraid=$BOSSRAID nsmraid=$SWRAID" > /opt/so/log/raid/status.log
-echo "osraid=1 nsmraid=$SWRAID" > /opt/so/log/raid/status.log
+echo "nsmraid=$SWRAID" > /opt/so/log/raid/status.log
  {%- elif grains['sosmodel'] in ['SOS1000F', 'SOS1000', 'SOSSN7200', 'SOS10K', 'SOS4000'] %}
 #check_boss_raid
 check_lsi_raid
-#echo "osraid=$BOSSRAID nsmraid=$LSIRAID" > /opt/so/log/raid/status.log
-echo "osraid=1 nsmraid=$LSIRAID" > /opt/so/log/raid/status.log
+echo "nsmraid=$LSIRAID" > /opt/so/log/raid/status.log
  {%- else %}
 exit 0
  {%- endif %}
@@ -23,6 +23,11 @@ TESTPCAP=$2

 . /usr/sbin/so-common

+if [ $# -lt 2 ]; then
+  echo "Usage: $0 <CustomRule> <TargetPCAP>"
+  exit 1
+fi
+
 echo ""
 echo "==============="
 echo "Running all.rules and $TESTRULE against the following pcap: $TESTPCAP" 
@@ -10,11 +10,10 @@ zeek_logs_enabled() {
 }

 whiptail_manager_adv_service_zeeklogs() {
-  BLOGS=$(whiptail --title "Security Onion Setup" --checklist "Please Select Logs to Send:" 24 78 12 \
+  BLOGS=$(whiptail --title "so-zeek-logs" --checklist "Please Select Logs to Send:" 24 78 12 \
  "conn" "Connection Logging" ON \
  "dce_rpc" "RPC Logs" ON \
  "dhcp" "DHCP Logs" ON \
-  "dhcpv6" "DHCP IPv6 Logs" ON \
  "dnp3" "DNP3 Logs" ON \
  "dns" "DNS Logs" ON \
  "dpd" "DPD Logs" ON \
@@ -25,25 +24,20 @@ whiptail_manager_adv_service_zeeklogs() {
  "irc" "IRC Chat Logs" ON \
  "kerberos" "Kerberos Logs" ON \
  "modbus" "MODBUS Logs" ON \
-  "mqtt" "MQTT Logs" ON \
  "notice" "Zeek Notice Logs" ON \
  "ntlm" "NTLM Logs" ON \
-  "openvpn" "OPENVPN Logs" ON \
  "pe" "PE Logs" ON \
  "radius" "Radius Logs" ON \
  "rfb" "RFB Logs" ON \
  "rdp" "RDP Logs" ON \
-  "signatures" "Signatures Logs" ON \
  "sip" "SIP Logs" ON \
  "smb_files" "SMB Files Logs" ON \
  "smb_mapping" "SMB Mapping Logs" ON \
  "smtp" "SMTP Logs" ON \
  "snmp" "SNMP Logs" ON \
-  "software" "Software Logs" ON \
  "ssh" "SSH Logs" ON \
  "ssl" "SSL Logs" ON \
  "syslog" "Syslog Logs" ON \
-  "telnet" "Telnet Logs" ON \
  "tunnel" "Tunnel Logs" ON \
  "weird" "Zeek Weird Logs" ON \
  "mysql" "MySQL Logs" ON \
@@ -61,10 +55,10 @@ whiptail_manager_adv_service_zeeklogs
 return_code=$?
 case $return_code in
  1)
-    whiptail --title "Security Onion Setup" --msgbox "Cancelling. No changes have been made." 8 75
+    whiptail --title "so-zeek-logs" --msgbox "Cancelling. No changes have been made." 8 75
    ;;
  255)
-    whiptail --title "Security Onion Setup" --msgbox "Whiptail error occured, exiting." 8 75
+    whiptail --title "so-zeek-logs" --msgbox "Whiptail error occured, exiting." 8 75
    ;;
  *)
    zeek_logs_enabled
@@ -20,10 +20,79 @@
 UPDATE_DIR=/tmp/sogh/securityonion
 INSTALLEDVERSION=$(cat /etc/soversion)
 POSTVERSION=$INSTALLEDVERSION
-INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk {'print $2'})
+INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk '{print $2}')
 BATCHSIZE=5
 SOUP_LOG=/root/soup.log
+INFLUXDB_MIGRATION_LOG=/opt/so/log/influxdb/soup_migration.log
 WHATWOULDYOUSAYYAHDOHERE=soup
+whiptail_title='Security Onion UPdater'
+
+check_err() {
+  local exit_code=$1
+  local err_msg="Unhandled error occured, please check $SOUP_LOG for details."
+
+  [[ $ERR_HANDLED == true ]] && exit $exit_code
+  if [[ $exit_code -ne 0 ]]; then
+    printf '%s' "Soup failed with error $exit_code: "
+    case $exit_code in
+      2)
+        echo 'No such file or directory'
+      ;;
+      5)
+        echo 'Interrupted system call'
+      ;;
+      12)
+        echo 'Out of memory'
+      ;;
+      28)
+        echo 'No space left on device'
+        echo 'Likely ran out of space on disk, please review hardware requirements for Security Onion: https://docs.securityonion.net/en/2.3/hardware.html'
+      ;;
+      30)
+        echo 'Read-only file system'
+      ;;
+      35)
+        echo 'Resource temporarily unavailable'
+      ;;
+      64)
+        echo 'Machine is not on the network'
+      ;;
+      67)
+        echo 'Link has been severed'
+      ;;
+      100)
+        echo 'Network is down'
+      ;;
+      101)
+        echo 'Network is unreachable'
+      ;;
+      102)
+        echo 'Network reset'
+      ;;
+      110)
+        echo 'Connection timed out'
+      ;;
+      111)
+        echo 'Connection refused'
+      ;;
+      112)
+        echo 'Host is down'
+      ;;
+      113)
+        echo 'No route to host'
+      ;;
+      *)
+        echo 'Unhandled error'
+        echo "$err_msg"
+      ;;
+    esac
+    if [[ $exit_code -ge 64 && $exit_code -le 113 ]]; then
+      echo "$err_msg"
+    fi
+    exit $exit_code
+  fi
+
+}

 add_common() {
  cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
@@ -39,15 +108,14 @@ airgap_mounted() {
    echo "The ISO is already mounted"
  else
    echo ""
-    echo "Looks like we need access to the upgrade content"
-    echo "" 
-    echo "If you just copied the .iso file over you can specify the path."
-    echo "If you burned the ISO to a disk the standard way you can specify the device."
-    echo "Example: /home/user/securityonion-2.X.0.iso"
-    echo "Example: /dev/sdx1"
-    echo ""
-    read -p 'Enter the location of the iso: ' ISOLOC
-    if [ -f $ISOLOC ]; then
+    cat << EOF
+In order for soup to proceed, the path to the downloaded Security Onion ISO file, or the path to the CD-ROM or equivalent device containing the ISO media must be provided. 
+For example, if you have copied the new Security Onion ISO file to your home directory, then the path might look like /home/myuser/securityonion-2.x.y.iso. 
+Or, if you have burned the new ISO onto an optical disk then the path might look like /dev/cdrom.
+
+EOF
+    read -rp 'Enter the path to the new Security Onion ISO content: ' ISOLOC
+    if [[ -f $ISOLOC ]]; then
      # Mounting the ISO image
      mkdir -p /tmp/soagupdate
      mount -t iso9660 -o loop $ISOLOC /tmp/soagupdate
@@ -59,7 +127,7 @@ airgap_mounted() {
      else
        echo "ISO has been mounted!"
      fi  
-    elif [ -f $ISOLOC/SecurityOnion/VERSION ]; then
+    elif [[ -f $ISOLOC/SecurityOnion/VERSION ]]; then
      ln -s $ISOLOC /tmp/soagupdate
      echo "Found the update content"
    else 
@@ -77,9 +145,9 @@ airgap_mounted() {
 }

 airgap_update_dockers() {
-  if [ $is_airgap -eq 0 ]; then
+  if [[ $is_airgap -eq 0 ]]; then
    # Let's copy the tarball
-    if [ ! -f $AGDOCKER/registry.tar ]; then
+    if [[ ! -f $AGDOCKER/registry.tar ]]; then
      echo "Unable to locate registry. Exiting"
      exit 1
    else
@@ -87,9 +155,9 @@ airgap_update_dockers() {
      docker stop so-dockerregistry
      docker rm so-dockerregistry
      echo "Copying the new dockers over"
-      tar xvf $AGDOCKER/registry.tar -C /nsm/docker-registry/docker
+      tar xvf "$AGDOCKER/registry.tar" -C /nsm/docker-registry/docker
      echo "Add Registry back"
-      docker load -i $AGDOCKER/registry_image.tar
+      docker load -i "$AGDOCKER/registry_image.tar"
    fi
  fi
 }
@@ -101,9 +169,9 @@ update_registry() {
 }

 check_sudoers() {
-	if grep -q "so-setup" /etc/sudoers; then
-		echo "There is an entry for so-setup in the sudoers file, this can be safely deleted using \"visudo\"."
-	fi
+  if grep -q "so-setup" /etc/sudoers; then
+    echo "There is an entry for so-setup in the sudoers file, this can be safely deleted using \"visudo\"."
+  fi
 }

 check_log_size_limit() {
@@ -177,7 +245,9 @@ check_os_updates() {
          echo "Continuing without updating packages"
      elif [[ "$confirm" == [uU] ]]; then
          echo "Applying Grid Updates"
-          salt \* -b 5 state.apply patch.os queue=True
+          set +e
+          run_check_net_err "salt '*' -b 5 state.apply patch.os queue=True" 'Could not apply OS updates, please check your network connection.'
+          set -e
      else
          echo "Exiting soup"
          exit 0
@@ -205,7 +275,9 @@ clone_to_tmp() {
  if [ -n "$BRANCH" ]; then
    SOUP_BRANCH="-b $BRANCH"
  fi
-  git clone $SOUP_BRANCH https://github.com/Security-Onion-Solutions/securityonion.git
+  set +e
+  run_check_net_err "git clone $SOUP_BRANCH https://github.com/Security-Onion-Solutions/securityonion.git" "Could not clone repo, please ensure network access to https://github.com"
+  set -e
  cd /tmp
  if [ ! -f $UPDATE_DIR/VERSION ]; then
    echo "Update was unable to pull from github. Please check your internet."
@@ -257,6 +329,7 @@ preupgrade_changes() {
    [[ "$INSTALLEDVERSION" == 2.3.0 || "$INSTALLEDVERSION" == 2.3.1 || "$INSTALLEDVERSION" == 2.3.2 || "$INSTALLEDVERSION" == 2.3.10 ]] && up_2.3.0_to_2.3.20
    [[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30
    [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]] && up_2.3.3X_to_2.3.50
+    true
 }

 postupgrade_changes() {
@@ -266,6 +339,8 @@ postupgrade_changes() {
    [[ "$POSTVERSION" =~ rc.1 ]] && post_rc1_to_rc2
    [[ "$POSTVERSION" == 2.3.20 || "$POSTVERSION" == 2.3.21 ]] && post_2.3.2X_to_2.3.30
    [[ "$POSTVERSION" == 2.3.30 ]] && post_2.3.30_to_2.3.40
+    [[ "$POSTVERSION" == 2.3.50 ]] && post_2.3.5X_to_2.3.60
+    true
 }

 post_rc1_to_2.3.21() {
@@ -286,6 +361,10 @@ post_2.3.30_to_2.3.40() {
  POSTVERSION=2.3.40
 }

+post_2.3.5X_to_2.3.60() {
+  POSTVERSION=2.3.60
+}
+

 rc1_to_rc2() {

@@ -419,7 +498,7 @@ up_2.3.2X_to_2.3.30() {
  sed -i "/  imagerepo: securityonion/c\  imagerepo: 'security-onion-solutions'" /opt/so/saltstack/local/pillar/global.sls 

  # Strelka rule repo pillar addition
-  if [ $is_airgap -eq 0 ]; then
+  if [[ $is_airgap -eq 0 ]]; then
      # Add manager as default Strelka YARA rule repo
      sed -i "/^strelka:/a \\  repos: \n    - https://$HOSTNAME/repo/rules/strelka" /opt/so/saltstack/local/pillar/global.sls;
  else
@@ -446,7 +525,7 @@ upgrade_to_2.3.50_repo() {
        rm -f "/etc/yum.repos.d/$DELREPO.repo"
      fi
    done
-    if [ $is_airgap -eq 1 ]; then
+    if [[ $is_airgap -eq 1 ]]; then
      # Copy the new repo file if not airgap
      cp $UPDATE_DIR/salt/repo/client/files/centos/securityonion.repo /etc/yum.repos.d/
      yum clean all
@@ -562,7 +641,7 @@ upgrade_check() {
  # Let's make sure we actually need to update.
  NEWVERSION=$(cat $UPDATE_DIR/VERSION)
  HOTFIXVERSION=$(cat $UPDATE_DIR/HOTFIX)
-  CURRENTHOTFIX=$(cat /etc/sohotfix 2>/dev/null)
+  [[ -f /etc/sohotfix ]] && CURRENTHOTFIX=$(cat /etc/sohotfix)
  if [ "$INSTALLEDVERSION" == "$NEWVERSION" ]; then
    echo "Checking to see if there are hotfixes needed"
    if [ "$HOTFIXVERSION" == "$CURRENTHOTFIX" ]; then
@@ -579,13 +658,14 @@ upgrade_check() {
 }

 upgrade_check_salt() {
-  NEWSALTVERSION=$(grep version: $UPDATE_DIR/salt/salt/master.defaults.yaml | awk {'print $2'})
+  NEWSALTVERSION=$(grep version: $UPDATE_DIR/salt/salt/master.defaults.yaml | awk '{print $2}')
  if [ "$INSTALLEDSALTVERSION" == "$NEWSALTVERSION" ]; then
    echo "You are already running the correct version of Salt for Security Onion."
  else
    UPGRADESALT=1
  fi
 }   
+
 upgrade_salt() {
  SALTUPGRADED=True
  echo "Performing upgrade of Salt from $INSTALLEDSALTVERSION to $NEWSALTVERSION."
@@ -597,7 +677,11 @@ upgrade_salt() {
    yum versionlock delete "salt-*"
    echo "Updating Salt packages and restarting services."
    echo ""
-    sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable "$NEWSALTVERSION"
+    set +e
+    run_check_net_err \
+    "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \
+    "Could not update salt, please check $SOUP_LOG for details."
+    set -e
    echo "Applying yum versionlock for Salt."
    echo ""
    yum versionlock add "salt-*"
@@ -610,7 +694,11 @@ upgrade_salt() {
    apt-mark unhold "salt-minion"
    echo "Updating Salt packages and restarting services."
    echo ""
-    sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable "$NEWSALTVERSION"
+    set +e
+    run_check_net_err \
+    "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable \"$NEWSALTVERSION\"" \
+    "Could not update salt, please check $SOUP_LOG for details."
+    set -e
    echo "Applying apt hold for Salt."
    echo ""
    apt-mark hold "salt-common"
@@ -635,222 +723,250 @@ verify_latest_update_script() {
    cp $UPDATE_DIR/salt/common/tools/sbin/soup $DEFAULT_SALT_DIR/salt/common/tools/sbin/
    cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
    cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
-    salt-call state.apply common queue=True
+    salt-call state.apply -l info common queue=True
    echo ""
    echo "soup has been updated. Please run soup again."
    exit 0
  fi
 }

-main () {
-echo "### Preparing soup at `date` ###"
-while getopts ":b" opt; do
-  case "$opt" in
-    b ) # process option b
-       shift
-       BATCHSIZE=$1
-       if ! [[ "$BATCHSIZE" =~ ^[0-9]+$ ]]; then
-         echo "Batch size must be a number greater than 0."
-         exit 1
-       fi
-    ;;
-    \? ) 
-      echo "Usage: cmd [-b]"
-    ;;
-  esac
-done
-
-echo "Checking to see if this is a manager."
-echo ""
-require_manager
-set_minionid
-echo "Checking to see if this is an airgap install"
-echo ""
-check_airgap
-echo "Found that Security Onion $INSTALLEDVERSION is currently installed."
-echo ""
-set_os
-set_palette
-check_elastic_license
-echo ""
-if [ $is_airgap -eq 0 ]; then
-  # Let's mount the ISO since this is airgap
-  airgap_mounted
-else
-  echo "Cloning Security Onion github repo into $UPDATE_DIR."
-  echo "Removing previous upgrade sources."
-  rm -rf $UPDATE_DIR
-  clone_to_tmp
-fi
-check_os_updates
-echo ""
-echo "Verifying we have the latest soup script."
-verify_latest_update_script
-echo ""
-
-echo "Generating new repo archive"
-generate_and_clean_tarballs
-if [ -f /usr/sbin/so-image-common ]; then
-  . /usr/sbin/so-image-common
-else 
-add_common
-fi
-
-echo "Let's see if we need to update Security Onion."
-upgrade_check
-upgrade_space
-
-echo "Checking for Salt Master and Minion updates."
-upgrade_check_salt
-
-
-if [ "$is_hotfix" == "true" ]; then
-  echo "Applying $HOTFIXVERSION" 
-  copy_new_files
-  echo ""
-  update_version
-  salt-call state.highstate -l info queue=True
+main() {
+  set -e
+  trap 'check_err $?' EXIT
  
-else
-  echo ""
-  echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION."
-  echo ""
+  echo "### Preparing soup at $(date) ###"
+  while getopts ":b" opt; do
+    case "$opt" in
+      b ) # process option b
+        shift
+        BATCHSIZE=$1
+        if ! [[ "$BATCHSIZE" =~ ^[0-9]+$ ]]; then
+          echo "Batch size must be a number greater than 0."
+          exit 1
+        fi
+      ;;
+      \? ) 
+        echo "Usage: cmd [-b]"
+      ;;
+    esac
+  done

-  echo "Updating dockers to $NEWVERSION."
-  if [ $is_airgap -eq 0 ]; then
-    airgap_update_dockers
-    update_centos_repo
-    yum clean all
-    check_os_updates
+  echo "Checking to see if this is a manager."
+  echo ""
+  require_manager
+  set_minionid
+  echo "Checking to see if this is an airgap install"
+  echo ""
+  check_airgap
+  echo "Found that Security Onion $INSTALLEDVERSION is currently installed."
+  echo ""
+  set_os
+  set_palette
+  check_elastic_license
+  echo ""
+  if [[ $is_airgap -eq 0 ]]; then
+    # Let's mount the ISO since this is airgap
+    airgap_mounted
  else
-    update_registry
-    update_docker_containers "soup"
+    echo "Cloning Security Onion github repo into $UPDATE_DIR."
+    echo "Removing previous upgrade sources."
+    rm -rf $UPDATE_DIR
+    clone_to_tmp
  fi
+  check_os_updates
+  echo ""
+  echo "Verifying we have the latest soup script."
+  verify_latest_update_script
+  echo ""

-  echo ""
-  echo "Stopping Salt Minion service."
-  systemctl stop salt-minion
-  echo "Killing any remaining Salt Minion processes."
-  pkill -9 -ef /usr/bin/salt-minion
-  echo ""
  echo "Stopping Salt Master service."
  systemctl stop salt-master
  echo ""

  upgrade_to_2.3.50_repo

-  # Does salt need upgraded. If so update it.
-  if [ "$UPGRADESALT" == "1" ]; then
-    echo "Upgrading Salt"
-    # Update the repo files so it can actually upgrade
-    upgrade_salt
+  echo "Generating new repo archive"
+  generate_and_clean_tarballs
+  if [ -f /usr/sbin/so-image-common ]; then
+    . /usr/sbin/so-image-common
+  else 
+  add_common
  fi

-  echo "Checking if Salt was upgraded."
-  echo ""
-  # Check that Salt was upgraded
-  SALTVERSIONPOSTUPGRADE=$(salt --versions-report | grep Salt: | awk {'print $2'})
-  if [[ "$SALTVERSIONPOSTUPGRADE" != "$NEWSALTVERSION" ]]; then
-    echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG."
-    echo "Once the issue is resolved, run soup again."
-    echo "Exiting."
+  echo "Let's see if we need to update Security Onion."
+  upgrade_check
+  upgrade_space
+
+  echo "Checking for Salt Master and Minion updates."
+  upgrade_check_salt
+
+
+  if [ "$is_hotfix" == "true" ]; then
+    echo "Applying $HOTFIXVERSION" 
+    copy_new_files
    echo ""
-    exit 1
+    update_version
+    salt-call state.highstate -l info queue=True
  else
-    echo "Salt upgrade success."
    echo ""
-  fi
-
-  preupgrade_changes
-  echo ""
-
-  if [ $is_airgap -eq 0 ]; then
-    echo "Updating Rule Files to the Latest."
-    update_airgap_rules
-  fi
-
-  # Only update the repo if its airgap
-  if [[ $is_airgap -eq 0 ]] && [[ "$UPGRADESALT" != "1" ]]; then
-  update_centos_repo
-  fi
-
-  echo ""
-  echo "Copying new Security Onion code from $UPDATE_DIR to $DEFAULT_SALT_DIR."
-  copy_new_files
-  echo ""
-  update_version
-
-  echo ""
-  echo "Locking down Salt Master for upgrade"
-  masterlock
-
-  echo ""
-  echo "Starting Salt Master service."
-  systemctl start salt-master
-
-  # Only regenerate osquery packages if Fleet is enabled
-  FLEET_MANAGER=$(lookup_pillar fleet_manager)
-  FLEET_NODE=$(lookup_pillar fleet_node)
-  if [[ "$FLEET_MANAGER" == "True" || "$FLEET_NODE" == "True" ]]; then
+    echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION."
    echo ""
-    echo "Regenerating Osquery Packages.... This will take several minutes."
-    salt-call state.apply fleet.event_gen-packages -l info queue=True
+
+    echo "Updating dockers to $NEWVERSION."
+    if [[ $is_airgap -eq 0 ]]; then
+      airgap_update_dockers
+      update_centos_repo
+      yum clean all
+      check_os_updates
+    else
+      update_registry
+      set +e
+      update_docker_containers "soup"
+      set -e
+    fi
+
+    echo ""
+    echo "Stopping Salt Minion service."
+    systemctl stop salt-minion
+    echo "Killing any remaining Salt Minion processes."
+    set +e
+    pkill -9 -ef /usr/bin/salt-minion
+    set -e
+    echo ""
+    echo "Stopping Salt Master service."
+    systemctl stop salt-master
    echo ""
-  fi

-  echo ""
-  echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes."
-  salt-call state.highstate -l info queue=True
-  echo ""
-  echo "Upgrade from $INSTALLEDVERSION to $NEWVERSION complete."
+    preupgrade_changes_2.3.50_repo

-  echo ""
-  echo "Stopping Salt Master to remove ACL"
-  systemctl stop salt-master
+    # Does salt need upgraded. If so update it.
+    if [[ $UPGRADESALT -eq 1 ]]; then
+      echo "Upgrading Salt"
+      # Update the repo files so it can actually upgrade
+      upgrade_salt
+    fi

-  masterunlock
-
-  echo ""
-  echo "Starting Salt Master service."
-  systemctl start salt-master
-  echo "Running a highstate. This could take several minutes."
-  salt-call state.highstate -l info queue=True
-  postupgrade_changes
-  unmount_update
-  thehive_maint
-
-  if [ "$UPGRADESALT" == "1" ]; then
-    if [ $is_airgap -eq 0 ]; then
+    echo "Checking if Salt was upgraded."
+    echo ""
+    # Check that Salt was upgraded
+    SALTVERSIONPOSTUPGRADE=$(salt --versions-report | grep Salt: | awk '{print $2}')
+    if [[ "$SALTVERSIONPOSTUPGRADE" != "$NEWSALTVERSION" ]]; then
+      echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG."
+      echo "Once the issue is resolved, run soup again."
+      echo "Exiting."
      echo ""
-      echo "Cleaning repos on remote Security Onion nodes."
-      salt -C 'not *_eval and not *_helixsensor and not *_manager and not *_managersearch and not *_standalone and G@os:CentOS' cmd.run "yum clean all"
+      exit 1
+    else
+      echo "Salt upgrade success."
      echo ""
    fi
-  fi

-  check_sudoers
+    preupgrade_changes
+    echo ""

-  if [[ -n $lsl_msg ]]; then
-    case $lsl_msg in
-      'distributed')
-        echo "[INFO] The value of log_size_limit in any heavy node minion pillars may be incorrect."
-        echo " -> We recommend checking and adjusting the values as necessary."
-        echo " -> Minion pillar directory: /opt/so/saltstack/local/pillar/minions/"
-      ;;
-      'single-node')
-        # We can assume the lsl_details array has been set if lsl_msg has this value
-        echo "[WARNING] The value of log_size_limit (${lsl_details[0]}) does not match the recommended value of ${lsl_details[1]}."
-        echo " -> We recommend checking and adjusting the value as necessary."
-        echo " -> File: /opt/so/saltstack/local/pillar/minions/${lsl_details[2]}.sls"
-      ;;
-    esac
-  fi
+    if [[ $is_airgap -eq 0 ]]; then
+      echo "Updating Rule Files to the Latest."
+      update_airgap_rules
+    fi

-  NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l)
+    # Only update the repo if its airgap
+    if [[ $is_airgap -eq 0 && $UPGRADESALT -ne 1 ]]; then
+      update_centos_repo
+    fi

-  if [ $NUM_MINIONS -gt 1 ]; then
+    echo ""
+    echo "Copying new Security Onion code from $UPDATE_DIR to $DEFAULT_SALT_DIR."
+    copy_new_files
+    echo ""
+    update_version

-    cat << EOF
+    echo ""
+    echo "Locking down Salt Master for upgrade"
+    masterlock
+
+    echo ""
+    echo "Starting Salt Master service."
+    systemctl start salt-master
+
+    # Testing that salt-master is up by checking that is it connected to itself
+    set +e
+    retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check $SOUP_LOG for details."
+    set -e
+
+    echo ""
+    echo "Ensuring python modules for Salt are installed and patched."
+    salt-call state.apply salt.python3-influxdb -l info queue=True
+    echo ""
+
+    # Only regenerate osquery packages if Fleet is enabled
+    FLEET_MANAGER=$(lookup_pillar fleet_manager)
+    FLEET_NODE=$(lookup_pillar fleet_node)
+    if [[ "$FLEET_MANAGER" == "True" || "$FLEET_NODE" == "True" ]]; then
+      echo ""
+      echo "Regenerating Osquery Packages.... This will take several minutes."
+      salt-call state.apply fleet.event_gen-packages -l info queue=True
+      echo ""
+    fi
+
+    echo ""
+    echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes."
+    salt-call state.highstate -l info queue=True
+    echo ""
+    echo "Upgrade from $INSTALLEDVERSION to $NEWVERSION complete."
+
+    echo ""
+    echo "Stopping Salt Master to remove ACL"
+    systemctl stop salt-master
+
+    masterunlock
+
+    echo ""
+    echo "Starting Salt Master service."
+    systemctl start salt-master
+
+    # Testing that salt-master is up by checking that is it connected to itself
+    set +e
+    retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check $SOUP_LOG for details."
+    set -e
+    
+    echo "Running a highstate. This could take several minutes."
+    salt-call state.highstate -l info queue=True
+    postupgrade_changes
+    [[ $is_airgap -eq 0 ]] && unmount_update
+    thehive_maint
+
+    if [[ $UPGRADESALT -eq 1 ]]; then
+      if [[ $is_airgap -eq 0 ]]; then
+        echo ""
+        echo "Cleaning repos on remote Security Onion nodes."
+        salt -C 'not *_eval and not *_helixsensor and not *_manager and not *_managersearch and not *_standalone and G@os:CentOS' cmd.run "yum clean all"
+        echo ""
+      fi
+    fi
+
+    check_sudoers
+
+    if [[ -n $lsl_msg ]]; then
+      case $lsl_msg in
+        'distributed')
+          echo "[INFO] The value of log_size_limit in any heavy node minion pillars may be incorrect."
+          echo " -> We recommend checking and adjusting the values as necessary."
+          echo " -> Minion pillar directory: /opt/so/saltstack/local/pillar/minions/"
+        ;;
+        'single-node')
+          # We can assume the lsl_details array has been set if lsl_msg has this value
+          echo "[WARNING] The value of log_size_limit (${lsl_details[0]}) does not match the recommended value of ${lsl_details[1]}."
+          echo " -> We recommend checking and adjusting the value as necessary."
+          echo " -> File: /opt/so/saltstack/local/pillar/minions/${lsl_details[2]}.sls"
+        ;;
+      esac
+    fi
+
+    NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l)
+
+    if [[ $NUM_MINIONS -gt 1 ]]; then
+
+      cat << EOF
    
  
  
@@ -864,10 +980,10 @@ For more information, please see https://docs.securityonion.net/en/2.3/soup.html

 EOF

+    fi
  fi
-fi

-echo "### soup has been served at `date` ###"
+  echo "### soup has been served at $(date) ###"
 }

 cat << EOF
@@ -882,6 +998,7 @@ Press Enter to continue or Ctrl-C to cancel.

 EOF

-read input
+read -r input

 main "$@" | tee -a $SOUP_LOG
+  
@@ -36,7 +36,7 @@ xpack.security.authc:
    roles: superuser
    authz_exception: true
 node.name: {{ grains.host }}
-script.max_compilations_rate: 1000/1m
+script.max_compilations_rate: 20000/1m
 {%- if TRUECLUSTER is sameas true %}
  {%- if grains.role == 'so-manager' %}
    {%- if salt['pillar.get']('nodestab', {}) %}
@@ -51,7 +51,7 @@
     },
     { "set":         { "field": "_index",        "value": "so-firewall", "override": true           } },
     { "set":         { "if": "ctx.network?.transport_id == '0'", "field": "network.transport",        "value": "icmp", "override": true           } },
-     {"community_id": { "if": "ctx.network?.transport != null", "field":["source.ip","source.port","destination.ip","destination.port","network.transport"],"target_field":"network.community_id"}},
+     {"community_id": {} },
     { "set":         { "field": "module",        "value": "pfsense", "override": true           } },
     { "set":         { "field": "dataset",        "value": "firewall", "override": true           } },
     { "remove":      { "field": ["real_message", "ip_sub_msg", "firewall.sub_message"],     "ignore_failure": true  } }
@@ -63,7 +63,8 @@
    { "rename":         { "field": "fields.module",              "target_field": "event.module",                  "ignore_failure": true, "ignore_missing": true  } },
    { "pipeline":      { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'",   "name": "sysmon"  }  },
    { "pipeline":      { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational'",  "name":"win.eventlogs" }  },
-    { "set":           { "if": "ctx.rule != null && ctx.rule.name != null",   "field": "event.dataset", "value": "alert", "override": true }  },
+    { "set":           { "if": "ctx.rule != null && ctx.rule.name != null",   "field": "event.dataset", "value": "ossec.alert", "override": true }  },
+    { "set":           { "if": "ctx.rule != null && ctx.rule.name != null",   "field": "event.kind", "value": "alert", "override": true }  },
    { "pipeline":       { "name": "common" } }    
  ]
 }
@@ -53,7 +53,8 @@
    { "set":         { "if": "ctx.exiftool?.FileDirectory != null", "field": "file.directory",        "value": "{{exiftool.FileDirectory}}", "ignore_failure": true }},
    { "set":         { "if": "ctx.exiftool?.Subsystem != null", "field": "host.subsystem",        "value": "{{exiftool.Subsystem}}", "ignore_failure": true }},
    { "set":         { "if": "ctx.scan?.yara?.matches != null", "field": "rule.name",        "value": "{{scan.yara.matches.0}}" }},
-    { "set":         { "if": "ctx.scan?.yara?.matches != null", "field": "dataset",        "value": "alert", "override": true }},
+    { "set":         { "if": "ctx.scan?.yara?.matches != null", "field": "dataset",        "value": "strelka.alert", "override": true }},
+    { "set":         { "if": "ctx.scan?.yara?.matches != null", "field": "event.kind",        "value": "alert", "override": true }},
    { "rename":         { "field": "file.flavors.mime", "target_field": "file.mime_type",        "ignore_missing": true }},
    { "set":         { "if": "ctx.rule?.name != null && ctx.rule?.score == null",   "field": "event.severity", "value": 3, "override": true }  },
    { "convert" :    { "if": "ctx.rule?.score != null", "field" : "rule.score","type": "integer"}},
@@ -1,7 +1,6 @@
 {
  "description" : "sysmon",
  "processors" : [
-    {"community_id":   {"if": "ctx.winlog.event_data?.Protocol != null", "field":["winlog.event_data.SourceIp","winlog.event_data.SourcePort","winlog.event_data.DestinationIp","winlog.event_data.DestinationPort","winlog.event_data.Protocol"],"target_field":"network.community_id"}},    
    { "set":           { "field": "event.code", "value": "{{winlog.event_id}}", "override": true }  },     
    { "set":           { "field": "event.module", "value": "sysmon", "override": true }  },   
    { "set":           { "if": "ctx.winlog?.computer_name != null",   "field": "observer.name", "value": "{{winlog.computer_name}}", "override": true }  },
@@ -64,6 +63,7 @@
    { "rename": 	   { "field": "winlog.event_data.SourceIp", 		          "target_field": "source.ip",		"ignore_missing": true 	} },
    { "rename": 	   { "field": "winlog.event_data.SourcePort", 		        "target_field": "source.port",		"ignore_missing": true 	} },   
    { "rename": 	   { "field": "winlog.event_data.targetFilename",		      "target_field": "file.target",	"ignore_missing": true 	} },
-    { "rename":      { "field": "winlog.event_data.TargetFilename",         "target_field": "file.target",    "ignore_missing": true  } }
+    { "rename":      { "field": "winlog.event_data.TargetFilename",         "target_field": "file.target",    "ignore_missing": true  } },
+    { "community_id": {} }
  ]
 }
@@ -8,11 +8,11 @@
    { "dot_expander":   { "field": "id.orig_p",                 "path": "message2",                     "ignore_failure": true  } },
    { "dot_expander":   { "field": "id.resp_h",                 "path": "message2",                     "ignore_failure": true  } },
    { "dot_expander":   { "field": "id.resp_p",                 "path": "message2",                     "ignore_failure": true  } },    
-    {"community_id": {"if": "ctx.network?.transport != null", "field":["message2.id.orig_h","message2.id.orig_p","message2.id.resp_h","message2.id.resp_p","network.transport"],"target_field":"network.community_id"}},
    { "rename":         { "field": "message2.id.orig_h",        "target_field": "source.ip",            "ignore_missing": true  } },
    { "rename":         { "field": "message2.id.orig_p",        "target_field": "source.port",          "ignore_missing": true  } },
    { "rename":         { "field": "message2.id.resp_h",        "target_field": "destination.ip",       "ignore_missing": true  } },
    { "rename":         { "field": "message2.id.resp_p",        "target_field": "destination.port",     "ignore_missing": true  } },
+    { "community_id": {} },
    { "set":         { "if": "ctx.source?.ip != null", "field": "client.ip",        "value": "{{source.ip}}"           } },
    { "set":         { "if": "ctx.source?.port != null", "field": "client.port",        "value": "{{source.port}}"       } },
    { "set":         { "if": "ctx.destination?.ip != null", "field": "server.ip",        "value": "{{destination.ip}}"  } },
@@ -1,5 +1,5 @@
 {
-  "index_patterns": ["so-ids-*", "so-firewall-*", "so-syslog-*", "so-zeek-*", "so-import-*", "so-ossec-*", "so-strelka-*", "so-beats-*", "so-osquery-*","so-playbook-*"],
+  "index_patterns": ["so-*"],
  "version":50001,
  "order":10,
  "settings":{
@@ -228,7 +228,11 @@
        "event":{
          "type":"object",
          "dynamic": true
-        }, 
+        },
+        "event_data":{
+          "type":"object",
+          "dynamic": true
+        },	
        "file":{
          "type":"object",
          "dynamic": true
@@ -316,7 +320,8 @@
          "type":"text",
          "fields":{  
            "keyword":{  
-              "type":"keyword"
+              "type":"keyword",
+	      "ignore_above": 32766
            }
          }
        },
@@ -522,11 +527,19 @@
            "version":{
              "type":"long"
            }
-        }
-      },
+          }
+        },
        "x509":{
          "type":"object",
          "dynamic": true
+        },
+	"suricata":{
+          "type":"object",
+          "dynamic": true
+        },
+	"zeek":{
+          "type":"object",
+          "dynamic": true
        }
     }
  }
@@ -71,7 +71,13 @@ logging.files:
 # Set to true to log messages in json format.
 #logging.json: false

+
+
 #==========================  Modules configuration ============================
+filebeat.config.modules:
+  enabled: true
+  path: ${path.config}/modules.d/*.yml
+
 filebeat.modules:
 #=========================== Filebeat prospectors =============================

@@ -183,7 +189,6 @@ filebeat.inputs:
  fields_under_root: true
  clean_removed: false
  close_removed: false
-
  {%- if STRELKAENABLED == 1 %}
 - type: log
  paths:
@@ -0,0 +1,10 @@
+{%- if grains['role'] in ['so-managersearch', 'so-heavynode', 'so-node'] %}
+{%- set MANAGER = salt['grains.get']('host' '') %}
+{%- else %}
+{%- set MANAGER = salt['grains.get']('master') %}
+{%- endif %}
+
+output.elasticsearch:
+  enabled: true
+  hosts: ["https://{{ MANAGER }}:9200"]
+  ssl.certificate_authorities: ["/usr/share/filebeat/intraca.crt"]
@@ -0,0 +1,16 @@
+# DO NOT EDIT THIS FILE
+{% for module in MODULES.modules.keys() %}
+- module: {{ module }}
+  {%- for fileset in MODULES.modules[module] %}
+  {{ fileset }}:
+    enabled: {{ MODULES.modules[module][fileset].enabled|string|lower }}
+    {#- only manage the settings if the fileset is enabled #}
+    {%- if MODULES.modules[module][fileset].enabled %}
+      {%- for var, value in MODULES.modules[module][fileset].items() %}
+        {%- if var|lower != 'enabled' %}
+    {{ var }}: {{ value }}
+        {%- endif %}
+      {%- endfor %}
+    {%- endif %}
+  {%- endfor %}
+{% endfor %}
@@ -20,18 +20,37 @@
 {% set LOCALHOSTIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] %}
 {% set MANAGER = salt['grains.get']('master') %}
 {% set MANAGERIP = salt['pillar.get']('global:managerip', '') %}
+{% from 'filebeat/map.jinja' import THIRDPARTY with context %}
+{% from 'filebeat/map.jinja' import SO with context %}
+{% set ES_INCLUDED_NODES = ['so-standalone'] %}
+
+#only include elastic state for certain nodes
+{% if grains.role in ES_INCLUDED_NODES %}
+include:
+  - elasticsearch
+{% endif %}
+
 filebeatetcdir:
  file.directory:
    - name: /opt/so/conf/filebeat/etc
    - user: 939
    - group: 939
    - makedirs: True
+
+filebeatmoduledir:
+  file.directory:
+    - name: /opt/so/conf/filebeat/modules
+    - user: root
+    - group: root
+    - makedirs: True
+
 filebeatlogdir:
  file.directory:
    - name: /opt/so/log/filebeat
    - user: 939
    - group: 939
    - makedirs: True
+
 filebeatpkidir:
  file.directory:
    - name: /opt/so/conf/filebeat/etc/pki
@@ -44,6 +63,7 @@ fileregistrydir:
    - user: 939
    - group: 939
    - makedirs: True
+
 # This needs to be owned by root
 filebeatconfsync:
  file.managed:
@@ -55,6 +75,32 @@ filebeatconfsync:
    - defaults:
        INPUTS: {{ salt['pillar.get']('filebeat:config:inputs', {}) }}
        OUTPUT: {{ salt['pillar.get']('filebeat:config:output', {}) }}
+
+# Filebeat module config file
+filebeatmoduleconfsync:
+  file.managed:
+    - name: /opt/so/conf/filebeat/etc/module-setup.yml
+    - source: salt://filebeat/etc/module-setup.yml
+    - user: root
+    - group: root
+    - template: jinja
+
+sodefaults_module_conf:
+  file.managed:
+    - name: /opt/so/conf/filebeat/modules/securityonion.yml
+    - source: salt://filebeat/etc/module_config.yml.jinja
+    - template: jinja
+    - defaults:
+        MODULES: {{ SO }}
+
+thirdparty_module_conf:
+  file.managed:
+    - name: /opt/so/conf/filebeat/modules/thirdparty.yml
+    - source: salt://filebeat/etc/module_config.yml.jinja
+    - template: jinja
+    - defaults:
+        MODULES: {{ THIRDPARTY }}
+    
 so-filebeat:
  docker_container.running:
    - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-filebeat:{{ VERSION }}
@@ -65,19 +111,40 @@ so-filebeat:
      - /nsm:/nsm:ro
      - /opt/so/log/filebeat:/usr/share/filebeat/logs:rw
      - /opt/so/conf/filebeat/etc/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
+      - /opt/so/conf/filebeat/etc/module-setup.yml:/usr/share/filebeat/module-setup.yml:ro
      - /nsm/wazuh/logs/alerts:/wazuh/alerts:ro
      - /nsm/wazuh/logs/archives:/wazuh/archives:ro
+      - /opt/so/conf/filebeat/modules:/usr/share/filebeat/modules.d
      - /opt/so/conf/filebeat/etc/pki/filebeat.crt:/usr/share/filebeat/filebeat.crt:ro
      - /opt/so/conf/filebeat/etc/pki/filebeat.key:/usr/share/filebeat/filebeat.key:ro
      - /opt/so/conf/filebeat/registry:/usr/share/filebeat/data/registry:rw
      - /etc/ssl/certs/intca.crt:/usr/share/filebeat/intraca.crt:ro
+      - /opt/so/log:/logs:ro
    - port_bindings:
        - 0.0.0.0:514:514/udp
        - 0.0.0.0:514:514/tcp
        - 0.0.0.0:5066:5066/tcp
+{% for module in THIRDPARTY.modules.keys() %}
+  {% for submodule in THIRDPARTY.modules[module] %}
+    {% if THIRDPARTY.modules[module][submodule].enabled %}
+        - {{ THIRDPARTY.modules[module][submodule].get("var.syslog_host", "0.0.0.0") }}:{{ THIRDPARTY.modules[module][submodule]["var.syslog_port"] }}:{{ THIRDPARTY.modules[module][submodule]["var.syslog_port"] }}/{{ THIRDPARTY.modules[module][submodule]["var.input"] }}
+    {% endif %}
+  {% endfor %}
+{% endfor %}
    - watch:
      - file: /opt/so/conf/filebeat/etc/filebeat.yml

+{% if grains.role in ES_INCLUDED_NODES %}
+run_module_setup:
+  cmd.run:
+    - name: /usr/sbin/so-filebeat-module-setup
+    - require:
+      - file: filebeatmoduleconfsync
+      - docker_container: so-filebeat
+    - onchanges_in:
+      - docker_container: so-elasticsearch
+{% endif %}
+
 append_so-filebeat_so-status.conf:
  file.append:
    - name: /opt/so/conf/so-status/so-status.conf
@@ -0,0 +1,6 @@
+{% import_yaml 'filebeat/thirdpartydefaults.yaml' as TPDEFAULTS %}
+{% set THIRDPARTY = salt['pillar.get']('filebeat:third_party_filebeat', default=TPDEFAULTS.third_party_filebeat, merge=True) %}
+
+{% import_yaml 'filebeat/securityoniondefaults.yaml' as SODEFAULTS %}
+{% set SO = SODEFAULTS.securityonion_filebeat %}
+{#% set SO = salt['pillar.get']('filebeat:third_party_filebeat', default=SODEFAULTS.third_party_filebeat, merge=True) %#}
@@ -0,0 +1,31 @@
+{%- set ZEEKVER = salt['pillar.get']('global:mdengine', '') %}
+{% set ZEEKLOGLOOKUP = {
+    'conn': 'connection',
+} %}
+securityonion_filebeat:
+  modules:
+  {%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone','so-node', 'so-hotnode', 'so-warmnode', 'so-heavynode']   %}
+    elasticsearch:
+      server:
+        enabled: true
+        var.paths: ["/logs/elasticsearch/*.log"]
+    logstash:
+      log:
+        enabled: true
+        var.paths: ["/logs/logstash.log"]
+  {%- endif %}
+  {%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone'] %}
+    kibana:
+      log:
+        enabled: true
+        var.paths: ["/logs/kibana/kibana.log"]
+  {%- endif %}
+  {%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-heavynode'] %}
+    redis:
+      log:
+        enabled: true
+        var.paths: ["/logs/redis.log"]
+      slowlog:
+        enabled: false
+  {%- endif %}
+        
@@ -0,0 +1,259 @@
+third_party_filebeat:
+  modules:
+    aws:
+      cloudtrail:
+        enabled: false
+      cloudwatch:
+        enabled: false
+      ec2:
+        enabled: false
+      elb:
+        enabled: false
+      s3access:
+        enabled: false
+      vpcflow:
+        enabled: false
+    azure:
+      activitylogs:
+        enabled: false
+      platformlogs:
+        enabled: false
+      auditlogs:
+        enabled: false
+      signinlogs:
+        enabled: false
+    barracuda:
+      waf:
+        enabled: false
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9503
+      spamfirewall:
+        enabled: false
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9524
+    bluecoat:
+      director:
+        enabled: false
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9505
+    cef:
+      log:
+        enabled: false
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9003
+    checkpoint:
+      firewall:
+        enabled: false
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9505
+    cisco:
+      asa:
+        enabled: false
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9001
+      ftd:
+        enabled: false
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9003
+      ios:
+        enabled: false
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9002
+      nexus:
+        enabled: false
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9506
+      meraki:
+        enabled: false
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9525
+      umbrella:
+        enabled: false
+      amp:
+        enabled: false
+    cyberark:
+      corepas:
+        enabled: false
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9527
+    cylance:
+      protect:
+        enabled: false
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9508
+    f5:
+      bigipapm:
+        enabled: false
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9504
+      bigipafm:
+        enabled: false
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9528    
+    fortinet:
+      firewall:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9004
+      clientendpoint:
+        enabled: false
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9510
+      fortimail:
+        enabled: false
+        var.input: udp
+        var.syslog_port: 9350
+    gcp:
+      vpcflow:
+        enabled: false
+      firewall:
+        enabled: false
+      audit:
+        enabled: false
+    google_workspace:
+      saml:
+        enabled: false
+      user_accounts:
+        enabled: false
+      login:
+        enabled: false
+      admin:
+        enabled: false
+      drive:
+        enabled: false
+      groups:
+        enabled: false
+    imperva:
+      securesphere:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9511
+    infoblox:
+      nios:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9512
+    juniper:
+      junos:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9513
+      netscreen:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9523
+      srx:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9006
+    microsoft:
+      defender_atp:
+        enabled: false 
+      m365_defender:
+        enabled: false
+      dhcp:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9515
+    misp:
+      threat:
+        enabled: false
+    netflow:
+      log:
+        enabled: false
+        var.netflow_host: 0.0.0.0
+        var.netflow_port: 2055
+        var.internal_networks:
+          - private
+    netscout:
+      sightline:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9502
+    o365:
+      audit:
+        enabled: false
+    okta:
+      system:
+        enabled: false
+    proofpoint:
+      emailsecurity:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9531
+    radware:
+      defensepro:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9518
+    snort:
+      log:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9532
+    snyk:
+      audit:
+        enabled: false
+      vulnerabilities:
+        enabled: false
+    sonicwall:
+      firewall:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9519
+    sophos:
+      xg:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9005
+      utm:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9533
+    squid:
+      log:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9520
+    tomcat:
+      log:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9501
+    zscaler:
+      zia:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9521
@@ -352,7 +352,7 @@
          ],
          "measurement": "zeekcaptureloss",
          "orderByTime": "ASC",
-          "policy": "autogen",
+          "policy": "default",
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
@@ -2176,7 +2176,7 @@
          ],
          "measurement": "docker_container_mem",
          "orderByTime": "ASC",
-          "policy": "autogen",
+          "policy": "default",
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
@@ -1647,7 +1647,7 @@
          ],
          "measurement": "influxsize",
          "orderByTime": "ASC",
-          "policy": "autogen",
+          "policy": "default",
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
@@ -1631,7 +1631,7 @@
          ],
          "measurement": "influxsize",
          "orderByTime": "ASC",
-          "policy": "autogen",
+          "policy": "default",
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
@@ -351,7 +351,7 @@
          ],
          "measurement": "zeekcaptureloss",
          "orderByTime": "ASC",
-          "policy": "autogen",
+          "policy": "default",
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
@@ -2866,7 +2866,7 @@
          ],
          "measurement": "healthcheck",
          "orderByTime": "ASC",
-          "policy": "autogen",
+          "policy": "default",
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
@@ -4486,7 +4486,7 @@
          ],
          "measurement": "zeekcaptureloss",
          "orderByTime": "ASC",
-          "policy": "autogen",
+          "policy": "default",
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
@@ -5107,7 +5107,7 @@
          ],
          "measurement": "influxsize",
          "orderByTime": "ASC",
-          "policy": "autogen",
+          "policy": "default",
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
@@ -0,0 +1,13 @@
+influxdb:
+  retention_policies:
+    so_short_term:
+      default: True
+      duration: 30d
+      shard_duration: 1d
+    so_long_term:
+      default: False
+      duration: 0d
+      shard_duration: 7d
+  downsample:
+    so_long_term:
+      resolution: 5m
@@ -2,12 +2,21 @@
 {% if sls in allowed_states %}

 {% set GRAFANA = salt['pillar.get']('manager:grafana', '0') %}
-{% set MANAGER = salt['grains.get']('master') %}
-{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
-{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}

 {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone'] or (grains.role == 'so-eval' and GRAFANA == 1) %}

+{% set MANAGER = salt['grains.get']('master') %}
+{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
+{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
+{% import_yaml 'influxdb/defaults.yaml' as default_settings %}
+{% set influxdb = salt['grains.filter_by'](default_settings, default='influxdb', merge=salt['pillar.get']('influxdb', {})) %}
+{% from 'salt/map.jinja' import PYTHON3INFLUX with context %}
+{% from 'salt/map.jinja' import  PYTHONINFLUXVERSION with context %}
+{% set PYTHONINFLUXVERSIONINSTALLED = salt['cmd.run']("python3 -c \"exec('try:import influxdb; print (influxdb.__version__)\\nexcept:print(\\'Module Not Found\\')')\"", python_shell=True) %}
+
+include:
+  - salt.minion
+  - salt.python3-influxdb
 # Influx DB
 influxconfdir:
  file.directory:
@@ -57,6 +66,70 @@ append_so-influxdb_so-status.conf:
    - name: /opt/so/conf/so-status/so-status.conf
    - text: so-influxdb

+# We have to make sure the influxdb module is the right version prior to state run since reload_modules is bugged
+{% if PYTHONINFLUXVERSIONINSTALLED == PYTHONINFLUXVERSION %}
+wait_for_influxdb:
+  http.query:
+    - name: 'https://{{MANAGER}}:8086/query?q=SHOW+DATABASES'
+    - ssl: True
+    - verify_ssl: False
+    - status: 200
+    - timeout: 30
+    - retry:
+        attempts: 5
+        interval: 60
+
+telegraf_database:
+  influxdb_database.present:
+    - name: telegraf
+    - database: telegraf
+    - ssl: True
+    - verify_ssl: /etc/pki/ca.crt
+    - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key']
+    - influxdb_host: {{ MANAGER }}
+    - require:
+      - docker_container: so-influxdb
+      - sls: salt.python3-influxdb
+      - http: wait_for_influxdb
+
+{% for rp in influxdb.retention_policies.keys() %}
+{{rp}}_retention_policy:
+  influxdb_retention_policy.present:
+    - name: {{rp}}
+    - database: telegraf
+    - duration: {{influxdb.retention_policies[rp].duration}}
+    - shard_duration: {{influxdb.retention_policies[rp].shard_duration}}
+    - replication: 1
+    - default: {{influxdb.retention_policies[rp].get('default', 'False')}}
+    - ssl: True
+    - verify_ssl: /etc/pki/ca.crt
+    - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key']
+    - influxdb_host: {{ MANAGER }}
+    - require:
+      - docker_container: so-influxdb
+      - influxdb_database: telegraf_database
+      - file: influxdb_retention_policy.present_patch
+      - sls: salt.python3-influxdb
+{% endfor %}
+
+{% for dest_rp in influxdb.downsample.keys() %}
+so_downsample_cq:
+  influxdb_continuous_query.present:
+    - name: so_downsample_cq
+    - database: telegraf
+    - query: SELECT mean(*) INTO "{{dest_rp}}".:MEASUREMENT FROM /.*/ GROUP BY time({{influxdb.downsample[dest_rp].resolution}}),*
+    - ssl: True
+    - verify_ssl: /etc/pki/ca.crt
+    - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key']
+    - influxdb_host: {{ MANAGER }}
+    - require:
+      - docker_container: so-influxdb
+      - influxdb_database: telegraf_database
+      - file: influxdb_continuous_query.present_patch
+      - sls: salt.python3-influxdb
+{% endfor %}
+
+{% endif %}
 {% endif %}

 {% else %}
@@ -1,10 +0,0 @@
-{ "attributes":
-        {
-          "defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29",
-          "defaultRoute":"/app/kibana#/dashboard/a8411b30-6d03-11ea-b301-3d6c35840645",
-          "discover:sampleSize":"100",
-          "dashboard:defaultDarkTheme":true,
-          "theme:darkMode":true,
-          "timepicker:timeDefaults":"{\n  \"from\": \"now-24h\",\n  \"to\": \"now\"\n}"
-        }
-}
@@ -460,7 +460,7 @@
 {"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\",\"default_field\":\"*\",\"time_zone\":\"America/New_York\"}},\"language\":\"lucene\"},\"filter\":[]}"},"optionsJSON":"{\"darkTheme\":true,\"useMargins\":true}","panelsJSON":"[{\"version\":\"7.9.0\",\"gridData\":{\"w\":8,\"h\":48,\"x\":0,\"y\":0,\"i\":\"1\"},\"panelIndex\":\"1\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_0\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":32,\"h\":8,\"x\":16,\"y\":0,\"i\":\"2\"},\"panelIndex\":\"2\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_1\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":48,\"h\":24,\"x\":0,\"y\":96,\"i\":\"6\"},\"panelIndex\":\"6\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":20,\"h\":28,\"x\":8,\"y\":20,\"i\":\"7\"},\"panelIndex\":\"7\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":20,\"h\":28,\"x\":28,\"y\":20,\"i\":\"8\"},\"panelIndex\":\"8\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_4\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":16,\"h\":24,\"x\":0,\"y\":72,\"i\":\"9\"},\"panelIndex\":\"9\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_5\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":48,\"h\":24,\"x\":0,\"y\":48,\"i\":\"10\"},\"panelIndex\":\"10\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_6\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":20,\"h\":12,\"x\":8,\"y\":8,\"i\":\"11\"},\"panelIndex\":\"11\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_7\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":32,\"h\":24,\"x\":16,\"y\":72,\"i\":\"12\"},\"panelIndex\":\"12\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_8\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":20,\"h\":12,\"x\":28,\"y\":8,\"i\":\"13\"},\"panelIndex\":\"13\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_9\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":48,\"h\":24,\"x\":0,\"y\":120,\"i\":\"14\"},\"panelIndex\":\"14\",\"embeddableConfig\":{\"columns\":[\"event_type\",\"source_ip\",\"source_port\",\"destination_ip\",\"destination_port\",\"_id\"],\"sort\":[\"@timestamp\",\"desc\"],\"enhancements\":{}},\"panelRefName\":\"panel_10\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":8,\"h\":8,\"x\":8,\"y\":0,\"i\":\"15\"},\"panelIndex\":\"15\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_11\"}]","timeRestore":false,"title":"z16.04 - Sysmon - Logs","version":1},"id":"6d189680-6d62-11e7-8ddb-e71eb260f4a3","migrationVersion":{"dashboard":"7.11.0"},"references":[{"id":"b3b449d0-3429-11e7-9d52-4f090484f59e","name":"panel_0","type":"visualization"},{"id":"8cfdeff0-6d6b-11e7-ad64-15aa071374a6","name":"panel_1","type":"visualization"},{"id":"0eb1fd80-6d70-11e7-b09b-f57b22df6524","name":"panel_2","type":"visualization"},{"id":"3072c750-6d71-11e7-b09b-f57b22df6524","name":"panel_3","type":"visualization"},{"id":"7bc74b40-6d71-11e7-b09b-f57b22df6524","name":"panel_4","type":"visualization"},{"id":"13ed0810-6d72-11e7-b09b-f57b22df6524","name":"panel_5","type":"visualization"},{"id":"3b6c92c0-6d72-11e7-b09b-f57b22df6524","name":"panel_6","type":"visualization"},{"id":"e09f6010-6d72-11e7-b09b-f57b22df6524","name":"panel_7","type":"visualization"},{"id":"29611940-6d75-11e7-b09b-f57b22df6524","name":"panel_8","type":"visualization"},{"id":"6b70b840-6d75-11e7-b09b-f57b22df6524","name":"panel_9","type":"visualization"},{"id":"248c1d20-6d6b-11e7-ad64-15aa071374a6","name":"panel_10","type":"search"},{"id":"AWDHHk1sxQT5EBNmq43Y","name":"panel_11","type":"visualization"}],"type":"dashboard","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQyLDRd"}
 {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"},"savedSearchRefName":"search_0","title":"SMB - Action (Pie Chart)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"SMB - Action (Pie Chart)\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"type\":\"pie\",\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"action.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\"}}]}"},"id":"6f883480-3aad-11e7-8b17-0d8709b02c80","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"19849f30-3aab-11e7-8b17-0d8709b02c80","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQzLDRd"}
 {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Security Onion - SSL - Subject","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Security Onion - SSL - Subject\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://PLACEHOLDER\",\"pathname\":\"/kibana/app/kibana\",\"basePath\":\"/kibana\"}}},\"params\":{},\"label\":\"ssl.certificate.subject.keyword: Descending\",\"aggType\":\"terms\"}]},\"showToolbar\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"ssl.certificate.subject.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Subject\"}}]}"},"id":"6fccb600-75ec-11ea-9565-7315f4ee5cac","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ0LDRd"}
-{"attributes":{"buildNum":33984,"dashboard:defaultDarkTheme":true,"defaultIndex":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute":"/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize":"100","theme:darkMode":true,"timepicker:timeDefaults":"{\n  \"from\": \"now-24h\",\n  \"to\": \"now\"\n}"},"id":"7.11.2","migrationVersion":{"config":"7.9.0"},"references":[],"type":"config","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ1LDRd"}
+{"attributes":{"buildNum":39457,"defaultIndex":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute":"/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize":100,"theme:darkMode":true,"timepicker:timeDefaults":"{\n  \"from\": \"now-24h\",\n  \"to\": \"now\"\n}"},"coreMigrationVersion":"7.13.1","id":"7.13.1","migrationVersion":{"config":"7.12.0"},"references":[],"type":"config","updated_at":"2021-04-29T21:42:52.430Z","version":"WzY3NTUsM10="}
 {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Strelka - File - MIME Flavors","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Strelka - File - MIME Flavors\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":0,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}],\"buckets\":[]},\"showToolbar\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"file.flavors.mime.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"70243970-772c-11ea-bee5-af7f7c7b8e05","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ2LDRd"}
 {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[]}"},"savedSearchRefName":"search_0","title":"Modbus - Log Count","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}","version":1,"visState":"{\"title\":\"Modbus - Log Count\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":\"30\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"bgFill\":\"#FB9E00\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{}}"},"id":"AWDG_9KpxQT5EBNmq4Oo","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"52dc9fe0-342e-11e7-9e93-53b62e1857b2","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ3LDRd"}
 {"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\",\"default_field\":\"*\"}},\"language\":\"lucene\"},\"filter\":[]}"},"optionsJSON":"{\"darkTheme\":true,\"useMargins\":true}","panelsJSON":"[{\"version\":\"7.9.0\",\"gridData\":{\"w\":8,\"h\":56,\"x\":0,\"y\":0,\"i\":\"2\"},\"panelIndex\":\"2\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_0\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":32,\"h\":8,\"x\":16,\"y\":0,\"i\":\"3\"},\"panelIndex\":\"3\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_1\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":12,\"h\":24,\"x\":8,\"y\":32,\"i\":\"5\"},\"panelIndex\":\"5\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":12,\"h\":24,\"x\":20,\"y\":32,\"i\":\"6\"},\"panelIndex\":\"6\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":48,\"h\":24,\"x\":0,\"y\":56,\"i\":\"8\"},\"panelIndex\":\"8\",\"embeddableConfig\":{\"columns\":[\"source_ip\",\"source_port\",\"destination_ip\",\"destination_port\",\"uid\",\"_id\"],\"sort\":[\"@timestamp\",\"desc\"],\"enhancements\":{}},\"panelRefName\":\"panel_4\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":16,\"h\":24,\"x\":32,\"y\":32,\"i\":\"9\"},\"panelIndex\":\"9\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_5\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":8,\"h\":8,\"x\":8,\"y\":0,\"i\":\"10\"},\"panelIndex\":\"10\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_6\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":40,\"h\":24,\"x\":8,\"y\":8,\"i\":\"11\"},\"panelIndex\":\"11\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_7\"}]","timeRestore":false,"title":"z16.04 - Bro - Modbus","version":1},"id":"70c005f0-3583-11e7-a588-05992195c551","migrationVersion":{"dashboard":"7.11.0"},"references":[{"id":"b3b449d0-3429-11e7-9d52-4f090484f59e","name":"panel_0","type":"visualization"},{"id":"0d168a30-363f-11e7-a6f7-4f44d7bf1c33","name":"panel_1","type":"visualization"},{"id":"20eabd60-380b-11e7-a1cc-ebc6a7e70e84","name":"panel_2","type":"visualization"},{"id":"3c65f500-380b-11e7-a1cc-ebc6a7e70e84","name":"panel_3","type":"visualization"},{"id":"52dc9fe0-342e-11e7-9e93-53b62e1857b2","name":"panel_4","type":"search"},{"id":"178209e0-6e1b-11e7-b553-7f80727663c1","name":"panel_5","type":"visualization"},{"id":"AWDG_9KpxQT5EBNmq4Oo","name":"panel_6","type":"visualization"},{"id":"453f8b90-4a58-11e8-9b0a-f1d33346f773","name":"panel_7","type":"visualization"}],"type":"dashboard","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ4LDRd"}
@@ -730,4 +730,4 @@
 {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Security Onion - Syslog - Severity (Donut)","uiStateJSON":"{}","version":1,"visState":"{\"type\":\"pie\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"syslog.severity.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":25,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Severity\"}}],\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100},\"dimensions\":{\"metric\":{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"},\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://PLACEHOLDER\",\"pathname\":\"/kibana/app/kibana\",\"basePath\":\"/kibana\"}}},\"params\":{},\"label\":\"syslog.severity.keyword: Descending\",\"aggType\":\"terms\"}]}},\"title\":\"Security Onion - Syslog - Severity (Donut)\"}"},"id":"fc8d41a0-777b-11ea-bee5-af7f7c7b8e05","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwNDExLDRd"}
 {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{}"},"savedSearchRefName":"search_0","title":"Security Onion - Connections - Top Source IPs","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Security Onion - Connections - Top Source IPs\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://PLACEHOLDER\",\"pathname\":\"/kibana/app/kibana\",\"basePath\":\"/kibana\"}}},\"params\":{},\"label\":\"source.ip: Descending\",\"aggType\":\"terms\"}]},\"showToolbar\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source.ip\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Source IP\"}}]}"},"id":"fd8b4640-6e9f-11ea-9266-1fd14ca6af34","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"9b333020-6e9f-11ea-9266-1fd14ca6af34","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwNDEyLDRd"}
 {"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"event.module:strelka\"},\"filter\":[]}"},"optionsJSON":"{\"hidePanelTitles\":false,\"useMargins\":true}","panelsJSON":"[{\"version\":\"7.6.1\",\"gridData\":{\"x\":0,\"y\":0,\"w\":8,\"h\":7,\"i\":\"a2e0a619-a5c5-40d9-8593-e60f13ae22bf\"},\"panelIndex\":\"a2e0a619-a5c5-40d9-8593-e60f13ae22bf\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_0\"},{\"version\":\"7.6.1\",\"gridData\":{\"x\":8,\"y\":0,\"w\":21,\"h\":7,\"i\":\"566a9d04-f2dc-4868-9625-97a19d985703\"},\"panelIndex\":\"566a9d04-f2dc-4868-9625-97a19d985703\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_1\"},{\"version\":\"7.6.1\",\"gridData\":{\"x\":29,\"y\":0,\"w\":19,\"h\":7,\"i\":\"f247ec64-c278-4e05-ac4d-983bea9dfb7d\"},\"panelIndex\":\"f247ec64-c278-4e05-ac4d-983bea9dfb7d\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2\"},{\"version\":\"7.6.1\",\"gridData\":{\"x\":0,\"y\":7,\"w\":12,\"h\":20,\"i\":\"6e80a142-ab0e-4fd3-891c-e495b78a1625\"},\"panelIndex\":\"6e80a142-ab0e-4fd3-891c-e495b78a1625\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3\"},{\"version\":\"7.6.1\",\"gridData\":{\"x\":12,\"y\":7,\"w\":11,\"h\":20,\"i\":\"292cc879-6bc0-4541-ba92-3b3c5f4e3368\"},\"panelIndex\":\"292cc879-6bc0-4541-ba92-3b3c5f4e3368\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_4\"},{\"version\":\"7.6.1\",\"gridData\":{\"x\":23,\"y\":7,\"w\":14,\"h\":20,\"i\":\"66979b2c-e7c1-4291-91ac-16537b7f9ec3\"},\"panelIndex\":\"66979b2c-e7c1-4291-91ac-16537b7f9ec3\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_5\"},{\"version\":\"7.6.1\",\"gridData\":{\"x\":37,\"y\":7,\"w\":11,\"h\":20,\"i\":\"8bb1cf98-0401-4a2d-9dd8-deca08205a22\"},\"panelIndex\":\"8bb1cf98-0401-4a2d-9dd8-deca08205a22\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_6\"},{\"version\":\"7.6.1\",\"gridData\":{\"x\":0,\"y\":27,\"w\":8,\"h\":20,\"i\":\"393f3cec-3ee0-4275-b319-f307e7a260c6\"},\"panelIndex\":\"393f3cec-3ee0-4275-b319-f307e7a260c6\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_7\"},{\"version\":\"7.6.1\",\"gridData\":{\"x\":8,\"y\":27,\"w\":15,\"h\":20,\"i\":\"0e8800a9-a6f5-4a79-8370-61713f584886\"},\"panelIndex\":\"0e8800a9-a6f5-4a79-8370-61713f584886\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_8\"},{\"version\":\"7.6.1\",\"gridData\":{\"x\":23,\"y\":27,\"w\":25,\"h\":20,\"i\":\"be9a0a2a-d8c6-4d15-b5d7-d5599d0482a3\"},\"panelIndex\":\"be9a0a2a-d8c6-4d15-b5d7-d5599d0482a3\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_9\"},{\"version\":\"7.6.1\",\"gridData\":{\"x\":0,\"y\":47,\"w\":48,\"h\":27,\"i\":\"40296d2b-cb6f-423f-989c-3fdaa82d2aad\"},\"panelIndex\":\"40296d2b-cb6f-423f-989c-3fdaa82d2aad\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_10\"}]","timeRestore":false,"title":"Security Onion - Strelka","version":1},"id":"ff689c50-75f3-11ea-9565-7315f4ee5cac","migrationVersion":{"dashboard":"7.11.0"},"references":[{"id":"8cfec8c0-6ec2-11ea-9266-1fd14ca6af34","name":"panel_0","type":"visualization"},{"id":"d04b5130-6e99-11ea-9266-1fd14ca6af34","name":"panel_1","type":"visualization"},{"id":"23ed13a0-6e9a-11ea-9266-1fd14ca6af34","name":"panel_2","type":"visualization"},{"id":"7a88adc0-75f0-11ea-9565-7315f4ee5cac","name":"panel_3","type":"visualization"},{"id":"49cfe850-772c-11ea-bee5-af7f7c7b8e05","name":"panel_4","type":"visualization"},{"id":"70243970-772c-11ea-bee5-af7f7c7b8e05","name":"panel_5","type":"visualization"},{"id":"ce9e03f0-772c-11ea-bee5-af7f7c7b8e05","name":"panel_6","type":"visualization"},{"id":"a7ebb450-772c-11ea-bee5-af7f7c7b8e05","name":"panel_7","type":"visualization"},{"id":"08c0b770-772e-11ea-bee5-af7f7c7b8e05","name":"panel_8","type":"visualization"},{"id":"e087c7d0-772d-11ea-bee5-af7f7c7b8e05","name":"panel_9","type":"visualization"},{"id":"8b6f3150-72a2-11ea-8dd2-9d8795a1200b","name":"panel_10","type":"search"}],"type":"dashboard","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwNDEzLDRd"}
-{"exportedCount":732,"missingRefCount":0,"missingReferences":[]}
+{"exportedCount":732,"missingRefCount":0,"missingReferences":[]}
@@ -3,4 +3,9 @@ input {
    port => "5044"
    tags => [ "beat-ext" ]
  }
+}
+filter {
+  mutate {
+    rename => {"@metadata" => "metadata"}
+  }
 }
@@ -0,0 +1,20 @@
+{%- if grains['role'] == 'so-eval' -%}
+{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
+{%- else %}
+{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
+{%- endif %}
+output {
+ if [metadata][pipeline] {
+   elasticsearch {
+     id => "filebeat_modules_metadata_pipeline"
+     pipeline => "%{[metadata][pipeline]}"
+     hosts => "{{ ES }}"
+     index => "so-%{[event][module]}-%{+YYYY.MM.dd}"
+     template_name => "so-common"
+     template => "/templates/so-common-template.json"
+     template_overwrite => true
+     ssl => true
+     ssl_certificate_verification => false
+   }
+ }
+}
@@ -15,4 +15,4 @@ output {
      ssl_certificate_verification => false
    }
  }
-}
+}
@@ -175,8 +175,8 @@ http {
 			auth_request_set      $userid $upstream_http_x_kratos_authenticated_identity_id;
 			proxy_set_header      x-user-id $userid;
 			proxy_pass            http://{{ manager_ip }}:9822/;
-			proxy_read_timeout    90;
-			proxy_connect_timeout 90;
+			proxy_read_timeout    300;
+			proxy_connect_timeout 300;
 			proxy_set_header      Host $host;
 			proxy_set_header      X-Real-IP $remote_addr;
 			proxy_set_header      X-Forwarded-For $proxy_add_x_forwarded_for;
@@ -261,8 +261,8 @@ http {
 			auth_request          /auth/sessions/whoami;
 			rewrite               /kibana/(.*) /$1 break;
 			proxy_pass            http://{{ manager_ip }}:5601/;
-			proxy_read_timeout    90;
-			proxy_connect_timeout 90;
+			proxy_read_timeout    300;
+			proxy_connect_timeout 300;
 			proxy_set_header      Host $host;
 			proxy_set_header      X-Real-IP $remote_addr;
 			proxy_set_header      X-Forwarded-For $proxy_add_x_forwarded_for;
@@ -341,8 +341,8 @@ http {
 		
 		location /soctopus/ {
 			proxy_pass            http://{{ manager_ip }}:7000/;
-			proxy_read_timeout    90;
-			proxy_connect_timeout 90;
+			proxy_read_timeout    300;
+			proxy_connect_timeout 300;
 			proxy_set_header      Host $host;
 			proxy_set_header      X-Real-IP $remote_addr;
 			proxy_set_header      X-Forwarded-For $proxy_add_x_forwarded_for;
@@ -0,0 +1,4 @@
+load_elastic_pipelines:
+  cmd.run: 
+    - name: /usr/sbin/so-filebeat-module-setup
+  
@@ -0,0 +1,4 @@
+60c60
+<             database, name, query, resample_time, coverage_period
+---
+>             database, name, query, resample_time, coverage_period, **client_args
@@ -0,0 +1,16 @@
+38c38
+<         hours = int(duration.split("h"))
+---
+>         hours = int(duration.split("h")[0])
+52c52
+< def present(name, database, duration="7d", replication=1, default=False, **client_args):
+---
+> def present(name, database, duration="7d", replication=1, default=False, shard_duration="1d", **client_args):
+77c77
+<             database, name, duration, replication, default, **client_args
+---
+>             database, name, duration, replication, shard_duration, default, **client_args
+119c119
+<                     database, name, duration, replication, default, **client_args
+---
+>                     database, name, duration, replication, shard_duration, default, **client_args
@@ -0,0 +1,16 @@
+427c427
+<     database, name, duration, replication, default=False, **client_args
+---
+>     database, name, duration, replication, shard_duration, default=False, **client_args
+462c462
+<     client.create_retention_policy(name, duration, replication, database, default)
+---
+>     client.create_retention_policy(name, duration, replication, database, default, shard_duration)
+468c468
+<     database, name, duration, replication, default=False, **client_args
+---
+>     database, name, duration, replication, shard_duration, default=False, **client_args
+504c504
+<     client.alter_retention_policy(name, database, duration, replication, default)
+---
+>     client.alter_retention_policy(name, database, duration, replication, default, shard_duration)
@@ -0,0 +1,3 @@
+patch_package:
+  pkg.installed:
+    - name: patch
@@ -5,10 +5,22 @@
  {% set SPLITCHAR = '+' %}
  {% set SALTNOTHELD = salt['cmd.run']('apt-mark showhold | grep -q salt ; echo $?', python_shell=True) %}
  {% set SALTPACKAGES = ['salt-common', 'salt-master', 'salt-minion'] %}
+  {% set SALT_STATE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/states' %}
+  {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/modules' %}
+  {% set PYTHONINFLUXVERSION = '5.3.1' %}
+  {% set PYTHON3INFLUX= 'influxdb == ' ~ PYTHONINFLUXVERSION  %}
+  {% set PYTHON3INFLUXDEPS= ['certifi', 'chardet', 'python-dateutil', 'pytz', 'requests'] %}
+  {% set PYTHONINSTALLER = 'pip' %}
 {% else %}
  {% set SPLITCHAR = '-' %}
  {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep -q salt ; echo $?', python_shell=True) %}
  {% set SALTPACKAGES = ['salt', 'salt-master', 'salt-minion'] %}
+  {% set SALT_STATE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/states' %}
+  {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/modules' %}
+  {% set PYTHONINFLUXVERSION = '5.3.1' %}
+  {% set PYTHON3INFLUX= 'securityonion-python3-influxdb' %}
+  {% set PYTHON3INFLUXDEPS= ['python36-certifi', 'python36-chardet', 'python36-dateutil', 'python36-pytz', 'python36-requests'] %}
+  {% set PYTHONINSTALLER = 'pkg' %}
 {% endif %}

 {% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %}
@@ -8,6 +8,7 @@

 include:
  - salt
+  - salt.helper-packages
  - systemd.reload

 {% if INSTALLEDSALTVERSION|string != SALTVERSION|string %}
@@ -82,4 +83,8 @@ salt_minion_service:
  service.running:
    - name: salt-minion
    - enable: True
-    - onlyif: test "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}"
+    - onlyif: test "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}"
+
+patch_pkg:
+  pkg.installed:
+    - name: patch
@@ -0,0 +1,70 @@
+{% from "salt/map.jinja" import SALT_STATE_CODE_PATH with context %}
+{% from "salt/map.jinja" import SALT_MODULE_CODE_PATH with context %}
+{% from "salt/map.jinja" import PYTHON3INFLUX with context %}
+{% from "salt/map.jinja" import PYTHON3INFLUXDEPS with context %}
+{% from "salt/map.jinja" import PYTHONINSTALLER with context %}
+
+include:
+  - salt.helper-packages
+
+python3_influxdb_dependencies:
+  {{PYTHONINSTALLER}}.installed:
+    - pkgs: {{ PYTHON3INFLUXDEPS }}
+
+python3_influxdb:
+  {{PYTHONINSTALLER}}.installed:
+    - name: {{ PYTHON3INFLUX }}
+
+# We circumvent the file.patch state putting ERROR in the log by using the unless and file.touch below
+# https://github.com/saltstack/salt/pull/47010 and https://github.com/saltstack/salt/issues/52329
+
+#https://github.com/saltstack/salt/issues/59766
+influxdb_continuous_query.present_patch:
+  file.patch:
+    - name: {{ SALT_STATE_CODE_PATH }}/influxdb_continuous_query.py
+    - source: salt://salt/files/influxdb_continuous_query.py.patch
+    - require:
+      - {{PYTHONINSTALLER}}: python3_influxdb
+      - pkg: patch_package
+    - unless: ls /opt/so/state/influxdb_continuous_query.py.patched
+
+influxdb_continuous_query.py.patched:
+  file.touch:
+    - name: /opt/so/state/influxdb_continuous_query.py.patched
+    - onchanges:
+      - file: influxdb_continuous_query.present_patch
+
+#https://github.com/saltstack/salt/issues/59761
+influxdb_retention_policy.present_patch:
+  file.patch:
+    - name: {{ SALT_STATE_CODE_PATH }}/influxdb_retention_policy.py
+    - source: salt://salt/files/influxdb_retention_policy.py.patch
+    - require:
+      - {{PYTHONINSTALLER}}: python3_influxdb
+      - pkg: patch_package
+    - unless: ls /opt/so/state/influxdb_retention_policy.py.patched
+
+influxdb_retention_policy.py.patched:
+  file.touch:
+    - name: /opt/so/state/influxdb_retention_policy.py.patched
+    - onchanges:
+      - file: influxdb_retention_policy.present_patch
+
+# We should be able to set reload_modules: True in this state in order to tell salt to reload its python modules due to us possibly installing
+# and possibly modifying modules in this state. This is bugged according to https://github.com/saltstack/salt/issues/24925
+influxdbmod.py_shard_duration_patch:
+  file.patch:
+    - name: {{ SALT_MODULE_CODE_PATH }}/influxdbmod.py
+    - source: salt://salt/files/influxdbmod.py.patch
+    - require:
+      - {{PYTHONINSTALLER}}: python3_influxdb
+      - pkg: patch_package
+    - unless: ls /opt/so/state/influxdbmod.py.patched
+
+influxdbmod.py.patched:
+  file.touch:
+    - name: /opt/so/state/influxdbmod.py.patched
+    - onchanges:
+      - file: influxdbmod.py_shard_duration_patch
+
+
@@ -1,29 +1 @@
-[
-  { "name": "actionHunt", "description": "actionHuntHelp", "icon": "fa-crosshairs", "target": "", 
-            "links": [
-              "/#/hunt?q=\"{value|escape}\" | groupby event.module event.dataset"
-            ]},
-  { "name": "actionCorrelate", "description": "actionCorrelateHelp", "icon": "fab fa-searchengin", "target": "",
-            "links": [
-              "/#/hunt?q=(\"{:log.id.fuid}\" OR \"{:log.id.uid}\" OR \"{:network.community_id}\") | groupby event.module event.dataset", 
-              "/#/hunt?q=(\"{:log.id.fuid}\" OR \"{:log.id.uid}\") | groupby event.module event.dataset", 
-              "/#/hunt?q=(\"{:log.id.fuid}\" OR \"{:network.community_id}\") | groupby event.module event.dataset", 
-              "/#/hunt?q=(\"{:log.id.uid}\" OR \"{:network.community_id}\") | groupby event.module event.dataset", 
-              "/#/hunt?q=\"{:log.id.fuid}\" | groupby event.module event.dataset",
-              "/#/hunt?q=\"{:log.id.uid}\" | groupby event.module event.dataset", 
-              "/#/hunt?q=\"{:network.community_id}\" | groupby event.module event.dataset"
-            ]},
-  { "name": "actionPcap", "description": "actionPcapHelp", "icon": "fa-stream", "target": "", 
-            "links": [
-              "/joblookup?esid={:soc_id}", 
-              "/joblookup?ncid={:network.community_id}"
-            ]},
-  { "name": "actionGoogle", "description": "actionGoogleHelp", "icon": "fab fa-google", "target": "_blank", 
-            "links": [
-              "https://www.google.com/search?q={value}"
-            ]},
-  { "name": "actionVirusTotal", "description": "actionVirusTotalHelp", "icon": "fa-external-link-alt", "target": "_blank", 
-            "links": [
-              "https://www.virustotal.com/gui/search/{value}"
-					  ]}
-]
+This file is no longer used. Please use menu.actions.json instead.
@@ -1,29 +1 @@
-[
-  { "name": "actionHunt", "description": "actionHuntHelp", "icon": "fa-crosshairs", "target": "", 
-            "links": [
-              "/#/hunt?q=\"{value|escape}\" | groupby event.module event.dataset"
-            ]},
-  { "name": "actionCorrelate", "description": "actionCorrelateHelp", "icon": "fab fa-searchengin", "target": "",
-            "links": [
-              "/#/hunt?q=(\"{:log.id.fuid}\" OR \"{:log.id.uid}\" OR \"{:network.community_id}\") | groupby event.module event.dataset", 
-              "/#/hunt?q=(\"{:log.id.fuid}\" OR \"{:log.id.uid}\") | groupby event.module event.dataset", 
-              "/#/hunt?q=(\"{:log.id.fuid}\" OR \"{:network.community_id}\") | groupby event.module event.dataset", 
-              "/#/hunt?q=(\"{:log.id.uid}\" OR \"{:network.community_id}\") | groupby event.module event.dataset", 
-              "/#/hunt?q=\"{:log.id.fuid}\" | groupby event.module event.dataset",
-              "/#/hunt?q=\"{:log.id.uid}\" | groupby event.module event.dataset", 
-              "/#/hunt?q=\"{:network.community_id}\" | groupby event.module event.dataset"
-            ]},
-  { "name": "actionPcap", "description": "actionPcapHelp", "icon": "fa-stream", "target": "", 
-            "links": [
-              "/joblookup?esid={:soc_id}", 
-              "/joblookup?ncid={:network.community_id}"
-            ]},
-  { "name": "actionGoogle", "description": "actionGoogleHelp", "icon": "fab fa-google", "target": "_blank", 
-            "links": [
-              "https://www.google.com/search?q={value}"
-            ]},
-  { "name": "actionVirusTotal", "description": "actionVirusTotalHelp", "icon": "fa-external-link-alt", "target": "_blank", 
-            "links": [
-              "https://www.virustotal.com/gui/search/{value}"
-					  ]}
-]
+This file is no longer used. Please use menu.actions.json instead.
@@ -63,4 +63,4 @@
          { "name": "x509", "description": "x.509 grouped by name and issuer", "query": "event.dataset:x509 | groupby x509.san_dns x509.certificate.issuer"},
          { "name": "x509", "description": "x.509 grouped by name and subject", "query": "event.dataset:x509 | groupby x509.san_dns x509.certificate.subject"},
          { "name": "Firewall", "description": "Firewall events grouped by action", "query": "event.dataset:firewall | groupby rule.action"}
-        ]
+        ]
@@ -0,0 +1,33 @@
+[
+  { "name": "actionHunt", "description": "actionHuntHelp", "icon": "fa-crosshairs", "target": "", 
+            "links": [
+              "/#/hunt?q=\"{value|escape}\" | groupby event.module event.dataset"
+            ]},
+  { "name": "actionCorrelate", "description": "actionCorrelateHelp", "icon": "fab fa-searchengin", "target": "",
+            "links": [
+              "/#/hunt?q=(\"{:log.id.fuid}\" OR \"{:log.id.uid}\" OR \"{:network.community_id}\") | groupby event.module event.dataset", 
+              "/#/hunt?q=(\"{:log.id.fuid}\" OR \"{:log.id.uid}\") | groupby event.module event.dataset", 
+              "/#/hunt?q=(\"{:log.id.fuid}\" OR \"{:network.community_id}\") | groupby event.module event.dataset", 
+              "/#/hunt?q=(\"{:log.id.uid}\" OR \"{:network.community_id}\") | groupby event.module event.dataset", 
+              "/#/hunt?q=\"{:log.id.fuid}\" | groupby event.module event.dataset",
+              "/#/hunt?q=\"{:log.id.uid}\" | groupby event.module event.dataset", 
+              "/#/hunt?q=\"{:network.community_id}\" | groupby event.module event.dataset"
+            ]},
+  { "name": "actionPcap", "description": "actionPcapHelp", "icon": "fa-stream", "target": "", 
+            "links": [
+              "/joblookup?esid={:soc_id}", 
+              "/joblookup?ncid={:network.community_id}"
+            ]},
+  { "name": "actionCyberChef", "description": "actionCyberChefHelp", "icon": "fas fa-bread-slice", "target": "_blank", 
+            "links": [
+              "/cyberchef/#input={value|base64}"
+            ]},
+  { "name": "actionGoogle", "description": "actionGoogleHelp", "icon": "fab fa-google", "target": "_blank", 
+            "links": [
+              "https://www.google.com/search?q={value}"
+            ]},
+  { "name": "actionVirusTotal", "description": "actionVirusTotalHelp", "icon": "fa-external-link-alt", "target": "_blank", 
+            "links": [
+              "https://www.virustotal.com/gui/search/{value}"
+					  ]}
+]
@@ -12,11 +12,10 @@
 {%- set CACHE_EXPIRATION = salt['pillar.get']('sensoroni:cache_expiration_ms', 0) %}
 {%- set ES_FIELDCAPS_CACHE = salt['pillar.get']('sensoroni:es_fieldcaps_cache_ms', '300000') %}
 {%- import_json "soc/files/soc/alerts.queries.json" as alerts_queries %}
-{%- import_json "soc/files/soc/alerts.actions.json" as alerts_actions %}
 {%- import_json "soc/files/soc/alerts.eventfields.json" as alerts_eventfields %}
 {%- import_json "soc/files/soc/hunt.queries.json" as hunt_queries %}
-{%- import_json "soc/files/soc/hunt.actions.json" as hunt_actions %}
 {%- import_json "soc/files/soc/hunt.eventfields.json" as hunt_eventfields %}
+{%- import_json "soc/files/soc/menu.actions.json" as menu_actions %}
 {%- import_json "soc/files/soc/tools.json" as tools %}
 {%- set DNET = salt['pillar.get']('global:dockernet', '172.17.0.0') %}

@@ -51,7 +50,8 @@
        "username": "",
        "password": "",
        "cacheMs": {{ ES_FIELDCAPS_CACHE }},
-        "verifyCert": false
+        "verifyCert": false,
+        "timeoutMs": {{ API_TIMEOUT }}
      },
      "influxdb": {
 {%- if grains['role'] in ['so-import'] or (grains['role'] == 'so-eval' and GRAFANA == 0) %}
@@ -123,8 +123,11 @@
        "queryBaseFilter": "",
        "queryToggleFilters": [],
        "queries": {{ hunt_queries | json }},
-        "actions": {{ hunt_actions | json }}           
+        "actions": {{ menu_actions | json }}           
      },
+      "job": {
+        "actions": {{ menu_actions | json }}        
+      }, 
      "alerts": {
        "advanced": false,
        "groupItemsPerPage": 50,
@@ -143,7 +146,7 @@
          { "name": "escalated", "filter": "event.escalated:true", "enabled": false, "exclusive": true, "enablesToggles":["acknowledged"] }
        ],
        "queries": {{ alerts_queries | json }},
-        "actions": {{ alerts_actions | json }}        
+        "actions": {{ menu_actions | json }}        
      }        
    }
  }
@@ -79,6 +79,7 @@ removeesp12dir:
    - signing_policy: influxdb
    - public_key: /etc/pki/influxdb.key
    - CN: {{ manager }}
+    - subjectAltName: DNS:{{ HOSTNAME }}
    - days_remaining: 0
    - days_valid: 820
    - backup: True
@@ -38,7 +38,6 @@ base:
    - patch.os.schedule
    - motd
    - salt.minion-check
-    - sensoroni
    - salt.lasthighstate
  
  '*_helixsensor and G@saltversion:{{saltversion}}':
@@ -47,6 +46,7 @@ base:
    - ca
    - ssl
    - registry
+    - sensoroni
    - telegraf
    - firewall
    - idstools
@@ -66,6 +66,7 @@ base:
    - match: compound
    - ca
    - ssl
+    - sensoroni
    - telegraf
    - firewall
    - nginx
@@ -92,6 +93,7 @@ base:
    - ca
    - ssl
    - registry
+    - sensoroni
    - manager
    - nginx
    - telegraf
@@ -153,6 +155,7 @@ base:
    - domainstats
    {%- endif %}
    - docker_clean
+    - pipeline.load

  '*_manager and G@saltversion:{{saltversion}}':
    - match: compound
@@ -160,6 +163,7 @@ base:
    - ca
    - ssl
    - registry
+    - sensoroni
    - nginx
    - telegraf
    - influxdb
@@ -213,6 +217,7 @@ base:
    - domainstats
    {%- endif %}
    - docker_clean
+    - pipeline.load

  '*_standalone and G@saltversion:{{saltversion}}':
    - match: compound
@@ -220,6 +225,7 @@ base:
    - ca
    - ssl
    - registry
+    - sensoroni
    - manager
    - nginx
    - telegraf
@@ -285,11 +291,13 @@ base:
    - domainstats
    {%- endif %}
    - docker_clean
+    - pipeline.load

  '*_searchnode and G@saltversion:{{saltversion}}':
    - match: compound
    - ca
    - ssl
+    - sensoroni
    - nginx
    - telegraf
    - firewall
@@ -313,6 +321,7 @@ base:
    {%- endif %}
    - schedule
    - docker_clean
+    - pipeline.load

  '*_managersearch and G@saltversion:{{saltversion}}':
    - match: compound
@@ -320,6 +329,7 @@ base:
    - ca
    - ssl
    - registry
+    - sensoroni
    - nginx
    - telegraf
    - influxdb
@@ -377,11 +387,13 @@ base:
    - domainstats
    {%- endif %}
    - docker_clean
+    - pipeline.load

  '*_heavynode and G@saltversion:{{saltversion}}':
    - match: compound
    - ca
    - ssl
+    - sensoroni
    - nginx
    - telegraf
    - firewall
@@ -419,11 +431,13 @@ base:
    {%- endif %}
    - schedule
    - docker_clean
+    - pipeline.load
  
  '*_fleet and G@saltversion:{{saltversion}}':
    - match: compound
    - ca
    - ssl
+    - sensoroni
    - nginx
    - telegraf
    - firewall
@@ -441,6 +455,7 @@ base:
    - ca
    - ssl
    - registry
+    - sensoroni
    - manager
    - nginx
    - soc
@@ -462,3 +477,4 @@ base:
    - zeek
    - schedule
    - docker_clean
+    - pipeline.load
@@ -76,9 +76,9 @@ zeekpolicysync:
 # Ensure the zeek spool tree (and state.db) ownership is correct
 zeekspoolownership:
  file.directory:
-    - name: /nsm/zeek/spool
+    - name: /nsm/zeek
    - user: 937
-    - max_depth: 0
+    - max_depth: 1
    - recurse:
      - user

@@ -183,6 +183,8 @@ so-zeek:
    - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-zeek:{{ VERSION }}
    - start: {{ START }}
    - privileged: True
+    - ulimits:
+      - core=0
    - binds:
      - /nsm/zeek/logs:/nsm/zeek/logs:rw
      - /nsm/zeek/spool:/nsm/zeek/spool:rw