Merge pull request #11040 from Security-Onion-Solutions/2.4/dev

2.4.10

DOWNLOAD_AND_VERIFY_ISO.md

@@ -1,18 +1,18 @@
-### 2.4.4-20230728 ISO image built on 2023/07/28
+### 2.4.10-20230815 ISO image released on 2023/08/15
 
 
 
 ### Download and Verify
 
-2.4.4-20230728 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.4.4-20230728.iso
-
-MD5: F63E76245F3E745B5BDE9E6E647A7CB6  
-SHA1: 6CE4E4A3399CD282D4F8592FB19D510388AB3EEA  
-SHA256: BF8FEB91B1D94B67C3D4A79D209B068F4A46FEC7C15EEF65B0FCE9851D7E6C9F 
+2.4.10-20230815 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-2.4.10-20230815.iso
+ 
+MD5: 97AEC929FB1FC22F106C0C93E3476FAB  
+SHA1: 78AF37FD19FDC34BA324C1A661632D19D1F2284A  
+SHA256: D04BA45D1664FC3CF7EA2188CB7E570642F6390C3959B4AFBB8222A853859394  
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.4-20230728.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.10-20230815.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS  
@@ -26,22 +26,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.4-20230728.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.10-20230815.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.4.4-20230728.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.4.10-20230815.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.4.4-20230728.iso.sig securityonion-2.4.4-20230728.iso
+gpg --verify securityonion-2.4.10-20230815.iso.sig securityonion-2.4.10-20230815.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Tue 11 Jul 2023 06:23:37 PM EDT using RSA key ID FE507013
+gpg: Signature made Sun 13 Aug 2023 05:30:29 PM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

README.md

@@ -1,6 +1,6 @@
-## Security Onion 2.4 Release Candidate 1 (RC1)
+## Security Onion 2.4
 
-Security Onion 2.4 Release Candidate 1 (RC1) is here!
+Security Onion 2.4 is here!
 
 ## Screenshots
 

VERSION

@@ -1 +1 @@
-2.4.4
+2.4.10

salt/common/packages.sls

@@ -17,6 +17,7 @@ commonpkgs:
       - netcat-openbsd
       - sqlite3
       - libssl-dev
+      - procps
       - python3-dateutil
       - python3-docker
       - python3-packaging
@@ -70,6 +71,7 @@ commonpkgs:
       - net-tools
       - nmap-ncat
       - openssl
+      - procps-ng
       - python3-dnf-plugin-versionlock
       - python3-docker
       - python3-m2crypto

salt/common/tools/sbin/so-common

@@ -5,7 +5,16 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-ELASTIC_AGENT_TARBALL_VERSION="8.7.1"
+# Elastic agent is not managed by salt. Because of this we must store this base information in a 
+# script that accompanies the soup system. Since so-common is one of those special soup files, 
+# and since this same logic is required during installation, it's included in this file.
+ELASTIC_AGENT_TARBALL_VERSION="8.8.2"
+ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
+ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
+ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
+ELASTIC_AGENT_MD5="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
+ELASTIC_AGENT_EXPANSION_DIR=/nsm/elastic-fleet/artifacts/beats/elastic-agent
+
 DEFAULT_SALT_DIR=/opt/so/saltstack/default
 DOC_BASE_URL="https://docs.securityonion.net/en/2.4"
 
@@ -161,6 +170,34 @@ disable_fastestmirror() {
 	sed -i 's/enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf
 }
 
+download_and_verify() {
+	source_url=$1
+	source_md5_url=$2
+	dest_file=$3
+	md5_file=$4
+	expand_dir=$5
+
+	if [[ -n "$expand_dir" ]]; then
+		mkdir -p "$expand_dir"
+	fi
+
+	if ! verify_md5_checksum "$dest_file" "$md5_file"; then
+		retry 15 10 "curl --fail --retry 5 --retry-delay 15 -L '$source_url' --output '$dest_file'" "" ""
+		retry 15 10 "curl --fail --retry 5 --retry-delay 15 -L '$source_md5_url' --output '$md5_file'"  "" ""
+
+		if verify_md5_checksum "$dest_file" "$md5_file"; then
+		  echo "Source file and checksum are good."
+		else
+		  echo "Unable to download and verify the source file and checksum."
+		  return 1
+		fi
+	fi
+
+	if [[ -n "$expand_dir" ]]; then
+		tar -xf "$dest_file" -C "$expand_dir"
+	fi
+}
+
 elastic_license() {
 
 read -r -d '' message <<- EOM
@@ -211,7 +248,7 @@ gpg_rpm_import() {
 	        echo "Imported $RPMKEY"
 	    done
 	elif [[ $is_rpm ]]; then
-		info "Importing the security onion GPG key"
+		echo "Importing the security onion GPG key"
 		rpm --import ../salt/repo/client/files/oracle/keys/securityonion.pub
 	fi
 }
@@ -225,12 +262,15 @@ init_monitor() {
 	
 	if [[ $MONITORNIC == "bond0" ]]; then 
 	  BIFACES=$(lookup_bond_interfaces)
+	  for i in rx tx sg tso ufo gso gro lro rx-vlan-offload tx-vlan-offload generic-receive-offload generic-segmentation-offload tcp-segmentation-offload; do
+	    ethtool -K "$MONITORNIC" "$i" off;
+  	  done
 	else
 	  BIFACES=$MONITORNIC
 	fi
 
 	for DEVICE_IFACE in $BIFACES; do
-	  for i in rx tx sg tso ufo gso gro lro; do
+	  for i in rx tx sg tso ufo gso gro lro rx-vlan-offload tx-vlan-offload generic-receive-offload generic-segmentation-offload tcp-segmentation-offload; do
 	    ethtool -K "$DEVICE_IFACE" "$i" off;
   	  done
 	  ip link set dev "$DEVICE_IFACE" arp off multicast off allmulticast off promisc on
@@ -467,6 +507,11 @@ has_uppercase() {
 		|| return 1
 }
 
+update_elastic_agent() {
+	echo "Checking if Elastic Agent update is necessary..."
+	download_and_verify "$ELASTIC_AGENT_URL" "$ELASTIC_AGENT_MD5_URL" "$ELASTIC_AGENT_FILE" "$ELASTIC_AGENT_MD5" "$ELASTIC_AGENT_EXPANSION_DIR"
+}
+
 valid_cidr() {
 	# Verify there is a backslash in the string
 	echo "$1" | grep -qP "^[^/]+/[^/]+$" || return 1
@@ -620,6 +665,23 @@ valid_username() {
 	echo "$user" | grep -qP '^[a-z_]([a-z0-9_-]{0,31}|[a-z0-9_-]{0,30}\$)$' && return 0 || return 1
 }
 
+verify_md5_checksum() {
+	data_file=$1
+	md5_file=${2:-${data_file}.md5}
+
+	if [[ ! -f "$dest_file" || ! -f "$md5_file" ]]; then
+		return 2
+	fi
+
+	SOURCEHASH=$(md5sum "$data_file" | awk '{ print $1 }')
+	HASH=$(cat "$md5_file")
+
+	if [[ "$HASH" == "$SOURCEHASH" ]]; then
+		return 0
+	fi
+	return 1
+}
+
 wait_for_web_response() {
 	url=$1
 	expected=$2

salt/common/tools/sbin/so-status

@@ -103,7 +103,7 @@ def output(options, console, code, data):
 def check_container_status(options, console):
   code = 0
   cli = "docker"
-  proc = subprocess.run([cli, 'ps', '--format', '{{json .}}'], stdout=subprocess.PIPE, encoding="utf-8")
+  proc = subprocess.run([cli, 'ps', '--format', 'json'], stdout=subprocess.PIPE, encoding="utf-8")
   if proc.returncode != 0:
     fail("Container system error; unable to obtain container process statuses")
 

salt/common/tools/sbin_jinja/so-desktop-install

@@ -5,15 +5,15 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
+source /usr/sbin/so-common
+doc_desktop_url="$DOC_BASE_URL/desktop.html"
 
-{# we only want the script to install the desktop if it is Rocky -#}
-{% if grains.os == 'Rocky' -%}
+{# we only want the script to install the desktop if it is OEL -#}
+{% if grains.os == 'OEL' -%}
 {#   if this is a manager -#}
 {%   if grains.master == grains.id.split('_')|first -%}
 
-source /usr/sbin/so-common
-doc_desktop_url="$DOC_BASE_URL/desktop.html"
-pillar_file="/opt/so/saltstack/local/pillar/minions/{{grains.id}}.sls"
+pillar_file="/opt/so/saltstack/local/pillar/minions/adv_{{grains.id}}.sls"
 
 if [ -f "$pillar_file" ]; then
   if ! grep -q "^desktop:$" "$pillar_file"; then
@@ -65,7 +65,7 @@ if [ -f "$pillar_file" ]; then
     fi
   else # desktop is already added
     echo "The desktop pillar already exists in $pillar_file."
-    echo "To enable/disable the gui, set 'desktop:gui:enabled' to true or false in $pillar_file."
+    echo "To enable/disable the gui, set 'desktop:gui:enabled' to true or false in $pillar_file. Alternatively, this can be set in the SOC UI under advanced."
     echo "Additional documentation can be found at $doc_desktop_url."
   fi
 else # if the pillar file doesn't exist
@@ -75,17 +75,22 @@ fi
 {#-  if this is not a manager #}
 {%   else -%}
 
-echo "Since this is not a manager, the pillar values to enable Security Onion Desktop must be set manually. Please view the documentation at $doc_desktop_url."
+echo "Since this is not a manager, the pillar values to enable Security Onion Desktop must be set manually. This can be enabled in the SOC UI under advanced by adding the following:"
+echo "desktop:"
+echo "  gui:"
+echo "    enabled: true"
+echo ""
+echo "Please view the documentation at $doc_desktop_url."
 
 {#- endif if this is a manager #}
 {%   endif -%}
 
-{#- if not Rocky #}
+{#- if not OEL #}
 {%- else %}
 
-echo "The Security Onion Desktop can only be installed on Rocky Linux. Please view the documentation at $doc_desktop_url."
+echo "The Security Onion Desktop can only be installed on Oracle Linux. Please view the documentation at $doc_desktop_url."
 
-{#- endif grains.os == Rocky #}
+{#- endif grains.os == OEL #}
 {% endif -%}
 
 exit 0

salt/common/tools/sbin_jinja/so-import-evtx

@@ -27,6 +27,8 @@ Imports one or more evtx files into Security Onion. The evtx files will be analy
 Options:
   --json      Outputs summary in JSON format. Implies --quiet.
   --quiet     Silences progress information to stdout.
+  --shift     Adds a time shift. Accepts a single argument that is intended to be the date of the last record, and shifts the dates of the previous records accordingly.
+              Ex. sudo so-import-evtx --shift "2023-08-01 01:01:01" example.evtx 
 EOF
 }
 
@@ -44,6 +46,10 @@ while [[ $# -gt 0 ]]; do
     --quiet)
       quiet=1
       ;;
+    --shift)
+      SHIFTDATE=$1
+      shift
+      ;;
     -*) 
       echo "Encountered unexpected parameter: $param"
       usage
@@ -68,8 +74,10 @@ function status {
 function evtx2es() {
   EVTX=$1
   HASH=$2
+  SHIFTDATE=$3
   
   docker run --rm \
+    -e "SHIFTTS=$SHIFTDATE" \
     -v "$EVTX:/tmp/data.evtx" \
     -v "/nsm/import/$HASH/evtx/:/tmp/evtx/" \
     -v "/nsm/import/evtx-end_newest:/tmp/newest" \
@@ -113,7 +121,9 @@ echo $END_NEWEST > /nsm/import/evtx-end_newest
 for EVTX in $INPUT_FILES; do
   EVTX=$(/usr/bin/realpath "$EVTX")
   status "Processing Import: ${EVTX}"
-
+  if ! [ -z "$SHIFTDATE" ]; then
+    status "- timeshifting logs to end date of $SHIFTDATE"
+  fi
   # generate a unique hash to assist with dedupe checks
   HASH=$(md5sum "${EVTX}" | awk '{ print $1 }')
   HASH_DIR=/nsm/import/${HASH}
@@ -136,7 +146,7 @@ for EVTX in $INPUT_FILES; do
 
     # import evtx and write them to import ingest pipeline
     status "- importing logs to Elasticsearch..."
-    evtx2es "${EVTX}" $HASH
+    evtx2es "${EVTX}" $HASH "$SHIFTDATE"
     if [[ $? -ne 0 ]]; then
       INVALID_EVTXS_COUNT=$((INVALID_EVTXS_COUNT + 1))
       status "- WARNING: This evtx file may not have fully imported successfully"
@@ -222,4 +232,4 @@ if [[ $json -eq 1 ]]; then
     }'''
 fi
 
-exit $RESULT
+exit $RESULT

salt/common/tools/sbin_jinja/so-raid-status

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
@@ -9,25 +9,26 @@
 
 . /usr/sbin/so-common
 
-appliance_check() {
-  {%- if salt['grains.get']('sosmodel', '') %}
-  APPLIANCE=1
-  {%- if grains['sosmodel'] in ['SO2AMI01', 'SO2GCI01', 'SO2AZI01'] %}
-  exit 0
-  {%- endif %}
-  DUDEYOUGOTADELL=$(dmidecode |grep Dell)
-  if [[ -n $DUDEYOUGOTADELL ]]; then
-  APPTYPE=dell
-  else
-  APPTYPE=sm
-  fi
-  mkdir -p /opt/so/log/raid
-
-  {%- else %}
-  echo "This is not an appliance"
-  exit 0
-  {%- endif %}
-}
+{%- if salt['grains.get']('sosmodel', '') %}
+{%- set model = salt['grains.get']('sosmodel') %}
+model={{ model }}
+# Don't need cloud images to use this
+if [[ $model =~ ^(SO2AMI01|SO2AZI01|SO2GCI01)$ ]]; then
+    exit 0
+fi
+{%- else %}
+echo "This is not an appliance"
+exit 0
+{%- endif %}
+if [[ $model =~ ^(SOS10K|SOS500|SOS1000|SOS1000F|SOS4000|SOSSN7200|SOSSNNV|SOSMN)$ ]]; then
+	is_bossraid=true
+fi
+if [[ $model =~ ^(SOSSNNV|SOSMN)$ ]]; then
+	is_swraid=true
+fi
+if [[ $model =~ ^(SOS10K|SOS500|SOS1000|SOS1000F|SOS4000|SOSSN7200)$ ]]; then
+	is_hwraid=true
+fi
 
 check_nsm_raid() {
   PERCCLI=$(/opt/raidtools/perccli/perccli64 /c0/v0 show|grep RAID|grep Optl)
@@ -49,61 +50,44 @@ check_nsm_raid() {
 check_boss_raid() {
   MVCLI=$(/usr/local/bin/mvcli info -o vd |grep status |grep functional)
 
-  if [[ -n $DUDEYOUGOTADELL ]]; then
-    if [[ -n $MVCLI ]]; then
-      BOSSRAID=0
-    else
-      BOSSRAID=1
-    fi
+  if [[ -n $MVCLI ]]; then
+    BOSSRAID=0
+  else
+    BOSSRAID=1
   fi
 }
 
 check_software_raid() {
-  if [[ -n $DUDEYOUGOTADELL ]]; then
-    SWRC=$(grep "_" /proc/mdstat)
-
-    if [[ -n $SWRC ]]; then
-        # RAID is failed in some way
-        SWRAID=1
-    else
-        SWRAID=0
-    fi
+  SWRC=$(grep "_" /proc/mdstat)
+  if [[ -n $SWRC ]]; then
+      # RAID is failed in some way
+      SWRAID=1
+  else
+      SWRAID=0
   fi
 }
 
-# This script checks raid status if you use SO appliances
+# Set everything to 0
+SWRAID=0
+BOSSRAID=0
+HWRAID=0
 
-# See if this is an appliance
-
-appliance_check
-check_nsm_raid
-check_boss_raid
-{%- if salt['grains.get']('sosmodel', '') %}
-{%- if grains['sosmodel'] in ['SOSMN', 'SOSSNNV'] %}
-check_software_raid
-{%- endif %}
-{%- endif %}
-
-if [[ -n $SWRAID ]]; then
-  if [[ $SWRAID == '0' && $BOSSRAID == '0' ]]; then
-    RAIDSTATUS=0
-  else
-    RAIDSTATUS=1
-  fi
-elif [[ -n $DUDEYOUGOTADELL ]]; then
-  if [[ $BOSSRAID == '0' && $HWRAID == '0' ]]; then
-    RAIDSTATUS=0
-  else
-    RAIDSTATUS=1
-  fi
-elif [[ "$APPTYPE" == 'sm' ]]; then
-  if [[ -n "$HWRAID" ]]; then
-    RAIDSTATUS=0
-  else
-    RAIDSTATUS=1
-  fi
+if [[ $is_hwraid ]]; then
+	check_nsm_raid
+fi
+if [[ $is_bossraid ]]; then
+	check_boss_raid
+fi
+if [[ $is_swraid ]]; then
+	check_software_raid
 fi
 
-echo "nsmraid=$RAIDSTATUS" > /opt/so/log/raid/status.log
+sum=$(($SWRAID + $BOSSRAID + $HWRAID))
 
+if [[ $sum == "0" ]]; then
+  RAIDSTATUS=0
+else
+  RAIDSTATUS=1
+fi
 
+echo "nsmraid=$RAIDSTATUS" > /opt/so/log/raid/status.log

salt/desktop/files/00-background

@@ -0,0 +1,8 @@
+# Specify the dconf path
+[org/gnome/desktop/background]
+
+# Specify the path to the desktop background image file
+picture-uri='file:///usr/local/share/backgrounds/so-wallpaper.jpg'
+
+# Specify one of the rendering options for the background image:
+picture-options='zoom'

salt/desktop/files/session.jinja

@@ -0,0 +1,7 @@
+# This file is managed by Salt in the desktop.xwindows state
+# It will not be overwritten if it already exists
+
+[User]
+Session=gnome-classic
+Icon=/home/{{USERNAME}}/.face
+SystemAccount=false

salt/desktop/packages.sls

@@ -3,7 +3,6 @@
 {# we only want this state to run it is CentOS #}
 {% if GLOBALS.os == 'OEL' %}
 
-
 desktop_packages:
   pkg.installed:
     - pkgs:
@@ -181,6 +180,7 @@ desktop_packages:
       - gstreamer1-plugins-good-gtk
       - gstreamer1-plugins-ugly-free
       - gtk-update-icon-cache
+      - gtk2
       - gtk3
       - gtk4
       - gtkmm30
@@ -295,6 +295,7 @@ desktop_packages:
       - mesa-vulkan-drivers
       - microcode_ctl
       - mobile-broadband-provider-info
+      - mono-devel
       - mpfr
       - mpg123-libs
       - mtdev
@@ -347,6 +348,7 @@ desktop_packages:
       - snappy
       - sound-theme-freedesktop
       - soundtouch
+      - securityonion-networkminer
       - speech-dispatcher
       - speech-dispatcher-espeak-ng
       - speex

salt/desktop/scripts/convert-gnome-classic.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+echo "Setting default session to gnome-classic"
+cp /usr/share/accountsservice/user-templates/standard /etc/accountsservice/user-templates/
+sed -i 's|Session=gnome|Session=gnome-classic|g' /etc/accountsservice/user-templates/standard

salt/desktop/trusted-ca.sls

@@ -31,6 +31,6 @@ update_ca_certs:
 
 desktop_trusted-ca_os_fail:
   test.fail_without_changes:
-    - comment: 'SO Desktop can only be installed on CentOS'
+    - comment: 'SO Desktop can only be installed on Oracle Linux'
 
 {% endif %}

salt/desktop/xwindows.sls

@@ -14,6 +14,44 @@ graphical_target:
     - require:
       - desktop_packages
 
+convert_gnome_classic:
+  cmd.script:
+    - name: salt://desktop/scripts/convert-gnome-classic.sh
+
+{%   for username in salt['file.find'](path='/home/',mindepth=1,maxdepth=1,type='d') %}
+{%     set username = username.split('/')[2] %}
+{%     if username != 'zeek' %}
+{%       if not salt['file.file_exists']('/var/lib/AccountsService/users/' ~ username) %}
+
+{{username}}_session:
+  file.managed:
+    - name: /var/lib/AccountsService/users/{{username}}
+    - source: salt://desktop/files/session.jinja
+    - template: jinja
+    - defaults:
+        USERNAME: {{username}}
+
+{%       endif %}
+{%     endif %}
+{%   endfor %}
+
+desktop_wallpaper:
+  file.managed:
+    - name: /usr/local/share/backgrounds/so-wallpaper.jpg
+    - source: salt://desktop/files/so-wallpaper.jpg
+    - makedirs: True
+
+set_wallpaper:
+  file.managed:
+    - name: /etc/dconf/db/local.d/00-background
+    - source: salt://desktop/files/00-background
+
+run_dconf_update:
+  cmd.run:
+    - name: 'dconf update'
+    - onchanges:
+      - file: set_wallpaper
+
 {% else %}
 
 desktop_xwindows_os_fail:

salt/elasticagent/config.sls

@@ -28,6 +28,13 @@ elasticagentconfdir:
     - group: 939
     - makedirs: True
 
+elasticagentlogdir:
+   file.directory:
+     - name: /opt/so/log/elasticagent
+     - user: 949
+     - group: 939
+     - makedirs: True
+
 elasticagent_sbin_jinja:
   file.recurse:
     - name: /usr/sbin

salt/elasticagent/enabled.sls

@@ -33,20 +33,25 @@ so-elastic-agent:
         {% endif %}
     - binds:
       - /opt/so/conf/elastic-agent/elastic-agent.yml:/usr/share/elastic-agent/elastic-agent.yml:ro
+      - /opt/so/log/elasticagent:/usr/share/elastic-agent/logs
       - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro 
       - /nsm:/nsm:ro
+      - /opt/so/log:/opt/so/log:ro
      {% if DOCKER.containers['so-elastic-agent'].custom_bind_mounts %}
         {% for BIND in DOCKER.containers['so-elastic-agent'].custom_bind_mounts %}
       - {{ BIND }}
         {% endfor %}
-      {% endif %}      
+      {% endif %}
     - environment:
       - FLEET_CA=/etc/pki/tls/certs/intca.crt
+      - LOGS_PATH=logs
       {% if DOCKER.containers['so-elastic-agent'].extra_env %}
         {% for XTRAENV in DOCKER.containers['so-elastic-agent'].extra_env %}
       - {{ XTRAENV }}
         {% endfor %}
       {% endif %}
+    - require:
+      - file: create-elastic-agent-config
     - watch:
       - file: create-elastic-agent-config
 

salt/elasticagent/files/elastic-agent.yml.jinja

@@ -3,7 +3,7 @@
 {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
 
 id: aea1ba80-1065-11ee-a369-97538913b6a9
-revision: 2
+revision: 1
 outputs:
   default:
     type: elasticsearch
@@ -22,56 +22,369 @@ agent:
     metrics: false
   features: {}
 inputs:
-  - id: logfile-logs-80ffa884-2cfc-459a-964a-34df25714d85
-    name: suricata-logs
-    revision: 1
+  - id: logfile-logs-fefef78c-422f-4cfa-8abf-4cd1b9428f62
+    name: import-evtx-logs
+    revision: 2
     type: logfile
     use_output: default
     meta:
       package:
         name: log
-        version: 
+        version:
     data_stream:
       namespace: so
-    package_policy_id: 80ffa884-2cfc-459a-964a-34df25714d85
+    package_policy_id: fefef78c-422f-4cfa-8abf-4cd1b9428f62
     streams:
-      - id: logfile-log.log-80ffa884-2cfc-459a-964a-34df25714d85
+      - id: logfile-log.log-fefef78c-422f-4cfa-8abf-4cd1b9428f62
+        data_stream:
+          dataset: import
+        paths:
+          - /nsm/import/*/evtx/*.json
+        processors:
+          - dissect:
+              field: log.file.path
+              tokenizer: '/nsm/import/%{import.id}/evtx/%{import.file}'
+              target_prefix: ''
+          - decode_json_fields:
+              fields:
+                - message
+              target: ''
+          - drop_fields:
+              ignore_missing: true
+              fields:
+                - host
+          - add_fields:
+              fields:
+                dataset: system.security
+                type: logs
+                namespace: default
+              target: data_stream
+          - add_fields:
+              fields:
+                dataset: system.security
+                module: system
+                imported: true
+              target: event
+          - then:
+              - add_fields:
+                  fields:
+                    dataset: windows.sysmon_operational
+                  target: data_stream
+              - add_fields:
+                  fields:
+                    dataset: windows.sysmon_operational
+                    module: windows
+                    imported: true
+                  target: event
+            if:
+              equals:
+                winlog.channel: Microsoft-Windows-Sysmon/Operational
+          - then:
+              - add_fields:
+                  fields:
+                    dataset: system.application
+                  target: data_stream
+              - add_fields:
+                  fields:
+                    dataset: system.application
+                  target: event
+            if:
+              equals:
+                winlog.channel: Application
+          - then:
+              - add_fields:
+                  fields:
+                    dataset: system.system
+                  target: data_stream
+              - add_fields:
+                  fields:
+                    dataset: system.system
+                  target: event
+            if:
+              equals:
+                winlog.channel: System
+          - then:
+              - add_fields:
+                  fields:
+                    dataset: windows.powershell_operational
+                  target: data_stream
+              - add_fields:
+                  fields:
+                    dataset: windows.powershell_operational
+                    module: windows
+                  target: event
+            if:
+              equals:
+                winlog.channel: Microsoft-Windows-PowerShell/Operational
+        tags:
+          - import
+  - id: logfile-redis-fc98c947-7d17-4861-a318-7ad075f6d1b0
+    name: redis-logs
+    revision: 2
+    type: logfile
+    use_output: default
+    meta:
+      package:
+        name: redis
+        version:
+    data_stream:
+      namespace: default
+    package_policy_id: fc98c947-7d17-4861-a318-7ad075f6d1b0
+    streams:
+      - id: logfile-redis.log-fc98c947-7d17-4861-a318-7ad075f6d1b0
+        data_stream:
+          dataset: redis.log
+          type: logs
+        exclude_files:
+          - .gz$
+        paths:
+          - /opt/so/log/redis/redis.log
+        tags:
+          - redis-log
+        exclude_lines:
+          - '^\s+[\-`(''.|_]'
+  - id: logfile-logs-3b56803d-5ade-4c93-b25e-9b37182f66b8
+    name: import-suricata-logs
+    revision: 2
+    type: logfile
+    use_output: default
+    meta:
+      package:
+        name: log
+        version:
+    data_stream:
+      namespace: so
+    package_policy_id: 3b56803d-5ade-4c93-b25e-9b37182f66b8
+    streams:
+      - id: logfile-log.log-3b56803d-5ade-4c93-b25e-9b37182f66b8
+        data_stream:
+          dataset: import
+        pipeline: suricata.common
+        paths:
+          - /nsm/import/*/suricata/eve*.json
+        processors:
+          - add_fields:
+              fields:
+                module: suricata
+                imported: true
+                category: network
+              target: event
+          - dissect:
+              field: log.file.path
+              tokenizer: '/nsm/import/%{import.id}/suricata/%{import.file}'
+              target_prefix: ''
+  - id: logfile-logs-c327e1a3-1ebe-449c-a8eb-f6f35032e69d
+    name: soc-server-logs
+    revision: 2
+    type: logfile
+    use_output: default
+    meta:
+      package:
+        name: log
+        version:
+    data_stream:
+      namespace: so
+    package_policy_id: c327e1a3-1ebe-449c-a8eb-f6f35032e69d
+    streams:
+      - id: logfile-log.log-c327e1a3-1ebe-449c-a8eb-f6f35032e69d
+        data_stream:
+          dataset: soc
+        pipeline: common
+        paths:
+          - /opt/so/log/soc/sensoroni-server.log
+        processors:
+          - decode_json_fields:
+              add_error_key: true
+              process_array: true
+              max_depth: 2
+              fields:
+                - message
+              target: soc
+          - add_fields:
+              fields:
+                module: soc
+                dataset_temp: server
+                category: host
+              target: event
+          - rename:
+              ignore_missing: true
+              fields:
+                - from: soc.fields.sourceIp
+                  to: source.ip
+                - from: soc.fields.status
+                  to: http.response.status_code
+                - from: soc.fields.method
+                  to: http.request.method
+                - from: soc.fields.path
+                  to: url.path
+                - from: soc.message
+                  to: event.action
+                - from: soc.level
+                  to: log.level
+        tags:
+          - so-soc
+  - id: logfile-logs-906e0d4c-9ec3-4c6a-bef6-e347ec9fd073
+    name: soc-sensoroni-logs
+    revision: 2
+    type: logfile
+    use_output: default
+    meta:
+      package:
+        name: log
+        version:
+    data_stream:
+      namespace: so
+    package_policy_id: 906e0d4c-9ec3-4c6a-bef6-e347ec9fd073
+    streams:
+      - id: logfile-log.log-906e0d4c-9ec3-4c6a-bef6-e347ec9fd073
+        data_stream:
+          dataset: soc
+        pipeline: common
+        paths:
+          - /opt/so/log/sensoroni/sensoroni.log
+        processors:
+          - decode_json_fields:
+              add_error_key: true
+              process_array: true
+              max_depth: 2
+              fields:
+                - message
+              target: sensoroni
+          - add_fields:
+              fields:
+                module: soc
+                dataset_temp: sensoroni
+                category: host
+              target: event
+          - rename:
+              ignore_missing: true
+              fields:
+                - from: sensoroni.fields.sourceIp
+                  to: source.ip
+                - from: sensoroni.fields.status
+                  to: http.response.status_code
+                - from: sensoroni.fields.method
+                  to: http.request.method
+                - from: sensoroni.fields.path
+                  to: url.path
+                - from: sensoroni.message
+                  to: event.action
+                - from: sensoroni.level
+                  to: log.level
+  - id: logfile-logs-df0d7f2c-221f-433b-b18b-d1cf83250515
+    name: soc-salt-relay-logs
+    revision: 2
+    type: logfile
+    use_output: default
+    meta:
+      package:
+        name: log
+        version:
+    data_stream:
+      namespace: so
+    package_policy_id: df0d7f2c-221f-433b-b18b-d1cf83250515
+    streams:
+      - id: logfile-log.log-df0d7f2c-221f-433b-b18b-d1cf83250515
+        data_stream:
+          dataset: soc
+        pipeline: common
+        paths:
+          - /opt/so/log/soc/salt-relay.log
+        processors:
+          - dissect:
+              field: message
+              tokenizer: '%{soc.ts} | %{event.action}'
+              target_prefix: ''
+          - add_fields:
+              fields:
+                module: soc
+                dataset_temp: salt_relay
+                category: host
+              target: event
+        tags:
+          - so-soc
+  - id: logfile-logs-74bd2366-fe52-493c-bddc-843a017fc4d0
+    name: soc-auth-sync-logs
+    revision: 2
+    type: logfile
+    use_output: default
+    meta:
+      package:
+        name: log
+        version:
+    data_stream:
+      namespace: so
+    package_policy_id: 74bd2366-fe52-493c-bddc-843a017fc4d0
+    streams:
+      - id: logfile-log.log-74bd2366-fe52-493c-bddc-843a017fc4d0
+        data_stream:
+          dataset: soc
+        pipeline: common
+        paths:
+          - /opt/so/log/soc/sync.log
+        processors:
+          - dissect:
+              field: message
+              tokenizer: '%{event.action}'
+              target_prefix: ''
+          - add_fields:
+              fields:
+                module: soc
+                dataset_temp: auth_sync
+                category: host
+              target: event
+        tags:
+          - so-soc
+  - id: logfile-logs-d151d9bf-ff2a-4529-9520-c99244bc0253
+    name: suricata-logs
+    revision: 2
+    type: logfile
+    use_output: default
+    meta:
+      package:
+        name: log
+        version:
+    data_stream:
+      namespace: so
+    package_policy_id: d151d9bf-ff2a-4529-9520-c99244bc0253
+    streams:
+      - id: logfile-log.log-d151d9bf-ff2a-4529-9520-c99244bc0253
         data_stream:
           dataset: suricata
+        pipeline: suricata.common
         paths:
           - /nsm/suricata/eve*.json
         processors:
           - add_fields:
-              target: event
               fields:
-                category: network
                 module: suricata
-        pipeline: suricata.common
-  - id: logfile-logs-90103ac4-f6bd-4a4a-b596-952c332390fc
+                category: network
+              target: event
+  - id: logfile-logs-31f94d05-ae75-40ee-b9c5-0e0356eff327
     name: strelka-logs
-    revision: 1
+    revision: 2
     type: logfile
     use_output: default
     meta:
       package:
         name: log
-        version: 
+        version:
     data_stream:
       namespace: so
-    package_policy_id: 90103ac4-f6bd-4a4a-b596-952c332390fc
+    package_policy_id: 31f94d05-ae75-40ee-b9c5-0e0356eff327
     streams:
-      - id: logfile-log.log-90103ac4-f6bd-4a4a-b596-952c332390fc
+      - id: logfile-log.log-31f94d05-ae75-40ee-b9c5-0e0356eff327
         data_stream:
           dataset: strelka
+        pipeline: strelka.file
         paths:
           - /nsm/strelka/log/strelka.log
         processors:
           - add_fields:
-              target: event
               fields:
-                category: file
                 module: strelka
-        pipeline: strelka.file
+                category: file
+              target: event
   - id: logfile-logs-6197fe84-9b58-4d9b-8464-3d517f28808d
     name: zeek-logs
     revision: 1

salt/elasticfleet/defaults.yaml

@@ -2,7 +2,7 @@ elasticfleet:
   enabled: False
   config:
     server:
-      custom_fqdn: ''
+      custom_fqdn: []
       enable_auto_configuration: True
       endpoints_enrollment: ''
       es_token: ''
@@ -28,8 +28,17 @@ elasticfleet:
     - aws
     - azure
     - cloudflare
+    - elasticsearch
     - endpoint
+    - fleet_server
     - fim
     - github
     - google_workspace
+    - log
+    - osquery_manager
+    - redis
+    - system
+    - tcp
+    - udp
+    - windows
     - 1password

salt/elasticfleet/enabled.sls

@@ -15,12 +15,14 @@
 include:
   - elasticfleet.config
   - elasticfleet.sostatus
+  - ssl
 
 # If enabled, automatically update Fleet Logstash Outputs
 {% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-import', 'so-eval', 'so-fleet'] %}
 so-elastic-fleet-auto-configure-logstash-outputs:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-outputs-update
+    - retry: True
 {% endif %}
 
 # If enabled, automatically update Fleet Server URLs & ES Connection
@@ -28,6 +30,7 @@ so-elastic-fleet-auto-configure-logstash-outputs:
 so-elastic-fleet-auto-configure-server-urls:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-urls-update
+    - retry: True
 {% endif %}
 
 # Automatically update Fleet Server Elasticsearch URLs
@@ -35,6 +38,7 @@ so-elastic-fleet-auto-configure-server-urls:
 so-elastic-fleet-auto-configure-elasticsearch-urls:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-es-url-update
+    - retry: True
 {% endif %}
 
 {%   if SERVICETOKEN != '' %}
@@ -61,11 +65,14 @@ so-elastic-fleet:
       - {{ BINDING }}
       {% endfor %}
     - binds:
-      - /etc/pki:/etc/pki:ro
+      - /etc/pki/elasticfleet-server.crt:/etc/pki/elasticfleet-server.crt:ro
+      - /etc/pki/elasticfleet-server.key:/etc/pki/elasticfleet-server.key:ro
+      - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro
       {% if GLOBALS.os_family == 'Debian' %}
-      - /etc/ssl:/etc/ssl:ro
+      - /etc/ssl/elasticfleet-server.crt:/etc/ssl/elasticfleet-server.crt:ro
+      - /etc/ssl/elasticfleet-server.key:/etc/ssl/elasticfleet-server.key:ro
+      - /etc/ssl/tls/certs/intca.crt:/etc/ssl/tls/certs/intca.crt:ro
       {% endif %}
-      #- /opt/so/conf/elastic-fleet/state:/usr/share/elastic-agent/state:rw
       - /opt/so/log/elasticfleet:/usr/share/elastic-agent/logs 
      {% if DOCKER.containers['so-elastic-fleet'].custom_bind_mounts %}
         {% for BIND in DOCKER.containers['so-elastic-fleet'].custom_bind_mounts %}
@@ -93,12 +100,20 @@ so-elastic-fleet:
       - {{ XTRAENV }}
         {% endfor %}
       {% endif %}
+    - watch:
+      - x509: etc_elasticfleet_key
+      - x509: etc_elasticfleet_crt
 {%   endif %}
 
 {%  if GLOBALS.role != "so-fleet" %}
 so-elastic-fleet-integrations:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-integration-policy-load
+
+so-elastic-agent-grid-upgrade:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-agent-grid-upgrade
+    - retry: True
 {%  endif %}
 
 delete_so-elastic-fleet_so-status.disabled:

salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json

@@ -13,7 +13,7 @@
     "logs-logfile": {
       "enabled": true,
       "streams": {
-        "log.log": {
+        "log.logs": {
           "enabled": true,
           "vars": {
             "paths": [

salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json

@@ -14,7 +14,7 @@
     "logs-logfile": {
       "enabled": true,
       "streams": {
-        "log.log": {
+        "log.logs": {
           "enabled": true,
           "vars": {
             "paths": [

salt/elasticfleet/files/integrations/endpoints-initial/elastic-defend-endpoints.json

@@ -5,17 +5,16 @@
 	"package": {
 		"name": "endpoint",
 		"title": "Elastic Defend",
-		"version": ""
+		"version": "8.8.0"
 	},
 	"enabled": true,
 	"policy_id": "endpoints-initial",
-	"vars": {},
 	"inputs": [{
-		"type": "endpoint",
+		"type": "ENDPOINT_INTEGRATION_CONFIG",
 		"enabled": true,
 		"streams": [],
 		"config": {
-			"integration_config": {
+			"_config": {
 				"value": {
 					"type": "endpoint",
 					"endpointConfig": {
@@ -25,4 +24,4 @@
 			}
 		}
 	}]
-}
+}

salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json

@@ -11,7 +11,7 @@
     "logs-logfile": {
       "enabled": true,
       "streams": {
-        "log.log": {
+        "log.logs": {
           "enabled": true,
           "vars": {
             "paths": [

salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json

@@ -12,7 +12,7 @@
     "logs-logfile": {
       "enabled": true,
       "streams": {
-        "log.log": {
+        "log.logs": {
           "enabled": true,
           "vars": {
             "paths": [

salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json

@@ -11,7 +11,7 @@
     "logs-logfile": {
       "enabled": true,
       "streams": {
-        "log.log": {
+        "log.logs": {
           "enabled": true,
           "vars": {
             "paths": [

salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json

@@ -11,7 +11,7 @@
     "logs-logfile": {
       "enabled": true,
       "streams": {
-        "log.log": {
+        "log.logs": {
           "enabled": true,
           "vars": {
             "paths": [

salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json

@@ -11,7 +11,7 @@
     "logs-logfile": {
       "enabled": true,
       "streams": {
-        "log.log": {
+        "log.logs": {
           "enabled": true,
           "vars": {
             "paths": [

salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json

@@ -11,7 +11,7 @@
     "logs-logfile": {
       "enabled": true,
       "streams": {
-        "log.log": {
+        "log.logs": {
           "enabled": true,
           "vars": {
             "paths": [

salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json

@@ -11,7 +11,7 @@
     "logs-logfile": {
       "enabled": true,
       "streams": {
-        "log.log": {
+        "log.logs": {
           "enabled": true,
           "vars": {
             "paths": [

salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json

@@ -11,7 +11,7 @@
     "logs-logfile": {
       "enabled": true,
       "streams": {
-        "log.log": {
+        "log.logs": {
           "enabled": true,
           "vars": {
             "paths": [

salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json

@@ -11,7 +11,7 @@
     "logs-logfile": {
       "enabled": true,
       "streams": {
-        "log.log": {
+        "log.logs": {
           "enabled": true,
           "vars": {
             "paths": [

salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json

@@ -11,7 +11,7 @@
     "logs-logfile": {
       "enabled": true,
       "streams": {
-        "log.log": {
+        "log.logs": {
           "enabled": true,
           "vars": {
             "paths": [

salt/elasticfleet/files/integrations/grid-nodes_heavy/elasticsearch-logs.json

@@ -1,106 +0,0 @@
-{
-  "package": {
-    "name": "elasticsearch",
-    "version": ""
-  },
-  "name": "elasticsearch-logs",
-  "namespace": "default",
-  "description": "Elasticsearch Logs",
-  "policy_id": "so-grid-nodes_heavy",
-  "inputs": {
-    "elasticsearch-logfile": {
-      "enabled": true,
-      "streams": {
-        "elasticsearch.audit": {
-          "enabled": false,
-          "vars": {
-            "paths": [
-              "/var/log/elasticsearch/*_audit.json"
-            ]
-          }
-        },
-        "elasticsearch.deprecation": {
-          "enabled": false,
-          "vars": {
-            "paths": [
-              "/var/log/elasticsearch/*_deprecation.json"
-            ]
-          }
-        },
-        "elasticsearch.gc": {
-          "enabled": false,
-          "vars": {
-            "paths": [
-              "/var/log/elasticsearch/gc.log.[0-9]*",
-              "/var/log/elasticsearch/gc.log"
-            ]
-          }
-        },
-        "elasticsearch.server": {
-          "enabled": true,
-          "vars": {
-            "paths": [
-              "/opt/so/log/elasticsearch/*.log"
-            ]
-          }
-        },
-        "elasticsearch.slowlog": {
-          "enabled": false,
-          "vars": {
-            "paths": [
-              "/var/log/elasticsearch/*_index_search_slowlog.json",
-              "/var/log/elasticsearch/*_index_indexing_slowlog.json"
-            ]
-          }
-        }
-      }
-    },
-    "elasticsearch-elasticsearch/metrics": {
-      "enabled": false,
-      "vars": {
-        "hosts": [
-          "http://localhost:9200"
-        ],
-        "scope": "node"
-      },
-      "streams": {
-        "elasticsearch.stack_monitoring.ccr": {
-          "enabled": false
-        },
-        "elasticsearch.stack_monitoring.cluster_stats": {
-          "enabled": false
-        },
-        "elasticsearch.stack_monitoring.enrich": {
-          "enabled": false
-        },
-        "elasticsearch.stack_monitoring.index": {
-          "enabled": false
-        },
-        "elasticsearch.stack_monitoring.index_recovery": {
-          "enabled": false,
-          "vars": {
-            "active.only": true
-          }
-        },
-        "elasticsearch.stack_monitoring.index_summary": {
-          "enabled": false
-        },
-        "elasticsearch.stack_monitoring.ml_job": {
-          "enabled": false
-        },
-        "elasticsearch.stack_monitoring.node": {
-          "enabled": false
-        },
-        "elasticsearch.stack_monitoring.node_stats": {
-          "enabled": false
-        },
-        "elasticsearch.stack_monitoring.pending_tasks": {
-          "enabled": false
-        },
-        "elasticsearch.stack_monitoring.shard": {
-          "enabled": false
-        }
-      }
-    }
-  }
-}

salt/elasticfleet/files/integrations/grid-nodes_heavy/kratos-logs.json

@@ -1,29 +0,0 @@
-{
-  "package": {
-    "name": "log",
-    "version": ""
-  },
-  "name": "kratos-logs",
-  "namespace": "so",
-  "description": "Kratos logs",
-  "policy_id": "so-grid-nodes_heavy",
-  "inputs": {
-    "logs-logfile": {
-      "enabled": true,
-      "streams": {
-        "log.log": {
-          "enabled": true,
-          "vars": {
-            "paths": [
-              "/opt/so/log/kratos/kratos.log"
-            ],
-            "data_stream.dataset": "kratos",
-            "tags": ["so-kratos"],
-            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: iam\n      module: kratos",
-            "custom": "pipeline: kratos"
-          }
-        }
-      }
-    }
-  }
-}

salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json

@@ -3,7 +3,7 @@
     "name": "osquery_manager",
     "version": ""
   },
-  "name": "osquery-grid-nodes",
+  "name": "osquery-grid-nodes_heavy",
   "namespace": "default",
   "policy_id": "so-grid-nodes_heavy",
   "inputs": {

salt/elasticfleet/files/integrations/grid-nodes_heavy/redis-logs.json

@@ -1,76 +0,0 @@
-{
-  "package": {
-    "name": "redis",
-    "version": ""
-  },
-  "name": "redis-logs",
-  "namespace": "default",
-  "description": "Redis logs",
-  "policy_id": "so-grid-nodes_heavy",
-  "inputs": {
-    "redis-logfile": {
-      "enabled": true,
-      "streams": {
-        "redis.log": {
-          "enabled": true,
-          "vars": {
-            "paths": [
-              "/opt/so/log/redis/redis.log"
-            ],
-            "tags": [
-              "redis-log"
-            ],
-            "preserve_original_event": false
-          }
-        }
-      }
-    },
-    "redis-redis": {
-      "enabled": false,
-      "streams": {
-        "redis.slowlog": {
-          "enabled": false,
-          "vars": {
-            "hosts": [
-              "127.0.0.1:6379"
-            ],
-            "password": ""
-          }
-        }
-      }
-    },
-    "redis-redis/metrics": {
-      "enabled": false,
-      "vars": {
-        "hosts": [
-          "127.0.0.1:6379"
-        ],
-        "idle_timeout": "20s",
-        "maxconn": 10,
-        "network": "tcp",
-        "password": ""
-      },
-      "streams": {
-        "redis.info": {
-          "enabled": false,
-          "vars": {
-            "period": "10s"
-          }
-        },
-        "redis.key": {
-          "enabled": false,
-          "vars": {
-            "key.patterns": "- limit: 20\n  pattern: *\n",
-            "period": "10s"
-          }
-        },
-        "redis.keyspace": {
-          "enabled": false,
-          "vars": {
-            "period": "10s"
-          }
-        }
-      }
-    }
-  }
-}

salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-auth-sync-logs.json

@@ -1,29 +0,0 @@
-{
-  "package": {
-    "name": "log",
-    "version": ""
-  },
-  "name": "soc-auth-sync-logs",
-  "namespace": "so",
-  "description": "Security Onion - Elastic Auth Sync - Logs",
-  "policy_id": "so-grid-nodes_heavy",
-  "inputs": {
-    "logs-logfile": {
-      "enabled": true,
-      "streams": {
-        "log.log": {
-          "enabled": true,
-          "vars": {
-            "paths": [
-              "/opt/so/log/soc/sync.log"
-            ],
-            "data_stream.dataset": "soc",
-            "tags": ["so-soc"],
-            "processors": "- dissect:\n    tokenizer: \"%{event.action}\"\n    field: \"message\"\n    target_prefix: \"\"\n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: auth_sync",
-            "custom": "pipeline: common"
-          }
-        }
-      }
-    }
-  }
-}

salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-salt-relay-logs.json

@@ -1,29 +0,0 @@
-{
-  "package": {
-    "name": "log",
-    "version": ""
-  },
-  "name": "soc-salt-relay-logs",
-  "namespace": "so",
-  "description": "Security Onion - Salt Relay - Logs",
-  "policy_id": "so-grid-nodes_heavy",
-  "inputs": {
-    "logs-logfile": {
-      "enabled": true,
-      "streams": {
-        "log.log": {
-          "enabled": true,
-          "vars": {
-            "paths": [
-              "/opt/so/log/soc/salt-relay.log"
-            ],
-            "data_stream.dataset": "soc",
-            "tags": ["so-soc"],
-            "processors": "- dissect:\n    tokenizer: \"%{soc.ts} | %{event.action}\"\n    field: \"message\"\n    target_prefix: \"\"\n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: salt_relay",
-            "custom": "pipeline: common"
-          }
-        }
-      }
-    }
-  }
-}

salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-sensoroni-logs.json

@@ -1,29 +0,0 @@
-{
-  "package": {
-    "name": "log",
-    "version": ""
-  },
-  "name": "soc-sensoroni-logs",
-  "namespace": "so",
-  "description": "Security Onion - Sensoroni - Logs",
-  "policy_id": "so-grid-nodes_heavy",
-  "inputs": {
-    "logs-logfile": {
-      "enabled": true,
-      "streams": {
-        "log.log": {
-          "enabled": true,
-          "vars": {
-            "paths": [
-              "/opt/so/log/sensoroni/sensoroni.log"
-            ],
-            "data_stream.dataset": "soc",
-            "tags": [],
-            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"sensoroni\"\n    process_array: true\n    max_depth: 2\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: sensoroni\n- rename:\n    fields:\n      - from: \"sensoroni.fields.sourceIp\"\n        to: \"source.ip\"\n      - from: \"sensoroni.fields.status\"\n        to: \"http.response.status_code\"\n      - from: \"sensoroni.fields.method\"\n        to: \"http.request.method\"\n      - from: \"sensoroni.fields.path\"\n        to: \"url.path\"\n      - from: \"sensoroni.message\"\n        to: \"event.action\"\n      - from: \"sensoroni.level\"\n        to: \"log.level\"\n    ignore_missing: true",
-            "custom": "pipeline: common"
-          }
-        }
-      }
-    }
-  }
-}

salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-server-logs.json

@@ -1,29 +0,0 @@
-{
-  "package": {
-    "name": "log",
-    "version": ""
-  },
-  "name": "soc-server-logs",
-  "namespace": "so",
-  "description": "Security Onion Console Logs",
-  "policy_id": "so-grid-nodes_heavy",
-  "inputs": {
-    "logs-logfile": {
-      "enabled": true,
-      "streams": {
-        "log.log": {
-          "enabled": true,
-          "vars": {
-            "paths": [
-              "/opt/so/log/soc/sensoroni-server.log"
-            ],
-            "data_stream.dataset": "soc",
-            "tags": ["so-soc"],
-            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"soc\"\n    process_array: true\n    max_depth: 2\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: server\n- rename:\n    fields:\n      - from: \"soc.fields.sourceIp\"\n        to: \"source.ip\"\n      - from: \"soc.fields.status\"\n        to: \"http.response.status_code\"\n      - from: \"soc.fields.method\"\n        to: \"http.request.method\"\n      - from: \"soc.fields.path\"\n        to: \"url.path\"\n      - from: \"soc.message\"\n        to: \"event.action\"\n      - from: \"soc.level\"\n        to: \"log.level\"\n    ignore_missing: true",
-            "custom": "pipeline: common"
-          }
-        }
-      }
-    }
-  }
-}

salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json

@@ -4,7 +4,7 @@
     "name": "system",
     "version": ""
   },
-  "name": "system-grid-nodes",
+  "name": "system-grid-nodes_heavy",
   "namespace": "default",
   "inputs": {
     "system-logfile": {

salt/elasticfleet/soc_elasticfleet.yaml

@@ -12,10 +12,11 @@ elasticfleet:
   config:
     server:
       custom_fqdn:
-        description: Custom FQDN for Agents to connect to.
+        description: Custom FQDN for Agents to connect to. One per line.
         global: True
         helpLink: elastic-fleet.html
         advanced: True
+        forcedType: "[]string"
       enable_auto_configuration:
         description: Enable auto-configuration of Logstash Outputs & Fleet Host URLs.
         global: True

salt/elasticfleet/tools/sbin/so-elastic-fleet-common

@@ -56,6 +56,11 @@ elastic_fleet_package_version_check() {
     curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages/$PACKAGE" | jq -r '.item.version'
 }
 
+elastic_fleet_package_latest_version_check() {
+    PACKAGE=$1
+    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages/$PACKAGE" | jq -r '.item.latestVersion'
+}
+
 elastic_fleet_package_install() {
     PKGKEY=$1
     curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' "localhost:5601/api/fleet/epm/packages/$PKGKEY"

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load

@@ -9,16 +9,17 @@
 RETURN_CODE=0
 
 if [ ! -f /opt/so/state/eaintegrations.txt ]; then
+  # First, check for any package upgrades
+  /usr/sbin/so-elastic-fleet-package-upgrade
+
   # Initial Endpoints
   for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/endpoints-initial/*.json
   do
     printf "\n\nInitial Endpoints Policy - Loading $INTEGRATION\n"
     elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION"
     if [ -n "$INTEGRATION_ID" ]; then
-      if [ "$NAME" != "elastic-defend-endpoints" ]; then
-        printf "\n\nIntegration $NAME exists - Updating integration\n"
-        elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"
-      fi
+      printf "\n\nIntegration $NAME exists - Updating integration\n"
+      elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"
     else
       printf "\n\nIntegration does not exist - Creating integration\n"
       elastic_fleet_integration_create "@$INTEGRATION"

salt/elasticfleet/tools/sbin/so-elastic-fleet-package-list

@@ -0,0 +1,15 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-elastic-fleet-common
+
+# Let's snag a cookie from Kibana
+SESSIONCOOKIE=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
+
+# List configured package policies
+curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages" -H 'kbn-xsrf: true' | jq
+
+echo

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers

@@ -11,6 +11,12 @@
 . /usr/sbin/so-common
 . /usr/sbin/so-elastic-fleet-common
 
+LOG="/opt/so/log/elasticfleet/so-elastic-agent-gen-installers.log"
+
+# Check to see if we are already running
+NUM_RUNNING=$(pgrep -cf "/bin/bash /sbin/so-elastic-agent-gen-installers")
+[ "$NUM_RUNNING" -gt 1 ] && echo "$(date) - $NUM_RUNNING gen installers script processes running...exiting." >>$LOG && exit 0
+
 for i in {1..30}
 do
    ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key')

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade

@@ -0,0 +1,38 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
+# this file except in compliance with the Elastic License 2.0.
+
+. /usr/sbin/so-common
+
+# Only run on Managers
+if ! is_manager_node; then
+    printf "Not a Manager Node... Exiting"
+    exit 0
+fi
+
+# Get current list of Grid Node Agents that need to be upgraded
+RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/agents?perPage=20&page=1&kuery=policy_id%20%3A%20so-grid-nodes_%2A&showInactive=false&showUpgradeable=true&getStatusSummary=true")
+
+# Check to make sure that the server responded with good data - else, bail from script
+CHECKSUM=$(jq -r '.page' <<< "$RAW_JSON")
+if [ "$CHECKSUM" -ne 1 ]; then
+ printf "Failed to query for current Grid Agents...\n"
+ exit 1
+fi
+
+# Generate list of Node Agents that need updates
+OUTDATED_LIST=$(jq -r '.items | map(.id) | (tojson)'  <<<  "$RAW_JSON")
+
+if [ "$OUTDATED_LIST" != '[]' ]; then
+   AGENTNUMBERS=$(jq -r '.total' <<< "$RAW_JSON")
+   printf "Initiating upgrades for $AGENTNUMBERS Agents to Elastic $ELASTIC_AGENT_TARBALL_VERSION...\n\n"
+
+   # Generate updated JSON payload
+   JSON_STRING=$(jq -n --arg ELASTICVERSION $ELASTIC_AGENT_TARBALL_VERSION --arg UPDATELIST $OUTDATED_LIST '{"version": $ELASTICVERSION,"agents": $UPDATELIST }')
+
+   # Update Node Agents
+   curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "http://localhost:5601/api/fleet/agents/bulk_upgrade" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+else
+    printf "No Agents need updates... Exiting\n\n"
+    exit 0
+fi

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update

@@ -12,9 +12,13 @@ if ! is_manager_node; then
 fi
 
 function update_es_urls() {
-	# Generate updated JSON payload
-    JSON_STRING=$(jq -n --arg UPDATEDLIST $NEW_LIST_JSON '{"name":"so-manager_elasticsearch","type":"elasticsearch","hosts": $UPDATEDLIST,"is_default":false,"is_default_monitoring":false,"config_yaml":""}')
-
+	
+    # Generate updated JSON payload
+{% if grains.role not in ['so-import', 'so-eval'] %}
+    JSON_STRING=$(jq -n --arg UPDATEDLIST $NEW_LIST_JSON '{"name":"so-manager_elasticsearch","type":"elasticsearch","hosts": $UPDATEDLIST,"config_yaml":""}')
+{%- else %}
+    JSON_STRING=$(jq -n --arg UPDATEDLIST $NEW_LIST_JSON '{"name":"so-manager_elasticsearch","type":"elasticsearch","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":""}')
+{%- endif %}
     # Update Fleet Elasticsearch URLs
     curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_elasticsearch" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
 }
@@ -42,6 +46,13 @@ NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "$
 NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}')
 
 # Compare the current & new list of URLs - if different, update the Fleet Elasticsearch URLs
+if [ "$1" = "--force" ]; then
+    printf "\nUpdating List, since --force was specified.\n"
+    printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n"
+    update_es_urls
+    exit 0
+fi
+
 if [ "$NEW_HASH" = "$CURRENT_HASH" ]; then
     printf "\nHashes match - no update needed.\n"
     printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n"

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update

@@ -2,7 +2,7 @@
 # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
 # this file except in compliance with the Elastic License 2.0.
 {% from 'vars/globals.map.jinja' import GLOBALS %}
-{% set CUSTOMFQDN = salt['pillar.get']('elasticfleet:config:server:custom_fqdn') %}
+{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
 
 . /usr/sbin/so-common
 
@@ -41,9 +41,14 @@ else
     NEW_LIST=("{{ GLOBALS.url_base }}:5055" "{{ GLOBALS.hostname }}:5055")
 fi
 
-{% if CUSTOMFQDN != "" %}
-# Add Custom Hostname to list
-NEW_LIST+=("{{ CUSTOMFQDN }}:5055")
+# Query for FQDN entries & add them to the list
+{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %}
+CUSTOMFQDNLIST=('{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(' ') }}')
+readarray -t -d ' ' CUSTOMFQDN < <(printf '%s' "$CUSTOMFQDNLIST")
+for CUSTOMNAME in "${CUSTOMFQDN[@]}"
+do
+ NEW_LIST+=("$CUSTOMNAME:5055")
+done
 {% endif %}
 
 # Query for the current Grid Nodes that are running Logstash

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
+# this file except in compliance with the Elastic License 2.0.
+{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
+{%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %}
+
+. /usr/sbin/so-elastic-fleet-common
+
+{%- for PACKAGE in SUPPORTED_PACKAGES %}
+echo "Upgrading {{ PACKAGE }} package..."
+VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}")
+elastic_fleet_package_install "{{ PACKAGE }}-$VERSION"
+echo
+{%- endfor %}
+echo

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update

@@ -2,7 +2,7 @@
 # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
 # this file except in compliance with the Elastic License 2.0.
 {% from 'vars/globals.map.jinja' import GLOBALS %}
-{% set CUSTOMFQDN = salt['pillar.get']('elasticfleet:config:server:custom_fqdn') %}
+{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
 
 . /usr/sbin/so-common
 
@@ -41,9 +41,14 @@ else
     NEW_LIST=("https://{{ GLOBALS.url_base }}:8220" "https://{{ GLOBALS.hostname }}:8220")
 fi
 
-{% if CUSTOMFQDN != "" %}
-# Add Custom Hostname to list
-NEW_LIST+=("https://{{ CUSTOMFQDN }}:8220")
+# Query for FQDN entries & add them to the list
+{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %}
+CUSTOMFQDNLIST=('{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(' ') }}')
+readarray -t -d ' ' CUSTOMFQDN < <(printf '%s' "$CUSTOMFQDNLIST")
+for CUSTOMNAME in "${CUSTOMFQDN[@]}"
+do
+ NEW_LIST+=("https://$CUSTOMNAME:8220")
+done
 {% endif %}
 
 # Query for the current Grid Nodes that are running Logstash (which includes Fleet Nodes)
@@ -62,7 +67,7 @@ fi
 NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${NEW_LIST[@]}")
 NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}')
 
-# Compare the current & new list of URLs - if different, update the Fleet Server URLs
+# Compare the current & new list of URLs - if different, update the Fleet Server URLs & regenerate the agent installer
 if [ "$NEW_HASH" = "$CURRENT_HASH" ]; then
     printf "\nHashes match - no update needed.\n"
     printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n"
@@ -71,4 +76,5 @@ else
     printf "\nHashes don't match - update needed.\n"
     printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n"
     update_fleet_urls
+    /sbin/so-elastic-agent-gen-installers >> /opt/so/log/elasticfleet/so-elastic-agent-gen-installers.log &
 fi

salt/elasticsearch/defaults.yaml

@@ -113,7 +113,7 @@ elasticsearch:
             name: elastic_agent
           managed_by: security_onion
           managed: true  
-    so-logs-system.auth:
+    so-logs-system_x_auth:
       index_sorting: False
       index_template:
         index_patterns:
@@ -132,7 +132,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-system.syslog:
+    so-logs-system_x_syslog:
       index_sorting: False
       index_template:
         index_patterns:
@@ -151,7 +151,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-system.system:
+    so-logs-system_x_system:
       index_sorting: False
       index_template:
         index_patterns:
@@ -170,7 +170,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-system.application:
+    so-logs-system_x_application:
       index_sorting: False
       index_template:
         index_patterns:
@@ -189,7 +189,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-system.security:
+    so-logs-system_x_security:
       index_sorting: False
       index_template:
         index_patterns:
@@ -208,7 +208,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-windows.forwarded:
+    so-logs-windows_x_forwarded:
       index_sorting: False
       index_template:
         index_patterns:
@@ -226,7 +226,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-windows.powershell:
+    so-logs-windows_x_powershell:
       index_sorting: False
       index_template:
         index_patterns:
@@ -244,7 +244,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-windows.powershell_operational:
+    so-logs-windows_x_powershell_operational:
       index_sorting: False
       index_template:
         index_patterns:
@@ -262,7 +262,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-windows.sysmon_operational:
+    so-logs-windows_x_sysmon_operational:
       index_sorting: False
       index_template:
         index_patterns:
@@ -280,7 +280,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-aws.cloudtrail:
+    so-logs-aws_x_cloudtrail:
       index_sorting: False
       index_template:
         index_patterns:
@@ -298,7 +298,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-aws.cloudwatch_logs:
+    so-logs-aws_x_cloudwatch_logs:
       index_sorting: False
       index_template:
         index_patterns:
@@ -316,7 +316,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-aws.ec2_logs:
+    so-logs-aws_x_ec2_logs:
       index_sorting: False
       index_template:
         index_patterns:
@@ -334,7 +334,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-aws.elb_logs:
+    so-logs-aws_x_elb_logs:
       index_sorting: False
       index_template:
         index_patterns:
@@ -352,7 +352,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-aws.firewall_logs:
+    so-logs-aws_x_firewall_logs:
       index_sorting: False
       index_template:
         index_patterns:
@@ -370,7 +370,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-aws.route53_public_logs:
+    so-logs-aws_x_route53_public_logs:
       index_sorting: False
       index_template:
         index_patterns:
@@ -388,7 +388,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-aws.route53_resolver_logs:
+    so-logs-aws_x_route53_resolver_logs:
       index_sorting: False
       index_template:
         index_patterns:
@@ -406,7 +406,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-aws.s3access:
+    so-logs-aws_x_s3access:
       index_sorting: False
       index_template:
         index_patterns:
@@ -424,7 +424,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-aws.vpcflow:
+    so-logs-aws_x_vpcflow:
       index_sorting: False
       index_template:
         index_patterns:
@@ -442,7 +442,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-aws.waf:
+    so-logs-aws_x_waf:
       index_sorting: False
       index_template:
         index_patterns:
@@ -460,7 +460,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-azure.activitylogs:
+    so-logs-azure_x_activitylogs:
       index_sorting: False
       index_template:
         index_patterns:
@@ -478,7 +478,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-azure.application_gateway:
+    so-logs-azure_x_application_gateway:
       index_sorting: False
       index_template:
         index_patterns:
@@ -496,7 +496,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-azure.auditlogs:
+    so-logs-azure_x_auditlogs:
       index_sorting: False
       index_template:
         index_patterns:
@@ -514,7 +514,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-azure.eventhub:
+    so-logs-azure_x_eventhub:
       index_sorting: False
       index_template:
         index_patterns:
@@ -532,7 +532,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-azure.firewall_logs:
+    so-logs-azure_x_firewall_logs:
       index_sorting: False
       index_template:
         index_patterns:
@@ -550,7 +550,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-azure.identity_protection:
+    so-logs-azure_x_identity_protection:
       index_sorting: False
       index_template:
         index_patterns:
@@ -568,7 +568,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-azure.platformlogs:
+    so-logs-azure_x_platformlogs:
       index_sorting: False
       index_template:
         index_patterns:
@@ -586,7 +586,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-azure.provisioning:
+    so-logs-azure_x_provisioning:
       index_sorting: False
       index_template:
         index_patterns:
@@ -604,7 +604,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-azure.signinlogs:
+    so-logs-azure_x_signinlogs:
       index_sorting: False
       index_template:
         index_patterns:
@@ -622,7 +622,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-azure.springcloudlogs:
+    so-logs-azure_x_springcloudlogs:
       index_sorting: False
       index_template:
         index_patterns:
@@ -640,7 +640,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-cloudflare.audit:
+    so-logs-cloudflare_x_audit:
       index_sorting: False
       index_template:
         index_patterns:
@@ -658,7 +658,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-cloudflare.logpull:
+    so-logs-cloudflare_x_logpull:
       index_sorting: False
       index_template:
         index_patterns:
@@ -676,7 +676,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-fim.event:
+    so-logs-fim_x_event:
       index_sorting: False
       index_template:
         index_patterns:
@@ -694,7 +694,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-github.audit:
+    so-logs-github_x_audit:
       index_sorting: False
       index_template:
         index_patterns:
@@ -712,7 +712,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-github.code_scanning:
+    so-logs-github_x_code_scanning:
       index_sorting: False
       index_template:
         index_patterns:
@@ -730,7 +730,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-github.dependabot:
+    so-logs-github_x_dependabot:
       index_sorting: False
       index_template:
         index_patterns:
@@ -748,7 +748,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-github.issues:
+    so-logs-github_x_issues:
       index_sorting: False
       index_template:
         index_patterns:
@@ -766,7 +766,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-github.secret_scanning:
+    so-logs-github_x_secret_scanning:
       index_sorting: False
       index_template:
         index_patterns:
@@ -784,7 +784,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-google_workspace.access_transparency:
+    so-logs-google_workspace_x_access_transparency:
       index_sorting: False
       index_template:
         index_patterns:
@@ -802,7 +802,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-google_workspace.admin:
+    so-logs-google_workspace_x_admin:
       index_sorting: False
       index_template:
         index_patterns:
@@ -820,7 +820,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-google_workspace.alert:
+    so-logs-google_workspace_x_alert:
       index_sorting: False
       index_template:
         index_patterns:
@@ -838,7 +838,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-google_workspace.context_aware_access:
+    so-logs-google_workspace_x_context_aware_access:
       index_sorting: False
       index_template:
         index_patterns:
@@ -856,7 +856,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-google_workspace.device:
+    so-logs-google_workspace_x_device:
       index_sorting: False
       index_template:
         index_patterns:
@@ -874,7 +874,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-google_workspace.drive:
+    so-logs-google_workspace_x_drive:
       index_sorting: False
       index_template:
         index_patterns:
@@ -892,7 +892,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-google_workspace.gcp:
+    so-logs-google_workspace_x_gcp:
       index_sorting: False
       index_template:
         index_patterns:
@@ -910,7 +910,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-google_workspace.group_enterprise:
+    so-logs-google_workspace_x_group_enterprise:
       index_sorting: False
       index_template:
         index_patterns:
@@ -928,7 +928,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-google_workspace.groups:
+    so-logs-google_workspace_x_groups:
       index_sorting: False
       index_template:
         index_patterns:
@@ -946,7 +946,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-google_workspace.login:
+    so-logs-google_workspace_x_login:
       index_sorting: False
       index_template:
         index_patterns:
@@ -964,7 +964,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-google_workspace.rules:
+    so-logs-google_workspace_x_rules:
       index_sorting: False
       index_template:
         index_patterns:
@@ -982,7 +982,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-google_workspace.saml:
+    so-logs-google_workspace_x_saml:
       index_sorting: False
       index_template:
         index_patterns:
@@ -1000,7 +1000,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-google_workspace.token:
+    so-logs-google_workspace_x_token:
       index_sorting: False
       index_template:
         index_patterns:
@@ -1018,7 +1018,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-google_workspace.user_accounts:
+    so-logs-google_workspace_x_user_accounts:
       index_sorting: False
       index_template:
         index_patterns:
@@ -1036,7 +1036,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-1password.item_usages:
+    so-logs-1password_x_item_usages:
       index_sorting: False
       index_template:
         index_patterns:
@@ -1054,7 +1054,7 @@ elasticsearch:
         data_stream:
           hidden: false
           allow_custom_routing: false
-    so-logs-1password.signin_attempts:
+    so-logs-1password_x_signin_attempts:
       index_sorting: False
       index_template:
         index_patterns:
@@ -1089,7 +1089,7 @@ elasticsearch:
             name: elastic_agent
           managed_by: security_onion
           managed: true
-    so-logs-osquery-manager-action.responses:
+    so-logs-osquery-manager-action_x_responses:
       index_sorting: False
       index_template:
         index_patterns:
@@ -1106,7 +1106,7 @@ elasticsearch:
             name: elastic_agent
           managed_by: security_onion
           managed: true
-    so-logs-elastic_agent.apm_server:
+    so-logs-elastic_agent_x_apm_server:
       index_sorting: False
       index_template:
         index_patterns:
@@ -1160,7 +1160,7 @@ elasticsearch:
             name: elastic_agent
           managed_by: security_onion
           managed: true
-    so-logs-elastic_agent.auditbeat:
+    so-logs-elastic_agent_x_auditbeat:
       index_sorting: False
       index_template:
         index_patterns:
@@ -1214,7 +1214,7 @@ elasticsearch:
             name: elastic_agent
           managed_by: security_onion
           managed: true
-    so-logs-elastic_agent.cloudbeat:
+    so-logs-elastic_agent_x_cloudbeat:
       index_sorting: False
       index_template:
         index_patterns:
@@ -1265,7 +1265,7 @@ elasticsearch:
             name: elastic_agent
           managed_by: security_onion
           managed: true
-    so-logs-elastic_agent.endpoint_security:
+    so-logs-elastic_agent_x_endpoint_security:
       index_sorting: False
       index_template:
         index_patterns:
@@ -1314,7 +1314,7 @@ elasticsearch:
             name: elastic_agent
           managed_by: security_onion
           managed: true
-    so-logs-endpoint.alerts:
+    so-logs-endpoint_x_alerts:
       index_sorting: False
       index_template:
         index_patterns:
@@ -1363,7 +1363,7 @@ elasticsearch:
             name: elastic_agent
           managed_by: security_onion
           managed: true
-    so-logs-endpoint.events.api:
+    so-logs-endpoint_x_events_x_api:
       index_sorting: False
       index_template:
         index_patterns:
@@ -1412,7 +1412,7 @@ elasticsearch:
             name: elastic_agent
           managed_by: security_onion
           managed: true
-    so-logs-endpoint.events.file:
+    so-logs-endpoint_x_events_x_file:
       index_sorting: False
       index_template:
         index_patterns:
@@ -1461,7 +1461,7 @@ elasticsearch:
             name: elastic_agent
           managed_by: security_onion
           managed: true
-    so-logs-endpoint.events.library:
+    so-logs-endpoint_x_events_x_library:
       index_sorting: False
       index_template:
         index_patterns:
@@ -1510,7 +1510,7 @@ elasticsearch:
             name: elastic_agent
           managed_by: security_onion
           managed: true
-    so-logs-endpoint.events.network:
+    so-logs-endpoint_x_events_x_network:
       index_sorting: False
       index_template:
         index_patterns:
@@ -1559,7 +1559,7 @@ elasticsearch:
             name: elastic_agent
           managed_by: security_onion
           managed: true
-    so-logs-endpoint.events.process:
+    so-logs-endpoint_x_events_x_process:
       index_sorting: False
       index_template:
         index_patterns:
@@ -1608,7 +1608,7 @@ elasticsearch:
             name: elastic_agent
           managed_by: security_onion
           managed: true
-    so-logs-endpoint.events.registry:
+    so-logs-endpoint_x_events_x_registry:
       index_sorting: False
       index_template:
         index_patterns:
@@ -1657,7 +1657,7 @@ elasticsearch:
             name: elastic_agent
           managed_by: security_onion
           managed: true
-    so-logs-endpoint.events.security:
+    so-logs-endpoint_x_events_x_security:
       index_sorting: False
       index_template:
         index_patterns:
@@ -1706,7 +1706,7 @@ elasticsearch:
             name: elastic_agent
           managed_by: security_onion
           managed: true
-    so-logs-elastic_agent.filebeat:
+    so-logs-elastic_agent_x_filebeat:
       index_sorting: False
       index_template:
         index_patterns:
@@ -1755,7 +1755,7 @@ elasticsearch:
             name: elastic_agent
           managed_by: security_onion
           managed: true
-    so-logs-elastic_agent.fleet_server:
+    so-logs-elastic_agent_x_fleet_server:
       index_sorting: False
       index_template:
         index_patterns:
@@ -1801,7 +1801,7 @@ elasticsearch:
             name: elastic_agent
           managed_by: security_onion
           managed: true
-    so-logs-elastic_agent.heartbeat:
+    so-logs-elastic_agent_x_heartbeat:
       index_sorting: False
       index_template:
         index_patterns:
@@ -1907,7 +1907,7 @@ elasticsearch:
             name: elastic_agent
           managed_by: security_onion
           managed: true
-    so-logs-elastic_agent.metricbeat:
+    so-logs-elastic_agent_x_metricbeat:
       index_sorting: False
       index_template:
         index_patterns:
@@ -1956,7 +1956,7 @@ elasticsearch:
             name: elastic_agent
           managed_by: security_onion
           managed: true
-    so-logs-elastic_agent.osquerybeat:
+    so-logs-elastic_agent_x_osquerybeat:
       index_sorting: False
       index_template:
         index_patterns:
@@ -2005,7 +2005,7 @@ elasticsearch:
             name: elastic_agent
           managed_by: security_onion
           managed: true
-    so-logs-elastic_agent.packetbeat:
+    so-logs-elastic_agent_x_packetbeat:
       index_sorting: False
       index_template:
         index_patterns:

salt/elasticsearch/files/ingest/.fleet_final_pipeline-1

@@ -78,6 +78,7 @@
     { "set":        { "if": "ctx.network?.direction == 'ingress'", "override": true, "field": "network.initiated", "value": "false" } },
     { "set":        { "if": "ctx.network?.type == 'ipv4'", "override": true, "field": "destination.ipv6", "value": "false" } },
     { "set":        { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } },
+    {"community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true }  },
     { "remove":     { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp" ], "ignore_missing": true, "ignore_failure": true } } 
   ],
   "on_failure": [

salt/elasticsearch/files/ingest/filterlog

@@ -49,11 +49,10 @@
                 "on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}]
         }
      },
-     { "set":         { "field": "_index",        "value": "so-firewall", "override": true           } },
      { "set":         { "if": "ctx.network?.transport_id == '0'", "field": "network.transport",        "value": "icmp", "override": true           } },
      { "community_id": {} },
-     { "set":         { "field": "module",	"value": "pfsense",	"override": true	} },
-     { "set":         { "field": "dataset",	"value": "firewall",	"override": true	} },
+     { "set":         { "field": "event.module",	"value": "pfsense",	"override": true	} },
+     { "set":         { "field": "event.dataset",	"value": "firewall",	"override": true	} },
      { "set":         { "field": "category",	"value": "network",	"override": true	} },
      { "remove":      { "field": ["real_message", "ip_sub_msg", "firewall.sub_message"],     "ignore_failure": true  } }
   ]

salt/elasticsearch/soc_elasticsearch.yaml

@@ -46,28 +46,26 @@ elasticsearch:
             description: Max number of boolean clauses per query.
             global: True
             helpLink: elasticsearch.html
-  index_settings: 
-    so-elasticsearch: &indexSettings
-      warm: 
-        description: Age (in days) of this index before it will move to warm storage, if warm nodes are present. Once moved, events on this index can take longer to fetch.
-        global: True
-        helpLink: elasticsearch.html
-      close: 
-        description: Age (in days) of this index before it will be closed. Once closed, events on this index cannot be retrieved without first re-opening the index.
-        global: True
-        helpLink: elasticsearch.html
-      delete: 
-        description: Age (in days) of this index before it will be deleted. Once deleted, events are permanently unrecoverable.
-        global: True
-        helpLink: elasticsearch.html
+  index_settings:
+    so-logs: &indexSettings
       index_sorting: 
         description: Sorts the index by event time, at the cost of additional processing resource consumption.
         global: True
         helpLink: elasticsearch.html
       index_template:
+        index_patterns:
+          description: Patterns for matching multiple indices or tables.
+          forceType: "[]string"
+          multiline: True
+          global: True
+          helpLink: elasticsearch.html
         template:
           settings:
             index:
+              number_of_replicas: 
+                description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs.
+                global: True
+                helpLink: elasticsearch.html
               mapping:
                 total_fields:
                   limit:
@@ -75,17 +73,59 @@ elasticsearch:
                     global: True
                     helpLink: elasticsearch.html
               refresh_interval: 
-                  description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation.
-                  global: True
-                  helpLink: elasticsearch.html
+                description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation.
+                global: True
+                helpLink: elasticsearch.html
               number_of_shards: 
-                  description: Number of shards required for this index. Using multiple shards increases fault tolerance, but also increases storage and network costs.
+                description: Number of shards required for this index. Using multiple shards increases fault tolerance, but also increases storage and network costs.
+                global: True
+                helpLink: elasticsearch.html
+              sort:
+                field:
+                  description: The field to sort by. Must set index_sorting to True.
                   global: True
                   helpLink: elasticsearch.html
-              number_of_replicas: 
-                  description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs.
+                order:
+                  description: The order to sort by. Must set index_sorting to True.
                   global: True
                   helpLink: elasticsearch.html
+          mappings:
+            _meta:
+              package:
+                name:
+                  description: Meta settings for the mapping.
+                  global: True
+                  helpLink: elasticsearch.html
+              managed_by:
+                  description: Meta settings for the mapping.
+                  global: True
+                  helpLink: elasticsearch.html
+              managed:
+                  description: Meta settings for the mapping.
+                  forcedType: bool
+                  global: True
+                  helpLink: elasticsearch.html
+        composed_of:
+          description: The index template is composed of these component templates.
+          forcedType: "[]string"
+          global: True
+          helpLink: elasticsearch.html
+        priority:
+          description: The priority of the index template.
+          forcedType: int
+          global: True
+          helpLink: elasticsearch.html
+        data_stream:
+          hidden:
+            description: Hide the data stream.
+            forcedType: bool
+            global: True
+            helpLink: elasticsearch.html
+          allow_custom_routing:
+            description: Allow custom routing for the data stream.
+            forcedType: bool
+            global: True
+            helpLink: elasticsearch.html
       policy:
         phases:
           hot:
@@ -97,6 +137,7 @@ elasticsearch:
               set_priority:
                 priority:
                   description: Priority of index. This is used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
+                  forcedType: int
                   global: True
                   helpLink: elasticsearch.html
               rollover:
@@ -117,19 +158,111 @@ elasticsearch:
               set_priority:
                 priority:
                   description: Used for index recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
+                  forcedType: int
                   global: True
                   helpLink: elasticsearch.html
           delete:
             min_age:
               description: Minimum age of index. This determines when the index should be deleted.
               global: True
-              helpLink: elastic
+              helpLink: elasticsearch.html
+        _meta:
+          package:
+            name:
+              description: Meta settings for the mapping.
+              global: True
+              helpLink: elasticsearch.html
+          managed_by:
+            description: Meta settings for the mapping.
+            global: True
+            helpLink: elasticsearch.html
+          managed:
+            description: Meta settings for the mapping.
+            forcedType: bool
+            global: True
+            helpLink: elasticsearch.html
+    so-logs-system_x_auth: *indexSettings
+    so-logs-system_x_syslog: *indexSettings
+    so-logs-system_x_system: *indexSettings
+    so-logs-system_x_application: *indexSettings
+    so-logs-system_x_security: *indexSettings
+    so-logs-windows_x_forwarded: *indexSettings
+    so-logs-windows_x_powershell: *indexSettings
+    so-logs-windows_x_powershell_operational: *indexSettings
+    so-logs-windows_x_sysmon_operational: *indexSettings
+    so-logs-aws_x_cloudtrail: *indexSettings
+    so-logs-aws_x_cloudwatch_logs: *indexSettings
+    so-logs-aws_x_ec2_logs: *indexSettings
+    so-logs-aws_x_elb_logs: *indexSettings
+    so-logs-aws_x_firewall_logs: *indexSettings
+    so-logs-aws_x_route53_public_logs: *indexSettings
+    so-logs-aws_x_route53_resolver_logs: *indexSettings
+    so-logs-aws_x_s3access: *indexSettings
+    so-logs-aws_x_vpcflow: *indexSettings
+    so-logs-aws_x_waf: *indexSettings
+    so-logs-azure_x_activitylogs: *indexSettings
+    so-logs-azure_x_application_gateway: *indexSettings
+    so-logs-azure_x_auditlogs: *indexSettings
+    so-logs-azure_x_eventhub: *indexSettings
+    so-logs-azure_x_firewall_logs: *indexSettings
+    so-logs-azure_x_identity_protection: *indexSettings
+    so-logs-azure_x_platformlogs: *indexSettings
+    so-logs-azure_x_provisioning: *indexSettings
+    so-logs-azure_x_signinlogs: *indexSettings
+    so-logs-azure_x_springcloudlogs: *indexSettings
+    so-logs-cloudflare_x_audit: *indexSettings
+    so-logs-cloudflare_x_logpull: *indexSettings
+    so-logs-fim_x_event: *indexSettings
+    so-logs-github_x_audit: *indexSettings
+    so-logs-github_x_code_scanning: *indexSettings
+    so-logs-github_x_dependabot: *indexSettings
+    so-logs-github_x_issues: *indexSettings
+    so-logs-github_x_secret_scanning: *indexSettings
+    so-logs-google_workspace_x_access_transparency: *indexSettings
+    so-logs-google_workspace_x_admin: *indexSettings
+    so-logs-google_workspace_x_alert: *indexSettings
+    so-logs-google_workspace_x_context_aware_access: *indexSettings
+    so-logs-google_workspace_x_device: *indexSettings
+    so-logs-google_workspace_x_drive: *indexSettings
+    so-logs-google_workspace_x_gcp: *indexSettings
+    so-logs-google_workspace_x_group_enterprise: *indexSettings
+    so-logs-google_workspace_x_groups: *indexSettings
+    so-logs-google_workspace_x_login: *indexSettings
+    so-logs-google_workspace_x_rules: *indexSettings
+    so-logs-google_workspace_x_saml: *indexSettings
+    so-logs-google_workspace_x_token: *indexSettings
+    so-logs-google_workspace_x_user_accounts: *indexSettings
+    so-logs-1password_x_item_usages: *indexSettings
+    so-logs-1password_x_signin_attempts: *indexSettings
+    so-logs-osquery-manager-actions: *indexSettings
+    so-logs-osquery-manager-action_x_responses: *indexSettings
+    so-logs-elastic_agent_x_apm_server: *indexSettings
+    so-logs-elastic_agent_x_auditbeat: *indexSettings
+    so-logs-elastic_agent_x_cloudbeat: *indexSettings
+    so-logs-elastic_agent_x_endpoint_security: *indexSettings
+    so-logs-endpoint_x_alerts: *indexSettings
+    so-logs-endpoint_x_events_x_api: *indexSettings
+    so-logs-endpoint_x_events_x_file: *indexSettings
+    so-logs-endpoint_x_events_x_library: *indexSettings
+    so-logs-endpoint_x_events_x_network: *indexSettings
+    so-logs-endpoint_x_events_x_process: *indexSettings
+    so-logs-endpoint_x_events_x_registry: *indexSettings
+    so-logs-endpoint_x_events_x_security: *indexSettings
+    so-logs-elastic_agent_x_filebeat: *indexSettings
+    so-logs-elastic_agent_x_fleet_server: *indexSettings
+    so-logs-elastic_agent_x_heartbeat: *indexSettings
+    so-logs-elastic_agent: *indexSettings
+    so-logs-elastic_agent_x_metricbeat: *indexSettings
+    so-logs-elastic_agent_x_osquerybeat: *indexSettings
+    so-logs-elastic_agent_x_packetbeat: *indexSettings
+    so-case: *indexSettings
+    so-common: *indexSettings
     so-endgame: *indexSettings
-    so-firewall: *indexSettings
+    so-idh: *indexSettings
+    so-suricata: *indexSettings
     so-import: *indexSettings
-    so-kibana: *indexSettings
+    so-kratos: *indexSettings
     so-logstash: *indexSettings
-    so-osquery: *indexSettings
     so-redis: *indexSettings
     so-strelka: *indexSettings
     so-syslog: *indexSettings

salt/elasticsearch/template.map.jinja

@@ -1,9 +1,11 @@
 {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS with context %}
-{%- set ES_INDEX_SETTINGS = salt['pillar.get']('elasticsearch:index_settings', default=ELASTICSEARCHDEFAULTS.elasticsearch.index_settings, merge=True) %}
-{% for index, settings in ES_INDEX_SETTINGS.items() %}
+{%- set ES_INDEX_SETTINGS_ORIG = salt['pillar.get']('elasticsearch:index_settings', default=ELASTICSEARCHDEFAULTS.elasticsearch.index_settings, merge=True) %}
+{% set ES_INDEX_SETTINGS = {} %}
+{% for index, settings in ES_INDEX_SETTINGS_ORIG.items() %}
   {% if settings.index_template is defined %}
     {% if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %}
       {% do settings.index_template.template.settings.index.pop('sort') %}
     {% endif %}
   {% endif %}
+  {% do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_ORIG[index]}) %}
 {% endfor %}

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load

@@ -6,8 +6,7 @@
 
 . /usr/sbin/so-common
 
-{%- import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
-{%- set ES_INDEX_SETTINGS = salt['pillar.get']('elasticsearch:index_settings', default=ELASTICSEARCHDEFAULTS.elasticsearch.index_settings, merge=True) %}
+{%- from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %}
 
 {%- for index, settings in ES_INDEX_SETTINGS.items() %}
 {%-   if settings.policy is defined %}

salt/firewall/defaults.yaml

@@ -20,12 +20,12 @@ firewall:
     managersearch: []
     receiver: []
     searchnode: []
-    securityonion_desktop: []
     self: []
     sensor: []
     standalone: []
     strelka_frontend: []
     syslog: []
+    desktop: []
     customhostgroup0: []
     customhostgroup1: []
     customhostgroup2: []
@@ -198,9 +198,6 @@ firewall:
               portgroups:
                 - redis
                 - elasticsearch_node
-            self:
-              portgroups:
-                - syslog
             beats_endpoint:
               portgroups:
                 - beats_5044
@@ -218,9 +215,6 @@ firewall:
             strelka_frontend:
               portgroups:
                 - strelka_frontend
-            syslog:
-              portgroups:
-                - syslog
             analyst:
               portgroups:
                 - nginx
@@ -255,6 +249,12 @@ firewall:
             localhost:
               portgroups:
                 - all
+            self:
+              portgroups:
+                - syslog
+            syslog:
+              portgroups:
+                - syslog
             customhostgroup0:
               portgroups: []
             customhostgroup1:
@@ -370,6 +370,7 @@ firewall:
                 - elastic_agent_data
                 - elastic_agent_update
                 - localrules
+                - sensoroni
             fleet:
               portgroups:
                 - elasticsearch_rest
@@ -383,6 +384,17 @@ firewall:
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
+            idh:
+              portgroups:
+                - docker_registry
+                - influxdb
+                - sensoroni
+                - yum
+                - beats_5044
+                - beats_5644
+                - elastic_agent_control
+                - elastic_agent_data
+                - elastic_agent_update
             sensor:
               portgroups:
                 - beats_5044
@@ -393,6 +405,7 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
+                - sensoroni
             searchnode:
               portgroups:
                 - redis
@@ -405,6 +418,7 @@ firewall:
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
+                - sensoroni
             heavynode:
               portgroups:
                 - redis
@@ -417,6 +431,7 @@ firewall:
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
+                - sensoroni
             receiver:
               portgroups:
                 - yum
@@ -425,12 +440,10 @@ firewall:
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
-            self:
+                - sensoroni
+            analyst:
               portgroups:
-                - syslog
-            syslog:
-              portgroups:
-                - syslog
+                - nginx
             beats_endpoint:
               portgroups:
                 - beats_5044
@@ -448,9 +461,9 @@ firewall:
             endgame:
               portgroups:
                 - endgame
-            analyst:
+            desktop:
               portgroups:
-                - nginx
+                - yum
             customhostgroup0:
               portgroups: []
             customhostgroup1:
@@ -482,6 +495,9 @@ firewall:
             fleet:
               portgroups:
                 - salt_manager
+            idh:
+              portgroups:
+                - salt_manager
             localhost:
               portgroups:
                 - all
@@ -497,6 +513,15 @@ firewall:
             receiver:
               portgroups:
                 - salt_manager
+            desktop:
+              portgroups:
+                - salt_manager
+            self:
+              portgroups:
+                - syslog
+            syslog:
+              portgroups:
+                - syslog
             customhostgroup0:
               portgroups: []
             customhostgroup1:
@@ -535,6 +560,7 @@ firewall:
                 - elastic_agent_data
                 - elastic_agent_update
                 - localrules
+                - sensoroni
             fleet:
               portgroups:
                 - elasticsearch_rest
@@ -548,6 +574,17 @@ firewall:
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
+            idh:
+              portgroups:
+                - docker_registry
+                - influxdb
+                - sensoroni
+                - yum
+                - beats_5044
+                - beats_5644
+                - elastic_agent_control
+                - elastic_agent_data
+                - elastic_agent_update
             sensor:
               portgroups:
                 - beats_5044
@@ -558,6 +595,7 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
+                - sensoroni
             searchnode:
               portgroups:
                 - redis
@@ -569,6 +607,7 @@ firewall:
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
+                - sensoroni
             heavynode:
               portgroups:
                 - redis
@@ -580,6 +619,7 @@ firewall:
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
+                - sensoroni
             receiver:
               portgroups:
                 - yum
@@ -588,9 +628,10 @@ firewall:
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
-            self:
+                - sensoroni
+            analyst:
               portgroups:
-                - syslog
+                - nginx
             beats_endpoint:
               portgroups:
                 - beats_5044
@@ -608,12 +649,9 @@ firewall:
             endgame:
               portgroups:
                 - endgame
-            syslog:
+            desktop:
               portgroups:
-                - syslog
-            analyst:
-              portgroups:
-                - nginx
+                - yum
             customhostgroup0:
               portgroups: []
             customhostgroup1:
@@ -645,6 +683,9 @@ firewall:
             fleet:
               portgroups:
                 - salt_manager
+            idh:
+              portgroups:
+                - salt_manager
             localhost:
               portgroups:
                 - all
@@ -660,6 +701,15 @@ firewall:
             receiver:
               portgroups:
                 - salt_manager
+            desktop:
+              portgroups:
+                - salt_manager
+            self:
+              portgroups:
+                - syslog
+            syslog:
+              portgroups:
+                - syslog
             customhostgroup0:
               portgroups: []
             customhostgroup1:
@@ -723,6 +773,17 @@ firewall:
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
+            idh:
+              portgroups:
+                - docker_registry
+                - influxdb
+                - sensoroni
+                - yum
+                - beats_5044
+                - beats_5644
+                - elastic_agent_control
+                - elastic_agent_data
+                - elastic_agent_update
             sensor:
               portgroups:
                 - docker_registry
@@ -760,9 +821,10 @@ firewall:
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
-            self:
+                - sensoroni
+            analyst:
               portgroups:
-                - syslog
+                - nginx
             beats_endpoint:
               portgroups:
                 - beats_5044
@@ -783,12 +845,9 @@ firewall:
             strelka_frontend:
               portgroups:
                 - strelka_frontend
-            syslog:
+            desktop:
               portgroups:
-                - syslog
-            analyst:
-              portgroups:
-                - nginx
+                - yum
             customhostgroup0:
               portgroups: []
             customhostgroup1:
@@ -819,7 +878,10 @@ firewall:
                 - all
             fleet:
               portgroups:
-                - salt_manager              
+                - salt_manager
+            idh:
+              portgroups:
+                - salt_manager
             localhost:
               portgroups:
                 - all
@@ -838,6 +900,15 @@ firewall:
             receiver:
               portgroups:
                 - salt_manager
+            desktop:
+              portgroups:
+                - salt_manager
+            self:
+              portgroups:
+                - syslog
+            syslog:
+              portgroups:
+                - syslog
             customhostgroup0:
               portgroups: []
             customhostgroup1:
@@ -866,6 +937,14 @@ firewall:
               portgroups:
                 - elasticsearch_node
                 - elasticsearch_rest
+            managersearch:
+              portgroups:
+                - elasticsearch_node
+                - elasticsearch_rest
+            standalone:
+              portgroups:
+                - elasticsearch_node
+                - elasticsearch_rest
             dockernet:
               portgroups:
                 - elasticsearch_node
@@ -876,9 +955,6 @@ firewall:
             searchnode:
               portgroups:
                 - elasticsearch_node
-            self:
-              portgroups:
-                - syslog
             customhostgroup0:
               portgroups: []
             customhostgroup1:
@@ -910,6 +986,12 @@ firewall:
             localhost:
               portgroups:
                 - all
+            self:
+              portgroups:
+                - syslog
+            syslog:
+              portgroups:
+                - syslog
             customhostgroup0:
               portgroups: []
             customhostgroup1:
@@ -934,9 +1016,6 @@ firewall:
       chain:
         DOCKER-USER:
           hostgroups:
-            self:
-              portgroups:
-                - syslog
             strelka_frontend:
               portgroups:
                 - strelka_frontend
@@ -971,6 +1050,12 @@ firewall:
             localhost:
               portgroups:
                 - all
+            self:
+              portgroups:
+                - syslog
+            syslog:
+              portgroups:
+                - syslog
             customhostgroup0:
               portgroups: []
             customhostgroup1:
@@ -1022,6 +1107,9 @@ firewall:
             strelka_frontend:
               portgroups:
                 - strelka_frontend
+            syslog:
+              portgroups:
+                - syslog
             customhostgroup0:
               portgroups: []
             customhostgroup1:
@@ -1111,6 +1199,9 @@ firewall:
             analyst:
               portgroups:
                 - nginx
+            desktop:
+              portgroups:
+                - yum
             customhostgroup0:
               portgroups: []
             customhostgroup1:
@@ -1181,11 +1272,7 @@ firewall:
             self:
               portgroups:
                 - redis
-                - syslog
                 - beats_5644
-            syslog:
-              portgroups:
-                - syslog
             beats_endpoint:
               portgroups:
                 - beats_5044
@@ -1226,6 +1313,12 @@ firewall:
             localhost:
               portgroups:
                 - all
+            self:
+              portgroups:
+                - syslog
+            syslog:
+              portgroups:
+                - syslog
             customhostgroup0:
               portgroups: []
             customhostgroup1:

salt/firewall/init.sls

@@ -1,15 +1,29 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls in allowed_states %}
+{%   from 'firewall/ipt.map.jinja' import iptmap %}
+
+install_iptables:
+  pkg.installed:
+    - name: {{ iptmap.iptpkg }}
+
+iptables_persist:
+  pkg.installed:
+    - name: {{ iptmap.persistpkg }}
+
+iptables_service:
+  service.running:
+    - name: {{ iptmap.service }}
+    - enable: True
 
 create_sysconfig_iptables:
   file.touch:
-    - name: /etc/sysconfig/iptables
+    - name: {{ iptmap.configfile }}
     - makedirs: True
-    - unless: 'ls /etc/sysconfig/iptables'
+    - unless: 'ls {{ iptmap.configfile }}'
 
 iptables_config:
   file.managed:
-    - name: /etc/sysconfig/iptables
+    - name: {{ iptmap.configfile }}
     - source: salt://firewall/iptables.jinja
     - template: jinja
 
@@ -24,11 +38,11 @@ disable_firewalld:
 
 iptables_restore:
   cmd.run:
-    - name: iptables-restore < /etc/sysconfig/iptables
+    - name: iptables-restore < {{ iptmap.configfile }}
     - require:
       - file: iptables_config
     - onlyif:
-      - iptables-restore --test /etc/sysconfig/iptables
+      - iptables-restore --test {{ iptmap.configfile }}
 
 {%   if grains.os_family == 'RedHat' %}
 enable_firewalld:

salt/firewall/ipt.map.jinja

@@ -0,0 +1,14 @@
+{% set iptmap = salt['grains.filter_by']({
+    'Debian': {
+        'service': 'netfilter-persistent',
+        'iptpkg': 'iptables',
+        'persistpkg': 'iptables-persistent',
+        'configfile': '/etc/iptables/rules.v4'
+    },
+    'RedHat': {
+        'service': 'iptables',
+        'iptpkg': 'iptables-nft',
+        'persistpkg': 'iptables-nft-services',
+        'configfile': '/etc/sysconfig/iptables'
+    },
+}) %}

salt/firewall/soc_firewall.yaml

@@ -39,12 +39,12 @@ firewall:
     managersearch: *hostgroupsettings
     receiver: *hostgroupsettings
     searchnode: *hostgroupsettings
-    securityonion_desktop: *hostgroupsettings
     self: *ROhostgroupsettingsadv
     sensor: *hostgroupsettings
     standalone: *hostgroupsettings
     strelka_frontend: *hostgroupsettings
     syslog: *hostgroupsettings
+    desktop: *hostgroupsettings
     customhostgroup0: &customhostgroupsettings
       description: List of IP or CIDR blocks to allow to this hostgroup.
       forcedType: "[]string"
@@ -191,6 +191,7 @@ firewall:
                 description: Portgroups to add access to the docker containers for this role.
                 advanced: True
                 multiline: True
+                forcedType: "[]string"
                 helpLink: firewall.html
             sensor:
               portgroups: *portgroupsdocker
@@ -214,6 +215,8 @@ firewall:
               portgroups: *portgroupsdocker
             analyst:
               portgroups: *portgroupsdocker
+            desktop:
+              portgroups: *portgroupsdocker
             customhostgroup0:
               portgroups: *portgroupsdocker
             customhostgroup1: 
@@ -241,6 +244,7 @@ firewall:
                 description: Portgroups to add access to the host.
                 advanced: True
                 multiline: True
+                forcedType: "[]string"
                 helpLink: firewall.html
             dockernet: 
               portgroups: *portgroupshost
@@ -336,7 +340,9 @@ firewall:
         DOCKER-USER:
           hostgroups:
             manager:
-              portgroups: *portgroupsdocker                
+              portgroups: *portgroupsdocker
+            idh:
+              portgroups: *portgroupsdocker
             sensor:
               portgroups: *portgroupsdocker
             searchnode:
@@ -359,6 +365,8 @@ firewall:
               portgroups: *portgroupsdocker
             analyst:
               portgroups: *portgroupsdocker
+            desktop:
+              portgroups: *portgroupsdocker
             customhostgroup0:
               portgroups: *portgroupsdocker
             customhostgroup1: 
@@ -387,12 +395,16 @@ firewall:
               portgroups: *portgroupshost
             localhost:
               portgroups: *portgroupshost
+            idh:
+              portgroups: *portgroupshost
             sensor:
               portgroups: *portgroupshost
             searchnode:
               portgroups: *portgroupshost
             heavynode:
               portgroups: *portgroupshost
+            desktop:
+              portgroups: *portgroupshost
             customhostgroup0:
               portgroups: *portgroupshost
             customhostgroup1: 
@@ -420,6 +432,8 @@ firewall:
           hostgroups:
             managersearch:
               portgroups: *portgroupsdocker
+            idh:
+              portgroups: *portgroupsdocker
             sensor:
               portgroups: *portgroupsdocker
             searchnode:
@@ -442,6 +456,8 @@ firewall:
               portgroups: *portgroupsdocker
             analyst:
               portgroups: *portgroupsdocker
+            desktop:
+              portgroups: *portgroupsdocker
             customhostgroup0:
               portgroups: *portgroupsdocker
             customhostgroup1: 
@@ -470,12 +486,16 @@ firewall:
               portgroups: *portgroupshost
             localhost:
               portgroups: *portgroupshost
+            idh:
+              portgroups: *portgroupshost
             sensor:
               portgroups: *portgroupshost
             searchnode:
               portgroups: *portgroupshost
             heavynode:
               portgroups: *portgroupshost
+            desktop:
+              portgroups: *portgroupshost
             customhostgroup0:
               portgroups: *portgroupshost
             customhostgroup1: 
@@ -507,6 +527,8 @@ firewall:
               portgroups: *portgroupsdocker
             fleet:
               portgroups: *portgroupsdocker
+            idh:
+              portgroups: *portgroupsdocker
             sensor:
               portgroups: *portgroupsdocker
             searchnode:
@@ -531,6 +553,8 @@ firewall:
               portgroups: *portgroupsdocker
             analyst:
               portgroups: *portgroupsdocker
+            desktop:
+              portgroups: *portgroupsdocker
             customhostgroup0:
               portgroups: *portgroupsdocker
             customhostgroup1: 
@@ -563,12 +587,16 @@ firewall:
               portgroups: *portgroupshost
             standalone:
               portgroups: *portgroupshost
+            idh:
+              portgroups: *portgroupshost
             sensor:
               portgroups: *portgroupshost
             searchnode:
               portgroups: *portgroupshost
             heavynode:
               portgroups: *portgroupshost
+            desktop:
+              portgroups: *portgroupshost
             customhostgroup0:
               portgroups: *portgroupshost
             customhostgroup1: 
@@ -793,6 +821,8 @@ firewall:
               portgroups: *portgroupsdocker
             analyst:
               portgroups: *portgroupsdocker
+            desktop:
+              portgroups: *portgroupsdocker
             customhostgroup0:
               portgroups: *portgroupsdocker
             customhostgroup1: 

salt/idh/enabled.sls

@@ -6,6 +6,7 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/docker.map.jinja' import DOCKER %}
 
 include:
   - idh.config

salt/idstools/enabled.sls

@@ -63,12 +63,22 @@ delete_so-idstools_so-status.disabled:
 
 so-rule-update:
   cron.present:
-    - name: /usr/sbin/so-rule-update > /opt/so/log/idstools/download.log 2>&1
+    - name: /usr/sbin/so-rule-update > /opt/so/log/idstools/download_cron.log 2>&1
     - identifier: so-rule-update
     - user: root
     - minute: '1'
     - hour: '7'
 
+# order this last to give so-idstools container time to be ready
+run_so-rule-update:
+  cmd.run:
+    - name: '/usr/sbin/so-rule-update > /opt/so/log/idstools/download_idstools_state.log 2>&1'
+    - require:
+      - docker_container: so-idstools
+    - onchanges:
+      - file: idstoolsetcsync
+    - order: last
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/idstools/sync_files.sls

@@ -26,6 +26,13 @@ rulesdir:
     - group: 939
     - makedirs: True
 
+SOrulesdir:
+  file.directory:
+    - name: /opt/so/rules/nids/sorules
+    - user: 939
+    - group: 939
+    - makedirs: True
+
 # Don't show changes because all.rules can be large
 synclocalnidsrules:
   file.recurse:
@@ -35,3 +42,13 @@ synclocalnidsrules:
     - group: 939
     - show_changes: False
     - include_pat: 'E@.rules'
+
+# Don't show changes because all.rules can be large
+syncnidsSOrules:
+  file.recurse:
+    - name: /opt/so/rules/nids/sorules
+    - source: salt://idstools/sorules/
+    - user: 939
+    - group: 939
+    - show_changes: False
+    - include_pat: 'E@.rules'

salt/idstools/tools/sbin_jinja/so-rule-update

@@ -1,32 +1,42 @@
 #!/bin/bash
-. /usr/sbin/so-common
+
+# if this script isn't already running
+if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
+
+    . /usr/sbin/so-common
 
 {%- from 'vars/globals.map.jinja' import GLOBALS %}
 {%- from 'idstools/map.jinja' import IDSTOOLSMERGED %}
-{%-   set proxy = salt['pillar.get']('manager:proxy') %}
 
-mkdir -p /nsm/rules/suricata
-chown -R socore:socore /nsm/rules/suricata
+{%- set proxy = salt['pillar.get']('manager:proxy') %}
+{%- set noproxy = salt['pillar.get']('manager:no_proxy', '') %}
+
+# Download the rules from the internet
+{%- if proxy %}
+    export http_proxy={{ proxy }}
+    export https_proxy={{ proxy }}
+    export no_proxy="{{ noproxy }}"
+{%- endif %}
+
+    mkdir -p /nsm/rules/suricata
+    chown -R socore:socore /nsm/rules/suricata
 # Download the rules from the internet
 {%- if GLOBALS.airgap != 'True' %}
-{%- if proxy %}
-export http_proxy={{ proxy }} 
-export https_proxy={{ proxy }} 
-export no_proxy= salt['pillar.get']('manager:no_proxy') 
-{%- endif %}
 {%-   if IDSTOOLSMERGED.config.ruleset == 'ETOPEN' %}
-docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force
+    docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force
 {%-   elif IDSTOOLSMERGED.config.ruleset == 'ETPRO' %}
-docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force --etpro={{ IDSTOOLSMERGED.config.oinkcode }}
+    docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force --etpro={{ IDSTOOLSMERGED.config.oinkcode }}
 {%-   elif IDSTOOLSMERGED.config.ruleset == 'TALOS' %}
-docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force --url=https://www.snort.org/rules/snortrules-snapshot-2983.tar.gz?oinkcode={{ IDSTOOLSMERGED.config.oinkcode }}
+    docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force --url=https://www.snort.org/rules/snortrules-snapshot-2983.tar.gz?oinkcode={{ IDSTOOLSMERGED.config.oinkcode }}
 {%-   endif %}
 {%- endif %}
 
 
-argstr=""
-for arg in "$@"; do
-    argstr="${argstr} \"${arg}\""
-done
+    argstr=""
+    for arg in "$@"; do
+        argstr="${argstr} \"${arg}\""
+    done
 
-docker exec so-idstools /bin/bash -c "cd /opt/so/idstools/etc && idstools-rulecat --force ${argstr}"
+    docker exec so-idstools /bin/bash -c "cd /opt/so/idstools/etc && idstools-rulecat --force ${argstr}"
+
+fi

salt/kibana/files/config_saved_objects.ndjson

@@ -1 +1 @@
-{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n  \"from\": \"now-24h\",\n  \"to\": \"now\"\n}"},"coreMigrationVersion": "8.7.1","id": "8.7.1","migrationVersion": {"config": "7.13.0"},"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="}
+{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n  \"from\": \"now-24h\",\n  \"to\": \"now\"\n}"},"coreMigrationVersion": "8.8.2","id": "8.8.2","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="}

salt/kibana/tools/sbin_jinja/so-kibana-config-load

@@ -63,7 +63,7 @@ update() {
 
     IFS=$'\r\n' GLOBIGNORE='*' command eval  'LINES=($(cat $1))'
     for i in "${LINES[@]}"; do
-      RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.7.1" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ")
+      RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.8.2" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ")
       echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi
     done
  

salt/logrotate/defaults.yaml

@@ -90,6 +90,26 @@ logrotate:
       - extension .log
       - dateext
       - dateyesterday
+    /opt/so/log/elasticagent/*_x_log:
+      - daily
+      - rotate 14
+      - missingok
+      - copytruncate
+      - compress
+      - create
+      - extension .log
+      - dateext
+      - dateyesterday
+    /opt/so/log/elasticagent/*_x_ndjson:
+      - daily
+      - rotate 14
+      - missingok
+      - copytruncate
+      - compress
+      - create
+      - extension .ndjson
+      - dateext
+      - dateyesterday
     /opt/so/log/elasticfleet/*_x_log:
       - daily
       - rotate 14

salt/logstash/enabled.sls

@@ -9,6 +9,11 @@
 {%   from 'docker/docker.map.jinja' import DOCKER %}
 {%   from 'logstash/map.jinja' import LOGSTASH_MERGED %}
 {%   from 'logstash/map.jinja' import REDIS_NODES %}
+{#   we append the manager here so that it is added to extra_hosts so the heavynode can resolve it #}
+{#   we cannont append in the logstash/map.jinja because then it would be added to the 0900_input_redis.conf #}
+{%   if GLOBALS.role == 'so-heavynode' %}
+{%     do REDIS_NODES.append({GLOBALS.manager:GLOBALS.manager_ip}) %}
+{%   endif %}
 {%   set lsheap = LOGSTASH_MERGED.settings.lsheap %}
 
 include:
@@ -17,6 +22,7 @@ include:
 {%   endif %}
   - logstash.config
   - logstash.sostatus
+  - ssl
 
 so-logstash:
   docker_container.running:
@@ -85,6 +91,10 @@ so-logstash:
         {% endfor %}
       {% endif %}
     - watch:
+      {% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-import', 'so-fleet', 'so-receiver'] %}
+      - x509: etc_elasticfleet_logstash_key
+      - x509: etc_elasticfleet_logstash_crt
+      {% endif %}
       - file: lsetcsync
       {% for assigned_pipeline in LOGSTASH_MERGED.assigned_pipelines.roles[GLOBALS.role.split('-')[1]] %}
       - file: ls_pipeline_{{assigned_pipeline}}

salt/manager/tools/sbin/so-firewall-minion

@@ -74,9 +74,12 @@ fi
 			so-firewall includehost heavynode "$IP" --apply
 			;;
 		'IDH')
-		    so-firewall includehost sensor "$IP" --apply
+		    so-firewall includehost idh "$IP" --apply
 			;;
 		'RECEIVER')
 			so-firewall includehost receiver "$IP" --apply
 			;;
-    esac
+		'DESKTOP')
+			so-firewall includehost desktop "$IP" --apply
+			;;
+    esac

salt/manager/tools/sbin/soup

@@ -179,12 +179,12 @@ update_registry() {
 
 check_airgap() {
   # See if this is an airgap install
-  AIRGAP=$(cat /opt/so/saltstack/local/pillar/global/soc_global.sls | grep airgap: | awk '{print $2}')
-  if [[ "$AIRGAP" == "True" ]]; then
+  AIRGAP=$(cat /opt/so/saltstack/local/pillar/global/soc_global.sls | grep airgap: | awk '{print $2}' | tr '[:upper:]' '[:lower:]')
+  if [[ "$AIRGAP" == "true" ]]; then
       is_airgap=0
       UPDATE_DIR=/tmp/soagupdate/SecurityOnion
       AGDOCKER=/tmp/soagupdate/docker
-      AGREPO=/tmp/soagupdate/Packages
+      AGREPO=/tmp/soagupdate/minimal/Packages
   else 
       is_airgap=1
   fi
@@ -346,7 +346,7 @@ clone_to_tmp() {
   # Make a temp location for the files
   mkdir -p /tmp/sogh
   cd /tmp/sogh
-  SOUP_BRANCH=""
+  SOUP_BRANCH="-b 2.4/main"
   if [ -n "$BRANCH" ]; then
     SOUP_BRANCH="-b $BRANCH"
   fi
@@ -391,6 +391,9 @@ preupgrade_changes() {
     echo "Checking to see if changes are needed."
 
     [[ "$INSTALLEDVERSION" == 2.4.2 ]] && up_to_2.4.3
+    [[ "$INSTALLEDVERSION" == 2.4.3 ]] && up_to_2.4.4
+    [[ "$INSTALLEDVERSION" == 2.4.4 ]] && up_to_2.4.5
+    [[ "$INSTALLEDVERSION" == 2.4.5 ]] && up_to_2.4.10
     true
 }
 
@@ -399,8 +402,9 @@ postupgrade_changes() {
     echo "Running post upgrade processes."
     
     [[ "$POSTVERSION" == 2.4.2 ]] && post_to_2.4.3
-    
-
+    [[ "$POSTVERSION" == 2.4.3 ]] && post_to_2.4.4
+    [[ "$POSTVERSION" == 2.4.4 ]] && post_to_2.4.5 
+    [[ "$POSTVERSION" == 2.4.5 ]] && post_to_2.4.10 
     true
 }
 
@@ -409,6 +413,22 @@ post_to_2.4.3() {
   POSTVERSION=2.4.3
 }
 
+post_to_2.4.4() {
+  echo "Nothing to apply"
+  POSTVERSION=2.4.4
+}
+
+post_to_2.4.5() {
+  echo "Regenerating Elastic Agent Installers"
+  /sbin/so-elastic-agent-gen-installers
+  POSTVERSION=2.4.5
+}
+
+post_to_2.4.10() {
+  echo "Updating Elastic Fleet ES URLs...."
+  /sbin/so-elastic-fleet-es-url-update --force
+  POSTVERSION=2.4.10
+}
 
 stop_salt_master() {
     # kill all salt jobs across the grid because the hang indefinitely if they are queued and salt-master restarts
@@ -423,7 +443,7 @@ stop_salt_master() {
 
     echo ""
     echo "Storing salt-master pid."
-    MASTERPID=$(pgrep salt-master | head -1)
+    MASTERPID=$(pgrep -f '/opt/saltstack/salt/bin/python3.10 /usr/bin/salt-master MainProcess')
     echo "Found salt-master PID $MASTERPID"
     systemctl_func "stop" "salt-master"
     timeout 30 tail --pid=$MASTERPID -f /dev/null || echo "salt-master still running at $(date +"%T.%6N") after waiting 30s. We cannot kill due to systemd restart option."
@@ -442,7 +462,7 @@ stop_salt_minion() {
     set -e
 
     echo "Storing salt-minion pid."
-    MINIONPID=$(pgrep salt-minion | head -1)
+    MINIONPID=$(pgrep -f '/opt/saltstack/salt/bin/python3.10 /usr/bin/salt-minion' | head -1)
     echo "Found salt-minion PID $MINIONPID"
     systemctl_func "stop" "salt-minion"
 
@@ -453,9 +473,40 @@ stop_salt_minion() {
 
 
 up_to_2.4.3() {
-   echo "Nothing to do for 2.4.3"
-   ##
-   INSTALLEDVERSION=2.3.140
+  echo "Nothing to do for 2.4.3"
+
+  INSTALLEDVERSION=2.4.3
+}
+
+up_to_2.4.4() {
+  echo "Nothing to do for 2.4.4"
+
+  INSTALLEDVERSION=2.4.4
+}
+
+up_to_2.4.5() {
+  determine_elastic_agent_upgrade
+
+  INSTALLEDVERSION=2.4.5
+}
+
+up_to_2.4.10() {
+  echo "Nothing to do for 2.4.10"
+  
+  INSTALLEDVERSION=2.4.10
+}
+
+determine_elastic_agent_upgrade() {
+  if [[ $is_airgap -eq 0 ]]; then
+    update_elastic_agent_airgap
+  else
+    update_elastic_agent
+  fi
+}
+
+update_elastic_agent_airgap() {
+  rsync -av /tmp/soagupdate/fleet/* /nsm/elastic-fleet/artifacts/
+  tar -xf "$ELASTIC_AGENT_FILE" -C "$ELASTIC_AGENT_EXPANSION_DIR"
 }
 
 verify_upgradespace() {
@@ -495,6 +546,7 @@ update_centos_repo() {
   echo "Syncing new updates to /nsm/repo"
   rsync -av $AGREPO/* /nsm/repo/
   echo "Creating repo"
+  dnf -y install yum-utils createrepo
   createrepo /nsm/repo
 }
 
@@ -510,7 +562,7 @@ update_version() {
   echo "Updating the Security Onion version file."
   echo $NEWVERSION > /etc/soversion
   echo $HOTFIXVERSION > /etc/sohotfix
-  sed -i "/  soversion:/c\  soversion: $NEWVERSION" /opt/so/saltstack/local/pillar/global/soc_global.sls
+  sed -i "s/soversion:.*/soversion: $NEWVERSION/" /opt/so/saltstack/local/pillar/global/soc_global.sls
 }
 
 upgrade_check() {
@@ -834,7 +886,7 @@ main() {
     set +e
 
     echo "Checking the number of minions."
-    NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l)
+    NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | grep -v adv_ | wc -l)
     if [[ $UPGRADESALT -eq 1 ]] && [[ $NUM_MINIONS -gt 1 ]]; then
       if [[ $is_airgap -eq 0 ]]; then
         echo ""
@@ -850,9 +902,6 @@ main() {
     echo "Checking sudoers file."
     check_sudoers
 
-    echo "Checking for necessary user migrations."
-    so-user migrate
-
     systemctl_func "start" "$cron_service_name"
 
     if [[ -n $lsl_msg ]]; then
@@ -938,6 +987,11 @@ while getopts ":b:f:y" opt; do
 done
 shift $((OPTIND - 1))
 
+if [ -f $SOUP_LOG ]; then
+  CURRENT_TIME=$(date +%Y%m%d.%H%M%S)
+  mv $SOUP_LOG $SOUP_LOG.$INSTALLEDVERSION.$CURRENT_TIME
+fi
+
 if [[ -z $UNATTENDED ]]; then
   cat << EOF
 

salt/manager/tools/sbin_jinja/so-yara-download

@@ -3,12 +3,13 @@ NOROOT=1
 . /usr/sbin/so-common
 
 {%- set proxy = salt['pillar.get']('manager:proxy') %}
+{%- set noproxy = salt['pillar.get']('manager:no_proxy', '') %}
 
 # Download the rules from the internet
 {%- if proxy %}
 export http_proxy={{ proxy }} 
 export https_proxy={{ proxy }} 
-export no_proxy= salt['pillar.get']('manager:no_proxy') 
+export no_proxy="{{ noproxy }}" 
 {%- endif %}
 
 repos="/opt/so/conf/strelka/repos.txt"

salt/nginx/etc/nginx.conf

@@ -296,7 +296,9 @@ http {
 		error_page 429 = @error429;
 
 		location @error401 {
-			add_header    Set-Cookie "AUTH_REDIRECT=$request_uri;Path=/;Max-Age=14400";
+			if ($request_uri ~* ^/(?!(^/api/.*))) {
+				add_header    Set-Cookie "AUTH_REDIRECT=$request_uri;Path=/;Max-Age=14400";
+			}
 			return        302 /auth/self-service/login/browser;
 		}
 

salt/sensor/files/99-so-checksum-offload-disable

@@ -0,0 +1,14 @@
+#!/bin/bash
+#
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
+
+. /usr/sbin/so-common
+
+{% set MNIC = salt['pillar.get']('sensor:interface') %}
+
+init_monitor {{ MNIC }}

salt/sensor/init.sls

@@ -0,0 +1,12 @@
+offload_script:
+  file.managed:
+    - name: /etc/NetworkManager/dispatcher.d/pre-up.d/99-so-checksum-offload-disable
+    - source: salt://sensor/files/99-so-checksum-offload-disable
+    - mode: 755
+    - template: jinja
+
+execute_checksum:
+  cmd.run:
+    - name: /etc/NetworkManager/dispatcher.d/pre-up.d/99-so-checksum-offload-disable
+    - onchanges:
+      - file: offload_script

salt/sensor/soc_sensor.yaml

@@ -1,7 +1,9 @@
 sensor:
   interface:
     description: Main sensor monitoring interface.
-    helpLink: sensor.html    
+    helpLink: network.html
+    readonly: True
   mtu:
-    description: Main IP address of the grid host.
-    helpLink: host.html
+    description: Maximum Transmission Unit (MTU) of the sensor monitoring interface.
+    helpLink: network.html
+    readonly: True

salt/soc/defaults.yaml

@@ -10,6 +10,14 @@ soc:
         target:
         links:
           - '/#/hunt?q="{value|escape}" | groupby event.module* event.dataset'
+      - name: actionAddToCase
+        description: actionAddToCaseHelp
+        icon: fa-briefcase
+        jsCall: openAddToCaseDialog
+        categories:
+          - hunt
+          - alerts
+          - dashboards
       - name: actionCorrelate
         description: actionCorrelateHelp
         icon: fab fa-searchengin
@@ -61,7 +69,7 @@ soc:
         - log.id.uid
         - network.community_id
         - event.dataset
-      ':kratos:kratos.audit':
+      ':kratos:audit':
         - soc_timestamp
         - http_request.headers.x-real-ip
         - identity_id
@@ -562,14 +570,13 @@ soc:
         - destination.geo.country_iso_code
         - user.name
         - source.ip
-      ':windows.sysmon_operational:':
+      '::sysmon_operational':
         - soc_timestamp
         - event.action
-        - process.executable
+        - winlog.computer_name
         - user.name
-        - file.target
-        - dns.question.name
-        - winlog.event_data.TargetObject
+        - process.executable
+        - process.pid
       '::network_connection':
         - soc_timestamp
         - source.ip
@@ -1132,7 +1139,7 @@ soc:
               showSubtitle: true
             - name: SOC - Auth
               description: Users authenticated to SOC grouped by IP address and identity
-              query: 'event.dataset:kratos.audit AND msg:authenticated | groupby http_request.headers.x-real-ip identity_id'
+              query: 'event.dataset:kratos.audit AND msg:*authenticated* | groupby http_request.headers.x-real-ip identity_id'
               showSubtitle: true
             - name: SOC - App
               description: Logs generated by the Security Onion Console (SOC) server and modules
@@ -1397,7 +1404,7 @@ soc:
               query: '* | groupby -sankey event.dataset event.category* | groupby -pie event.category | groupby -bar event.module* | groupby event.dataset | groupby event.module* | groupby event.category | groupby observer.name | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name'
             - name: SOC Auth
               description: SOC (Security Onion Console) authentication logs
-              query: 'event.dataset:kratos.audit AND msg:authenticated | groupby -sankey http_request.headers.x-real-ip identity_id | groupby http_request.headers.x-real-ip | groupby identity_id | groupby http_request.headers.user-agent'
+              query: 'event.dataset:kratos.audit AND msg:*authenticated* | groupby -sankey http_request.headers.x-real-ip identity_id | groupby http_request.headers.x-real-ip | groupby identity_id | groupby http_request.headers.user-agent'
             - name: Elastalerts
               description: Elastalert logs
               query: '_index: "*:elastalert*" | groupby rule_name | groupby alert_info.type'

salt/soc/files/soc/motd.md

@@ -8,6 +8,10 @@ If you're ready to dive in, take a look at the [Alerts](/#/alerts) interface to
 
 To see all the latest features and fixes in this version of Security Onion, click the upper-right menu and then click the [What's New](/docs/release-notes.html) link.
 
+## Enterprise Appliances
+
+Want the best hardware for your enterprise deployment? Check out our [enterprise appliances](https://securityonionsolutions.com/hardware/)!
+
 ## Customize This Space
 
 Make this area your own by customizing the content in the [Config](/#/config?s=soc.files.soc.motd__md) interface.

salt/soc/soc_soc.yaml

@@ -45,9 +45,10 @@ soc:
     actions:
       description: A list of actions a user can take from the SOC UI against a hunt, alert, and other records. The action must be defined in JSON object format, and contain a "name" key and "links" key. The links is a list of URLs, where the most suitable URL in the list will be the selected URL when the user clicks the action.
       global: True
+      forcedType: "[]{}"
     eventFields:
       default:
-        description: The list of fields to show as columns in the Hunt/Dashboards event table, when no other specific mapping applies. Mappings are defined by the format ":event.module:event.dataset".
+        description: Event fields mappings are defined by the format ":event.module:event.dataset". For example, to customize which fields show for 'syslog' events originating from 'zeek', find the eventField item in the left panel that looks like ':zeek:syslog'. This 'default' entry is used for all events that do not match an existing mapping defined in the list to the left.
         global: True
         advanced: True
     server:
@@ -139,6 +140,7 @@ soc:
           description: List of available external tools visible in the SOC UI. Each tool is defined in JSON object notation, and must include the "name" key and "link" key, where the link is the tool's URL.
           global: True
           advanced: True
+          forcedType: "[]{}"
         hunt: &appSettings
           groupItemsPerPage:
             description: Default number of aggregations to show per page. Larger values consume more vertical area in the SOC UI.
@@ -164,6 +166,12 @@ soc:
           queries:
             description: List of default queries to show in the query list. Each query is represented in JSON object notation, and must include the "name" key and "query" key.
             global: True
+            forcedType: "[]{}"
+          queryToggleFilters:
+            description: Customize togglable query filters that apply to all queries. Exclusive toggles will invert the filter if toggled off rather than omitting the filter from the query.
+            global: True
+            advanced: True
+            forcedType: "[]{}"
         alerts: *appSettings
         cases: *appSettings
         dashboards: *appSettings

salt/ssl/init.sls

@@ -7,7 +7,7 @@
 {% if sls in allowed_states %}
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 
-{% set CUSTOMFQDN = salt['pillar.get']('elasticfleet:config:server:custom_fqdn') %}
+{%   from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
 
 {% set global_ca_text = [] %}
 {% set global_ca_server = [] %}
@@ -153,8 +153,8 @@ etc_elasticfleet_crt:
     - ca_server: {{ ca_server }}
     - signing_policy: elasticfleet
     - private_key: /etc/pki/elasticfleet-server.key
-    - CN: {{ GLOBALS.url_base }}
-    - subjectAltName: DNS:{{ GLOBALS.hostname }},IP:{{ GLOBALS.node_ip }} {% if CUSTOMFQDN != "" %},DNS:{{ CUSTOMFQDN }}{% endif %}
+    - CN: {{ GLOBALS.hostname }}
+    - subjectAltName: DNS:{{ GLOBALS.hostname }},DNS:{{ GLOBALS.url_base }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %}
     - days_remaining: 0
     - days_valid: 820
     - backup: True
@@ -210,8 +210,8 @@ etc_elasticfleet_logstash_crt:
     - ca_server: {{ ca_server }}
     - signing_policy: elasticfleet
     - private_key: /etc/pki/elasticfleet-logstash.key
-    - CN: {{ GLOBALS.url_base }}
-    - subjectAltName: DNS:{{ GLOBALS.hostname }},IP:{{ GLOBALS.node_ip }} {% if CUSTOMFQDN != "" %},DNS:{{ CUSTOMFQDN }}{% endif %}
+    - CN: {{ GLOBALS.hostname }}
+    - subjectAltName: DNS:{{ GLOBALS.hostname }},DNS:{{ GLOBALS.url_base }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %}
     - days_remaining: 0
     - days_valid: 820
     - backup: True

salt/strelka/filestream/config.sls

@@ -6,6 +6,7 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'strelka/map.jinja' import STRELKAMERGED %}
+{%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'strelka/map.jinja' import filecheck_runas %}
 
 include:
@@ -78,6 +79,46 @@ filecheck_script:
     - group: 939
     - mode: 755
 
+filecheck.log:
+  file.managed:
+    - name: /opt/so/log/strelka/filecheck.log
+    - user: {{ filecheck_runas }}
+    - group: {{ filecheck_runas }}
+
+filecheck_stdout.log:
+  file.managed:
+    - name: /opt/so/log/strelka/filecheck_stdout.log
+    - user: {{ filecheck_runas }}
+    - group: {{ filecheck_runas }}
+
+{% if GLOBALS.md_engine == 'ZEEK' %}
+
+filecheck_run_socore:
+  cron.present:
+    - name: 'ps -ef | grep filecheck | grep -v grep > /dev/null 2>&1 || python3 /opt/so/conf/strelka/filecheck >> /opt/so/log/strelka/filecheck_stdout.log 2>&1 &'
+    - identifier: filecheck_run_socore
+    - user: socore
+
+remove_filecheck_run_suricata:
+  cron.absent:
+    - identifier: filecheck_run_suricata
+    - user: suricata
+
+{% elif GLOBALS.md_engine == 'SURICATA'%}
+
+filecheck_run_suricata:
+  cron.present:
+    - name: 'ps -ef | grep filecheck | grep -v grep > /dev/null 2>&1 || python3 /opt/so/conf/strelka/filecheck >> /opt/so/log/strelka/filecheck_stdout.log 2>&1 &'
+    - identifier: filecheck_run_suricata
+    - user: suricata
+
+remove_filecheck_run_socore:
+  cron.absent:
+    - identifier: filecheck_run_socore
+    - user: socore
+
+{% endif %}
+
 filecheck_restart:
   cmd.run:
     - name: pkill -f "python3 /opt/so/conf/strelka/filecheck"
@@ -85,12 +126,7 @@ filecheck_restart:
     - success_retcodes: [0,1]
     - onchanges:
       - file: filecheck_script
-
-filecheck_run:
-  cron.present:
-    - name: 'ps -ef | grep filecheck | grep -v grep > /dev/null 2>&1 || python3 /opt/so/conf/strelka/filecheck >> /opt/so/log/strelka/filecheck_stdout.log 2>&1 &'
-    - identifier: filecheck_run
-    - user: {{ filecheck_runas }}
+      - file: filecheck_conf
 
 filcheck_history_clean:
   cron.present:

salt/strelka/tools/sbin_jinja/so-yara-download

@@ -8,7 +8,7 @@ NOROOT=1
 {%- if proxy %}
 export http_proxy={{ proxy }} 
 export https_proxy={{ proxy }} 
-export no_proxy= salt['pillar.get']('manager:no_proxy') 
+export no_proxy=salt['pillar.get']('manager:no_proxy') 
 {%- endif %}
 
 mkdir -p /tmp/yara

salt/suricata/defaults.yaml

@@ -416,7 +416,6 @@ suricata:
         enabled: "yes"
         filename: keyword_perf.log
         append: "yes"
-    
       prefilter:
         enabled: "yes"
         filename: prefilter_perf.log

salt/suricata/map.jinja

@@ -11,7 +11,7 @@
 {# suricata.config.af-packet has to be rewritten here since we cant display '- interface' in the ui #}
 {# we are limited to only one iterface #}
 {% load_yaml as afpacket %}
-- interface: {{ SURICATAMERGED.config['af-packet'].interface }}
+- interface: {{ GLOBALS.sensor.interface }}
   cluster-id: {{ SURICATAMERGED.config['af-packet']['cluster-id'] }}
   cluster-type: {{ SURICATAMERGED.config['af-packet']['cluster-type'] }}
   defrag: {{ SURICATAMERGED.config['af-packet'].defrag }}

salt/suricata/soc_suricata.yaml

@@ -14,7 +14,9 @@ suricata:
   config:
     af-packet:
       interface:
-        description: The network interface that Suricata will monitor.
+        description: The network interface that Suricata will monitor. This is set under sensor > interface.
+        advanced: True
+        readonly: True
         helpLink: suricata.html
       cluster-id:
         advanced: True

salt/telegraf/config.sls

@@ -32,17 +32,16 @@ tgrafetsdir:
     - name: /opt/so/conf/telegraf/scripts
     - makedirs: True
 
-tgrafsyncscripts:
-  file.recurse:
-    - name: /opt/so/conf/telegraf/scripts
+{% for script in TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] %}
+tgraf_sync_script_{{script}}:
+  file.managed:
+    - name: /opt/so/conf/telegraf/scripts/{{script}}
     - user: root
     - group: 939
-    - file_mode: 770
+    - mode: 770
     - template: jinja
-    - source: salt://telegraf/scripts
-    {% if GLOBALS.md_engine == 'SURICATA' %}
-    - exclude_pat: zeekcaptureloss.sh
-    {% endif %}
+    - source: salt://telegraf/scripts/{{script}}
+{% endfor %}
 
 telegraf_sbin:
   file.recurse:

salt/telegraf/defaults.yaml

@@ -9,3 +9,82 @@ telegraf:
     flush_jitter: '0s'
     debug: 'false'
     quiet: 'false'
+  scripts:
+    eval:
+      - beatseps.sh
+      - checkfiles.sh
+      - influxdbsize.sh
+      - oldpcap.sh
+      - raid.sh
+      - redis.sh
+      - sostatus.sh
+      - stenoloss.sh
+      - suriloss.sh
+      - zeekcaptureloss.sh
+      - zeekloss.sh
+    standalone:
+      - beatseps.sh
+      - checkfiles.sh
+      - eps.sh
+      - influxdbsize.sh
+      - oldpcap.sh
+      - raid.sh
+      - redis.sh
+      - sostatus.sh
+      - stenoloss.sh
+      - suriloss.sh
+      - zeekcaptureloss.sh
+      - zeekloss.sh
+    manager:
+      - beatseps.sh
+      - influxdbsize.sh
+      - raid.sh
+      - redis.sh
+      - sostatus.sh
+    managersearch:
+      - beatseps.sh
+      - eps.sh
+      - influxdbsize.sh
+      - raid.sh
+      - redis.sh
+      - sostatus.sh
+    import:
+      - sostatus.sh
+    sensor:
+      - beatseps.sh
+      - checkfiles.sh
+      - oldpcap.sh
+      - raid.sh
+      - sostatus.sh
+      - stenoloss.sh
+      - suriloss.sh
+      - zeekcaptureloss.sh
+      - zeekloss.sh
+    heavynode:
+      - beatseps.sh
+      - checkfiles.sh
+      - eps.sh
+      - oldpcap.sh
+      - raid.sh
+      - redis.sh
+      - sostatus.sh
+      - stenoloss.sh
+      - suriloss.sh
+      - zeekcaptureloss.sh
+      - zeekloss.sh
+    idh:
+      - sostatus.sh
+    searchnode:
+      - beatseps.sh
+      - eps.sh
+      - raid.sh
+      - sostatus.sh
+    receiver:
+      - beatseps.sh
+      - eps.sh
+      - raid.sh
+      - redis.sh
+      - sostatus.sh
+    fleet:
+      - sostatus.sh
+    desktop: []

salt/telegraf/enabled.sls

@@ -7,6 +7,7 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKER %}
+{%   from 'telegraf/map.jinja' import TELEGRAFMERGED %}
 
 
 include:
@@ -67,8 +68,10 @@ so-telegraf:
     {% endif %}
     - watch:
       - file: tgrafconf
-      - file: tgrafsyncscripts
       - file: node_config
+    {% for script in TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] %}
+      - file: tgraf_sync_script_{{script}}
+    {% endfor %}
     - require: 
       - file: tgrafconf
       - file: node_config

salt/telegraf/etc/telegraf.conf

@@ -193,7 +193,7 @@
   username = "{{ ES_USER }}"
   password = "{{ ES_PASS }}"
   insecure_skip_verify = true
-{%- elif grains['role'] in ['so-searchnode', 'so-hotnode', 'so-warmnode']  %}
+{%- elif grains['role'] in ['so-searchnode']  %}
 [[inputs.elasticsearch]]
   servers = ["https://{{ NODEIP }}:9200"]
   cluster_stats = false
@@ -244,6 +244,8 @@
 {%- endif %}
 
 # # Read metrics from one or more commands that can output to stdout
+{%- if 'sostatus.sh' in TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] %}
+{%-   do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('sostatus.sh') %}
 [[inputs.exec]]
    commands = [
       "/scripts/sostatus.sh"
@@ -251,122 +253,26 @@
    data_format = "influx"
    timeout = "15s"
    interval = "60s"
+{%- endif %}
 
-#   ## Commands array
-{% if grains['role'] in ['so-manager'] %}
+{%- if TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] | length > 0 %}
 [[inputs.exec]]
    commands = [
-      "/scripts/redis.sh",
-      "/scripts/influxdbsize.sh",
-      "/scripts/raid.sh",
-      "/scripts/beatseps.sh"
+{%-   for script in TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] %}
+      "/scripts/{{script}}"{% if not loop.last %},{% endif %}
+{%-   endfor %}
    ]
   data_format = "influx"
   ## Timeout for each command to complete.
   timeout = "15s"
-{% elif grains['role'] in ['so-managersearch'] %}
-[[inputs.exec]]
-   commands = [
-      "/scripts/redis.sh",
-      "/scripts/influxdbsize.sh",
-      "/scripts/eps.sh",
-      "/scripts/raid.sh",
-      "/scripts/beatseps.sh"
-   ]
-  data_format = "influx"
-  ## Timeout for each command to complete.
-  timeout = "15s"
-{% elif grains['role'] in ['so-searchnode', 'so-receiver'] %}
-[[inputs.exec]]
-   commands = [
-      "/scripts/eps.sh",
-      "/scripts/raid.sh",
-  {% if grains.role == 'so-receiver' %}
-      "/scripts/redis.sh",
-  {% endif %}
-      "/scripts/beatseps.sh"
-   ]
-  data_format = "influx"
-  ## Timeout for each command to complete.
-  timeout = "15s"
-{% elif grains['role'] == 'so-sensor' %}
-[[inputs.exec]]
-   commands = [
-      "/scripts/stenoloss.sh",
-      "/scripts/suriloss.sh",
-      "/scripts/checkfiles.sh",
-  {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED %}
-      "/scripts/zeekloss.sh",
-      "/scripts/zeekcaptureloss.sh",
-  {%- endif %}
-      "/scripts/oldpcap.sh",
-      "/scripts/raid.sh",
-      "/scripts/beatseps.sh"
-   ]
-  data_format = "influx"
-  timeout = "15s"
-{% elif grains['role'] == 'so-heavynode' %}
-[[inputs.exec]]
-   commands = [
-      "/scripts/redis.sh",
-      "/scripts/stenoloss.sh",
-      "/scripts/suriloss.sh",
-      "/scripts/checkfiles.sh",
-  {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED %}
-      "/scripts/zeekloss.sh",
-      "/scripts/zeekcaptureloss.sh",
-  {%- endif %}
-      "/scripts/oldpcap.sh",
-      "/scripts/eps.sh",
-      "/scripts/raid.sh",
-      "/scripts/beatseps.sh"
-   ]
-  data_format = "influx"
-  timeout = "15s"
-{% elif grains['role'] == 'so-standalone' %}
-[[inputs.exec]]
-   commands = [
-      "/scripts/redis.sh",
-      "/scripts/influxdbsize.sh",
-      "/scripts/stenoloss.sh",
-      "/scripts/suriloss.sh",
-      "/scripts/checkfiles.sh",
-  {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED %}
-      "/scripts/zeekloss.sh",
-      "/scripts/zeekcaptureloss.sh",
-  {%- endif %}
-      "/scripts/oldpcap.sh",
-      "/scripts/eps.sh",
-      "/scripts/raid.sh",
-      "/scripts/beatseps.sh"
-   ]
-  data_format = "influx"
-  timeout = "15s"
-{% elif grains['role'] == 'so-eval' %}
-[[inputs.exec]]
-   commands = [
-      "/scripts/redis.sh",
-      "/scripts/stenoloss.sh",
-      "/scripts/suriloss.sh",
-      "/scripts/checkfiles.sh",
-  {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED %}
-      "/scripts/zeekloss.sh",
-      "/scripts/zeekcaptureloss.sh",
-  {%- endif %}
-      "/scripts/oldpcap.sh",
-      "/scripts/influxdbsize.sh",
-      "/scripts/raid.sh",
-      "/scripts/beatseps.sh"
-   ]
-  data_format = "influx"
-  timeout = "15s"
-{% endif %}
+{%- endif %}
 
 {%- if salt['pillar.get']('healthcheck:enabled', False) %}
 [[inputs.file]]
   files = ["/host/nsm/zeek/logs/zeek_restart.log"]
   data_format = "influx"
 {%- endif %}
+
 [[inputs.file]]
   files = ["/etc/telegraf/node_config.json"]
   name_override = "node_config"

salt/telegraf/map.jinja

@@ -2,6 +2,16 @@
    or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
    https://securityonion.net/license; you may not use this file except in compliance with the
    Elastic License 2.0. #}
-   
+
+{% from 'vars/globals.map.jinja' import GLOBALS %}
 {% import_yaml 'telegraf/defaults.yaml' as TELEGRAFDEFAULTS %}
 {% set TELEGRAFMERGED = salt['pillar.get']('telegraf', TELEGRAFDEFAULTS.telegraf, merge=True) %}
+
+{% if GLOBALS.role in ['so-eval', 'so-standalone', 'so-sensor', 'so-heavynode'] %}
+{%   from 'zeek/config.map.jinja' import ZEEKMERGED %}
+{#   if the md engine isn't zeek or zeek is disabled, dont run the zeek scripts for telegraf #}
+{%   if GLOBALS.md_engine != 'ZEEK' or not ZEEKMERGED.enabled %}
+{%     do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('zeekloss.sh') %}
+{%     do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('zeekcaptureloss.sh') %}
+{%   endif %}
+{% endif %}

salt/telegraf/scripts/zeekcaptureloss.sh

@@ -5,16 +5,18 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-
-
-
 # This script returns the average of all the workers average capture loss to telegraf / influxdb in influx format include nanosecond precision timestamp
 
 # if this script isn't already running
+{%- from 'zeek/config.map.jinja' import ZEEKMERGED %}
 if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
 
     if [ -d "/host/nsm/zeek/spool/logger" ]; then
-      WORKERS={{ salt['pillar.get']('sensor:zeek_lbprocs', salt['pillar.get']('sensor:zeek_pins') | length) }}
+{%- if ZEEKMERGED.config.node.pins %}
+      WORKERS={{ ZEEKMERGED.config.node.pins | length }}
+{%- else %}
+      WORKERS={{ ZEEKMERGED.config.node.lb_procs }}
+{%- endif %}
       ZEEKLOG=/host/nsm/zeek/spool/logger/capture_loss.log
     elif [ -d "/host/nsm/zeek/spool/zeeksa" ]; then
       WORKERS=1

salt/telegraf/soc_telegraf.yaml

@@ -42,4 +42,21 @@ telegraf:
       global: True
       advanced: True
       helpLink: telegraf.html
-    
+  scripts:
+    eval: &telegrafscripts
+      description: List of input.exec scripts to run for this node type. The script must be present in salt/telegraf/scripts.
+      forcedType: "[]string"
+      multiline: True
+      advanced: True
+      helpLink: telegraf.html
+    standalone: *telegrafscripts
+    manager: *telegrafscripts
+    managersearch: *telegrafscripts
+    import: *telegrafscripts
+    sensor: *telegrafscripts
+    heavynode: *telegrafscripts
+    idh: *telegrafscripts
+    searchnode: *telegrafscripts
+    receiver: *telegrafscripts
+    fleet: *telegrafscripts
+    desktop: *telegrafscripts

salt/top.sls

@@ -36,6 +36,7 @@ base:
   
   '*_sensor and G@saltversion:{{saltversion}}':
     - match: compound
+    - sensor
     - ssl
     - sensoroni
     - telegraf
@@ -52,6 +53,7 @@ base:
   '*_eval and G@saltversion:{{saltversion}}':
     - match: compound
     - salt.master
+    - sensor
     - ca
     - ssl
     - registry
@@ -118,6 +120,7 @@ base:
   '*_standalone and G@saltversion:{{saltversion}}':
     - match: compound
     - salt.master
+    - sensor
     - ca
     - ssl
     - registry
@@ -196,6 +199,7 @@ base:
 
   '*_heavynode and G@saltversion:{{saltversion}}':
     - match: compound
+    - sensor
     - ssl
     - sensoroni
     - nginx
@@ -216,6 +220,7 @@ base:
   '*_import and G@saltversion:{{saltversion}}':
     - match: compound
     - salt.master
+    - sensor
     - ca
     - ssl
     - registry
@@ -272,10 +277,10 @@ base:
     - schedule
     - docker_clean
 
-  'J@desktop:gui:enabled:^[Tt][Rr][Uu][Ee]$ and ( G@saltversion:{{saltversion}} and G@os:Rocky )':
+  'J@desktop:gui:enabled:^[Tt][Rr][Uu][Ee]$ and ( G@saltversion:{{saltversion}} and G@os:OEL )':
     - match: compound
     - desktop
 
-  'J@desktop:gui:enabled:^[Ff][Aa][Ll][Ss][Ee]$ and ( G@saltversion:{{saltversion}} and G@os:Rocky )':
+  'J@desktop:gui:enabled:^[Ff][Aa][Ll][Ss][Ee]$ and ( G@saltversion:{{saltversion}} and G@os:OEL )':
     - match: compound
     - desktop.remove_gui

setup/so-functions

@@ -85,12 +85,13 @@ analyze_system() {
 
 desktop_salt_local() {
 
+	SALTVERSION=$(egrep 'version: [0-9]{4}' ../salt/salt/master.defaults.yaml | sed 's/^.*version: //')
 	# Install everything using local salt
 	# Set the repo
 	securityonion_repo
 	gpg_rpm_import
 	# Install salt
-	logCmd "yum -y install salt-minion-3004.1 httpd-tools python3 python36-docker python36-dateutil python36-m2crypto python36-mysql python36-packaging python36-lxml yum-utils device-mapper-persistent-data lvm2 openssl jq"
+	logCmd "yum -y install salt-minion-$SALTVERSION httpd-tools python3 python36-docker python36-dateutil python36-m2crypto python36-mysql python36-packaging python36-lxml yum-utils device-mapper-persistent-data lvm2 openssl jq"
 	logCmd "yum -y update --exclude=salt*"
 
 	logCmd "salt-call state.apply desktop --local --file-root=../salt/ -l info"
@@ -116,7 +117,7 @@ desktop_pillar() {
 		"  mainint: '$MNIC'"\
 		"desktop:"\
 		"  gui:"\
-		"    enabled: true" >> "$pillar_file"\
+		"    enabled: true"\
 		"sensoroni:"\
 		"  config:"\
 		"    node_description: '${NODE_DESCRIPTION//\'/''}'" > $pillar_file
@@ -1014,25 +1015,9 @@ detect_os() {
 }
 
 download_elastic_agent_artifacts() {
-	agentArchive=/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz
-	agentMd5=/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5
-	beatsDir=/nsm/elastic-fleet/artifacts/beats/elastic-agent
-	logCmd "mkdir -p $beatsDir"
-	if [[ ! -f "$agentArchive" ]]; then
-		retry 15 10 "curl --fail --retry 5 --retry-delay 15 -L https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz --output $agentArchive"  "" ""
-		retry 15 10 "curl --fail --retry 5 --retry-delay 15 -L https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5 --output $agentMd5"  "" ""
-			
-		SOURCEHASH=$(md5sum $agentArchive | awk '{ print $1 }')
-		HASH=$(cat $agentMd5)
-
-		if [[ "$HASH" == "$SOURCEHASH" ]]; then
-		  info "Elastic Agent source hash is good."
-		else
-		  info "Unable to download the Elastic Agent source files."
-		  fail_setup
-		fi
+	if ! update_elastic_agent 2>&1 | tee -a "$setup_log"; then
+		fail_setup
 	fi
-	logCmd "tar -xf $agentArchive -C $beatsDir"
 }
 
 installer_progress_loop() {
@@ -1897,7 +1882,9 @@ securityonion_repo() {
 	if [[ $is_oracle ]]; then
 		logCmd "dnf -v clean all"
 		logCmd "mkdir -vp /root/oldrepos"
-		logCmd "mv -v /etc/yum.repos.d/* /root/oldrepos/"
+		if [ -n "$(ls -A /etc/yum.repos.d/ 2>/dev/null)" ]; then
+			logCmd "mv -v /etc/yum.repos.d/* /root/oldrepos/"
+		fi
 		if [[ $is_desktop_iso ]]; then
 			gpg_rpm_import
 			if [[ ! $is_airgap ]]; then
@@ -2315,6 +2302,15 @@ set_default_log_size() {
 	log_size_limit=$( echo "$disk_size_gb" "$percentage" | awk '{printf("%.0f", $1 * ($2/100))}')
 }
 
+set_desktop_background() {
+
+	logCmd "mkdir /usr/local/share/backgrounds"
+	logCmd "cp ../salt/desktop/files/so-wallpaper.jpg /usr/local/share/backgrounds/so-wallpaper.jpg"
+	logCmd "cp ../salt/desktop/files/00-background /etc/dconf/db/local.d/00-background"
+	logCmd "dconf update"
+
+}
+
 set_hostname() {
 
 	logCmd "hostnamectl set-hostname --static $HOSTNAME"

setup/so-setup

@@ -341,6 +341,8 @@ if [[ $is_desktop ]]; then
 			securityonion_repo
 			info "Enabling graphical interface and setting it to load at boot"
 			systemctl set-default graphical.target
+			info "Setting desktop background"
+			set_desktop_background
 			echo "Desktop Install Complete!"
 			echo ""
 			echo "Please reboot to start graphical interface."
@@ -661,6 +663,7 @@ if ! [[ -f $install_opt_file ]]; then
 		logCmd "salt-call state.show_top"
 		sleep 2 # Debug RSA Key format errors
 		logCmd "salt-key -ya $MINION_ID"
+		logCmd "salt-call saltutil.sync_all"
 
 		logCmd "salt-call state.apply common.packages"
 		logCmd "salt-call state.apply common"
@@ -694,9 +697,11 @@ if ! [[ -f $install_opt_file ]]; then
 			logCmd "so-rule-update"
 			title "Downloading YARA rules"
 			logCmd "su socore -c '/usr/sbin/so-yara-download'"
-			if [[ $monints ]]; then
+			if [[ $monints || $is_import ]]; then
 				title "Restarting Suricata to pick up the new rules"
 				logCmd "so-suricata-restart"
+			fi
+			if [[ $monints ]]; then
 				title "Restarting Strelka to use new rules"
 				logCmd "so-strelka-restart"
 			fi

setup/so-verify

@@ -51,6 +51,7 @@ log_has_errors() {
         grep -vE "/nsm/rules/sigma*" | \
         grep -vE "/nsm/rules/yara*" | \
         grep -vE "Failed to restart snapd" | \
+        grep -vE "Login Failed Details" | \
         grep -vE "Running scope as unit" &> "$error_log"
     
     if [[ $? -eq 0 ]]; then

setup/so-whiptail

@@ -1012,9 +1012,9 @@ whiptail_manager_unreachable() {
 
 	local msg
 	read -r -d '' msg <<- EOM
-	Setup is unable to access the manager at this time.
+	Setup is unable to access the manager. This most likely means that you need to allow this machine to connect through the manager's firewall. 
 
-	Run the following on the manager:
+	You can either go to SOC --> Administration --> Configuration and choose the correct firewall option from the list OR you can run the following command on the manager:
 	
 	sudo so-firewall-minion --role=$install_type --ip=$MAINIP