Merge branch '2.4/dev' into jertel/redircurl

salt/global/soc_global.yaml

@@ -6,8 +6,13 @@ global:
   managerip:
     description: The IP address of the grid manager.
     global: True
+    advanced: True
+    regex: ^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?)?$
+    regexFailureMessage: You must enter a valid IP address or CIDR.
   mdengine:
     description: What engine to use for meta data generation. Options are ZEEK and SURICATA.
+    regex: ^(ZEEK|SURICATA)$
+    regexFailureMessage: You must enter either ZEEK or SURICATA.
     global: True
   ids:
     description: Which IDS engine to use. Currently only Suricata is supported.

salt/suricata/defaults.yaml

@@ -4,10 +4,12 @@ suricata:
     threading:
       set-cpu-affinity: "no"
       cpu-affinity:
-        - management-cpu-set:
+        management-cpu-set:
-            cpu: []
+          cpu:
-        - worker-cpu-set:
+            - 1
-            cpu: []
+        worker-cpu-set:
+          cpu:
+            - 2-3
           mode: exclusive
           prio:
             default: high
@@ -22,32 +24,61 @@ suricata:
       ring-size: 5000
     vars:
       address-groups:
-        HOME_NET: "[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]"
+        HOME_NET:
-        EXTERNAL_NET: "any"
+          - 192.168.0.0/16
-        HTTP_SERVERS: "$HOME_NET"
+          - 10.0.0.0/8
-        SMTP_SERVERS: "$HOME_NET"
+          - 172.16.0.0/12
-        SQL_SERVERS: "$HOME_NET"
+        EXTERNAL_NET:
-        DNS_SERVERS: "$HOME_NET"
+          - any
-        TELNET_SERVERS: "$HOME_NET"
+        HTTP_SERVERS:
-        AIM_SERVERS: "$EXTERNAL_NET"
+          - $HOME_NET
-        DC_SERVERS: "$HOME_NET"
+        SMTP_SERVERS:
-        DNP3_SERVER: "$HOME_NET"
+          - $HOME_NET
-        DNP3_CLIENT: "$HOME_NET"
+        SQL_SERVERS:
-        MODBUS_CLIENT: "$HOME_NET"
+          - $HOME_NET
-        MODBUS_SERVER: "$HOME_NET"
+        DNS_SERVERS:
-        ENIP_CLIENT: "$HOME_NET"
+          - $HOME_NET
-        ENIP_SERVER: "$HOME_NET"  
+        TELNET_SERVERS:
+          - $HOME_NET
+        AIM_SERVERS:
+          - $EXTERNAL_NET
+        DC_SERVERS:
+          - $HOME_NET
+        DNP3_SERVER:
+          - $HOME_NET
+        DNP3_CLIENT:
+          - $HOME_NET
+        MODBUS_CLIENT:
+          - $HOME_NET
+        MODBUS_SERVER:
+          - $HOME_NET
+        ENIP_CLIENT:
+          - $HOME_NET
+        ENIP_SERVER:
+          - $HOME_NET
       port-groups:
-        HTTP_PORTS: "80"
+        HTTP_PORTS:
-        SHELLCODE_PORTS: "!80"
+          - 80
-        ORACLE_PORTS: "1521"
+        SHELLCODE_PORTS:
-        SSH_PORTS: "22"
+          - "!80"
-        DNP3_PORTS: "20000"
+        ORACLE_PORTS:
-        MODBUS_PORTS: "502"
+          - 1521
-        FILE_DATA_PORTS: "[$HTTP_PORTS,110,143]"
+        SSH_PORTS:
-        FTP_PORTS: "21"
+          - 22
-        VXLAN_PORTS: "4789"
+        DNP3_PORTS:
-        TEREDO_PORTS: "3544"
+          - 20000
+        MODBUS_PORTS:
+          - 502
+        FILE_DATA_PORTS:
+          - $HTTP_PORTS
+          - 110
+          - 143
+        FTP_PORTS:
+          - 21
+        VXLAN_PORTS:
+          - 4789
+        TEREDO_PORTS:
+          - 3544
     default-log-dir: /var/log/suricata/
     stats:
       enabled: "yes"
@@ -66,7 +97,7 @@ suricata:
         community-id: true
         community-id-seed: 0
         types:
-          - alert:
+          alert:
             payload: "no"
             payload-buffer-size: 4kb
             payload-printable: "yes"

salt/suricata/map.jinja

@@ -23,6 +23,45 @@
 {% do SURICATAMERGED.config.pop('af-packet') %}
 {% do SURICATAMERGED.config.update({'af-packet': afpacket}) %}
 
+{# eve-log.types is a list but we convert to dict in defaults to work with ui #}
+{# below they are converted back to lists #}
+{% load_yaml as evelogtypes %}
+{% for le, ld in SURICATAMERGED.config.outputs['eve-log'].types.items() %}
+  - {{ le }}: {{ ld }}
+{% endfor %}
+{% endload %}
+{% do SURICATAMERGED.config.outputs['eve-log'].pop('types') %}
+{% do SURICATAMERGED.config.outputs['eve-log'].update({'types': evelogtypes}) %}
+
+{# threading.cpu-affinity is a list but we convert to dict in defaults to work with ui #}
+{# below they are converted back to lists #}
+{% load_yaml as cpuaffinity %}
+{% for le, ld in SURICATAMERGED.config.threading['cpu-affinity'].items() %}
+  - {{ le }}: {{ ld }}
+{% endfor %}
+{% endload %}
+{% do SURICATAMERGED.config.threading.pop('cpu-affinity') %}
+{% do SURICATAMERGED.config.threading.update({'cpu-affinity': cpuaffinity}) %}
+
+{# Find the index of eve-log and file-store in suricata_mdengine.suricata.config.outputs #}
+{# update outputs eve-log.types and filestore with config for Suricata metadata engine #}
+{% if GLOBALS.md_engine == 'SURICATA' %}
+{%   for li in suricata_mdengine.suricata.config.outputs %}
+{%     if 'eve-log' in li.keys() %}
+{%       do surimeta_evelog_index.append(loop.index0) %}
+{%     endif %}
+{%     if 'file-store' in li.keys() %}
+{%       do surimeta_filestore_index.append(loop.index0) %}
+{%     endif %}
+{%   endfor %}
+{%   set surimeta_evelog_index = surimeta_evelog_index[0] %}
+{%   set surimeta_filestore_index = surimeta_filestore_index[0] %}
+{%   do SURICATAMERGED.config.outputs['eve-log'].types.extend(suricata_mdengine.suricata.config.outputs[surimeta_evelog_index]['eve-log'].types) %}
+{%   do SURICATAMERGED.config.outputs['file-store'].update({'enabled':suricata_mdengine.suricata.config.outputs[surimeta_filestore_index]['file-store']['enabled']}) %}
+{% endif %}
+
+{# outputs is a list but we convert to dict in defaults to work with ui #}
+{# below they are converted back to lists #}
 {% load_yaml as outputs %}
 {% for le, ld in SURICATAMERGED.config.outputs.items() %}
   - {{ le }}: {{ ld }}
@@ -31,31 +70,22 @@
 {% do SURICATAMERGED.config.pop('outputs') %}
 {% do SURICATAMERGED.config.update({'outputs': outputs}) %}
 
-{# Find the index of eve-log so it can be updated later #}
+{# change address-groups vars from list to comma seperated string #}
-{% for li in SURICATAMERGED.config.outputs %}
+{% for k, v in SURICATAMERGED.config.vars['address-groups'].items() %}
-  {% if 'eve-log' in li.keys() %}
+{#   if address-group value is a list #}
-    {% do default_evelog_index.append(loop.index0) %}
+{%   if v is iterable and (v is not string and v is not mapping and v | length > 1) %}
-  {% endif %}
+{%     do SURICATAMERGED.config.vars['address-groups'].update({k: '[' ~ v | join(',') ~ ']'}) %}
-  {% if 'file-store' in li.keys() %}
+{%   else %}
-    {% do default_filestore_index.append(loop.index0) %}
+{%     do SURICATAMERGED.config.vars['address-groups'].update({k: v[0]}) %}
-  {% endif %}
-{% endfor %}
-{% set default_evelog_index = default_evelog_index[0]  %}
-{% set default_filestore_index = default_filestore_index[0]  %}
-
-{# Find the index of eve-log so it can be grabbed later #}
-{% for li in suricata_mdengine.suricata.config.outputs %}
-  {% if 'eve-log' in li.keys() %}
-    {% do surimeta_evelog_index.append(loop.index0) %}
-  {% endif %}
-  {% if 'file-store' in li.keys() %}
-    {% do surimeta_filestore_index.append(loop.index0) %}
-  {% endif %}
-{% endfor %}
-{% set surimeta_evelog_index = surimeta_evelog_index[0] %}
-{% set surimeta_filestore_index = surimeta_filestore_index[0] %}
-
-{% if GLOBALS.md_engine == 'SURICATA' %}
-  {% do SURICATAMERGED.config.outputs[default_evelog_index]['eve-log'].types.extend(suricata_mdengine.suricata.config.outputs[surimeta_evelog_index]['eve-log'].types) %}
-  {% do SURICATAMERGED.config.outputs[default_filestore_index]['file-store'].update({'enabled':suricata_mdengine.suricata.config.outputs[surimeta_filestore_index]['file-store']['enabled']}) %}
 {%   endif %}
+{% endfor %}
+
+{# change port-groups vars from list to comma seperated string #}
+{% for k, v in SURICATAMERGED.config.vars['port-groups'].items() %}
+{#   if address-group value is a list #}
+{%   if v is iterable and (v is not string and v is not mapping and v | length > 1) %}
+{%     do SURICATAMERGED.config.vars['port-groups'].update({k: '[' ~ v | join(',') ~ ']'}) %}
+{%   else %}
+{%     do SURICATAMERGED.config.vars['port-groups'].update({k: v[0]}) %}
+{%   endif %}
+{% endfor %}

salt/suricata/soc_suricata.yaml

@@ -12,10 +12,54 @@ suricata:
       title: SIDS
       helpLink: suricata.html
   config:
+    af-packet:
+      interface:
+        description: The network interface that Suricata will monitor.
+        helpLink: suricata.html
+      cluster-id:
+        advanced: True
+      cluster-type:
+        advanced: True
+        regex: ^(cluster_flow|cluster_qm)$
+      defrag:
+        advanced: True
+        regex: ^(yes|no)$
+      use-mmap:
+        advanced: True
+        readonly: True
+      threads:
+        description: The amount of worker threads.
+        helpLink: suricata.html
+        forcedType: int
+      tpacket-v3:
+        advanced: True
+        readonly: True
+      ring-size:
+        description: Buffer size for packets per thread.
+        forcedType: int
+        helpLink: suricata.html
+    threading:
+      set-cpu-affinity:
+        description: Bind(yes) or unbind(no) management and worker threads to a core or range of cores.
+        regex: ^(yes|no)$
+        helpLink: suricata.html
+      cpu-affinity:
+        management-cpu-set:
+          cpu:
+            description: Bind management threads to a core or range of cores. This can be a sigle core, list of cores, or list of range of cores. set-cpu-affinity must be set to 'yes' for this to be used.
+            forcedType: "[]string"
+            helpLink: suricata.html
+        worker-cpu-set:
+          cpu:
+            description: Bind worker threads to a core or range of cores. This can be a sigle core, list of cores, or list of range of cores. set-cpu-affinity must be set to 'yes' for this to be used.
+            forcedType: "[]string"
+            helpLink: suricata.html
     vars:
       address-groups:
         HOME_NET:
           description: List of hosts or networks.
+          regex: ^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?)?$
+          regexFailureMessage: You must enter a valid IP address or CIDR.
           helpLink: suricata.html
         EXTERNAL_NET: 
           description: List of hosts or networks.
@@ -92,6 +136,8 @@ suricata:
           helpLink: suricata.html
     outputs:
       eve-log:
+        types:
+          alert:
             xff:
               enabled:
                 description: Enable X-Forward-For support.

setup/so-functions

@@ -28,6 +28,12 @@ title() {
 	echo -e "\n-----------------------------\n $1\n-----------------------------\n" >> "$setup_log" 2>&1
 }
 
+fail_setup() {
+	error "Setup encounted an unrecoverable failure, exiting"
+	touch /root/failure
+	exit 1
+}
+
 logCmd() {
 	cmd=$1
 	info "Executing command: $cmd"
@@ -796,7 +802,7 @@ compare_main_nic_ip() {
 			EOM
 
 		  [[ -n $TESTING ]] || whiptail --title "$whiptail_title" --msgbox "$message" 11 75
-		  kill -SIGINT "$(ps --pid $$ -oppid=)"; exit 1
+		  kill -SIGINT "$(ps --pid $$ -oppid=)"; fail_setup
 	  fi
 	else
 		# Setup uses MAINIP, but since we ignore the equality condition when using a VPN 
@@ -921,9 +927,10 @@ create_repo() {
 
 detect_cloud() {
 	info "Testing if setup is running on a cloud instance..."
-	if dmidecode -s bios-version | grep -q amazon || \
+	if [ -f /etc/SOCLOUD ] || \
-		dmidecode -s bios-vendor | grep -q Amazon || \
+	    dmidecode -s bios-version 2>&1 | grep -q amazon || \
-		dmidecode -s bios-vendor | grep -q Google || \
+		dmidecode -s bios-vendor 2>&1 | grep -q Amazon || \
+		dmidecode -s bios-vendor 2>&1 | grep -q Google || \
 		[ -f /var/log/waagent.log ]; then
 
 		info "Detected a cloud installation..."
@@ -943,7 +950,7 @@ detect_os() {
 			pkgman="dnf"
 		else
 			info "We do not support the operating system you are trying to use."
-			exit 1
+			fail_setup
 		fi
 
 	elif [ -f /etc/os-release ]; then
@@ -953,12 +960,12 @@ detect_os() {
 			is_ubuntu=true
 		else
 			info "We do not support your current version of Ubuntu."
-			exit 1
+			fail_setup
 		fi
 
 	else
 		info "We were unable to determine if you are using a supported OS."
-		exit 1
+		fail_setup
 	fi
 
 	info "Found OS: $OS $OSVER"
@@ -981,7 +988,7 @@ download_elastic_agent_artifacts() {
 		  info "Elastic Agent source hash is good."
 		else
 		  info "Unable to download the Elastic Agent source files."
-		  exit 1
+		  fail_setup
 		fi
 
 		  logCmd "tar -xf /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz -C /nsm/elastic-fleet/artifacts/beats/elastic-agent/"
@@ -1012,18 +1019,18 @@ installer_prereq_packages() {
 	if [ "$OS" == ubuntu ]; then
 		# Print message to stdout so the user knows setup is doing something
 		info "Running apt-get update"
-		retry 150 10 "apt-get update" "" "Err:" >> "$setup_log" 2>&1 || exit 1
+		retry 150 10 "apt-get update" "" "Err:" >> "$setup_log" 2>&1 || fail_setup
 		# Install network manager so we can do interface stuff
 		if ! command -v nmcli > /dev/null 2>&1; then
 			info "Installing network-manager"
-			retry 150 10 "apt-get -y install network-manager" >> "$setup_log" 2>&1 || exit 1
+			retry 150 10 "apt-get -y install network-manager" >> "$setup_log" 2>&1 || fail_setup
 			{
 				systemctl enable NetworkManager
 				systemctl start NetworkManager
 			} >> "$setup_log" 2<&1
 		fi
 		if ! command -v curl > /dev/null 2>&1; then
-			retry 150 10 "apt-get -y install curl"  >> "$setup_log" 2>&1 || exit 1
+			retry 150 10 "apt-get -y install curl"  >> "$setup_log" 2>&1 || fail_setup
 		fi
 	fi
 }
@@ -1746,7 +1753,7 @@ proxy_validate() {
 		error "Received error: $proxy_test_err"
 		if [[ -n $TESTING ]]; then
 			error "Exiting setup"
-			kill -SIGINT "$(ps --pid $$ -oppid=)"; exit 1
+			kill -SIGINT "$(ps --pid $$ -oppid=)"; fail_setup
 		fi
 	fi
 	return $ret
@@ -1817,7 +1824,7 @@ reinstall_init() {
 
 					# Stop the systemctl process trying to kill the service, show user a message, then exit setup
 					kill -9 $pid
-					exit 1
+					fail_setup
 				fi
 				
 				sleep 5
@@ -2020,7 +2027,7 @@ saltify() {
 	SALTVERSION=$(egrep 'version: [0-9]{4}' ../salt/salt/master.defaults.yaml | sed 's/^.*version: //')
 	if [[ $is_ubuntu ]]; then
 
-		DEBIAN_FRONTEND=noninteractive retry 150 20 "apt-get -y -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" upgrade" >> "$setup_log" 2>&1 || exit 1
+		DEBIAN_FRONTEND=noninteractive retry 150 20 "apt-get -y -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" upgrade" >> "$setup_log" 2>&1 || fail_setup
 		update-alternatives --install /usr/bin/python python /usr/bin/python3.8 10 
 		local pkg_arr=(
 			'apache2-utils'
@@ -2032,7 +2039,7 @@ saltify() {
 			'netcat'
 			'jq'
 		)
-		retry 150 20 "apt-get -y install ${pkg_arr[*]}" || exit 1
+		retry 150 20 "apt-get -y install ${pkg_arr[*]}" || fail_setup
 
 		logCmd "mkdir -vp /etc/apt/keyrings"
 		#logCmd "wget -q --inet4-only -O /opt/so/gpg/SALTSTACK-GPG-KEY.pub https://repo.securityonion.net/file/securityonion-repo/ubuntu/20.04/amd64/salt/SALTSTACK-GPG-KEY.pub"
@@ -2053,9 +2060,9 @@ saltify() {
 		
 		# Ain't nothing but a GPG 
 
-		retry 150 20 "apt-get update" "" "Err:" || exit 1
+		retry 150 20 "apt-get update" "" "Err:" || fail_setup
-		retry 150 20 "apt-get -y install salt-common-$SALTVERSION salt-minion-$SALTVERSION" || exit 1
+		retry 150 20 "apt-get -y install salt-common-$SALTVERSION salt-minion-$SALTVERSION" || fail_setup
-		retry 150 20 "apt-mark hold salt-minion salt-common" || exit 1
+		retry 150 20 "apt-mark hold salt-minion salt-common" || fail_setup
 		#retry 150 20 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb python3-packaging python3-influxdb python3-lxml" || exit 1
 
 	fi
@@ -2122,7 +2129,7 @@ set_main_ip() {
 			info "MAINIP=$MAINIP"
 			info "MNIC_IP=$MNIC_IP"
 			whiptail_error_message "The management IP could not be determined. Please check the log at /root/sosetup.log and verify the network configuration. Select OK to exit."
-			exit 1
+			fail_setup
 		fi
 		sleep 1
 	done
@@ -2372,13 +2379,13 @@ ubuntu_check() {
 	if [[ $OS == "ubuntu" ]]; then
 		if [[ $waitforstate ]]; then
 			whiptail_ubuntu_notsupported
-			exit 1
+			fail_setup
 		else
 		    if [[ $UBUNTUINSTALL == "needtoupgrade" ]]; then
 				whiptail_ubuntu_warning
 			else
 				whiptail_ubuntu_notsupported
-				exit 1
+				fail_setup
 			fi
 		fi
 	fi
@@ -2397,9 +2404,9 @@ update_packages() {
 		logCmd "dnf -y update --allowerasing --exclude=salt*,wazuh*,docker*,containerd*"
 	else
 		info "Running apt-get update"
-		retry 150 10 "apt-get -y update" "" "Err:" >> "$setup_log" 2>&1 || exit 1
+		retry 150 10 "apt-get -y update" "" "Err:" >> "$setup_log" 2>&1 || fail_setup
 		info "Running apt-get upgrade"
-		retry 150 10 "apt-get -y upgrade" >> "$setup_log" 2>&1 || exit 1
+		retry 150 10 "apt-get -y upgrade" >> "$setup_log" 2>&1 || fail_setup
 	fi
 }
 
@@ -2445,7 +2452,7 @@ wait_for_file() {
 }
 
 wait_for_salt_minion() {
-	retry 60 5 "journalctl -u salt-minion.service | grep 'Minion is ready to receive requests'" >> "$setup_log" 2>&1 || exit 1
+	retry 60 5 "journalctl -u salt-minion.service | grep 'Minion is ready to receive requests'" >> "$setup_log" 2>&1 || fail_setup
 }
 
 verify_setup() {

setup/so-setup

@@ -10,13 +10,13 @@
 uid="$(id -u)"
 if [ "$uid" -ne 0 ]; then
 	echo "This script must be run using sudo!"
-	exit 1
+	fail_setup
 fi
 
 # Save the original argument array since we modify it
 original_args=("$@")
 
-cd "$(dirname "$0")" || exit 255
+cd "$(dirname "$0")" || fail_setup
 
 echo "Getting started..."
 
@@ -82,7 +82,7 @@ if [[ "$setup_type" == 'iso' ]]; then
 		is_iso=true
 	else
 		echo "Only use 'so-setup iso' for an ISO install on Security Onion ISO images. Please run 'so-setup network' instead."
-		exit 1
+		fail_setup
 	fi
 fi
 
@@ -161,7 +161,7 @@ catch() {
 	info "Fatal error occurred at $1 in so-setup, failing setup."
 	grep --color=never "ERROR" "$setup_log" > "$error_log"
 	whiptail_setup_failed
-	exit 1
+	fail_setup
 }
 
 # Add the progress function for manager node type installs
@@ -236,7 +236,7 @@ case "$setup_type" in
 		;;
 	*)
 		error "Invalid install type, must be 'iso', 'network' or 'analyst'."
-		exit 1
+		fail_setup
 		;;
 esac