Merge pull request #3116 from Security-Onion-Solutions/issue/3115

Issue/3115

salt/common/tools/sbin/so-image-common

@@ -151,7 +151,7 @@ update_docker_containers() {
 
     # Pull down the trusted docker image
     local image=$i:$VERSION$IMAGE_TAG_SUFFIX
-    docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image >> "$LOG_FILE" 2>&1 
+    retry 50 10 "docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image" >> "$LOG_FILE" 2>&1 
     
     # Get signature
     retry 50 10 "curl -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" >> "$LOG_FILE" 2>&1

salt/common/tools/sbin/so-monitor-add

@@ -2,4 +2,22 @@
 
 . /usr/sbin/so-common
 
+usage() {
+  read -r -d '' message <<- EOM
+		usage: so-monitor-add [-h] NIC
+
+    positional arguments:
+      NIC           The interface to add to the monitor bond (ex: eth2)
+
+    optional arguments:
+      -h, --help    Show this help message and exit
+	EOM
+  echo "$message"
+  exit 1
+}
+
+if [[ $# -eq 0 || $# -gt 1 ]] || [[ $1 == '-h' || $1 == '--help' ]]; then
+  usage
+fi
+
 add_interface_bond0 "$1"

salt/common/tools/sbin/so-rule

@@ -58,7 +58,7 @@ def check_apply(args: dict, prompt: bool = True):
 
 
 def find_minion_pillar() -> str:
-  regex = '^.*_(manager|standalone|import|eval)\.sls$'
+  regex = '^.*_(manager|managersearch|standalone|import|eval)\.sls$'
   
   result = []
   for root, _, files in os.walk(minion_pillar_dir):
@@ -67,7 +67,7 @@ def find_minion_pillar() -> str:
         result.append(os.path.join(root, f_minion_id))
   
   if len(result) == 0:
-    print_err('Could not find manager-type pillar (eval, standalone, manager, import). Are you running this script on the manager?')
+    print_err('Could not find manager-type pillar (eval, standalone, manager, managersearch, import). Are you running this script on the manager?')
     sys.exit(3)
   elif len(result) > 1:
     res_str = ', '.join(f'\"{result}\"')

salt/common/tools/sbin/soup

@@ -196,6 +196,9 @@ playbook() {
     rm -f /opt/so/rules/elastalert/playbook/*.yaml
     so-playbook-ruleupdate >> /root/soup_playbook_rule_update.log 2>&1 &
   fi
+  if [[ "$INSTALLEDVERSION" != 2.3.30 ]]; then
+    so-playbook-sigma-refresh >> /root/soup_playbook_sigma_refresh.log 2>&1 &
+  fi
 }
 
 pillar_changes() {
@@ -379,6 +382,10 @@ unmount_update() {
   umount /tmp/soagupdate
 }
 
+update_airgap_rules() {
+  # Copy the rules over to update them for airgap.
+  rsync -av $UPDATE_DIR/agrules/* /nsm/repo/rules/
+}
 
 update_centos_repo() {
   # Update the files in the repo
@@ -588,6 +595,11 @@ echo "Making pillar changes."
 pillar_changes
 echo ""
 
+if [ $is_airgap -eq 0 ]; then
+  echo "Updating Rule Files to the Latest."
+  update_airgap_rules
+fi
+
 # Only update the repo if its airgap
 if [[ $is_airgap -eq 0 ]] && [[ "$UPGRADESALT" != "1" ]]; then
 update_centos_repo

salt/idstools/sorules/extraction.rules

@@ -7,20 +7,20 @@ alert smb any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document";
 alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:1100004; rev:1;)
 alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:1100005; rev:1;)
 alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:1100006; rev:1;)
-alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:11000007; rev:1;)
+alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:1100007; rev:1;)
 alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:1100008; rev:1;)
 alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:1100009; rev:1;)
 alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:1100010; rev:1;)
-alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:11000011; rev:1;)
+alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:1100011; rev:1;)
 
 # Extract all Zip files
 alert http any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; sid:1100012; rev:1;)
-alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"Zip"; filestore; sid:1100013; rev:1;)
-alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"Zip"; filestore; sid:1100014; rev:1;)
-alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"Zip"; filestore; sid:11000015; rev:1;)
+alert smtp any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; sid:1100013; rev:1;)
+alert nfs any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; sid:1100014; rev:1;)
+alert smb any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; sid:1100015; rev:1;)
 
 # Extract Word Docs
-alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100016; rev:1;)
-alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100017; rev:1;)
-alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100018; rev:1;)
-alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100019; rev:1;)
+alert http any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100016; rev:1;)
+alert smtp any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100017; rev:1;)
+alert nfs any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100018; rev:1;)
+alert smb any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100019; rev:1;)

salt/idstools/sorules/filters.rules

@@ -8,4 +8,4 @@
 # Example of filtering out  Google's certificate from being in the ssl log.
 #config tls any any -> any any (tls.fingerprint; content:"4f:a4:5e:58:7e:d9:db:20:09:d7:b6:c7:ff:58:c4:7b:dc:3f:55:b4"; config: logging disable, type tx, scope tx; sid:1200003;)
 # Example of filtering out a md5 of a file from being in the files log.
-#config fileinfo any any -> any any (fileinfo.filemd5; content:"google.com"; config: logging disable, type tx, scope tx; sid:1200004;)
+#config fileinfo any any -> any any (fileinfo.filemd5; content:"7a125dc69c82d5caf94d3913eecde4b5"; config: logging disable, type tx, scope tx; sid:1200004;)

salt/salt/minion.sls

@@ -23,6 +23,15 @@ salt_minion_package:
     - hold: True
     - onlyif: test "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}"
 
+set_log_levels:
+  file.append:
+    - name: /etc/salt/minion
+    - text:
+      - "log_level: info"
+      - "log_level_log_file: info"
+    - listen_in:
+      - service: salt_minion_service
+
 salt_minion_service:
   service.running:
     - name: salt-minion

setup/so-functions

@@ -905,9 +905,10 @@ detect_cloud() {
 }
 
 detect_os() {
+	local log=${1:-${setup_log}}
 
 	# Detect Base OS
-	echo "Detecting Base OS" >> "$setup_log" 2>&1
+	echo "Detecting Base OS" >> "$log" 2>&1
 	if [ -f /etc/redhat-release ]; then
 		OS=centos
 		if grep -q "CentOS Linux release 7" /etc/redhat-release; then
@@ -937,7 +938,7 @@ detect_os() {
 		exit 1
 	fi
 
-	echo "Found OS: $OS $OSVER" >> "$setup_log" 2>&1
+	echo "Found OS: $OS $OSVER" >> "$log" 2>&1
 
 }
 

setup/so-preflight

@@ -0,0 +1,137 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+source ../salt/common/tools/sbin/so-common
+source ./so-functions
+
+preflight_log='/root/preflight.log'
+
+check_default_repos() {
+	local ret_code=0
+
+	printf '  Checking OS default repos with ' | tee -a "$preflight_log"
+	if [[ $OS == 'centos' ]]; then
+		printf '%s' 'yum update.' | tee -a "$preflight_log"
+		echo "" >> "$preflight_log"
+		yum -y update >> $preflight_log 2>&1
+		ret_code=$?
+	else
+		printf '%s' 'apt update.' | tee -a "$preflight_log"
+		echo "" >> "$preflight_log"
+		retry 50 10 "apt-get -y update" >> $preflight_log 2>&1
+		ret_code=$?
+	fi
+
+	[[ $ret_code == 0 ]] && printf '%s\n' '  SUCCESS' || printf '%s\n' '  FAILURE'
+	return $ret_code
+}
+
+check_new_repos() {
+	printf '  Checking repo URLs added by setup.' | tee -a "$preflight_log"
+
+	if [[ $OS == 'centos' ]]; then
+		local repo_arr=( 
+			"https://download.docker.com/linux/centos/docker-ce.repo"
+			"https://repo.saltstack.com/py3/redhat/7/x86_64/archive/3002.2/SALTSTACK-GPG-KEY.pub"
+			"https://repo.saltstack.com/py3/ubuntu/18.04/amd64/archive/3002.2/SALTSTACK-GPG-KEY.pub"
+			"https://download.docker.com/linux/ubuntu/gpg"
+			"https://packages.wazuh.com/key/GPG-KEY-WAZUH"
+			"https://packages.wazuh.com/3.x/yum/"
+			)
+	else
+		local ubuntu_version
+		ubuntu_version=$(grep VERSION_ID /etc/os-release 2> /dev/null | awk -F '[ "]' '{print $2}')
+		if [ "$OSVER" != "xenial" ]; then local py_ver_url_path="/py3"; else local py_ver_url_path="/apt"; fi
+		local repo_arr=(
+			"https://download.docker.com/linux/ubuntu/gpg"
+			"https://download.docker.com/linux/ubuntu"
+			"https://repo.saltstack.com$py_ver_url_path/ubuntu/$ubuntu_version/amd64/archive/3002.2/SALTSTACK-GPG-KEY.pub"
+			"https://packages.wazuh.com/key/GPG-KEY-WAZUH"
+			"https://packages.wazuh.com"
+		)
+	fi
+
+	__check_url_arr "${repo_arr[@]}"
+	local ret_code=$?
+	[[ $ret_code == 0 ]] && printf '%s\n' '  SUCCESS' || printf '%s\n' '  FAILURE'
+	return $ret_code
+}
+
+check_misc_urls() {
+	printf '  Checking various other URLs used by setup.' | tee -a "$preflight_log"
+
+	local so_version=$(cat ../VERSION)
+	local url_arr=(
+		"https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS"
+		"https://github.com/Neo23x0/signature-base"
+		"https://sigs.securityonion.net/$so_version/securityonion-$so_version.iso.sig"
+		"https://ghcr.io/"
+		"https://rules.emergingthreats.net/open/"
+		"https://rules.emergingthreatspro.com/"
+	)
+
+	__check_url_arr "${url_arr[@]}"
+	local ret_code=$?
+	[[ $ret_code == 0 ]] && printf '%s\n' '  SUCCESS' || printf '%s\n' '  FAILURE'
+	return $ret_code
+}
+
+__check_url_arr() {
+	local ret_code=0
+	echo "" >> "$preflight_log"
+	for url in "$@"; do
+		local status
+		status=$(curl -s -o /dev/null -w "%{http_code}" -L "$url" 2> /dev/null)
+		local ret=$?
+		if [[ $ret == 0 ]]; then
+			printf '%s' "    - Successfully reached $url" >> "$preflight_log"
+			if [[ $status -ge 400 ]]; then
+				printf '%s\n' " but server responded with error code $status" >> "$preflight_log"
+			else
+				printf '\n' >> "$preflight_log"
+			fi
+		else
+			ret_code=1
+			echo "    - [ERROR]: Could not reach $url" >> "$preflight_log"
+		fi
+	done
+	echo "" >> "$preflight_log"
+	return $ret_code
+}
+
+main() {
+	detect_os "$preflight_log"
+
+	[[ -f $preflight_log ]] || touch "$preflight_log"
+	echo "Beginning pre-flight checks." | tee "$preflight_log"
+	check_default_repos &&\
+	check_new_repos &&\
+	check_misc_urls
+
+	local success=$?
+	
+	echo ""
+	if [[ $success == 0 ]]; then
+		echo -e "Pre-flight checks completed successfully!\n" | tee -a "$preflight_log"
+	else
+		echo -e "Pre-flight checks could not complete." | tee -a "$preflight_log"
+		echo -e "  Check $preflight_log for details.\n"
+		exit 1
+	fi
+}
+
+main