From 3b32eb539f4f95e9b9e88d0169c05373d6b9dccd Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 23 Feb 2021 11:21:23 -0500 Subject: [PATCH 01/12] Copy latest rules when using airgaps --- salt/common/tools/sbin/soup | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 31b1219f1..accc615a8 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -379,6 +379,10 @@ unmount_update() { umount /tmp/soagupdate } +update_airgap_rules() { + # Copy the rules over to update them for airgap. + rsync -av $UPDATE_DIR/SecurityOnion/agrules/* /nsm/repo/rules/ +} update_centos_repo() { # Update the files in the repo @@ -588,6 +592,11 @@ echo "Making pillar changes." pillar_changes echo "" +if [ $is_airgap -eq 0 ]; then + echo "Updating Rule Files to the Latest." + update_airgap_rules +fi + # Only update the repo if its airgap if [[ $is_airgap -eq 0 ]] && [[ "$UPGRADESALT" != "1" ]]; then update_centos_repo From 186710964b17e8a06fa63eda07781016397e5b70 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 23 Feb 2021 13:07:23 -0500 Subject: [PATCH 02/12] Fix Airgap Rule Path --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index accc615a8..68fc93466 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -381,7 +381,7 @@ unmount_update() { update_airgap_rules() { # Copy the rules over to update them for airgap. - rsync -av $UPDATE_DIR/SecurityOnion/agrules/* /nsm/repo/rules/ + rsync -av $UPDATE_DIR/agrules/* /nsm/repo/rules/ } update_centos_repo() { From 25698dafe3ead49c662b20597538ab93e1301874 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 23 Feb 2021 13:25:54 -0500 Subject: [PATCH 03/12] Add initial pre-flight check script --- setup/so-functions | 5 +- setup/so-preflight | 135 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+), 2 deletions(-) create mode 100644 setup/so-preflight diff --git a/setup/so-functions b/setup/so-functions index 40c1127de..4410e7443 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -905,9 +905,10 @@ detect_cloud() { } detect_os() { + local log=${1:-${setup_log}} # Detect Base OS - echo "Detecting Base OS" >> "$setup_log" 2>&1 + echo "Detecting Base OS" >> "$log" 2>&1 if [ -f /etc/redhat-release ]; then OS=centos if grep -q "CentOS Linux release 7" /etc/redhat-release; then @@ -937,7 +938,7 @@ detect_os() { exit 1 fi - echo "Found OS: $OS $OSVER" >> "$setup_log" 2>&1 + echo "Found OS: $OS $OSVER" >> "$log" 2>&1 } diff --git a/setup/so-preflight b/setup/so-preflight new file mode 100644 index 000000000..c2bbb7988 --- /dev/null +++ b/setup/so-preflight @@ -0,0 +1,135 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +source ../salt/common/tools/sbin/so-common +source ./so-functions + +preflight_log='/root/preflight.log' + +check_default_repos() { + local ret_code=0 + + printf ' Checking OS default repos with ' | tee -a "$preflight_log" + if [[ $OS == 'centos' ]]; then + printf '%s' 'yum update.' | tee -a "$preflight_log" + echo "" >> "$preflight_log" + yum -y update >> $preflight_log 2>&1 + ret_code=$? + else + printf '%s' 'apt update.' | tee -a "$preflight_log" + echo "" >> "$preflight_log" + retry 50 10 "apt-get -y update" >> $preflight_log 2>&1 + ret_code=$? + fi + + [[ $ret_code == 0 ]] && printf '%s\n' ' SUCCESS' || printf '%s\n' ' FAILURE' + return $ret_code +} + +check_new_repos() { + printf ' Checking repo URLs added by setup.' | tee -a "$preflight_log" + + if [[ $OS == 'centos' ]]; then + local repo_arr=( + "https://download.docker.com/linux/centos/docker-ce.repo" + "https://repo.saltstack.com/py3/redhat/7/x86_64/archive/3002.2/SALTSTACK-GPG-KEY.pub" + "https://repo.saltstack.com/py3/ubuntu/18.04/amd64/archive/3002.2/SALTSTACK-GPG-KEY.pub" + "https://download.docker.com/linux/ubuntu/gpg" + "https://packages.wazuh.com/key/GPG-KEY-WAZUH" + "https://packages.wazuh.com/3.x/yum/" + ) + else + local ubuntu_version + ubuntu_version=$(grep -q VERSION_ID /etc/os-release 2> /dev/null | awk -F '[ "]' '{print $2}') + if [ "$OSVER" != "xenial" ]; then local py_ver_url_path="/py3"; else local py_ver_url_path="/apt"; fi + local repo_arr=( + "https://download.docker.com/linux/ubuntu/gpg" + "https://download.docker.com/linux/ubuntu" + "https://repo.saltstack.com$py_ver_url_path/ubuntu/$ubuntu_version/amd64/archive/3002.2/SALTSTACK-GPG-KEY.pub" + "https://packages.wazuh.com/key/GPG-KEY-WAZUH" + "https://packages.wazuh.com" + ) + fi + + __check_url_arr "${repo_arr[@]}" + local ret_code=$? + [[ $ret_code == 0 ]] && printf '%s\n' ' SUCCESS' || printf '%s\n' ' FAILURE' + return $ret_code +} + +check_misc_urls() { + printf ' Checking various other URLs used by setup.' | tee -a "$preflight_log" + + local so_version=$(cat ../VERSION) + local url_arr=( + "https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS" + "https://github.com/Neo23x0/signature-base" + "https://sigs.securityonion.net/$so_version/securityonion-$so_version.iso.sig" + "https://ghcr.io/" + ) + + __check_url_arr "${url_arr[@]}" + local ret_code=$? + [[ $ret_code == 0 ]] && printf '%s\n' ' SUCCESS' || printf '%s\n' ' FAILURE' + return $ret_code +} + +__check_url_arr() { + local ret_code=0 + echo "" >> "$preflight_log" + for url in "$@"; do + local status + status=$(curl -s -o /dev/null -w "%{http_code}" -L "$url" 2> /dev/null) + local ret=$? + if [[ $ret == 0 ]]; then + printf '%s' " - Successfully reached $url" >> "$preflight_log" + if [[ $status -ge 400 ]]; then + printf '%s\n' " but server responded with error code $status" >> "$preflight_log" + else + printf '\n' >> "$preflight_log" + fi + else + ret_code=1 + echo " - [ERROR]: Could not reach $url" >> "$preflight_log" + fi + done + echo "" >> "$preflight_log" + return $ret_code +} + +main() { + detect_os "$preflight_log" + + [[ -f $preflight_log ]] || touch "$preflight_log" + echo "Beginning pre-flight checks." | tee "$preflight_log" + check_default_repos &&\ + check_new_repos &&\ + check_misc_urls + + local success=$? + + echo "" + if [[ $success == 0 ]]; then + echo -e "Pre-flight checks completed successfully!\n" | tee -a "$preflight_log" + else + echo -e "Pre-flight checks could not complete." | tee -a "$preflight_log" + echo -e " Check $preflight_log for details.\n" + exit 1 + fi +} + +main From 61a23509a1e0b74941b563e5faa4d9b0cabde6e6 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 23 Feb 2021 13:43:10 -0500 Subject: [PATCH 04/12] [fix] grep -q doesn't give output to parse, so remove the flag --- setup/so-preflight | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-preflight b/setup/so-preflight index c2bbb7988..05ff75826 100644 --- a/setup/so-preflight +++ b/setup/so-preflight @@ -54,7 +54,7 @@ check_new_repos() { ) else local ubuntu_version - ubuntu_version=$(grep -q VERSION_ID /etc/os-release 2> /dev/null | awk -F '[ "]' '{print $2}') + ubuntu_version=$(grep VERSION_ID /etc/os-release 2> /dev/null | awk -F '[ "]' '{print $2}') if [ "$OSVER" != "xenial" ]; then local py_ver_url_path="/py3"; else local py_ver_url_path="/apt"; fi local repo_arr=( "https://download.docker.com/linux/ubuntu/gpg" From 5a2fa26d72f6d6faf69cee08c70c9bd579261aa1 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 23 Feb 2021 13:47:52 -0500 Subject: [PATCH 05/12] Add ET OPEN/PRO URLs --- setup/so-preflight | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup/so-preflight b/setup/so-preflight index 05ff75826..69435446a 100644 --- a/setup/so-preflight +++ b/setup/so-preflight @@ -80,6 +80,8 @@ check_misc_urls() { "https://github.com/Neo23x0/signature-base" "https://sigs.securityonion.net/$so_version/securityonion-$so_version.iso.sig" "https://ghcr.io/" + "https://rules.emergingthreats.net/open/" + "https://rules.emergingthreatspro.com/" ) __check_url_arr "${url_arr[@]}" From d5477b47216b3e9241d2f29b834579751192786f Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 23 Feb 2021 13:48:54 -0500 Subject: [PATCH 06/12] Add usage/help message to so-monitor-add --- salt/common/tools/sbin/so-monitor-add | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/salt/common/tools/sbin/so-monitor-add b/salt/common/tools/sbin/so-monitor-add index 8b351558f..b58cc4ee7 100644 --- a/salt/common/tools/sbin/so-monitor-add +++ b/salt/common/tools/sbin/so-monitor-add @@ -2,4 +2,22 @@ . /usr/sbin/so-common +usage() { + read -r -d '' message <<- EOM + usage: so-monitor-add [-h] NIC + + positional arguments: + NIC The interface you want to add to the monitor bond + + optional arguments: + -h, --help Show this help message and exit + EOM + echo "$message" + exit 1 +} + +if [[ $# -eq 0 || $# -gt 1 ]] || [[ $1 == '-h' || $1 == '--help' ]]; then + usage +fi + add_interface_bond0 "$1" From 9cf15cdae58ab4e5b828a89918f1d5a2d2df89cf Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 23 Feb 2021 13:55:18 -0500 Subject: [PATCH 07/12] [fix] Reword so-monitor-add help message --- salt/common/tools/sbin/so-monitor-add | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-monitor-add b/salt/common/tools/sbin/so-monitor-add index b58cc4ee7..256ce48b4 100644 --- a/salt/common/tools/sbin/so-monitor-add +++ b/salt/common/tools/sbin/so-monitor-add @@ -7,7 +7,7 @@ usage() { usage: so-monitor-add [-h] NIC positional arguments: - NIC The interface you want to add to the monitor bond + NIC The interface to add to the monitor bond (ex: eth2) optional arguments: -h, --help Show this help message and exit From a361ca0e19192c26204eb037f85fcaa57f157140 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 23 Feb 2021 14:15:17 -0500 Subject: [PATCH 08/12] [fix] Add managersearch node type to so-rule pillar search --- salt/common/tools/sbin/so-rule | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-rule b/salt/common/tools/sbin/so-rule index 9654456b4..8be26fcdf 100644 --- a/salt/common/tools/sbin/so-rule +++ b/salt/common/tools/sbin/so-rule @@ -58,7 +58,7 @@ def check_apply(args: dict, prompt: bool = True): def find_minion_pillar() -> str: - regex = '^.*_(manager|standalone|import|eval)\.sls$' + regex = '^.*_(manager|managersearch|standalone|import|eval)\.sls$' result = [] for root, _, files in os.walk(minion_pillar_dir): @@ -67,7 +67,7 @@ def find_minion_pillar() -> str: result.append(os.path.join(root, f_minion_id)) if len(result) == 0: - print_err('Could not find manager-type pillar (eval, standalone, manager, import). Are you running this script on the manager?') + print_err('Could not find manager-type pillar (eval, standalone, manager, managersearch, import). Are you running this script on the manager?') sys.exit(3) elif len(result) > 1: res_str = ', '.join(f'\"{result}\"') From 5baa4cb6a5560f549d71c9a29b73e049b5bdbaa6 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 23 Feb 2021 15:42:58 -0500 Subject: [PATCH 09/12] Clean up on sid numbers --- salt/idstools/sorules/extraction.rules | 18 +++++++++--------- salt/idstools/sorules/filters.rules | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/salt/idstools/sorules/extraction.rules b/salt/idstools/sorules/extraction.rules index ae18edcb9..bccfc69d6 100644 --- a/salt/idstools/sorules/extraction.rules +++ b/salt/idstools/sorules/extraction.rules @@ -7,20 +7,20 @@ alert smb any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:1100004; rev:1;) alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:1100005; rev:1;) alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:1100006; rev:1;) -alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:11000007; rev:1;) +alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:1100007; rev:1;) alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:1100008; rev:1;) alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:1100009; rev:1;) alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:1100010; rev:1;) -alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:11000011; rev:1;) +alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:1100011; rev:1;) # Extract all Zip files alert http any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; sid:1100012; rev:1;) -alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"Zip"; filestore; sid:1100013; rev:1;) -alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"Zip"; filestore; sid:1100014; rev:1;) -alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"Zip"; filestore; sid:11000015; rev:1;) +alert smtp any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; sid:1100013; rev:1;) +alert nfs any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; sid:1100014; rev:1;) +alert smb any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; sid:1100015; rev:1;) # Extract Word Docs -alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100016; rev:1;) -alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100017; rev:1;) -alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100018; rev:1;) -alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100019; rev:1;) \ No newline at end of file +alert http any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100016; rev:1;) +alert smtp any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100017; rev:1;) +alert nfs any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100018; rev:1;) +alert smb any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100019; rev:1;) \ No newline at end of file diff --git a/salt/idstools/sorules/filters.rules b/salt/idstools/sorules/filters.rules index 9243e0f04..051d1913f 100644 --- a/salt/idstools/sorules/filters.rules +++ b/salt/idstools/sorules/filters.rules @@ -8,4 +8,4 @@ # Example of filtering out Google's certificate from being in the ssl log. #config tls any any -> any any (tls.fingerprint; content:"4f:a4:5e:58:7e:d9:db:20:09:d7:b6:c7:ff:58:c4:7b:dc:3f:55:b4"; config: logging disable, type tx, scope tx; sid:1200003;) # Example of filtering out a md5 of a file from being in the files log. -#config fileinfo any any -> any any (fileinfo.filemd5; content:"google.com"; config: logging disable, type tx, scope tx; sid:1200004;) +#config fileinfo any any -> any any (fileinfo.filemd5; content:"7a125dc69c82d5caf94d3913eecde4b5"; config: logging disable, type tx, scope tx; sid:1200004;) From a7eb3cd38d3ac44b2406bf805500e12d5cb33d8e Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 23 Feb 2021 15:43:09 -0500 Subject: [PATCH 10/12] Add so-playbook-sigma-refresh --- salt/common/tools/sbin/soup | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 31b1219f1..39e679c82 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -196,6 +196,9 @@ playbook() { rm -f /opt/so/rules/elastalert/playbook/*.yaml so-playbook-ruleupdate >> /root/soup_playbook_rule_update.log 2>&1 & fi + if [[ "$INSTALLEDVERSION" != 2.3.30 ]]; then + so-playbook-sigma-refresh >> /root/soup_playbook_sigma_refresh.log 2>&1 & + fi } pillar_changes() { From 050058a959a54025dc55298eaf1692d91a0076a2 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 24 Feb 2021 09:34:14 -0500 Subject: [PATCH 11/12] Add retry support for 'docker pull' command --- salt/common/tools/sbin/so-image-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 4e5aedc5f..402ae97f3 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -151,7 +151,7 @@ update_docker_containers() { # Pull down the trusted docker image local image=$i:$VERSION$IMAGE_TAG_SUFFIX - docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image >> "$LOG_FILE" 2>&1 + retry 50 10 "docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image" >> "$LOG_FILE" 2>&1 # Get signature retry 50 10 "curl -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" >> "$LOG_FILE" 2>&1 From 3552abfca15422ee44a7766f29596413632471ca Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 24 Feb 2021 11:50:08 -0500 Subject: [PATCH 12/12] ensure info log level - --- salt/salt/minion.sls | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index de85693c6..4b0bbcd3f 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -23,6 +23,15 @@ salt_minion_package: - hold: True - onlyif: test "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}" +set_log_levels: + file.append: + - name: /etc/salt/minion + - text: + - "log_level: info" + - "log_level_log_file: info" + - listen_in: + - service: salt_minion_service + salt_minion_service: service.running: - name: salt-minion