From 61611b8de288b429f25ef0bad986d0d9f88d8435 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 2 Mar 2021 10:23:04 -0500 Subject: [PATCH 01/27] Fix Elasticsearch disk space prompt Resolves #3205 --- setup/so-whiptail | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 390cd70a5..0d976558c 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -631,11 +631,23 @@ whiptail_invalid_hostname() { whiptail_log_size_limit() { [ -n "$TESTING" ] && return - - log_size_limit=$(whiptail --title "Security Onion Setup" --inputbox \ - "Please specify the amount of disk space (in GB) you would like to allocate for Elasticsearch data storage: \n\ - By default, this is set to 80% of the disk space allotted for /nsm." 10 75 "$log_size_limit" 3>&1 1>&2 2>&3) + case $install_type in + STANDALONE | EVAL | HEAVYNODE) + percentage=50 + ;; + *) + percentage=80 + ;; + esac + + read -r -d '' message <<- EOM + Please specify the amount of disk space (in GB) you would like to allocate for Elasticsearch data storage. + + By default, this is set to ${percentage}% of the disk space allotted for /nsm. + EOM + + log_size_limit=$(whiptail --title "Security Onion Setup" --inputbox "$message" 11 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus From 497938460a83de8d86e9b23393ba9f53ecd773ae Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 2 Mar 2021 16:47:49 -0500 Subject: [PATCH 02/27] [fix] manager:log_size_limit is no longer used, remove generation --- setup/files/.curlrc | 0 setup/files/.wgetrc | 0 setup/so-functions | 1 - 3 files changed, 1 deletion(-) create mode 100644 setup/files/.curlrc create mode 100644 setup/files/.wgetrc diff --git a/setup/files/.curlrc b/setup/files/.curlrc new file mode 100644 index 000000000..e69de29bb diff --git a/setup/files/.wgetrc b/setup/files/.wgetrc new file mode 100644 index 000000000..e69de29bb diff --git a/setup/so-functions b/setup/so-functions index 21602f320..f19f60e72 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1446,7 +1446,6 @@ manager_pillar() { printf '%s\n'\ " elastalert: 1"\ " es_port: $node_es_port"\ - " log_size_limit: $log_size_limit"\ " cur_close_days: $CURCLOSEDAYS"\ " grafana: $GRAFANA"\ " osquery: $OSQUERY"\ From 4df53b3c70cebc2c8c5b0c27e62e972ad908ce48 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 2 Mar 2021 17:38:17 -0500 Subject: [PATCH 03/27] Unify log_size_limit variable value in so-curator-closed-delete-delete --- salt/curator/files/bin/so-curator-closed-delete-delete | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/curator/files/bin/so-curator-closed-delete-delete b/salt/curator/files/bin/so-curator-closed-delete-delete index 2a9165921..58433ee1a 100755 --- a/salt/curator/files/bin/so-curator-closed-delete-delete +++ b/salt/curator/files/bin/so-curator-closed-delete-delete @@ -4,12 +4,11 @@ {%- if grains['role'] in ['so-node', 'so-heavynode'] %} {%- set ELASTICSEARCH_HOST = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- set ELASTICSEARCH_PORT = salt['pillar.get']('elasticsearch:es_port', '') -%} - {%- set LOG_SIZE_LIMIT = salt['pillar.get']('elasticsearch:log_size_limit', '') -%} {%- elif grains['role'] in ['so-eval', 'so-managersearch', 'so-standalone'] %} {%- set ELASTICSEARCH_HOST = salt['pillar.get']('manager:mainip', '') -%} {%- set ELASTICSEARCH_PORT = salt['pillar.get']('manager:es_port', '') -%} - {%- set LOG_SIZE_LIMIT = salt['pillar.get']('manager:log_size_limit', '') -%} {%- endif -%} +{%- set LOG_SIZE_LIMIT = salt['pillar.get']('elasticsearch:log_size_limit', '') -%} # Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC # From de77d3ebc99733cac0eb304507f745e4071198c9 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 2 Mar 2021 17:41:49 -0500 Subject: [PATCH 04/27] [wip] Initial work for setting up proxy on manager --- salt/common/tools/sbin/so-common | 20 ++++++++++++ setup/so-functions | 54 ++++++++++++++++++++++++++++++-- setup/so-variables | 2 ++ setup/so-whiptail | 15 +++++++++ 4 files changed, 89 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index b76ad05ff..cdc8b4772 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -340,6 +340,26 @@ valid_int() { # {% raw %} +valid_proxy() { + local proxy=$1 + local url_prefixes=( 'http://' 'https://' ) + + local has_prefix=false + for prefix in "${url_prefixes[@]}"; do + echo "$proxy" | grep "$prefix" && has_prefix=true && proxy=${proxy#"$prefix"} + done + + local url + readarray -t url -d ':' <<< "$proxy" + + local valid_url=true + if ! valid_ip4 "$proxy" && ! valid_fqdn "$proxy"; then + valid_url=false + fi + + [[ $has_prefix == true ]] && [[ $valid_url ]] && return 0 || return 1 +} + valid_string() { local str=$1 local min_length=${2:-1} diff --git a/setup/so-functions b/setup/so-functions index f19f60e72..d457e7005 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -535,6 +535,17 @@ collect_patch_schedule_name_import() { done } +collect_proxy() { + if whiptail_proxy_ask; then + whiptail_proxy_addr + + while ! valid_proxy "$proxy_addr"; do + whiptail_invalid_input + whiptail_proxy_addr + done + fi +} + collect_redirect_host() { whiptail_set_redirect_host "$HOSTNAME" @@ -1511,7 +1522,7 @@ manager_global() { " hnmanager: '$HNMANAGER'"\ " ntpserver: '$NTPSERVER'"\ " dockernet: '$DOCKERNET'"\ - " proxy: '$PROXY'"\ + " proxy: '$proxy_addr'"\ " mdengine: '$ZEEKVERSION'"\ " ids: '$NIDS'"\ " url_base: '$REDIRECTIT'"\ @@ -2184,7 +2195,46 @@ set_main_ip() { # Add /usr/sbin to everyone's path set_path() { - echo "complete -cf sudo" > /etc/profile.d/securityonion.sh + echo "complete -cf sudo" >> "$profile_d_config_file" +} + +set_proxy() { + # Don't proxy localhost, local ip, and management ip + local no_proxy_string="localhost, 127.0.0.1, ${MAINIP}" + + # Set proxy environment variables used by curl, wget, docker, and others + { + echo "export use_proxy=on" + echo "export http_proxy=\"${proxy_addr}\"" + echo "export https_proxy=\"\$http_addr\"" + echo "export ftp_proxy=\"\$http_addr\"" + echo "export no_proxy=\"${no_proxy_string}\"" + } >> "$profile_d_config_file" + + # Create proxy config for dockerd + printf '%s\n'\ + "[Service]"\ + "Environment=\"HTTP_PROXY=${proxy_addr}\""\ + "Environment=\"HTTPS_PROXY=${proxy_addr}\""\ + "Environment=\"NO_PROXY=${no_proxy_string}\"" > /etc/systemd/system/docker.service.d/http-proxy.conf + + systemctl daemon-reload + systemctl restart docker + + # Set proxy for package manager + if [ "$OS" = 'centos' ]; then + echo "proxy=$proxy_addr" >> /etc/yum.conf + else + # Set it up so the updates roll through the manager + printf '%s\n'\ + "Acquire::http::Proxy \"$proxy_addr\";"\ + "Acquire::https::Proxy \"$proxy_addr\";" > /etc/apt/apt.conf.d/00-proxy.conf + fi + + # Set global git proxy + printf '%s\n'\ + "[http]"\ + " proxy = ${proxy_addr}" > /etc/gitconfig } setup_salt_master_dirs() { diff --git a/setup/so-variables b/setup/so-variables index a2fdf03c6..cd6478fa7 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -72,3 +72,5 @@ export install_opt_file net_init_file=/root/net_init export net_init_file + +export profile_d_config_file='/etc/profile.d/securityonion.sh' diff --git a/setup/so-whiptail b/setup/so-whiptail index 7bb265de1..6baa143ad 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1216,6 +1216,21 @@ whiptail_patch_schedule_select_hours() { } +whiptail_proxy_ask() { + [ -n "$TESTING" ] && return + + whiptail --title "Security Onion Setup" --yesno "Do you want to use a proxy server to complete setup?" 7 60 +} + +whiptail_proxy_addr() { + [ -n "$TESTING" ] && return + + proxy_addr=$(whiptail --title "Security Onion Setup" --inputbox "Please input the proxy server you wish to use, including the URL prefix (ex: https://your.proxy.com:1234):" 8 60 3>&1 1>&2 2>&3) + + local exitstatus=$? + whiptail_check_exitstatus $exitstatus +} + whiptail_requirements_error() { local requirement_needed=$1 From 2b94fa366ed785750382edaa57a936253d0cc34f Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 3 Mar 2021 09:51:38 -0500 Subject: [PATCH 05/27] [wip] Add auth inputs for proxy settings, fix some broken logic --- salt/common/tools/sbin/so-common | 4 ++-- setup/so-functions | 29 +++++++++++++++++++++++++++++ setup/so-whiptail | 18 ++++++++++++++++++ 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index cdc8b4772..438fe9329 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -346,7 +346,7 @@ valid_proxy() { local has_prefix=false for prefix in "${url_prefixes[@]}"; do - echo "$proxy" | grep "$prefix" && has_prefix=true && proxy=${proxy#"$prefix"} + echo "$proxy" | grep "$prefix" && has_prefix=true && proxy=${proxy#"$prefix"} && break done local url @@ -357,7 +357,7 @@ valid_proxy() { valid_url=false fi - [[ $has_prefix == true ]] && [[ $valid_url ]] && return 0 || return 1 + [[ $has_prefix == true ]] && [[ $valid_url == true ]] && return 0 || return 1 } valid_string() { diff --git a/setup/so-functions b/setup/so-functions index d457e7005..c8adbc9f5 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -537,12 +537,41 @@ collect_patch_schedule_name_import() { collect_proxy() { if whiptail_proxy_ask; then + whiptail_proxy_addr while ! valid_proxy "$proxy_addr"; do whiptail_invalid_input whiptail_proxy_addr done + + if whiptail_proxy_auth_ask; then + whiptail_proxy_auth_user + + while ! valid_string "$proxy_user"; do + whiptail_invalid_input + whiptail_proxy_auth_user + done + + whiptail_proxy_auth_pass + + while ! valid_string "$proxy_pass"; do + whiptail_invalid_input + whiptail_proxy_auth_pass + done + + local url_prefixes=( 'http://' 'https://' ) + for prefix in "${url_prefixes[@]}"; do + if echo "$proxy_addr" | grep "$prefix"; then + proxy=${proxy_addr#"$prefix"} + proxy="${prefix}${proxy_user}:${proxy_pass}@${proxy}" + break + fi + done + else + proxy="$proxy_addr" + fi + export proxy fi } diff --git a/setup/so-whiptail b/setup/so-whiptail index 6baa143ad..205bf6f98 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1231,6 +1231,24 @@ whiptail_proxy_addr() { whiptail_check_exitstatus $exitstatus } +whiptail_proxy_auth_ask() { + [ -n "$TESTING" ] && return + + whiptail --title "Security Onion Setup" --yesno "Does your proxy require authentication?" 7 60 +} + +whiptail_proxy_auth_user() { + [ -n "$TESTING" ] && return + + proxy_user=$(whiptail --title "Security Onion Setup" --inputbox "Please input the proxy user:" 8 60 3>&1 1>&2 2>&3) +} + +whiptail_proxy_auth_pass() { + [ -n "$TESTING" ] && return + + proxy_pass=$(whiptail --title "Security Onion Setup" --inputbox "Please input the proxy password:" 8 60 3>&1 1>&2 2>&3) +} + whiptail_requirements_error() { local requirement_needed=$1 From 82018a206c356a3cd048960ca01ecc8d63e3cdaf Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 3 Mar 2021 09:56:14 -0500 Subject: [PATCH 06/27] [wip] Don't validate user+pass for proxy, use new variable --- setup/so-functions | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index c8adbc9f5..1ae0d092c 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -548,18 +548,8 @@ collect_proxy() { if whiptail_proxy_auth_ask; then whiptail_proxy_auth_user - while ! valid_string "$proxy_user"; do - whiptail_invalid_input - whiptail_proxy_auth_user - done - whiptail_proxy_auth_pass - while ! valid_string "$proxy_pass"; do - whiptail_invalid_input - whiptail_proxy_auth_pass - done - local url_prefixes=( 'http://' 'https://' ) for prefix in "${url_prefixes[@]}"; do if echo "$proxy_addr" | grep "$prefix"; then @@ -1551,7 +1541,7 @@ manager_global() { " hnmanager: '$HNMANAGER'"\ " ntpserver: '$NTPSERVER'"\ " dockernet: '$DOCKERNET'"\ - " proxy: '$proxy_addr'"\ + " proxy: '$proxy'"\ " mdengine: '$ZEEKVERSION'"\ " ids: '$NIDS'"\ " url_base: '$REDIRECTIT'"\ @@ -2234,7 +2224,7 @@ set_proxy() { # Set proxy environment variables used by curl, wget, docker, and others { echo "export use_proxy=on" - echo "export http_proxy=\"${proxy_addr}\"" + echo "export http_proxy=\"${proxy}\"" echo "export https_proxy=\"\$http_addr\"" echo "export ftp_proxy=\"\$http_addr\"" echo "export no_proxy=\"${no_proxy_string}\"" @@ -2243,8 +2233,8 @@ set_proxy() { # Create proxy config for dockerd printf '%s\n'\ "[Service]"\ - "Environment=\"HTTP_PROXY=${proxy_addr}\""\ - "Environment=\"HTTPS_PROXY=${proxy_addr}\""\ + "Environment=\"HTTP_PROXY=${proxy}\""\ + "Environment=\"HTTPS_PROXY=${proxy}\""\ "Environment=\"NO_PROXY=${no_proxy_string}\"" > /etc/systemd/system/docker.service.d/http-proxy.conf systemctl daemon-reload @@ -2252,18 +2242,18 @@ set_proxy() { # Set proxy for package manager if [ "$OS" = 'centos' ]; then - echo "proxy=$proxy_addr" >> /etc/yum.conf + echo "proxy=$proxy" >> /etc/yum.conf else # Set it up so the updates roll through the manager printf '%s\n'\ - "Acquire::http::Proxy \"$proxy_addr\";"\ - "Acquire::https::Proxy \"$proxy_addr\";" > /etc/apt/apt.conf.d/00-proxy.conf + "Acquire::http::Proxy \"$proxy\";"\ + "Acquire::https::Proxy \"$proxy\";" > /etc/apt/apt.conf.d/00-proxy.conf fi # Set global git proxy printf '%s\n'\ "[http]"\ - " proxy = ${proxy_addr}" > /etc/gitconfig + " proxy = ${proxy}" > /etc/gitconfig } setup_salt_master_dirs() { From 279a5b60b84bd9a74cad558cb4ec30e6eabc6940 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 3 Mar 2021 11:58:10 -0500 Subject: [PATCH 07/27] Soup indent fixes --- salt/common/tools/sbin/soup | 213 ++++++++++++++++++------------------ 1 file changed, 106 insertions(+), 107 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 6eb01a694..e33d0cc41 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -105,12 +105,12 @@ check_airgap() { # See if this is an airgap install AIRGAP=$(cat /opt/so/saltstack/local/pillar/global.sls | grep airgap: | awk '{print $2}') if [[ "$AIRGAP" == "True" ]]; then - is_airgap=0 - UPDATE_DIR=/tmp/soagupdate/SecurityOnion - AGDOCKER=/tmp/soagupdate/docker - AGREPO=/tmp/soagupdate/Packages + is_airgap=0 + UPDATE_DIR=/tmp/soagupdate/SecurityOnion + AGDOCKER=/tmp/soagupdate/docker + AGREPO=/tmp/soagupdate/Packages else - is_airgap=1 + is_airgap=1 fi } @@ -256,14 +256,14 @@ playbook() { } pillar_changes() { - # This function is to add any new pillar items if needed. - echo "Checking to see if pillar changes are needed." - - [[ "$INSTALLEDVERSION" =~ rc.1 ]] && rc1_to_rc2 - [[ "$INSTALLEDVERSION" =~ rc.2 ]] && rc2_to_rc3 - [[ "$INSTALLEDVERSION" =~ rc.3 ]] && rc3_to_2.3.0 - [[ "$INSTALLEDVERSION" == 2.3.0 || "$INSTALLEDVERSION" == 2.3.1 || "$INSTALLEDVERSION" == 2.3.2 || "$INSTALLEDVERSION" == 2.3.10 ]] && up_2.3.0_to_2.3.20 - [[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30 + # This function is to add any new pillar items if needed. + echo "Checking to see if pillar changes are needed." + + [[ "$INSTALLEDVERSION" =~ rc.1 ]] && rc1_to_rc2 + [[ "$INSTALLEDVERSION" =~ rc.2 ]] && rc2_to_rc3 + [[ "$INSTALLEDVERSION" =~ rc.3 ]] && rc3_to_2.3.0 + [[ "$INSTALLEDVERSION" == 2.3.0 || "$INSTALLEDVERSION" == 2.3.1 || "$INSTALLEDVERSION" == 2.3.2 || "$INSTALLEDVERSION" == 2.3.10 ]] && up_2.3.0_to_2.3.20 + [[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30 } rc1_to_rc2() { @@ -296,15 +296,14 @@ rc1_to_rc2() { done > /opt/so/saltstack/local/pillar/global.sls sed -i 's/playbook:/playbook_db:/' /opt/so/saltstack/local/pillar/secrets.sls @@ -385,7 +384,6 @@ up_2.3.0_to_2.3.20(){ fi INSTALLEDVERSION=2.3.20 - } up_2.3.2X_to_2.3.30() { @@ -395,11 +393,11 @@ up_2.3.2X_to_2.3.30() { sed -i -r "s/ (\{\{.*}})$/ '\1'/g" "$pillar" done -# Change the IMAGEREPO + # Change the IMAGEREPO sed -i "/ imagerepo: 'securityonion'/c\ imagerepo: 'security-onion-solutions'" /opt/so/saltstack/local/pillar/global.sls sed -i "/ imagerepo: securityonion/c\ imagerepo: 'security-onion-solutions'" /opt/so/saltstack/local/pillar/global.sls -# Strelka rule repo pillar addition + # Strelka rule repo pillar addition if [ $is_airgap -eq 0 ]; then # Add manager as default Strelka YARA rule repo sed -i "/^strelka:/a \\ repos: \n - https://$HOSTNAME/repo/rules/strelka" /opt/so/saltstack/local/pillar/global.sls; @@ -414,10 +412,10 @@ space_check() { # Check to see if there is enough space CURRENTSPACE=$(df -BG / | grep -v Avail | awk '{print $4}' | sed 's/.$//') if [ "$CURRENTSPACE" -lt "10" ]; then - echo "You are low on disk space. Upgrade will try and clean up space."; - clean_dockers + echo "You are low on disk space. Upgrade will try and clean up space."; + clean_dockers else - echo "Plenty of space for upgrading" + echo "Plenty of space for upgrading" fi } @@ -427,16 +425,16 @@ thehive_maint() { COUNT=0 THEHIVE_CONNECTED="no" while [[ "$COUNT" -le 240 ]]; do - curl --output /dev/null --silent --head --fail -k "https://localhost/thehive/api/alert" - if [ $? -eq 0 ]; then - THEHIVE_CONNECTED="yes" - echo "connected!" - break - else - ((COUNT+=1)) - sleep 1 - echo -n "." - fi + curl --output /dev/null --silent --head --fail -k "https://localhost/thehive/api/alert" + if [ $? -eq 0 ]; then + THEHIVE_CONNECTED="yes" + echo "connected!" + break + else + ((COUNT+=1)) + sleep 1 + echo -n "." + fi done if [ "$THEHIVE_CONNECTED" == "yes" ]; then echo "Migrating thehive databases if needed." @@ -471,80 +469,80 @@ update_version() { } upgrade_check() { - # Let's make sure we actually need to update. - NEWVERSION=$(cat $UPDATE_DIR/VERSION) - if [ "$INSTALLEDVERSION" == "$NEWVERSION" ]; then - echo "You are already running the latest version of Security Onion." - exit 0 - fi + # Let's make sure we actually need to update. + NEWVERSION=$(cat $UPDATE_DIR/VERSION) + if [ "$INSTALLEDVERSION" == "$NEWVERSION" ]; then + echo "You are already running the latest version of Security Onion." + exit 0 + fi } upgrade_check_salt() { - NEWSALTVERSION=$(grep version: $UPDATE_DIR/salt/salt/master.defaults.yaml | awk {'print $2'}) - if [ "$INSTALLEDSALTVERSION" == "$NEWSALTVERSION" ]; then - echo "You are already running the correct version of Salt for Security Onion." - else - UPGRADESALT=1 - fi + NEWSALTVERSION=$(grep version: $UPDATE_DIR/salt/salt/master.defaults.yaml | awk {'print $2'}) + if [ "$INSTALLEDSALTVERSION" == "$NEWSALTVERSION" ]; then + echo "You are already running the correct version of Salt for Security Onion." + else + UPGRADESALT=1 + fi } upgrade_salt() { - SALTUPGRADED=True - echo "Performing upgrade of Salt from $INSTALLEDSALTVERSION to $NEWSALTVERSION." - echo "" - # If CentOS - if [ "$OS" == "centos" ]; then - echo "Removing yum versionlock for Salt." - echo "" - yum versionlock delete "salt-*" - echo "Updating Salt packages and restarting services." - echo "" - if [ $is_airgap -eq 0 ]; then - sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable "$NEWSALTVERSION" - else - sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable "$NEWSALTVERSION" - fi - echo "Applying yum versionlock for Salt." - echo "" - yum versionlock add "salt-*" - # Else do Ubuntu things - elif [ "$OS" == "ubuntu" ]; then - echo "Removing apt hold for Salt." - echo "" - apt-mark unhold "salt-common" - apt-mark unhold "salt-master" - apt-mark unhold "salt-minion" - echo "Updating Salt packages and restarting services." - echo "" - sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable "$NEWSALTVERSION" - echo "Applying apt hold for Salt." - echo "" - apt-mark hold "salt-common" - apt-mark hold "salt-master" - apt-mark hold "salt-minion" - fi + SALTUPGRADED=True + echo "Performing upgrade of Salt from $INSTALLEDSALTVERSION to $NEWSALTVERSION." + echo "" + # If CentOS + if [ "$OS" == "centos" ]; then + echo "Removing yum versionlock for Salt." + echo "" + yum versionlock delete "salt-*" + echo "Updating Salt packages and restarting services." + echo "" + if [ $is_airgap -eq 0 ]; then + sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable "$NEWSALTVERSION" + else + sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable "$NEWSALTVERSION" + fi + echo "Applying yum versionlock for Salt." + echo "" + yum versionlock add "salt-*" + # Else do Ubuntu things + elif [ "$OS" == "ubuntu" ]; then + echo "Removing apt hold for Salt." + echo "" + apt-mark unhold "salt-common" + apt-mark unhold "salt-master" + apt-mark unhold "salt-minion" + echo "Updating Salt packages and restarting services." + echo "" + sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable "$NEWSALTVERSION" + echo "Applying apt hold for Salt." + echo "" + apt-mark hold "salt-common" + apt-mark hold "salt-master" + apt-mark hold "salt-minion" + fi } verify_latest_update_script() { - # Check to see if the update scripts match. If not run the new one. - CURRENTSOUP=$(md5sum /opt/so/saltstack/default/salt/common/tools/sbin/soup | awk '{print $1}') - GITSOUP=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/soup | awk '{print $1}') - CURRENTCMN=$(md5sum /opt/so/saltstack/default/salt/common/tools/sbin/so-common | awk '{print $1}') - GITCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-common | awk '{print $1}') - CURRENTIMGCMN=$(md5sum /opt/so/saltstack/default/salt/common/tools/sbin/so-image-common | awk '{print $1}') - GITIMGCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-image-common | awk '{print $1}') + # Check to see if the update scripts match. If not run the new one. + CURRENTSOUP=$(md5sum /opt/so/saltstack/default/salt/common/tools/sbin/soup | awk '{print $1}') + GITSOUP=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/soup | awk '{print $1}') + CURRENTCMN=$(md5sum /opt/so/saltstack/default/salt/common/tools/sbin/so-common | awk '{print $1}') + GITCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-common | awk '{print $1}') + CURRENTIMGCMN=$(md5sum /opt/so/saltstack/default/salt/common/tools/sbin/so-image-common | awk '{print $1}') + GITIMGCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-image-common | awk '{print $1}') - if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" ]]; then - echo "This version of the soup script is up to date. Proceeding." - else - echo "You are not running the latest soup version. Updating soup and its components. Might take multiple runs to complete" - cp $UPDATE_DIR/salt/common/tools/sbin/soup $DEFAULT_SALT_DIR/salt/common/tools/sbin/ - cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ - cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ - salt-call state.apply common queue=True - echo "" - echo "soup has been updated. Please run soup again." - exit 0 - fi + if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" ]]; then + echo "This version of the soup script is up to date. Proceeding." + else + echo "You are not running the latest soup version. Updating soup and its components. Might take multiple runs to complete" + cp $UPDATE_DIR/salt/common/tools/sbin/soup $DEFAULT_SALT_DIR/salt/common/tools/sbin/ + cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ + cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ + salt-call state.apply common queue=True + echo "" + echo "soup has been updated. Please run soup again." + exit 0 + fi } main () { @@ -557,9 +555,10 @@ while getopts ":b" opt; do echo "Batch size must be a number greater than 0." exit 1 fi - ;; - \? ) echo "Usage: cmd [-b]" - ;; + ;; + \? ) + echo "Usage: cmd [-b]" + ;; esac done From 83791d87c7a46053939199fd5ecbe787bb4a0d72 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 3 Mar 2021 11:58:45 -0500 Subject: [PATCH 08/27] [wip][fix] Use passwordbox for proxy password --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 205bf6f98..cc08acf01 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1246,7 +1246,7 @@ whiptail_proxy_auth_user() { whiptail_proxy_auth_pass() { [ -n "$TESTING" ] && return - proxy_pass=$(whiptail --title "Security Onion Setup" --inputbox "Please input the proxy password:" 8 60 3>&1 1>&2 2>&3) + proxy_pass=$(whiptail --title "Security Onion Setup" --passwordbox "Please input the proxy password:" 8 60 3>&1 1>&2 2>&3) } whiptail_requirements_error() { From 26b1da744c08bfe2a9bbb1d1a7bd47ff9bb74d62 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 3 Mar 2021 12:01:15 -0500 Subject: [PATCH 09/27] [wip] Reword proxy yesno prompt --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index cc08acf01..2f2ff2f78 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1219,7 +1219,7 @@ whiptail_patch_schedule_select_hours() { whiptail_proxy_ask() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --yesno "Do you want to use a proxy server to complete setup?" 7 60 + whiptail --title "Security Onion Setup" --yesno "Do you want to set a proxy server for this installation?" 7 60 } whiptail_proxy_addr() { From e951e9d9c5aff18b00d63767494d093b1247daf2 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 3 Mar 2021 12:19:14 -0500 Subject: [PATCH 10/27] [wip] Further proxy changes * Remove unused docker.conf template * Rename proxy variable to avoid name collision * Reword address prompt to specify users should not include user:pass in their input * Actually call the collect_proxy function --- setup/proxies/docker.conf | 2 -- setup/so-functions | 27 ++++++++++++++++----------- setup/so-whiptail | 8 +++++++- 3 files changed, 23 insertions(+), 14 deletions(-) delete mode 100644 setup/proxies/docker.conf diff --git a/setup/proxies/docker.conf b/setup/proxies/docker.conf deleted file mode 100644 index 9ab2c4b4c..000000000 --- a/setup/proxies/docker.conf +++ /dev/null @@ -1,2 +0,0 @@ -[Service] -ExecStart=/usr/bin/dockerd /usr/bin/dockerd -H fd:// --registry-mirror "$proxy_addr" diff --git a/setup/so-functions b/setup/so-functions index 1ae0d092c..f0b139f6d 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -553,13 +553,13 @@ collect_proxy() { local url_prefixes=( 'http://' 'https://' ) for prefix in "${url_prefixes[@]}"; do if echo "$proxy_addr" | grep "$prefix"; then - proxy=${proxy_addr#"$prefix"} - proxy="${prefix}${proxy_user}:${proxy_pass}@${proxy}" + local proxy=${proxy_addr#"$prefix"} + so_proxy="${prefix}${proxy_user}:${proxy_pass}@${proxy}" break fi done else - proxy="$proxy_addr" + so_proxy="$proxy_addr" fi export proxy fi @@ -1541,7 +1541,7 @@ manager_global() { " hnmanager: '$HNMANAGER'"\ " ntpserver: '$NTPSERVER'"\ " dockernet: '$DOCKERNET'"\ - " proxy: '$proxy'"\ + " proxy: '$so_proxy'"\ " mdengine: '$ZEEKVERSION'"\ " ids: '$NIDS'"\ " url_base: '$REDIRECTIT'"\ @@ -1719,6 +1719,10 @@ network_init() { if [[ "$setup_type" == 'iso' ]]; then set_management_interface fi + # Check if the proxy variable was set then set proxy for various apps if so + if [[ -n "$so_proxy" ]]; then + set_proxy + fi } network_init_whiptail() { @@ -1742,6 +1746,7 @@ network_init_whiptail() { whiptail_management_nic ;; esac + collect_proxy } network_setup() { @@ -2224,7 +2229,7 @@ set_proxy() { # Set proxy environment variables used by curl, wget, docker, and others { echo "export use_proxy=on" - echo "export http_proxy=\"${proxy}\"" + echo "export http_proxy=\"${so_proxy}\"" echo "export https_proxy=\"\$http_addr\"" echo "export ftp_proxy=\"\$http_addr\"" echo "export no_proxy=\"${no_proxy_string}\"" @@ -2233,8 +2238,8 @@ set_proxy() { # Create proxy config for dockerd printf '%s\n'\ "[Service]"\ - "Environment=\"HTTP_PROXY=${proxy}\""\ - "Environment=\"HTTPS_PROXY=${proxy}\""\ + "Environment=\"HTTP_PROXY=${so_proxy}\""\ + "Environment=\"HTTPS_PROXY=${so_proxy}\""\ "Environment=\"NO_PROXY=${no_proxy_string}\"" > /etc/systemd/system/docker.service.d/http-proxy.conf systemctl daemon-reload @@ -2242,18 +2247,18 @@ set_proxy() { # Set proxy for package manager if [ "$OS" = 'centos' ]; then - echo "proxy=$proxy" >> /etc/yum.conf + echo "proxy=$so_proxy" >> /etc/yum.conf else # Set it up so the updates roll through the manager printf '%s\n'\ - "Acquire::http::Proxy \"$proxy\";"\ - "Acquire::https::Proxy \"$proxy\";" > /etc/apt/apt.conf.d/00-proxy.conf + "Acquire::http::Proxy \"$so_proxy\";"\ + "Acquire::https::Proxy \"$so_proxy\";" > /etc/apt/apt.conf.d/00-proxy.conf fi # Set global git proxy printf '%s\n'\ "[http]"\ - " proxy = ${proxy}" > /etc/gitconfig + " proxy = ${so_proxy}" > /etc/gitconfig } setup_salt_master_dirs() { diff --git a/setup/so-whiptail b/setup/so-whiptail index 2f2ff2f78..ffa03e128 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1224,8 +1224,14 @@ whiptail_proxy_ask() { whiptail_proxy_addr() { [ -n "$TESTING" ] && return + local message + read -r -d '' message <<- EOM + Please input the proxy server you wish to use, including the URL prefix (ex: https://your.proxy.com:1234). - proxy_addr=$(whiptail --title "Security Onion Setup" --inputbox "Please input the proxy server you wish to use, including the URL prefix (ex: https://your.proxy.com:1234):" 8 60 3>&1 1>&2 2>&3) + If your proxy requires a username and password do not include them in your input. Setup will ask for those values next. + EOM + + proxy_addr=$(whiptail --title "Security Onion Setup" --inputbox "$message" 13 60 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus From 4ffa0fbc131393b790ec0669fd9d07c48ddff0bc Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 3 Mar 2021 14:09:59 -0500 Subject: [PATCH 11/27] [wip] Fix proxy validation --- salt/common/tools/sbin/so-common | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 438fe9329..25dfc2fab 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -346,14 +346,14 @@ valid_proxy() { local has_prefix=false for prefix in "${url_prefixes[@]}"; do - echo "$proxy" | grep "$prefix" && has_prefix=true && proxy=${proxy#"$prefix"} && break + echo "$proxy" | grep -q "$prefix" && has_prefix=true && proxy=${proxy#"$prefix"} && break done - local url - readarray -t url -d ':' <<< "$proxy" + local url_arr + mapfile -t url_arr <<< "$(echo "$proxy" | tr ":" "\n")" local valid_url=true - if ! valid_ip4 "$proxy" && ! valid_fqdn "$proxy"; then + if ! valid_ip4 "${url_arr[0]}" && ! valid_fqdn "${url_arr[0]}"; then valid_url=false fi From 1ea3cb1c615fc876b1429ada48bca2c81f62ecd7 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 3 Mar 2021 14:20:26 -0500 Subject: [PATCH 12/27] [wip] Change when proxy is set up so main ip is known * Also only restart docker if the command exists (i.e. docker is installed) --- setup/so-functions | 12 ++++++++---- setup/so-setup | 4 +--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index f0b139f6d..d61470d30 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -774,10 +774,10 @@ compare_main_nic_ip() { if ! [[ $MNIC =~ ^(tun|wg|vpn).*$ ]]; then if [[ "$MAINIP" != "$MNIC_IP" ]]; then read -r -d '' message <<- EOM - The IP being routed by Linux is not the IP address assigned to the management interface ($MNIC). + The IP being routed by Linux is not the IP address assigned to the management interface ($MNIC). This is not a supported configuration, please remediate and rerun setup. - EOM + EOM whiptail --title "Security Onion Setup" --msgbox "$message" 10 75 kill -SIGINT "$(ps --pid $$ -oppid=)"; exit 1 fi @@ -1719,9 +1719,11 @@ network_init() { if [[ "$setup_type" == 'iso' ]]; then set_management_interface fi + set_main_ip >> $setup_log 2>&1 + compare_main_nic_ip # Check if the proxy variable was set then set proxy for various apps if so if [[ -n "$so_proxy" ]]; then - set_proxy + set_proxy >> $setup_log 2>&1 fi } @@ -2235,6 +2237,8 @@ set_proxy() { echo "export no_proxy=\"${no_proxy_string}\"" } >> "$profile_d_config_file" + [[ -d '/etc/systemd/system/docker.service.d' ]] || mkdir -p '/etc/systemd/system/docker.service.d' + # Create proxy config for dockerd printf '%s\n'\ "[Service]"\ @@ -2243,7 +2247,7 @@ set_proxy() { "Environment=\"NO_PROXY=${no_proxy_string}\"" > /etc/systemd/system/docker.service.d/http-proxy.conf systemctl daemon-reload - systemctl restart docker + command -v docker &> /dev/null && systemctl restart docker # Set proxy for package manager if [ "$OS" = 'centos' ]; then diff --git a/setup/so-setup b/setup/so-setup index 65fbbe16e..f16699e58 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -537,9 +537,7 @@ catch() { exit } -# This block sets REDIRECTIT which is used by a function outside the below subshell -set_main_ip >> $setup_log 2>&1 -compare_main_nic_ip +# Set REDIRECTIT variable, which is used by a function outside the below subshell set_redirect >> $setup_log 2>&1 # Begin install From 3d5cf128ae579e35920f271926c4057fa5ac2ba9 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 3 Mar 2021 15:02:21 -0500 Subject: [PATCH 13/27] [wip] Test proxy before using it --- setup/so-functions | 18 ++++++++++++++++-- setup/so-setup | 17 +++++++++++++++++ setup/so-whiptail | 15 ++++++++++++++- 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index d61470d30..861dd2e14 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -536,7 +536,8 @@ collect_patch_schedule_name_import() { } collect_proxy() { - if whiptail_proxy_ask; then + local ask=${1:-true} + if [[ $ask == true ]] && whiptail_proxy_ask; then whiptail_proxy_addr @@ -1748,7 +1749,6 @@ network_init_whiptail() { whiptail_management_nic ;; esac - collect_proxy } network_setup() { @@ -1813,6 +1813,17 @@ print_salt_state_apply() { echo "Applying $state Salt state" } +proxy_validate() { + local test_url="https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS" + curl "$test_url" --proxy "$so_proxy" &> /dev/null + local ret=$? + + if [[ $ret != 0 ]]; then + error "Could not reach $test_url using proxy $so_proxy" + fi + return $ret +} + reserve_group_ids() { # This is a hack to fix CentOS from taking group IDs that we need groupadd -g 928 kratos @@ -2225,6 +2236,7 @@ set_path() { } set_proxy() { + # Don't proxy localhost, local ip, and management ip local no_proxy_string="localhost, 127.0.0.1, ${MAINIP}" @@ -2237,6 +2249,8 @@ set_proxy() { echo "export no_proxy=\"${no_proxy_string}\"" } >> "$profile_d_config_file" + source "$profile_d_config_file" + [[ -d '/etc/systemd/system/docker.service.d' ]] || mkdir -p '/etc/systemd/system/docker.service.d' # Create proxy config for dockerd diff --git a/setup/so-setup b/setup/so-setup index f16699e58..18bec9983 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -198,6 +198,14 @@ if ! [[ -f $install_opt_file ]]; then printf '%s\n' \ "MNIC=$MNIC" \ "HOSTNAME=$HOSTNAME" > "$net_init_file" + if [[ $is_manager ]]; then + collect_proxy + while ! proxy_validate; do + whiptail_invalid_proxy + collect_proxy no_ask + done + [[ -n "$proxy_addr" ]] && set_proxy >> $setup_log 2>&1 + fi whiptail_net_setup_complete else whiptail_install_type @@ -289,6 +297,15 @@ if ! [[ -f $install_opt_file ]]; then if [[ $reinit_networking ]] || ! [[ -f $net_init_file ]]; then network_init fi + + if [[ $is_manager ]]; then + collect_proxy + while ! proxy_validate; do + whiptail_invalid_proxy + collect_proxy no_ask + done + [[ -n "$proxy_addr" ]] && set_proxy >> $setup_log 2>&1 + fi if [[ -n "$TURBO" ]]; then use_turbo_proxy diff --git a/setup/so-whiptail b/setup/so-whiptail index ffa03e128..56c0ecc5f 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -588,8 +588,21 @@ whiptail_invalid_input() { # TODO: This should accept a list of arguments to spe } +whiptail_invalid_proxy() { + [ -n "$TESTING" ] && return + + local message + read -r -d '' message <<- EOM + Could not reach test url using proxy ${so_proxy}. + + Check log for details. + EOM + + whiptail --title "Security Onion Setup" --msgbox "$message" 10 60 +} + whiptail_invalid_string() { - [ -n "$TESTING" ] && return + [ -n "$TESTING" ] && return whiptail --title "Security Onion Setup" --msgbox "Invalid input, please try again.\n\nThe $1 cannot contain spaces." 9 45 From 7ee0fd63752a551139f84a6dc20082a8f199f42a Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 4 Mar 2021 11:31:22 -0500 Subject: [PATCH 14/27] [wip] Specify setup log location to user when directing them to it --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 56c0ecc5f..eaa8e8af1 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -595,7 +595,7 @@ whiptail_invalid_proxy() { read -r -d '' message <<- EOM Could not reach test url using proxy ${so_proxy}. - Check log for details. + Check log (${setup_log}) for details. EOM whiptail --title "Security Onion Setup" --msgbox "$message" 10 60 From 5a97341d332f222407b63d07dc98ca44339f17f2 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 4 Mar 2021 11:41:36 -0500 Subject: [PATCH 15/27] [wip] Fix how collect_proxy function works on retry --- setup/so-functions | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 861dd2e14..2b757bad6 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -537,7 +537,15 @@ collect_patch_schedule_name_import() { collect_proxy() { local ask=${1:-true} - if [[ $ask == true ]] && whiptail_proxy_ask; then + local use_proxy + if [[ $ask != true ]]; then + use_proxy=0 + else + whiptail_proxy_ask + use_proxy=$? + fi + + if [[ $use_proxy == 0 ]]; then whiptail_proxy_addr From 13dc8221978cc8b7e9811d938d2b216d0af486af Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 4 Mar 2021 11:53:08 -0500 Subject: [PATCH 16/27] [wip] Ask user if they want to re-enter the proxy --- setup/so-functions | 5 ++--- setup/so-setup | 8 ++++++-- setup/so-whiptail | 8 ++++---- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 2b757bad6..a3db35e41 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -551,12 +551,11 @@ collect_proxy() { while ! valid_proxy "$proxy_addr"; do whiptail_invalid_input - whiptail_proxy_addr + whiptail_proxy_addr "$proxy_addr" done if whiptail_proxy_auth_ask; then - whiptail_proxy_auth_user - + whiptail_proxy_auth_user "$proxy_user" whiptail_proxy_auth_pass local url_prefixes=( 'http://' 'https://' ) diff --git a/setup/so-setup b/setup/so-setup index 18bec9983..66542c6f3 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -301,8 +301,12 @@ if ! [[ -f $install_opt_file ]]; then if [[ $is_manager ]]; then collect_proxy while ! proxy_validate; do - whiptail_invalid_proxy - collect_proxy no_ask + if whiptail_invalid_proxy; then + collect_proxy no_ask + else + proxy_addr="" + break + fi done [[ -n "$proxy_addr" ]] && set_proxy >> $setup_log 2>&1 fi diff --git a/setup/so-whiptail b/setup/so-whiptail index eaa8e8af1..b7f6fad6c 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -593,12 +593,12 @@ whiptail_invalid_proxy() { local message read -r -d '' message <<- EOM - Could not reach test url using proxy ${so_proxy}. + Could not reach test url using proxy ${proxy_addr}. Check log (${setup_log}) for details. EOM - whiptail --title "Security Onion Setup" --msgbox "$message" 10 60 + whiptail --title "Security Onion Setup" --yesno "$message" --yestext "Enter Again" --notext "Skip" 10 60 } whiptail_invalid_string() { @@ -1244,7 +1244,7 @@ whiptail_proxy_addr() { If your proxy requires a username and password do not include them in your input. Setup will ask for those values next. EOM - proxy_addr=$(whiptail --title "Security Onion Setup" --inputbox "$message" 13 60 3>&1 1>&2 2>&3) + proxy_addr=$(whiptail --title "Security Onion Setup" --inputbox "$message" 13 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -1259,7 +1259,7 @@ whiptail_proxy_auth_ask() { whiptail_proxy_auth_user() { [ -n "$TESTING" ] && return - proxy_user=$(whiptail --title "Security Onion Setup" --inputbox "Please input the proxy user:" 8 60 3>&1 1>&2 2>&3) + proxy_user=$(whiptail --title "Security Onion Setup" --inputbox "Please input the proxy user:" 8 60 "$1" 3>&1 1>&2 2>&3) } whiptail_proxy_auth_pass() { From 2e56252f54ed327407b6ca88ff0179da6b733758 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 4 Mar 2021 11:54:21 -0500 Subject: [PATCH 17/27] [wip] Syntax fixes --- setup/so-functions | 2 +- setup/so-whiptail | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index a3db35e41..426d34b0d 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -556,7 +556,7 @@ collect_proxy() { if whiptail_proxy_auth_ask; then whiptail_proxy_auth_user "$proxy_user" - whiptail_proxy_auth_pass + whiptail_proxy_auth_pass # Don't pass in existing pass since it's obfuscated local url_prefixes=( 'http://' 'https://' ) for prefix in "${url_prefixes[@]}"; do diff --git a/setup/so-whiptail b/setup/so-whiptail index b7f6fad6c..f62d4c32b 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -598,7 +598,7 @@ whiptail_invalid_proxy() { Check log (${setup_log}) for details. EOM - whiptail --title "Security Onion Setup" --yesno "$message" --yestext "Enter Again" --notext "Skip" 10 60 + whiptail --title "Security Onion Setup" --yesno "$message" --yes-button "Enter Again" --no-button "Skip" 10 60 } whiptail_invalid_string() { From d19ca943ccaacbe204847d5fdcb073e0441df68c Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 4 Mar 2021 11:57:16 -0500 Subject: [PATCH 18/27] [fix][wip] Only setup proxy early on configure network setup --- setup/so-functions | 4 ---- setup/so-setup | 9 +++++---- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 426d34b0d..e082852e6 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1729,10 +1729,6 @@ network_init() { fi set_main_ip >> $setup_log 2>&1 compare_main_nic_ip - # Check if the proxy variable was set then set proxy for various apps if so - if [[ -n "$so_proxy" ]]; then - set_proxy >> $setup_log 2>&1 - fi } network_init_whiptail() { diff --git a/setup/so-setup b/setup/so-setup index 66542c6f3..0e00b1a60 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -308,7 +308,6 @@ if ! [[ -f $install_opt_file ]]; then break fi done - [[ -n "$proxy_addr" ]] && set_proxy >> $setup_log 2>&1 fi if [[ -n "$TURBO" ]]; then @@ -357,14 +356,14 @@ minion_type=$(get_minion_type) set_default_log_size >> $setup_log 2>&1 if [[ $is_helix ]]; then - RULESETUP=${RULESETUP:-ETOPEN} + RULESETUP=${RULESETUP:-ETOPEN} NSMSETUP=${NSMSETUP:-BASIC} HNSENSOR=${HNSENSOR:-inherit} MANAGERUPDATES=${MANAGERUPDATES:-0} fi if [[ $is_helix || ( $is_manager && $is_node ) ]]; then - RULESETUP=${RULESETUP:-ETOPEN} + RULESETUP=${RULESETUP:-ETOPEN} NSMSETUP=${NSMSETUP:-BASIC} fi @@ -384,7 +383,7 @@ fi if [[ $is_import ]]; then PATCHSCHEDULENAME=${PATCHSCHEDULENAME:-auto} MTU=${MTU:-1500} - RULESETUP=${RULESETUP:-ETOPEN} + RULESETUP=${RULESETUP:-ETOPEN} NSMSETUP=${NSMSETUP:-BASIC} HNSENSOR=${HNSENSOR:-inherit} MANAGERUPDATES=${MANAGERUPDATES:-0} @@ -560,6 +559,8 @@ catch() { # Set REDIRECTIT variable, which is used by a function outside the below subshell set_redirect >> $setup_log 2>&1 +[[ -n "$proxy_addr" ]] && set_proxy >> $setup_log 2>&1 + # Begin install { From 4339ded17f4ed27c0b39a62e8313a17f5624682a Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 4 Mar 2021 12:10:14 -0500 Subject: [PATCH 19/27] [wip][fix] Don't add logic to so-setup, create wrapper function in so-functions --- setup/so-functions | 12 ++++++++++++ setup/so-setup | 16 ++-------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index e082852e6..ff58bfc09 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -536,6 +536,18 @@ collect_patch_schedule_name_import() { } collect_proxy() { + collect_proxy_details + while ! proxy_validate; do + if whiptail_invalid_proxy; then + collect_proxy_details no_ask + else + so_proxy="" + break + fi + done +} + +collect_proxy_details() { local ask=${1:-true} local use_proxy if [[ $ask != true ]]; then diff --git a/setup/so-setup b/setup/so-setup index 0e00b1a60..8b42ca7b3 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -200,11 +200,7 @@ if ! [[ -f $install_opt_file ]]; then "HOSTNAME=$HOSTNAME" > "$net_init_file" if [[ $is_manager ]]; then collect_proxy - while ! proxy_validate; do - whiptail_invalid_proxy - collect_proxy no_ask - done - [[ -n "$proxy_addr" ]] && set_proxy >> $setup_log 2>&1 + [[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 fi whiptail_net_setup_complete else @@ -300,14 +296,6 @@ if ! [[ -f $install_opt_file ]]; then if [[ $is_manager ]]; then collect_proxy - while ! proxy_validate; do - if whiptail_invalid_proxy; then - collect_proxy no_ask - else - proxy_addr="" - break - fi - done fi if [[ -n "$TURBO" ]]; then @@ -559,7 +547,7 @@ catch() { # Set REDIRECTIT variable, which is used by a function outside the below subshell set_redirect >> $setup_log 2>&1 -[[ -n "$proxy_addr" ]] && set_proxy >> $setup_log 2>&1 +[[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1 # Begin install From b26585464478cbdf2870497bd92d34fd30866f16 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 4 Mar 2021 12:10:42 -0500 Subject: [PATCH 20/27] [wip] Move proxy config to separate file --- setup/so-functions | 6 +++--- setup/so-variables | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index ff58bfc09..26593cdaa 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2247,7 +2247,7 @@ set_main_ip() { # Add /usr/sbin to everyone's path set_path() { - echo "complete -cf sudo" >> "$profile_d_config_file" + echo "complete -cf sudo" >> /etc/profile.d/securityonion.sh } set_proxy() { @@ -2262,9 +2262,9 @@ set_proxy() { echo "export https_proxy=\"\$http_addr\"" echo "export ftp_proxy=\"\$http_addr\"" echo "export no_proxy=\"${no_proxy_string}\"" - } >> "$profile_d_config_file" + } > /etc/profile.d/so-proxy.sh - source "$profile_d_config_file" + source /etc/profile.d/so-proxy.sh [[ -d '/etc/systemd/system/docker.service.d' ]] || mkdir -p '/etc/systemd/system/docker.service.d' diff --git a/setup/so-variables b/setup/so-variables index cd6478fa7..a2fdf03c6 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -72,5 +72,3 @@ export install_opt_file net_init_file=/root/net_init export net_init_file - -export profile_d_config_file='/etc/profile.d/securityonion.sh' From 2ee8c7ad1c7fe00d66e48904a26d0ff85b577ce5 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 4 Mar 2021 12:16:23 -0500 Subject: [PATCH 21/27] [fix] Always pass $proxy_addr since we retry the surrounding function --- setup/so-functions | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 26593cdaa..0986c8899 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -558,8 +558,7 @@ collect_proxy_details() { fi if [[ $use_proxy == 0 ]]; then - - whiptail_proxy_addr + whiptail_proxy_addr "$proxy_addr" while ! valid_proxy "$proxy_addr"; do whiptail_invalid_input From a435ea77e830ef10762a7bb5b09aa7e397678f6f Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 4 Mar 2021 12:43:42 -0500 Subject: [PATCH 22/27] [fix] Also add hostname to no_proxy list --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 0986c8899..ea5720a03 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2252,7 +2252,7 @@ set_path() { set_proxy() { # Don't proxy localhost, local ip, and management ip - local no_proxy_string="localhost, 127.0.0.1, ${MAINIP}" + local no_proxy_string="localhost, 127.0.0.1, ${MAINIP}, ${HOSTNAME}" # Set proxy environment variables used by curl, wget, docker, and others { From b3c7760ad4210d7c63f43493bdfffeac951c8470 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 4 Mar 2021 14:08:21 -0500 Subject: [PATCH 23/27] [fix] Use correct variable in so-proxy.sh --- setup/so-functions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index ea5720a03..0b35c6e5f 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2258,8 +2258,8 @@ set_proxy() { { echo "export use_proxy=on" echo "export http_proxy=\"${so_proxy}\"" - echo "export https_proxy=\"\$http_addr\"" - echo "export ftp_proxy=\"\$http_addr\"" + echo "export https_proxy=\"\$http_proxy\"" + echo "export ftp_proxy=\"\$http_proxy\"" echo "export no_proxy=\"${no_proxy_string}\"" } > /etc/profile.d/so-proxy.sh From e0d0baafcc8453011dc21a61eb62477099c11008 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 4 Mar 2021 16:40:32 -0500 Subject: [PATCH 24/27] [fix] Permanently set proxy for yum using template --- salt/yum/etc/yum.conf.jinja | 2 ++ setup/so-functions | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/yum/etc/yum.conf.jinja b/salt/yum/etc/yum.conf.jinja index 5e1d30510..710b3968b 100644 --- a/salt/yum/etc/yum.conf.jinja +++ b/salt/yum/etc/yum.conf.jinja @@ -14,4 +14,6 @@ clean_requirements_on_remove=1 {% if (grains['role'] not in ['so-eval','so-managersearch', 'so-manager', 'so-standalone']) and salt['pillar.get']('global:managerupdate', '0') %} proxy=http://{{ salt['pillar.get']('yum:config:proxy', salt['config.get']('master')) }}:3142 +{% elif salt['pillar.get']('manager:proxy') is not None %} +proxy={{salt['pillar.get']('manager:proxy')}} {% endif %} diff --git a/setup/so-functions b/setup/so-functions index 0b35c6e5f..6d1c8cf27 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1481,6 +1481,7 @@ manager_pillar() { "manager:"\ " mainip: '$MAINIP'"\ " mainint: '$MNIC'"\ + " proxy: '$so_proxy'"\ " esheap: '$ES_HEAP_SIZE'"\ " esclustername: '{{ grains.host }}'"\ " freq: 0"\ @@ -1560,7 +1561,6 @@ manager_global() { " hnmanager: '$HNMANAGER'"\ " ntpserver: '$NTPSERVER'"\ " dockernet: '$DOCKERNET'"\ - " proxy: '$so_proxy'"\ " mdengine: '$ZEEKVERSION'"\ " ids: '$NIDS'"\ " url_base: '$REDIRECTIT'"\ From 33b2bd33fe7826a5a9c4966d0ebc83c4219a8bf9 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 4 Mar 2021 17:12:10 -0500 Subject: [PATCH 25/27] [fix] Also create config.json so containers use proxy --- setup/so-functions | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 6d1c8cf27..8057500be 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2265,7 +2265,7 @@ set_proxy() { source /etc/profile.d/so-proxy.sh - [[ -d '/etc/systemd/system/docker.service.d' ]] || mkdir -p '/etc/systemd/system/docker.service.d' + [[ -d '/etc/systemd/system/docker.service.d' ]] || mkdir -p /etc/systemd/system/docker.service.d # Create proxy config for dockerd printf '%s\n'\ @@ -2277,6 +2277,22 @@ set_proxy() { systemctl daemon-reload command -v docker &> /dev/null && systemctl restart docker + # Create config.json for docker containers + [[ -d /root/.docker ]] || mkdir /root/.docker + printf '%s\n'\ + "{"\ + " \"proxies\":"\ + " {"\ + " \"default\":"\ + " {"\ + " \"httpProxy\":\"${so_proxy}\","\ + " \"httpsProxy\":\"${so_proxy}\","\ + " \"ftpProxy\":\"${so_proxy}\","\ + " \"noProxy\":\"${no_proxy_string}\""\ + " }"\ + " }"\ + "}" > /root/.docker/config.json + # Set proxy for package manager if [ "$OS" = 'centos' ]; then echo "proxy=$so_proxy" >> /etc/yum.conf From ec04145d1525ea7be484c466849dc3b731299dd3 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 5 Mar 2021 11:34:31 -0500 Subject: [PATCH 26/27] [fix] Set proxy for idstools container manually --- salt/idstools/init.sls | 7 +++++++ setup/so-functions | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/salt/idstools/init.sls b/salt/idstools/init.sls index 2d3128797..f65f93a7a 100644 --- a/salt/idstools/init.sls +++ b/salt/idstools/init.sls @@ -19,6 +19,7 @@ {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} {% set ENGINE = salt['pillar.get']('global:mdengine', '') %} +{% set proxy = salt['pillar.get']('manager:proxy') %} # IDSTools Setup idstoolsdir: file.directory: @@ -71,6 +72,12 @@ so-idstools: - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-idstools:{{ VERSION }} - hostname: so-idstools - user: socore + {% if proxy is not none %} + - environment: + - http_proxy={{ proxy }} + - https_proxy={{ proxy }} + - no_proxy={{ salt['pillar.get']('manager:no_proxy') }} + {% endif %} - binds: - /opt/so/conf/idstools/etc:/opt/so/idstools/etc:ro - /opt/so/rules/nids:/opt/so/rules/nids:rw diff --git a/setup/so-functions b/setup/so-functions index 8057500be..54f22cc44 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1482,6 +1482,7 @@ manager_pillar() { " mainip: '$MAINIP'"\ " mainint: '$MNIC'"\ " proxy: '$so_proxy'"\ + " no_proxy: '$no_proxy_string'"\ " esheap: '$ES_HEAP_SIZE'"\ " esclustername: '{{ grains.host }}'"\ " freq: 0"\ @@ -2252,7 +2253,7 @@ set_path() { set_proxy() { # Don't proxy localhost, local ip, and management ip - local no_proxy_string="localhost, 127.0.0.1, ${MAINIP}, ${HOSTNAME}" + no_proxy_string="localhost, 127.0.0.1, ${MAINIP}, ${HOSTNAME}" # Set proxy environment variables used by curl, wget, docker, and others { From 245902326f5d21740a0d3f1682bfdce7495bd028 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 5 Mar 2021 12:53:20 -0500 Subject: [PATCH 27/27] [wip] Add automation support for proxy settings --- setup/automation/standalone-iso-proxy | 78 +++++++++++++++++++++++++++ setup/so-functions | 4 ++ 2 files changed, 82 insertions(+) create mode 100644 setup/automation/standalone-iso-proxy diff --git a/setup/automation/standalone-iso-proxy b/setup/automation/standalone-iso-proxy new file mode 100644 index 000000000..98bb6c2c6 --- /dev/null +++ b/setup/automation/standalone-iso-proxy @@ -0,0 +1,78 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +TESTING=true + +address_type=DHCP +ADMINUSER=onionuser +ADMINPASS1=onionuser +ADMINPASS2=onionuser +ALLOW_CIDR=0.0.0.0/0 +ALLOW_ROLE=a +BASICZEEK=2 +BASICSURI=2 +# BLOGS= +BNICS=eth1 +ZEEKVERSION=ZEEK +# CURCLOSEDAYS= +# EVALADVANCED=BASIC +GRAFANA=1 +# HELIXAPIKEY= +HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 +HNSENSOR=inherit +HOSTNAME=standalone +install_type=STANDALONE +# LSINPUTBATCHCOUNT= +# LSINPUTTHREADS= +# LSPIPELINEBATCH= +# LSPIPELINEWORKERS= +MANAGERADV=BASIC +MANAGERUPDATES=1 +# MDNS= +# MGATEWAY= +# MIP= +# MMASK= +MNIC=eth0 +# MSEARCH= +# MSRV= +# MTU= +NIDS=Suricata +# NODE_ES_HEAP_SIZE= +# NODE_LS_HEAP_SIZE= +NODESETUP=NODEBASIC +NSMSETUP=BASIC +NODEUPDATES=MANAGER +# OINKCODE= +OSQUERY=1 +# PATCHSCHEDULEDAYS= +# PATCHSCHEDULEHOURS= +PATCHSCHEDULENAME=auto +PLAYBOOK=1 +proxy_addr=http://10.66.166.30:3128 +# REDIRECTHOST= +REDIRECTINFO=IP +RULESETUP=ETOPEN +# SHARDCOUNT= +# SKIP_REBOOT= +SOREMOTEPASS1=onionuser +SOREMOTEPASS2=onionuser +STRELKA=1 +THEHIVE=1 +WAZUH=1 +WEBUSER=onionuser@somewhere.invalid +WEBPASSWD1=0n10nus3r +WEBPASSWD2=0n10nus3r diff --git a/setup/so-functions b/setup/so-functions index 54f22cc44..d677fe9cd 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1835,6 +1835,10 @@ proxy_validate() { if [[ $ret != 0 ]]; then error "Could not reach $test_url using proxy $so_proxy" + if [[ -n $TESTING ]]; then + error "Exiting setup" + kill -SIGINT "$(ps --pid $$ -oppid=)"; exit 1 + fi fi return $ret }