From 79c7af9a31c50058dc13e089654c33419f0d27a5 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 15 Mar 2021 10:48:24 -0400 Subject: [PATCH 01/10] soup for 2.3.40 --- salt/common/tools/sbin/soup | 60 ++++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 21 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 264f1b603..996f0354f 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -19,6 +19,7 @@ UPDATE_DIR=/tmp/sogh/securityonion INSTALLEDVERSION=$(cat /etc/soversion) +POSTVERSION=$INSTALLEDVERSION INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk {'print $2'}) DEFAULT_SALT_DIR=/opt/so/saltstack/default BATCHSIZE=5 @@ -243,18 +244,6 @@ masterunlock() { fi } -playbook() { - echo "Applying playbook settings" - if [[ "$INSTALLEDVERSION" =~ rc.1 ]]; then - salt-call state.apply playbook.OLD_db_init - rm -f /opt/so/rules/elastalert/playbook/*.yaml - so-playbook-ruleupdate >> /root/soup_playbook_rule_update.log 2>&1 & - fi - if [[ "$INSTALLEDVERSION" != 2.3.30 ]]; then - so-playbook-sigma-refresh >> /root/soup_playbook_sigma_refresh.log 2>&1 & - fi -} - pillar_changes() { # This function is to add any new pillar items if needed. echo "Checking to see if pillar changes are needed." @@ -266,6 +255,26 @@ pillar_changes() { [[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30 } +post_changes() { + # This function is to add any new pillar items if needed. + echo "Checking to see if pillar changes are needed." + + [[ "$POSTVERSION" =~ rc.1 ]] && post_rc1_to_rc2 + [[ "$POSTVERSION" == 2.3.20 || "$POSTVERSION" == 2.3.21 ]] && post_2.3.2X_to_2.3.30 +} + +post_rc1_to_2.3.21() { + salt-call state.apply playbook.OLD_db_init + rm -f /opt/so/rules/elastalert/playbook/*.yaml + so-playbook-ruleupdate >> /root/soup_playbook_rule_update.log 2>&1 & + POSTVERSION=2.3.21 +} + +post_2.3.2X_to_2.3.30() { + so-playbook-sigma-refresh >> /root/soup_playbook_sigma_refresh.log 2>&1 & + POSTVERSION=2.3.30 +} + rc1_to_rc2() { # Move the static file to global.sls @@ -410,16 +419,25 @@ up_2.3.2X_to_2.3.30() { check_log_size_limit } -space_check() { - # Check to see if there is enough space +verify_upgradespace() { CURRENTSPACE=$(df -BG / | grep -v Avail | awk '{print $4}' | sed 's/.$//') if [ "$CURRENTSPACE" -lt "10" ]; then - echo "You are low on disk space. Upgrade will try and clean up space."; - clean_dockers + echo "You are low on disk space."; + return 1 else - echo "Plenty of space for upgrading" - fi - + return 0 +} + +upgrade_space() { + if ! verify_upgradespace; then + clean_dockers + if ! verify_upgradespace; then + echo "There is not enough space to perform the upgrade. Please free up space and try again" + exit 1 + fi + else + echo "You have enough space for upgrade. Proceeding with soup." + fi } thehive_maint() { @@ -599,7 +617,7 @@ fi echo "Let's see if we need to update Security Onion." upgrade_check -space_check +upgrade_space echo "Checking for Salt Master and Minion updates." upgrade_check_salt @@ -704,7 +722,7 @@ echo "Starting Salt Master service." systemctl start salt-master echo "Running a highstate. This could take several minutes." salt-call state.highstate -l info queue=True -playbook +post_changes unmount_update thehive_maint From ab75d0e5634ad13582a006f14a6ff0dad5a556bb Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 15 Mar 2021 10:51:31 -0400 Subject: [PATCH 02/10] soup for 2.3.40 --- salt/common/tools/sbin/soup | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 996f0354f..c70726210 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -244,10 +244,10 @@ masterunlock() { fi } -pillar_changes() { +preupgrade_changes() { # This function is to add any new pillar items if needed. - echo "Checking to see if pillar changes are needed." - + echo "Checking to see if changes are needed." + [[ "$INSTALLEDVERSION" =~ rc.1 ]] && rc1_to_rc2 [[ "$INSTALLEDVERSION" =~ rc.2 ]] && rc2_to_rc3 [[ "$INSTALLEDVERSION" =~ rc.3 ]] && rc3_to_2.3.0 @@ -255,9 +255,9 @@ pillar_changes() { [[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30 } -post_changes() { +postupgrade_changes() { # This function is to add any new pillar items if needed. - echo "Checking to see if pillar changes are needed." + echo "Running post upgrade processes." [[ "$POSTVERSION" =~ rc.1 ]] && post_rc1_to_rc2 [[ "$POSTVERSION" == 2.3.20 || "$POSTVERSION" == 2.3.21 ]] && post_2.3.2X_to_2.3.30 @@ -667,8 +667,7 @@ else echo "" fi -echo "Making pillar changes." -pillar_changes +preupgrade_changes echo "" if [ $is_airgap -eq 0 ]; then @@ -722,7 +721,7 @@ echo "Starting Salt Master service." systemctl start salt-master echo "Running a highstate. This could take several minutes." salt-call state.highstate -l info queue=True -post_changes +postupgrade_changes unmount_update thehive_maint From 67e0d450e4c896a48d8fef869d8edd3a402ed83b Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 15 Mar 2021 17:32:36 -0400 Subject: [PATCH 03/10] Add Elastic License Prompts --- salt/common/tools/sbin/so-common | 58 ++++++++++++++++++++++++++++++++ salt/common/tools/sbin/soup | 15 ++------- setup/so-setup | 5 ++- 3 files changed, 62 insertions(+), 16 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index b76ad05ff..a62e654cc 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -86,6 +86,19 @@ add_interface_bond0() { fi } +check_airgap() { + # See if this is an airgap install + AIRGAP=$(cat /opt/so/saltstack/local/pillar/global.sls | grep airgap: | awk '{print $2}') + if [[ "$AIRGAP" == "True" ]]; then + is_airgap=0 + UPDATE_DIR=/tmp/soagupdate/SecurityOnion + AGDOCKER=/tmp/soagupdate/docker + AGREPO=/tmp/soagupdate/Packages + else + is_airgap=1 + fi +} + check_container() { docker ps | grep "$1:" > /dev/null 2>&1 return $? @@ -97,6 +110,45 @@ check_password() { return $? } +check_elastic_license() { + + [ -n "$TESTING" ] && return + + # See if the user has already accepted the license + if [ ! -f /opt/so/state/yeselastic.txt ] + elastic_license + else + echo "Elastic License has already been accepted" + fi +} + +elastic_license() { + +read -r -d '' message <<- EOM +\n +Security Onion now uses the Elastic Stack binaries that fall under the Elastic license. + +Please review the Elastic license: +https://raw.githubusercontent.com/elastic/elasticsearch/master/licenses/ELASTIC-LICENSE.txt + +Do you agree to the terms of the Elastic license? + +If so, type AGREE to accept the Elastic license and continue. Otherwise, press Enter to exit this program without making any changes. +EOM + +AGREED=$(whiptail --title "Security Onion Setup" --inputbox \ +"$message" 20 75) + +if [ "$AGREED" = 'AGREE' ]; then + mkdir -p /opt/so/state + touch /opt/so/state/yeselastic.txt +else + echo "Starting in 2.3.40 you must accept the Elastic license if you want to run Security Onion." + exit 1 +fi + +} + fail() { msg=$1 echo "ERROR: $msg" @@ -250,6 +302,12 @@ set_minionid() { MINIONID=$(lookup_grain id) } +set_palette() { + if [ "$OS" == ubuntu ]; then + update-alternatives --set newt-palette /etc/newt/palette.original + fi +} + set_version() { CURRENTVERSION=0.0.0 if [ -f /etc/soversion ]; then diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index c70726210..2a121d602 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -102,19 +102,6 @@ update_registry() { salt-call state.apply registry queue=True } -check_airgap() { - # See if this is an airgap install - AIRGAP=$(cat /opt/so/saltstack/local/pillar/global.sls | grep airgap: | awk '{print $2}') - if [[ "$AIRGAP" == "True" ]]; then - is_airgap=0 - UPDATE_DIR=/tmp/soagupdate/SecurityOnion - AGDOCKER=/tmp/soagupdate/docker - AGREPO=/tmp/soagupdate/Packages - else - is_airgap=1 - fi -} - check_sudoers() { if grep -q "so-setup" /etc/sudoers; then echo "There is an entry for so-setup in the sudoers file, this can be safely deleted using \"visudo\"." @@ -591,6 +578,8 @@ check_airgap echo "Found that Security Onion $INSTALLEDVERSION is currently installed." echo "" set_os +set_palette +check_elastic_license echo "" if [ $is_airgap -eq 0 ]; then # Let's mount the ISO since this is airgap diff --git a/setup/so-setup b/setup/so-setup index 0af49af53..eaf7c7657 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -156,10 +156,8 @@ export PATH=$PATH:$local_sbin installer_prereq_packages && detect_cloud set_network_dev_status_list +set_palette >> $setup_log 2>&1 -if [ "$OS" == ubuntu ]; then - update-alternatives --set newt-palette /etc/newt/palette.original >> $setup_log 2>&1 -fi # Kernel messages can overwrite whiptail screen #812 # https://github.com/Security-Onion-Solutions/securityonion/issues/812 @@ -251,6 +249,7 @@ fi # Check if this is an airgap install if [[ ( $is_manager || $is_import ) && $is_iso ]]; then + check_elastic_license whiptail_airgap if [[ "$INTERWEBS" == 'AIRGAP' ]]; then is_airgap=true From 72759de97fdccc0ad0fda504d4deaced971f520f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 15 Mar 2021 17:37:44 -0400 Subject: [PATCH 04/10] Fix so-common syntax --- salt/common/tools/sbin/so-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index a62e654cc..99b41e659 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -115,7 +115,7 @@ check_elastic_license() { [ -n "$TESTING" ] && return # See if the user has already accepted the license - if [ ! -f /opt/so/state/yeselastic.txt ] + if [ ! -f /opt/so/state/yeselastic.txt ]; then elastic_license else echo "Elastic License has already been accepted" From 12501e00796904513881980ce43e5f7fbeef66a5 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 15 Mar 2021 17:41:45 -0400 Subject: [PATCH 05/10] Add check license to its own logic --- setup/so-setup | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index eaf7c7657..e9039f164 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -249,13 +249,16 @@ fi # Check if this is an airgap install if [[ ( $is_manager || $is_import ) && $is_iso ]]; then - check_elastic_license whiptail_airgap if [[ "$INTERWEBS" == 'AIRGAP' ]]; then is_airgap=true fi fi +if [[ $is_manager || $is_import ]]; then + check_elastic_license +fi + if ! [[ -f $install_opt_file ]]; then if [[ $is_manager && $is_sensor ]]; then check_requirements "standalone" From 9a4c4448f31e4f7efa9f20122671afc0f2d36fd8 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 15 Mar 2021 17:45:44 -0400 Subject: [PATCH 06/10] Fix whiptail display --- salt/common/tools/sbin/so-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 99b41e659..22a21de95 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -137,7 +137,7 @@ If so, type AGREE to accept the Elastic license and continue. Otherwise, press EOM AGREED=$(whiptail --title "Security Onion Setup" --inputbox \ -"$message" 20 75) +"$message" 20 75 3>&1 1>&2 2>&3) if [ "$AGREED" = 'AGREE' ]; then mkdir -p /opt/so/state From 8d23518f9010f91d4c7beb87b401c7c9a08b4779 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 15 Mar 2021 17:50:06 -0400 Subject: [PATCH 07/10] Update Elastic Link --- salt/common/tools/sbin/so-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 22a21de95..1f8b3d54c 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -129,7 +129,7 @@ read -r -d '' message <<- EOM Security Onion now uses the Elastic Stack binaries that fall under the Elastic license. Please review the Elastic license: -https://raw.githubusercontent.com/elastic/elasticsearch/master/licenses/ELASTIC-LICENSE.txt +https://www.elastic.co/licensing/elastic-license Do you agree to the terms of the Elastic license? From 00025e5c743c3a040b0485670d0b78f090da9ef6 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 16 Mar 2021 09:34:53 -0400 Subject: [PATCH 08/10] Fix Syntax Error --- salt/common/tools/sbin/soup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 2a121d602..1a877c2e4 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -409,10 +409,11 @@ up_2.3.2X_to_2.3.30() { verify_upgradespace() { CURRENTSPACE=$(df -BG / | grep -v Avail | awk '{print $4}' | sed 's/.$//') if [ "$CURRENTSPACE" -lt "10" ]; then - echo "You are low on disk space."; + echo "You are low on disk space." return 1 else return 0 + fi } upgrade_space() { From 2851840e76c55a660d84ca44faff386b8c754ce4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 16 Mar 2021 12:18:01 -0400 Subject: [PATCH 09/10] Fix Logging --- salt/common/tools/sbin/soup | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 1a877c2e4..701cebafc 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -25,8 +25,6 @@ DEFAULT_SALT_DIR=/opt/so/saltstack/default BATCHSIZE=5 SOUP_LOG=/root/soup.log -exec 3>&1 1>${SOUP_LOG} 2>&1 - add_common() { cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ @@ -554,6 +552,7 @@ verify_latest_update_script() { } main () { +echo "### Preparing soup at `date` ###" while getopts ":b" opt; do case "$opt" in b ) # process option b @@ -758,7 +757,7 @@ For more information, please see https://docs.securityonion.net/en/2.3/soup.html EOF fi - +echo "### soup has been served at `date` ###" } -main "$@" | tee /dev/fd/3 +main "$@" | tee -a $SOUP_LOG From e528d84ebee1bdd84bedf3f81db854b8261e33bc Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 16 Mar 2021 12:56:06 -0400 Subject: [PATCH 10/10] Update Elastic License Text --- salt/common/tools/sbin/so-common | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 1f8b3d54c..4a934d099 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -126,14 +126,15 @@ elastic_license() { read -r -d '' message <<- EOM \n -Security Onion now uses the Elastic Stack binaries that fall under the Elastic license. +Starting in Elastic Stack version 7.11, the Elastic Stack binaries are only available under the Elastic License: +https://blog.securityonion.net/2021/02/elastic-license-changes-and-security.html -Please review the Elastic license: +Please review the Elastic License: https://www.elastic.co/licensing/elastic-license -Do you agree to the terms of the Elastic license? +Do you agree to the terms of the Elastic License? -If so, type AGREE to accept the Elastic license and continue. Otherwise, press Enter to exit this program without making any changes. +If so, type AGREE to accept the Elastic License and continue. Otherwise, press Enter to exit this program without making any changes. EOM AGREED=$(whiptail --title "Security Onion Setup" --inputbox \