From 9d674d6d3a55b34418da525b63872cae3aa5eaa6 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 4 Jan 2021 13:35:14 -0500 Subject: [PATCH 01/56] [feat] Add so-monitor-add script --- salt/common/tools/sbin/so-common | 47 ++++++++++++++++++ salt/common/tools/sbin/so-monitor-add | 7 +++ setup/so-functions | 69 +++++++-------------------- setup/so-setup | 12 ++--- 4 files changed, 76 insertions(+), 59 deletions(-) create mode 100644 salt/common/tools/sbin/so-monitor-add diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index c71e9150c..a012f2ed9 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -21,6 +21,53 @@ if [ "$(id -u)" -ne 0 ]; then exit 1 fi +add_interface_bond0() { + local BNIC=$1 + if [[ -z $MTU ]]; then + local MTU + MTU=$(lookup_pillar "mtu" "sensor") + fi + local nic_error=0 + + # Check if specific offload features are able to be disabled + for string in "generic-segmentation-offload" "generic-receive-offload" "tcp-segmentation-offload"; do + if ethtool -k "$BNIC" | grep $string | grep -q "on [fixed]"; then + echo "The hardware or driver for interface ${BNIC} is not supported, packet capture may not work as expected." + ((nic_error++)) + break + fi + done + + for i in rx tx sg tso ufo gso gro lro; do + ethtool -K "$BNIC" $i off + done + # Check if the bond slave connection has already been created + nmcli -f name,uuid -p con | grep -q "bond0-slave-$BNIC" + local found_int=$? + + if [[ $found_int != 0 ]]; then + # Create the slave interface and assign it to the bond + nmcli con add type ethernet ifname "$BNIC" con-name "bond0-slave-$BNIC" master bond0 -- \ + ethernet.mtu "$MTU" \ + connection.autoconnect "yes" + else + local int_uuid + int_uuid=$(nmcli -f name,uuid -p con | sed -n "s/bond0-slave-$BNIC //p" | tr -d ' ') + + nmcli con mod "$int_uuid" \ + ethernet.mtu "$MTU" \ + connection.autoconnect "yes" + fi + + ip link set dev "$BNIC" arp off multicast off allmulticast off promisc on + + nmcli con up "bond0-slave-$BNIC" # Bring the slave interface up + + if [ "$nic_error" != 0 ]; then + return "$nic_error" + fi +} + # Define a banner to separate sections banner="=========================================================================" diff --git a/salt/common/tools/sbin/so-monitor-add b/salt/common/tools/sbin/so-monitor-add new file mode 100644 index 000000000..7eb100ee8 --- /dev/null +++ b/salt/common/tools/sbin/so-monitor-add @@ -0,0 +1,7 @@ +#!/bin/bash + +. /usr/sbin/so-common + +set -e + +add_interface_bond0 "$1" diff --git a/setup/so-functions b/setup/so-functions index d0e502941..402afa298 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -572,31 +572,27 @@ compare_versions() { configure_network_sensor() { echo "Setting up sensor interface" >> "$setup_log" 2>&1 - local nic_error=0 - - # Set the MTU - if [[ $NSMSETUP != 'ADVANCED' ]]; then - if [[ $is_cloud ]]; then MTU=1575; else MTU=1500; fi - fi if [[ $is_cloud ]]; then INTERFACE=${BNICS[0]} - local nmcli_con_arg="type ethernet" + local nmcli_con_args=( "type" "ethernet" ) else INTERFACE='bond0' - local nmcli_con_arg="type bond mode 0" + local nmcli_con_args=( "type" "bond" "mode" "0" ) fi + local MTU + MTU=$(lookup_pillar "mtu" "sensor") + # Create the bond interface only if it doesn't already exist - nmcli -f name,uuid -p con | grep -q "$INTERFACE" >> "$setup_log" 2>&1 local found_int=$? if [[ $found_int != 0 ]]; then - nmcli con add ifname "$INTERFACE" con-name "$INTERFACE" $nmcli_con_arg -- \ + nmcli con add ifname "$INTERFACE" con-name "$INTERFACE" "${nmcli_con_args[@]}" -- \ ipv4.method disabled \ ipv6.method ignore \ - ethernet.mtu $MTU \ + ethernet.mtu "$MTU" \ connection.autoconnect "yes" >> "$setup_log" 2>&1 else local int_uuid @@ -605,53 +601,14 @@ configure_network_sensor() { nmcli con mod "$int_uuid" \ ipv4.method disabled \ ipv6.method ignore \ - ethernet.mtu $MTU \ + ethernet.mtu "$MTU" \ connection.autoconnect "yes" >> "$setup_log" 2>&1 fi for BNIC in "${BNICS[@]}"; do - # Check if specific offload features are able to be disabled - for string in "generic-segmentation-offload" "generic-receive-offload" "tcp-segmentation-offload"; do - if ethtool -k "$BNIC" | grep $string | grep -q "on [fixed]"; then - echo "The hardware or driver for interface ${BNIC} is not supported, packet capture may not work as expected." >> "$setup_log" 2>&1 - nic_error=1 - break - fi - done - - # Turn off various offloading settings for the interface - for i in rx tx sg tso ufo gso gro lro; do - ethtool -K "$BNIC" $i off >> "$setup_log" 2>&1 - done - - if [[ $is_cloud ]]; then - nmcli con up "$BNIC" >> "$setup_log" 2>&1 - else - # Check if the bond slave connection has already been created - nmcli -f name,uuid -p con | grep -q "bond0-slave-$BNIC" >> "$setup_log" 2>&1 - local found_int=$? - - if [[ $found_int != 0 ]]; then - # Create the slave interface and assign it to the bond - nmcli con add type ethernet ifname "$BNIC" con-name "bond0-slave-$BNIC" master bond0 -- \ - ethernet.mtu $MTU \ - connection.autoconnect "yes" >> "$setup_log" 2>&1 - else - local int_uuid - int_uuid=$(nmcli -f name,uuid -p con | sed -n "s/bond0-slave-$BNIC //p" | tr -d ' ') - - nmcli con mod "$int_uuid" \ - ethernet.mtu $MTU \ - connection.autoconnect "yes" >> "$setup_log" 2>&1 - fi - - nmcli con up "bond0-slave-$BNIC" >> "$setup_log" 2>&1 # Bring the slave interface up - fi + add_interface_bond0 "$BNIC" >> "$setup_log" 2>&1 + return $? done - - if [ $nic_error != 0 ]; then - return 1 - fi } copy_salt_master_config() { @@ -1980,6 +1937,11 @@ sensor_pillar() { local pillar_file=$temp_install_dir/pillar/minions/$MINION_ID.sls + # Set the MTU + if [[ $NSMSETUP != 'ADVANCED' ]]; then + if [[ $is_cloud ]]; then MTU=1575; else MTU=1500; fi + fi + # Create the sensor pillar printf '%s\n'\ "sensor:"\ @@ -2012,6 +1974,7 @@ sensor_pillar() { if [ "$HNSENSOR" != 'inherit' ]; then echo " hnsensor: $HNSENSOR" >> "$pillar_file" fi + } set_default_log_size() { diff --git a/setup/so-setup b/setup/so-setup index 686ae52e6..c792dbc55 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -559,19 +559,19 @@ set_redirect >> $setup_log 2>&1 set_progress_str 2 'Updating packages' update_packages >> $setup_log 2>&1 - if [[ $is_sensor || $is_helix ]]; then - set_progress_str 3 'Configuring sensor interface' - configure_network_sensor >> $setup_log 2>&1 - fi - if [[ $is_sensor || $is_helix || $is_import ]]; then - set_progress_str 4 'Generating sensor pillar' + set_progress_str 3 'Generating sensor pillar' sensor_pillar >> $setup_log 2>&1 if [[ $is_sensor || $is_helix ]]; then steno_pillar >> $setup_log fi fi + if [[ $is_sensor || $is_helix ]]; then + set_progress_str 4 'Configuring sensor interface' + configure_network_sensor >> $setup_log 2>&1 + fi + set_progress_str 5 'Installing Salt and dependencies' saltify 2>> $setup_log From fb28faa4e3f3194d34420808f011f0f0d23faba2 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 29 Dec 2020 10:34:27 -0500 Subject: [PATCH 02/56] Monitor interface will not always be bond0 - pull correct value from pillar; Replay test data after automated test installations complete. --- salt/common/tools/sbin/so-tcpreplay | 6 +++--- setup/so-setup | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/so-tcpreplay b/salt/common/tools/sbin/so-tcpreplay index 4cd473c0c..a5c75c267 100755 --- a/salt/common/tools/sbin/so-tcpreplay +++ b/salt/common/tools/sbin/so-tcpreplay @@ -20,9 +20,11 @@ . /usr/sbin/so-common . /usr/sbin/so-image-common -REPLAYIFACE=${REPLAYIFACE:-bond0} +REPLAYIFACE=${REPLAYIFACE:-$(lookup_pillar interface sensor)} REPLAYSPEED=${REPLAYSPEED:-10} +mkdir -p /opt/so/samples + if [[ $# -lt 1 ]]; then echo "Replays one or more PCAP sample files to the Security Onion monitoring interface." echo @@ -48,8 +50,6 @@ if ! docker ps | grep -q so-tcpreplay; then TRUSTED_CONTAINERS=("so-tcpreplay") update_docker_containers "tcpreplay" so-tcpreplay-start || fail "Unable to initialize tcpreplay" - mkdir -p /opt/so/samples - docker cp so-tcpreplay:/opt/samples/* /opt/so/samples fi echo "Replaying PCAP(s) at ${REPLAYSPEED} Mbps on interface ${REPLAYIFACE}..." diff --git a/setup/so-setup b/setup/so-setup index c792dbc55..37e167d1d 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -840,6 +840,8 @@ if [[ -n $SO_ERROR ]]; then else echo "Successfully completed setup! Continuing with post-installation steps" >> $setup_log 2>&1 { + [ -n "$TESTING" ] && so-test + export percentage=95 # set to last percentage used in previous subshell if [[ -n $ALLOW_ROLE && -n $ALLOW_CIDR ]]; then set_progress_str 97 "Running so-allow -${ALLOW_ROLE} for ${ALLOW_CIDR}" From 229657f7d22940c42c776ecf3296bfee53a99763 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 29 Dec 2020 11:16:57 -0500 Subject: [PATCH 03/56] Use AMI's public IP for external access --- setup/automation/aws_standalone_defaults | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/automation/aws_standalone_defaults b/setup/automation/aws_standalone_defaults index db199986b..6c00cc789 100644 --- a/setup/automation/aws_standalone_defaults +++ b/setup/automation/aws_standalone_defaults @@ -62,7 +62,7 @@ OSQUERY=1 # PATCHSCHEDULEHOURS= PATCHSCHEDULENAME=auto PLAYBOOK=1 -REDIRECTHOST=securityonion +REDIRECTHOST=$(curl http://169.254.169.254/latest/meta-data/public-ipv4) REDIRECTINFO=OTHER RULESETUP=ETOPEN # SHARDCOUNT= From fc4447451974868e53368c16b8fc907daf4c9278 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 29 Dec 2020 16:15:10 -0500 Subject: [PATCH 04/56] Add eval automation --- setup/automation/pm_eval_defaults | 77 +++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 setup/automation/pm_eval_defaults diff --git a/setup/automation/pm_eval_defaults b/setup/automation/pm_eval_defaults new file mode 100644 index 000000000..a2acf0457 --- /dev/null +++ b/setup/automation/pm_eval_defaults @@ -0,0 +1,77 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +TESTING=true + +address_type=DHCP +ADMINUSER=onionuser +ADMINPASS1=onionuser +ADMINPASS2=onionuser +ALLOW_CIDR=0.0.0.0/0 +ALLOW_ROLE=a +BASICZEEK=7 +BASICSURI=7 +# BLOGS= +BNICS=eth1 +ZEEKVERSION=ZEEK +# CURCLOSEDAYS= +# EVALADVANCED=BASIC +GRAFANA=1 +# HELIXAPIKEY= +HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 +HNSENSOR=inherit +HOSTNAME=eval +install_type=EVAL +# LSINPUTBATCHCOUNT= +# LSINPUTTHREADS= +# LSPIPELINEBATCH= +# LSPIPELINEWORKERS= +MANAGERADV=BASIC +MANAGERUPDATES=1 +# MDNS= +# MGATEWAY= +# MIP= +# MMASK= +MNIC=eth0 +# MSEARCH= +# MSRV= +# MTU= +NIDS=Suricata +# NODE_ES_HEAP_SIZE= +# NODE_LS_HEAP_SIZE= +NODESETUP=NODEBASIC +NSMSETUP=BASIC +NODEUPDATES=MANAGER +# OINKCODE= +OSQUERY=1 +# PATCHSCHEDULEDAYS= +# PATCHSCHEDULEHOURS= +PATCHSCHEDULENAME=auto +PLAYBOOK=1 +# REDIRECTHOST= +REDIRECTINFO=IP +RULESETUP=ETOPEN +# SHARDCOUNT= +SKIP_REBOOT=1 +SOREMOTEPASS1=onionuser +SOREMOTEPASS2=onionuser +STRELKA=1 +THEHIVE=1 +WAZUH=1 +WEBUSER=onionuser@somewhere.invalid +WEBPASSWD1=0n10nus3r +WEBPASSWD2=0n10nus3r From 9b40318bfe1ea3dc1e76442a9e95f6b59cf5fed2 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 29 Dec 2020 17:25:53 -0500 Subject: [PATCH 05/56] Ensure so-test is logged --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 37e167d1d..01f8991e4 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -840,7 +840,7 @@ if [[ -n $SO_ERROR ]]; then else echo "Successfully completed setup! Continuing with post-installation steps" >> $setup_log 2>&1 { - [ -n "$TESTING" ] && so-test + [ -n "$TESTING" ] && logCmd so-setup export percentage=95 # set to last percentage used in previous subshell if [[ -n $ALLOW_ROLE && -n $ALLOW_CIDR ]]; then From f20feabda27f6b1c2c27ae61c11f53aefc79c710 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 29 Dec 2020 20:42:50 -0500 Subject: [PATCH 06/56] Reboot to ensure thehive falls in line before kicking off the test --- setup/automation/aws_eval_defaults | 2 +- setup/automation/aws_standalone_defaults | 2 +- setup/automation/pm_eval_defaults | 2 +- setup/automation/pm_standalone_defaults | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/setup/automation/aws_eval_defaults b/setup/automation/aws_eval_defaults index e038bf29d..2c5a8a52d 100644 --- a/setup/automation/aws_eval_defaults +++ b/setup/automation/aws_eval_defaults @@ -66,7 +66,7 @@ PLAYBOOK=1 REDIRECTINFO=HOSTNAME RULESETUP=ETOPEN # SHARDCOUNT= -SKIP_REBOOT=0 +# SKIP_REBOOT=0 SOREMOTEPASS1=onionuser SOREMOTEPASS2=onionuser STRELKA=1 diff --git a/setup/automation/aws_standalone_defaults b/setup/automation/aws_standalone_defaults index 6c00cc789..d32e1fad7 100644 --- a/setup/automation/aws_standalone_defaults +++ b/setup/automation/aws_standalone_defaults @@ -66,7 +66,7 @@ REDIRECTHOST=$(curl http://169.254.169.254/latest/meta-data/public-ipv4) REDIRECTINFO=OTHER RULESETUP=ETOPEN # SHARDCOUNT= -SKIP_REBOOT=0 +# SKIP_REBOOT= SOREMOTEPASS1=onionuser SOREMOTEPASS2=onionuser STRELKA=1 diff --git a/setup/automation/pm_eval_defaults b/setup/automation/pm_eval_defaults index a2acf0457..6e5560028 100644 --- a/setup/automation/pm_eval_defaults +++ b/setup/automation/pm_eval_defaults @@ -66,7 +66,7 @@ PLAYBOOK=1 REDIRECTINFO=IP RULESETUP=ETOPEN # SHARDCOUNT= -SKIP_REBOOT=1 +# SKIP_REBOOT= SOREMOTEPASS1=onionuser SOREMOTEPASS2=onionuser STRELKA=1 diff --git a/setup/automation/pm_standalone_defaults b/setup/automation/pm_standalone_defaults index d7bc1ea1f..0561a2883 100644 --- a/setup/automation/pm_standalone_defaults +++ b/setup/automation/pm_standalone_defaults @@ -66,7 +66,7 @@ PLAYBOOK=1 REDIRECTINFO=IP RULESETUP=ETOPEN # SHARDCOUNT= -SKIP_REBOOT=1 +# SKIP_REBOOT= SOREMOTEPASS1=onionuser SOREMOTEPASS2=onionuser STRELKA=1 From ebb0e615b9fefd5b95f3403bc0e11160fe0fe51f Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 30 Dec 2020 10:31:04 -0500 Subject: [PATCH 07/56] Fix script typo to correctly run the so-test --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 01f8991e4..e76510292 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -840,7 +840,7 @@ if [[ -n $SO_ERROR ]]; then else echo "Successfully completed setup! Continuing with post-installation steps" >> $setup_log 2>&1 { - [ -n "$TESTING" ] && logCmd so-setup + [ -n "$TESTING" ] && logCmd so-test export percentage=95 # set to last percentage used in previous subshell if [[ -n $ALLOW_ROLE && -n $ALLOW_CIDR ]]; then From 10d04f760da27d950bdab2ce460916c8c92f25e6 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 30 Dec 2020 12:02:42 -0500 Subject: [PATCH 08/56] Use manager internal IP for intra-service comms --- salt/soctopus/files/SOCtopus.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soctopus/files/SOCtopus.conf b/salt/soctopus/files/SOCtopus.conf index 29f31f95f..4f58ecf83 100644 --- a/salt/soctopus/files/SOCtopus.conf +++ b/salt/soctopus/files/SOCtopus.conf @@ -1,4 +1,4 @@ -{%- set MANAGER = salt['pillar.get']('global:url_base', '') %} +{%- set MANAGER = salt['pillar.get']('manager:mainip', '') %} {%- set URLBASE = salt['pillar.get']('global:url_base', '') %} {%- set HIVEKEY = salt['pillar.get']('global:hivekey', '') %} {%- set CORTEXKEY = salt['pillar.get']('global:cortexorguserkey', '') %} From bdbc63785234646afb5b7c3113ff4443a6a2a3e6 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 30 Dec 2020 16:33:46 -0500 Subject: [PATCH 09/56] Stop SOC prior to opening the firewall for analysts, this ensures no outside requests can be processed prior to the server rebooting --- setup/so-setup | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-setup b/setup/so-setup index e76510292..d9bc03444 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -843,6 +843,7 @@ else [ -n "$TESTING" ] && logCmd so-test export percentage=95 # set to last percentage used in previous subshell + so-soc-stop # Stop SOC so it doesn't accept external requests prior to the reboot if [[ -n $ALLOW_ROLE && -n $ALLOW_CIDR ]]; then set_progress_str 97 "Running so-allow -${ALLOW_ROLE} for ${ALLOW_CIDR}" IP=$ALLOW_CIDR so-allow -$ALLOW_ROLE >> $setup_log 2>&1 From 7d97e3590ce63a2aaaf0e2b5e0abdceb3a6e3241 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 30 Dec 2020 18:48:56 -0500 Subject: [PATCH 10/56] Redirect tcpreplay init output to file --- salt/common/tools/sbin/so-tcpreplay | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-tcpreplay b/salt/common/tools/sbin/so-tcpreplay index a5c75c267..3f5c0aead 100755 --- a/salt/common/tools/sbin/so-tcpreplay +++ b/salt/common/tools/sbin/so-tcpreplay @@ -48,7 +48,8 @@ if ! docker ps | grep -q so-tcpreplay; then echo TRUSTED_CONTAINERS=("so-tcpreplay") - update_docker_containers "tcpreplay" + mkdir -p /opt/so/log/tcpreplay + update_docker_containers "tcpreplay" "" "" "/opt/so/log/tcpreplay/init.log" so-tcpreplay-start || fail "Unable to initialize tcpreplay" fi From bedbd39b82ad9d965dde07396edf4055593dabd7 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 30 Dec 2020 22:02:19 -0500 Subject: [PATCH 11/56] tcpreplay doesn't need an interactive terminal to run, remove 'it' --- salt/common/tools/sbin/so-tcpreplay | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-tcpreplay b/salt/common/tools/sbin/so-tcpreplay index 3f5c0aead..fa992bdd8 100755 --- a/salt/common/tools/sbin/so-tcpreplay +++ b/salt/common/tools/sbin/so-tcpreplay @@ -54,6 +54,6 @@ if ! docker ps | grep -q so-tcpreplay; then fi echo "Replaying PCAP(s) at ${REPLAYSPEED} Mbps on interface ${REPLAYIFACE}..." -docker exec -it so-tcpreplay /usr/bin/bash -c "/usr/local/bin/tcpreplay -i ${REPLAYIFACE} -M${REPLAYSPEED} $@" +docker exec so-tcpreplay /usr/bin/bash -c "/usr/local/bin/tcpreplay -i ${REPLAYIFACE} -M${REPLAYSPEED} $@" echo "Replay completed. Warnings shown above are typically expected." From 7dcd9342699d32603d424301f70b402b5ebc6fb1 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 31 Dec 2020 10:52:59 -0500 Subject: [PATCH 12/56] so-fleet-setup doesn't need an interactive terminal to run, remove 'it' --- salt/common/tools/sbin/so-fleet-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-fleet-setup b/salt/common/tools/sbin/so-fleet-setup index 8de83b118..a3aa013a2 100755 --- a/salt/common/tools/sbin/so-fleet-setup +++ b/salt/common/tools/sbin/so-fleet-setup @@ -16,7 +16,7 @@ if [ ! "$(docker ps -q -f name=so-fleet)" ]; then fi docker exec so-fleet fleetctl config set --address https://127.0.0.1:8080 --tls-skip-verify --url-prefix /fleet -docker exec -it so-fleet bash -c 'while [[ "$(curl -s -o /dev/null --insecure -w ''%{http_code}'' https://127.0.0.1:8080/fleet)" != "301" ]]; do sleep 5; done' +docker exec so-fleet bash -c 'while [[ "$(curl -s -o /dev/null --insecure -w ''%{http_code}'' https://127.0.0.1:8080/fleet)" != "301" ]]; do sleep 5; done' docker exec so-fleet fleetctl setup --email $1 --password $2 docker exec so-fleet fleetctl apply -f /packs/palantir/Fleet/Endpoints/MacOS/osquery.yaml From 1cc8a78aa5e89ab77c997a034d445f55ebdbef35 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 31 Dec 2020 15:09:22 -0500 Subject: [PATCH 13/56] Only stop SOC if is_manager or is_import --- ...{aws_forwardnode_defaults => distributed_forwardnode_ami} | 0 .../{aws_manager_defaults => distributed_manager_ami} | 0 .../{aws_searchnode_defaults => distributed_searchnode_ami} | 0 setup/automation/{aws_eval_defaults => eval_ami} | 0 setup/automation/{pm_eval_defaults => eval_iso} | 0 setup/automation/{aws_standalone_defaults => standalone_ami} | 0 setup/automation/{pm_standalone_defaults => standalone_iso} | 0 setup/so-setup | 5 ++++- 8 files changed, 4 insertions(+), 1 deletion(-) rename setup/automation/{aws_forwardnode_defaults => distributed_forwardnode_ami} (100%) rename setup/automation/{aws_manager_defaults => distributed_manager_ami} (100%) rename setup/automation/{aws_searchnode_defaults => distributed_searchnode_ami} (100%) rename setup/automation/{aws_eval_defaults => eval_ami} (100%) rename setup/automation/{pm_eval_defaults => eval_iso} (100%) rename setup/automation/{aws_standalone_defaults => standalone_ami} (100%) rename setup/automation/{pm_standalone_defaults => standalone_iso} (100%) diff --git a/setup/automation/aws_forwardnode_defaults b/setup/automation/distributed_forwardnode_ami similarity index 100% rename from setup/automation/aws_forwardnode_defaults rename to setup/automation/distributed_forwardnode_ami diff --git a/setup/automation/aws_manager_defaults b/setup/automation/distributed_manager_ami similarity index 100% rename from setup/automation/aws_manager_defaults rename to setup/automation/distributed_manager_ami diff --git a/setup/automation/aws_searchnode_defaults b/setup/automation/distributed_searchnode_ami similarity index 100% rename from setup/automation/aws_searchnode_defaults rename to setup/automation/distributed_searchnode_ami diff --git a/setup/automation/aws_eval_defaults b/setup/automation/eval_ami similarity index 100% rename from setup/automation/aws_eval_defaults rename to setup/automation/eval_ami diff --git a/setup/automation/pm_eval_defaults b/setup/automation/eval_iso similarity index 100% rename from setup/automation/pm_eval_defaults rename to setup/automation/eval_iso diff --git a/setup/automation/aws_standalone_defaults b/setup/automation/standalone_ami similarity index 100% rename from setup/automation/aws_standalone_defaults rename to setup/automation/standalone_ami diff --git a/setup/automation/pm_standalone_defaults b/setup/automation/standalone_iso similarity index 100% rename from setup/automation/pm_standalone_defaults rename to setup/automation/standalone_iso diff --git a/setup/so-setup b/setup/so-setup index d9bc03444..785d5241f 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -843,8 +843,11 @@ else [ -n "$TESTING" ] && logCmd so-test export percentage=95 # set to last percentage used in previous subshell - so-soc-stop # Stop SOC so it doesn't accept external requests prior to the reboot if [[ -n $ALLOW_ROLE && -n $ALLOW_CIDR ]]; then + if [[ $is_manager || $is_import ]]; then + set_progress_str 96 "Stopping SOC prior to adjusting firewall rules" + so-soc-stop # Stop SOC so it doesn't accept external requests prior to the reboot + fi set_progress_str 97 "Running so-allow -${ALLOW_ROLE} for ${ALLOW_CIDR}" IP=$ALLOW_CIDR so-allow -$ALLOW_ROLE >> $setup_log 2>&1 fi From 6ba11f835dfe8b1c758b41cc4b573f95d82ed96b Mon Sep 17 00:00:00 2001 From: William Wernert Date: Sat, 2 Jan 2021 21:03:15 -0500 Subject: [PATCH 14/56] [fix] Remove condition for stopping SOC, since the parent condition covers what's tested --- setup/so-setup | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 785d5241f..b84986c36 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -844,10 +844,9 @@ else export percentage=95 # set to last percentage used in previous subshell if [[ -n $ALLOW_ROLE && -n $ALLOW_CIDR ]]; then - if [[ $is_manager || $is_import ]]; then - set_progress_str 96 "Stopping SOC prior to adjusting firewall rules" - so-soc-stop # Stop SOC so it doesn't accept external requests prior to the reboot - fi + set_progress_str 96 "Stopping SOC prior to adjusting firewall rules" + so-soc-stop # Stop SOC so it doesn't accept external requests prior to the reboot + set_progress_str 97 "Running so-allow -${ALLOW_ROLE} for ${ALLOW_CIDR}" IP=$ALLOW_CIDR so-allow -$ALLOW_ROLE >> $setup_log 2>&1 fi From 958635b0126f4078beea42a25ab35601219a215a Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 4 Jan 2021 10:18:32 -0500 Subject: [PATCH 15/56] Remove old Strelka cron job --- salt/manager/init.sls | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 4136b276d..502c89579 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -88,6 +88,13 @@ append_so-aptcacherng_so-status.conf: {% endif %} +strelka_yara_update_old: + cron.absent: + - user: root + - name: '/usr/sbin/so-yara-update > /dev/null 2>&1' + - hour: '7' + - minute: '1' + strelka_yara_update: cron.present: - user: root From ac35a345ffccdb5a5482fc748b58e178718ecc90 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 4 Jan 2021 11:58:25 -0500 Subject: [PATCH 16/56] [fix] Don't prompt to only set up network and then skip if network was previously configured --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index b84986c36..334500453 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -177,7 +177,7 @@ if ! [[ -f $install_opt_file ]]; then fi if [[ $setup_type == 'iso' ]]; then whiptail_first_menu_iso - if [[ $option == "Configure Network" ]] && ! [[ -f $net_init_file ]]; then + if [[ $option == "Configure Network" ]]; then network_init_whiptail whiptail_management_interface_setup network_init From 80a3d8dcf8e44d7b5641d8c184a8f2400c06b9df Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 4 Jan 2021 14:46:48 -0500 Subject: [PATCH 17/56] [fix] Fix automation compatibility --- setup/so-setup | 6 ++---- setup/so-whiptail | 7 +++++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 334500453..a436c25c1 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -175,7 +175,7 @@ if ! [[ -f $install_opt_file ]]; then echo "User cancelled setup." | tee -a "$setup_log" whiptail_cancel fi - if [[ $setup_type == 'iso' ]]; then + if [[ $setup_type == 'iso' ]] && [ "$automated" == no ]; then whiptail_first_menu_iso if [[ $option == "Configure Network" ]]; then network_init_whiptail @@ -184,9 +184,7 @@ if ! [[ -f $install_opt_file ]]; then printf '%s\n' \ "MNIC=$MNIC" \ "HOSTNAME=$HOSTNAME" > "$net_init_file" - whiptail --title "Security Onion Setup" \ - --msgbox "Successfully set up networking, setup will now exit." 7 75 - exit 0 + whiptail_net_setup_complete else whiptail_install_type fi diff --git a/setup/so-whiptail b/setup/so-whiptail index b034ab679..6af5b701f 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -794,6 +794,13 @@ whiptail_management_interface_setup() { whiptail_check_exitstatus $exitstatus } +whiptail_net_setup_complete() { + [ -n "$TESTING" ] && return + + whiptail --title "Security Onion Setup" \ + --msgbox "Successfully set up networking, setup will now exit." 7 75 + exit 0 +} whiptail_management_server() { From c864cc607f8aa051f7ae99c0ade1d99cd637fb21 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 4 Jan 2021 16:29:32 -0500 Subject: [PATCH 18/56] Remove multiple old so-yara-update cron jobs, if needed --- salt/manager/init.sls | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 502c89579..597ca3c43 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -88,7 +88,14 @@ append_so-aptcacherng_so-status.conf: {% endif %} -strelka_yara_update_old: +strelka_yara_update_old_1: + cron.absent: + - user: root + - name: '[ -d /opt/so/saltstack/default/salt/strelka/rules/ ] && /usr/sbin/so-yara-update > /dev/null 2>&1' + - hour: '7' + - minute: '1' + +strelka_yara_update_old_2: cron.absent: - user: root - name: '/usr/sbin/so-yara-update > /dev/null 2>&1' From 9d8fb79d9f66b2627dd4a006ffb0379bf8df0e32 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 4 Jan 2021 16:40:08 -0500 Subject: [PATCH 19/56] [feat] Reorder network-only prompt --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 6af5b701f..f3e612f70 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -662,8 +662,8 @@ whiptail_first_menu_iso() { [ -n "$TESTING" ] && return option=$(whiptail --title "Security Onion Setup" --menu "Select an option" 10 75 2 \ - "Configure Network" "Configure networking only " \ "Security Onion Installer" "Run the standard Security Onion installation " \ + "Configure Network" "Configure networking only " \ 3>&1 1>&2 2>&3 ) local exitstatus=$? From d37023e0f577ffc7d063e00ecd317179048c6272 Mon Sep 17 00:00:00 2001 From: Masaya-A <68965261+Masaya-A@users.noreply.github.com> Date: Thu, 17 Dec 2020 22:14:03 +0900 Subject: [PATCH 20/56] Make yum removing unneeded packages Reference: https://www.stigviewer.com/stig/red_hat_enterprise_linux_7/2020-09-03/finding/V-204452 --- salt/yum/etc/yum.conf.jinja | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/yum/etc/yum.conf.jinja b/salt/yum/etc/yum.conf.jinja index bef9c2128..5e1d30510 100644 --- a/salt/yum/etc/yum.conf.jinja +++ b/salt/yum/etc/yum.conf.jinja @@ -10,6 +10,7 @@ plugins=1 installonly_limit={{ salt['pillar.get']('yum:config:installonly_limit', 2) }} bugtracker_url=http://bugs.centos.org/set_project.php?project_id=23&ref=http://bugs.centos.org/bug_report_page.php?category=yum distroverpkg=centos-release +clean_requirements_on_remove=1 {% if (grains['role'] not in ['so-eval','so-managersearch', 'so-manager', 'so-standalone']) and salt['pillar.get']('global:managerupdate', '0') %} proxy=http://{{ salt['pillar.get']('yum:config:proxy', salt['config.get']('master')) }}:3142 From 7d25e8a08b1b89caebda058c9a8e896727343bff Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 5 Jan 2021 13:56:56 -0500 Subject: [PATCH 21/56] Remove ERSPAN so log doesn't show a warning --- salt/suricata/defaults.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/salt/suricata/defaults.yaml b/salt/suricata/defaults.yaml index 9f34c0871..49a25917c 100644 --- a/salt/suricata/defaults.yaml +++ b/salt/suricata/defaults.yaml @@ -460,8 +460,6 @@ suricata: enabled: true ports: $VXLAN_PORTS erspan: - typeI: - enabled: false detect: profile: medium custom-values: From e7db1a99bde5911ec0f1a789d8c7e6f1f7710f98 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 6 Jan 2021 16:47:35 +0000 Subject: [PATCH 22/56] Set @timestamp to winlog.systemTime --- salt/elasticsearch/files/ingest/win.eventlogs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/elasticsearch/files/ingest/win.eventlogs b/salt/elasticsearch/files/ingest/win.eventlogs index 3137e6bb5..2644be7a2 100644 --- a/salt/elasticsearch/files/ingest/win.eventlogs +++ b/salt/elasticsearch/files/ingest/win.eventlogs @@ -4,6 +4,8 @@ { "set": { "if": "ctx.winlog?.channel != null", "field": "event.module", "value": "windows_eventlog", "override": false, "ignore_failure": true } }, { "set": { "if": "ctx.winlog?.channel != null", "field": "event.dataset", "value": "{{winlog.channel}}", "override": true } }, { "set": { "if": "ctx.winlog?.computer_name != null", "field": "observer.name", "value": "{{winlog.computer_name}}", "override": true } }, + { "rename": { "if": "ctx.winlog?.systemTime != null", "field": "@timestamp", "target_field": "ingest.timestamp", "ignore_missing": true } }, + { "set": { "if": "ctx.winlog?.systemTime != null", "field": "@timestamp", "value": "{{winlog.systemTime}}", "override": true } }, { "set": { "field": "event.code", "value": "{{winlog.event_id}}", "override": true } }, { "set": { "field": "event.category", "value": "host", "override": true } }, { "rename": { "field": "winlog.event_data.SubjectUserName", "target_field": "user.name", "ignore_failure": true, "ignore_missing": true } }, From 7f64d571111d4434af5d27d6271d21f0051fd0b8 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 6 Jan 2021 15:52:10 +0000 Subject: [PATCH 23/56] Reserve port for Wazuh API and check if port is already in use --- salt/wazuh/init.sls | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/salt/wazuh/init.sls b/salt/wazuh/init.sls index 99f16cb8a..3cad6c367 100644 --- a/salt/wazuh/init.sls +++ b/salt/wazuh/init.sls @@ -96,6 +96,16 @@ wazuhmgrwhitelist: - mode: 755 - template: jinja +# Reserve OS port for Wazuh API +wazuhreserveport: + cmd.run: + - name: grep -q 55000 /proc/sys/net/ipv4/ip_local_reserved_ports || sysctl -w net.ipv4.ip_local_reserved_ports="55000" > /dev/null && echo "55000" >> /proc/sys/net/ipv4/ip_local_reserved_ports + +# Check to see if Wazuh API port is available +wazuhportavailable: + cmd.run: + - name: netstat -anp | grep 55000 | grep -qv docker && PROCESS=$(netstat -anp | grep 55000 | awk '{print $NF}' | uniq) && echo "Another process ($PROCESS) appears to be using port 55000. Please terminate this process, or reboot to ensure a clean state so that the Wazuh API can start properly." && exit 1 || exit 0 + so-wazuh: docker_container.running: - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-wazuh:{{ VERSION }} @@ -158,4 +168,4 @@ wazuh_state_not_allowed: test.fail_without_changes: - name: wazuh_state_not_allowed -{% endif %} \ No newline at end of file +{% endif %} From fa06a38a3b9589cd8919fb7ae9f37ffeabe589e2 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 7 Jan 2021 10:36:01 -0500 Subject: [PATCH 24/56] [refactor] Remove duplicate function --- setup/so-whiptail | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index f3e612f70..49dc49b22 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -255,17 +255,6 @@ whiptail_create_web_user_password2() { } -whiptail_fleet_custom_hostname() { - - [ -n "$TESTING" ] && return - - FLEETCUSTOMHOSTNAME=$(whiptail --title "Security Onion Install" --inputbox \ - "What FQDN should osquery clients use for connections to this Fleet node? Leave blank if the local system hostname will be used." 10 60 3>&1 1>&2 2>&3) - - local exitstatus=$? - whiptail_check_exitstatus $exitstatus -} - whiptail_requirements_error() { local requirement_needed=$1 From c1e32ed68054c8a7486a6768d5ce0f95be1bc9e8 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 7 Jan 2021 10:36:32 -0500 Subject: [PATCH 25/56] [refactor] Rename MD tool function to be more clear --- setup/so-whiptail | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 49dc49b22..0a72f1e3b 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -65,18 +65,6 @@ whiptail_basic_zeek() { whiptail_check_exitstatus $exitstatus } -whiptail_zeek_version() { - - [ -n "$TESTING" ] && return - - ZEEKVERSION=$(whiptail --title "Security Onion Setup" --radiolist "What tool would you like to use to generate metadata?" 20 75 4 \ - "ZEEK" "Zeek (formerly known as Bro)" ON \ - "SURICATA" "Suricata" OFF 3>&1 1>&2 2>&3) - - local exitstatus=$? - whiptail_check_exitstatus $exitstatus -} - whiptail_bond_nics_mtu() { [ -n "$TESTING" ] && return @@ -500,6 +488,8 @@ whiptail_helix_apikey() { } +#TODO: Combine these two functions + whiptail_homenet_manager() { [ -n "$TESTING" ] && return @@ -1572,14 +1562,16 @@ whiptail_zeek_pins() { IFS=' ' read -ra ZEEKPINS <<< "$ZEEKPINS" } -whiptail_zeek_version() { +whiptail_metadata_tool() { [ -n "$TESTING" ] && return - ZEEKVERSION=$(whiptail --title "Security Onion Setup" --radiolist "What tool would you like to use to generate metadata?" 20 75 4 "ZEEK" "Zeek (formerly known as Bro)" ON \ + # Legacy variable naming + ZEEKVERSION=$(whiptail --title "Security Onion Setup" --radiolist "What tool would you like to use to generate metadata?" 20 75 4 \ + "ZEEK" "Zeek (formerly known as Bro)" ON \ "SURICATA" "Suricata" OFF 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus - } + From 1ec45fb4ae978e3a47a3e3df43c0abb0715a501a Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 7 Jan 2021 10:37:25 -0500 Subject: [PATCH 26/56] [fix] Only show Zeek prompts if Zeek was selected as the MD tool Resolves #900 --- setup/so-setup | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index a436c25c1..1a8a5d223 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -392,10 +392,11 @@ if [[ $is_manager && ! $is_eval ]]; then whiptail_manager_adv_escluster fi fi + whiptail_metadata_tool - if [ "$MANAGERADV" = 'ADVANCED' ] && [ "$ZEEKVERSION" != 'SURICATA' ]; then - whiptail_manager_adv_service_zeeklogs - fi + + [[ $MANAGERADV == "ADVANCED" ]] && [[ $ZEEKVERSION == "ZEEK" ]] && whiptail_manager_adv_service_zeeklogs + # Don't run this function for now since Snort is not yet supported # whiptail_nids NIDS=Suricata @@ -441,11 +442,11 @@ if [[ $is_sensor && ! $is_eval ]]; then whiptail_homenet_sensor whiptail_sensor_config if [ $NSMSETUP == 'ADVANCED' ]; then - whiptail_zeek_pins + [[ $ZEEKVERSION == "ZEEK" ]] && whiptail_zeek_pins whiptail_suricata_pins whiptail_bond_nics_mtu else - whiptail_basic_zeek + [[ $ZEEKVERSION == "ZEEK" ]] && whiptail_basic_zeek whiptail_basic_suri fi fi From ef7a934b9d274919459dc47e437f8362d773a243 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 12 Jan 2021 11:01:04 -0500 Subject: [PATCH 27/56] [feat] Add functions for input validation --- salt/common/tools/sbin/so-common | 83 ++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index a012f2ed9..b3b621a68 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -233,6 +233,89 @@ __check_apt_lock() { return $lock } +valid_cidr() { + # Verify there is a backslash in the string + echo "$1" | grep -qP "^[^/]+/[^/]+$" || return 1 + + local cidr + local ip + + cidr=$(echo "$1" | sed 's/.*\///') + ip=$(echo "$1" | sed 's/\/.*//' ) + + if valid_ip4 "$ip"; then + [[ $cidr =~ ([0-9]|[1-2][0-9]|3[0-2]) ]] && return 0 || return 1 + else + return 1 + fi +} + +valid_cidr_list() { + local all_valid=0 + + IFS="," read -r -a net_arr <<< "$1" + + for net in "${net_arr[@]}"; do + valid_cidr "$net" || all_valid=1 + done + + return $all_valid +} + +valid_dns_list() { + local all_valid=0 + + read -r -a dns_arr <<< "$1" + + for addr in "${dns_arr[@]}"; do + valid_ip4 "$addr" || all_valid=1 + done + + return $all_valid +} + +valid_fqdn() { + local fqdn=$1 + + echo "$fqdn" | grep -qP '(?=^.{4,253}$)(^((?!-)[a-zA-Z0-9-]{0,62}[a-zA-Z0-9]\.)+[a-zA-Z]{2,63}$)' \ + && return 0 \ + || return 1 +} + +valid_hostname() { + local hostname=$1 + + [[ $hostname =~ ^[a-zA-Z0-9\-]+$ ]] && [[ $hostname != 'localhost' ]] && return 0 || return 1 +} + +valid_ip4() { + local ip=$1 + + echo "$ip" | grep -qP '^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$' && return 0 || return 1 +} + +valid_int() { + local num=$1 + local min=${2:-1} + local max=${3:-1000} + + [[ $num =~ ^[0-9]*$ ]] && [[ $num -ge $min ]] && [[ $num -le $max ]] && return 0 || return 1 +} + +valid_string() { + local str=$1 + local min_length=${2:-1} + local max_length=${3:-64} + + echo "$str" | grep -qP '^\S+$' && [[ ${#str} -ge $min_length ]] && [[ ${#str} -le $max_length ]] && return 0 || return 1 +} + +valid_username() { + local user=$1 + + echo "$user" | grep -qP '^[a-z_]([a-z0-9_-]{0,31}|[a-z0-9_-]{0,30}\$)$' && return 0 || return 1 +} + wait_for_web_response() { url=$1 expected=$2 From 332c6877b89c6dc9597dadf76ba1170748ae88d6 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 12 Jan 2021 11:01:25 -0500 Subject: [PATCH 28/56] [fix] Add extra arg to printf instead of using echo --- salt/common/tools/sbin/so-common | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index b3b621a68..9fd4ef7dc 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -21,6 +21,9 @@ if [ "$(id -u)" -ne 0 ]; then exit 1 fi +# Define a banner to separate sections +banner="=========================================================================" + add_interface_bond0() { local BNIC=$1 if [[ -z $MTU ]]; then @@ -68,12 +71,8 @@ add_interface_bond0() { fi } -# Define a banner to separate sections -banner="=========================================================================" - header() { - echo - printf '%s\n' "$banner" "$*" "$banner" + printf '%s\n' "" "$banner" " $*" "$banner" } lookup_salt_value() { From 0dc0780e28a58ed220702a6eaed9b2f5db6a17d8 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 12 Jan 2021 11:02:00 -0500 Subject: [PATCH 29/56] [feat] Add unit tests for input validation --- tests/validation.sh | 187 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 tests/validation.sh diff --git a/tests/validation.sh b/tests/validation.sh new file mode 100644 index 000000000..04cdaa7c7 --- /dev/null +++ b/tests/validation.sh @@ -0,0 +1,187 @@ +#!/bin/bash + +. ../salt/common/tools/sbin/so-common + +script_ret=0 + +GREEN="\e[1;32m" +RED="\e[1;31m" +RESET="\e[0m" + +test_fun() { + local expected_result=$1 + shift + + local fun=$1 + shift + + $fun "$@" + local ret=$? + [[ $ret -eq 0 ]] && res="O" || res="X" + + [[ $ret -ne $expected_result ]] && script_ret=1 + + local prefix=$1 + [[ -n $2 ]] && prefix="$prefix, min=$2" + [[ -n $3 ]] && prefix="$prefix, max=$3" + + [[ $prefix == "" ]] && prefix="[EMPTY]" + + [[ $ret -eq $expected_result ]] \ + && printf "${GREEN}%b${RESET}" " $res" \ + || printf "${RED}%b${RESET}" " $res" + + printf "%s\n" " - $prefix" +} + +header "FQDN" + +test_fun 0 valid_fqdn "rwwiv.com" + +test_fun 0 valid_fqdn "ddns.rwwiv.com" + +test_fun 1 valid_fqdn ".com" + +test_fun 1 valid_fqdn "rwwiv." + +test_fun 1 valid_fqdn "" + +sleep 0.15s + +header "ip4" + +test_fun 0 valid_ip4 "192.168.1.1" + +test_fun 0 valid_ip4 "192.168.1.255" + +test_fun 1 valid_ip4 "192.168.1.256" + +test_fun 1 valid_ip4 "192.168..1" + +test_fun 1 valid_ip4 "192.168.1.1." + +test_fun 1 valid_ip4 "" + +sleep 0.15s + +header "CIDR (ipv4)" + +test_fun 0 valid_cidr "192.168.1.0/24" + +test_fun 0 valid_cidr "192.168.1.0/12" + +test_fun 1 valid_cidr "192.168.1.0" + +test_fun 1 valid_ip4 "192.168.1.0/" + +test_fun 1 valid_ip4 "/24" + +test_fun 1 valid_cidr "" + +sleep 0.15s + +header "CIDR list" + +test_fun 0 valid_cidr_list "10.0.0.0/8,192.168.0.0/16,172.16.0.0/12" + +test_fun 0 valid_cidr_list "10.0.0.0/8" + +test_fun 1 valid_cidr_list "10.0.0.0/8,192.168.0.0/16172.16.0.0/12" + +test_fun 1 valid_cidr_list "10.0.0.0" + +sleep 0.15s + +header "DNS" + +test_fun 0 valid_dns_list "8.8.8.8 8.8.4.4" + +test_fun 0 valid_dns_list "8.8.8.8" + +test_fun 1 valid_dns_list "8.8.8.8,8.8.4.4" + +test_fun 1 valid_dns_list "8.8.8. 8.8.4.4" + +test_fun 1 valid_dns_list "192.168.9." + +sleep 0.15s + +header "int (default min: 1, default max: 1000)" + +test_fun 0 valid_int "24" + +test_fun 0 valid_int "1" + +test_fun 0 valid_int "2" "2" + +test_fun 0 valid_int "1000" + +test_fun 1 valid_int "10001" + +test_fun 1 valid_int "24" "" "20" + +test_fun 1 valid_int "-1" + +test_fun 1 valid_int "1" "2" + +test_fun 1 valid_int "257" "" "256" + +test_fun 1 valid_int "not_a_num" + +test_fun 1 valid_int "" + +sleep 0.15s + +header "hostname" + +test_fun 0 valid_hostname "so-sensor01" + +test_fun 0 valid_hostname "so" + +test_fun 1 valid_hostname "so_sensor01" + +test_fun 1 valid_hostname "so.sensor01" + +test_fun 1 valid_hostname "localhost" + +test_fun 1 valid_hostname "" + +sleep 0.15s + +header "string (default min_length: 1, default max_length: 64)" + +test_fun 0 valid_string "string" + +test_fun 0 valid_string "s" + +test_fun 0 valid_string "very_long_string_64_sdhkjashasdfkajjagskfjhgkslfkjhlaskfhlaskjhf" + +test_fun 0 valid_string "12" + +test_fun 1 valid_string "string with spaces" + +test_fun 1 valid_string "very_long_string_<64_sdhflkjashasdfkajshfgkjsahgfkjagskfjhgkslfkjhlaskfhlaskjhf" + +test_fun 1 valid_string "too_short" "12" + +test_fun 1 valid_string "too_long" "" "4" + +test_fun 1 valid_string "" + +sleep 0.15s + +header "Linux user" + +test_fun 0 valid_username "so_user_01" + +test_fun 0 valid_username "onionuser" + +test_fun 1 valid_username "12fa" + +test_fun 1 valid_username "so.user.01" + +test_fun 1 valid_username "very_long_username_asdflashfsafasdfasdfkahsgkjahfdkjhsg" + +echo + +exit $script_ret From 5d077d278e5f6493bc496a00871397a51b606f4e Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 12 Jan 2021 11:02:33 -0500 Subject: [PATCH 30/56] [feat] Add input validation to inputbox whiptail prompts --- setup/so-functions | 330 ++++++++++++++++++++++++++++++++++++++++++--- setup/so-setup | 43 +++--- setup/so-whiptail | 194 +++++++++----------------- 3 files changed, 393 insertions(+), 174 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 402afa298..65e9a3a83 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -360,18 +360,14 @@ clear_manager() { } -collect_soremote_inputs() { - whiptail_create_soremote_user - SCMATCH=no - while [[ $SCMATCH != yes ]]; do - whiptail_create_soremote_user_password1 - whiptail_create_soremote_user_password2 - check_soremote_pass - done -} - collect_adminuser_inputs() { whiptail_create_admin_user + + while ! valid_username "$ADMINUSER"; do + whiptail_invalid_input + whiptail_create_admin_user + done + APMATCH=no while [[ $APMATCH != yes ]]; do whiptail_create_admin_user_password1 @@ -380,8 +376,80 @@ collect_adminuser_inputs() { done } +collect_cur_close_days() { + whiptail_cur_close_days + + while ! valid_int "$CURCLOSEDAYS" "1"; do + whiptail_invalid_input + whiptail_cur_close_days + done +} + +collect_dns() { + whiptail_management_interface_dns + + while ! valid_dns_list "$MDNS"; do + whiptail_invalid_input + whiptail_management_interface_dns + done +} + +collect_dns_domain() { + whiptail_management_interface_dns_search + + while ! valid_fqdn "$MSEARCH"; do + whiptail_invalid_input + whiptail_management_interface_dns_search + done +} + +collect_dockernet() { + if whiptail_dockernet_check; then + whiptail_dockernet_net + + while ! valid_ip4 "$DOCKERNET"; do + whiptail_invalid_input + whiptail_dockernet_net + done + fi +} + +collect_es_cluster_name() { + if whiptail_manager_adv_escluster; then + whiptail_manager_adv_escluster_name + + while ! valid_string "$ESCLUSTERNAME"; do + whiptail_invalid_input + whiptail_manager_adv_escluster_name + done + fi +} + +collect_es_shard_count() { + whiptail_shard_count + + while ! valid_int "$SHARDCOUNT"; do + whiptail_invalid_input + whiptail_shard_count + done +} + +collect_es_space_limit() { + whiptail_log_size_limit + + while ! valid_int "$log_size_limit" "1"; do # Upper/lower bounds? + whiptail_invalid_input + whiptail_log_size_limit + done +} + collect_fleet_custom_hostname_inputs() { whiptail_fleet_custom_hostname + + while ! valid_fqdn "$FLEETCUSTOMHOSTNAME" || [[ $FLEETCUSTOMHOSTNAME != "" ]]; do + whiptail_invalid_input + whiptail_fleet_custom_hostname + done } collect_fleetuser_inputs() { @@ -408,6 +476,218 @@ collect_fleetuser_inputs() { done } +collect_gateway() { + whiptail_management_interface_gateway + + while ! valid_ip4 "$MGATEWAY"; do + whiptail_invalid_input + whiptail_management_interface_gateway + done +} + +collect_helix_key() { + whiptail_helix_apikey # validate? +} + +collect_homenet_mngr() { + whiptail_homenet_manager + + while ! __validate_cidr_arr "$HNMANAGER"; do + whiptail_invalid_input + whiptail_homenet_manager + done +} + +collect_homenet_snsr() { + if whiptail_homenet_sensor_inherit; then + export HNSENSOR=inherit + else + whiptail_homenet_sensor + + while ! __validate_cidr_arr "$HNSENSOR"; do + whiptail_invalid_input + whiptail_homenet_sensor + done + fi +} + +collect_hostname() { + HOSTNAME=$(cat /etc/hostname) + if [[ "$HOSTNAME" == *'localhost'* ]]; then HOSTNAME=securityonion; fi + + whiptail_set_hostname + + while ! valid_hostname "$HOSTNAME"; do + whiptail_invalid_hostname + whiptail_set_hostname + done +} + +collect_int_ip_mask() { + whiptail_management_interface_ip_mask + + while ! valid_cidr "$manager_ip_mask"; do + whiptail_invalid_input + whiptail_management_interface_ip_mask + done + + MIP=$(echo "$manager_ip_mask" | sed 's/\/.*//' ) + MMASK=$(echo "$manager_ip_mask" | sed 's/.*\///') +} + +collect_mngr_hostname() { + whiptail_management_server + + while ! valid_hostname "$MSRV"; do + whiptail_invalid_hostname + whiptail_management_server + done + + if ! getent hosts "$MSRV"; then + add_manager_hostfile + else + MSRVIP=$(getent hosts "$MSRV" | awk 'NR==1{print $1}') + fi +} + +collect_mtu() { + whiptail_bond_nics_mtu + + while ! valid_int "$MTU" "68"; do + whiptail_invalid_input + whiptail_bond_nics_mtu + done +} + +collect_node_es_heap() { + whiptail_node_es_heap + + while ! valid_int "$NODE_ES_HEAP_SIZE"; do + whiptail_invalid_input + whiptail_node_es_heap + done +} + +collect_node_ls_heap() { + whiptail_node_ls_heap + + while ! valid_int "$NODE_LS_HEAP_SIZE"; do + whiptail_invalid_input + whiptail_node_ls_heap + done +} + +collect_node_ls_input() { + whiptail_node_ls_input_threads + while ! valid_int "$LSINPUTTHREADS"; do + whiptail_invalid_input + whiptail_node_ls_input_threads + done +} + +collect_node_ls_pipeline_batch_size() { + whiptail_node_ls_pipline_batchsize + + while ! valid_int "$LSPIPELINEBATCH"; do + whiptail_invalid_input + whiptail_node_ls_pipline_batchsize + done +} + +collect_node_ls_pipeline_worker_count() { + whiptail_node_ls_pipeline_worker + + while ! valid_int "$LSPIPELINEWORKERS"; do + whiptail_invalid_input + whiptail_node_ls_pipeline_worker + done +} + +collect_oinkcode() { + whiptail_oinkcode + while ! valid_string "$OINKCODE" "" "128"; do #TODO: verify max length here + whiptail_invalid_input + whiptail_oinkcode + done +} + +collect_patch_schedule() { + whiptail_patch_schedule + + case $patch_schedule in + 'New Schedule') + whiptail_patch_schedule_select_days + whiptail_patch_schedule_select_hours + collect_patch_schedule_name_new + patch_schedule_os_new + ;; + 'Import Schedule') + collect_patch_schedule_name_import + ;; + 'Automatic') + PATCHSCHEDULENAME='auto' + ;; + 'Manual') + PATCHSCHEDULENAME='manual' + ;; + esac +} + +collect_patch_schedule_name_new() { + whiptail_patch_name_new_schedule + + while ! valid_string "$PATCHSCHEDULENAME"; do + whiptail_invalid_input + whiptail_patch_name_new_schedule + done +} + +collect_patch_schedule_name_import() { + whiptail_patch_schedule_import + + while ! valid_string "$PATCHSCHEDULENAME"; do + whiptail_invalid_input + whiptail_patch_schedule_import + done +} + +collect_redirect_host() { + whiptail_set_redirect_host + + while ! valid_ip4 "$REDIRECTHOST" && ! valid_hostname "$REDIRECTHOST" && ! valid_fqdn "$REDIRECTHOST"; do + whiptail_invalid_input + whiptail_set_redirect_host + done +} + +collect_so_allow() { + if whiptail_so_allow_yesno; then + whiptail_so_allow + while ! valid_cidr "$ALLOW_CIDR" && ! valid_ip4 "$ALLOW_CIDR"; do + whiptail_invalid_input + whiptail_so_allow + done + fi +} + +collect_soremote_inputs() { + whiptail_create_soremote_user + SCMATCH=no + while [[ $SCMATCH != yes ]]; do + whiptail_create_soremote_user_password1 + whiptail_create_soremote_user_password2 + check_soremote_pass + done +} + +collect_suri() { + whiptail_basic_suri + + while ! valid_int "$BASICSURI"; do + whiptail_invalid_input + whiptail_basic_suri + done +} collect_webuser_inputs() { # Get a password for the web admin user @@ -425,9 +705,9 @@ collect_webuser_inputs() { while [[ $WPMATCH != yes ]]; do whiptail_create_web_user_password1 while ! check_password "$WEBPASSWD1"; do - whiptail_invalid_pass_characters_warning - whiptail_create_web_user_password1 - done + whiptail_invalid_pass_characters_warning + whiptail_create_web_user_password1 + done if echo "$WEBPASSWD1" | so-user valpass >> "$setup_log" 2>&1; then whiptail_create_web_user_password2 check_web_pass @@ -437,6 +717,15 @@ collect_webuser_inputs() { done } +collect_zeek() { + whiptail_basic_zeek + + while ! valid_int "$BASICZEEK"; do + whiptail_invalid_input + whiptail_basic_zeek + done +} + configure_minion() { local minion_type=$1 echo "Configuring minion type as $minion_type" >> "$setup_log" 2>&1 @@ -1019,7 +1308,7 @@ generate_repo_tarball() { get_redirect() { whiptail_set_redirect if [ "$REDIRECTINFO" = "OTHER" ]; then - whiptail_set_redirect_host + collect_redirect_host fi } @@ -1345,22 +1634,21 @@ network_init() { network_init_whiptail() { case "$setup_type" in 'iso') - whiptail_set_hostname + collect_hostname whiptail_management_nic whiptail_dhcp_or_static if [ "$address_type" != 'DHCP' ]; then - whiptail_management_interface_ip - whiptail_management_interface_mask - whiptail_management_interface_gateway - whiptail_management_interface_dns - whiptail_management_interface_dns_search + collect_int_ip_mask + collect_gateway + collect_dns + collect_dns_domain fi ;; 'network') whiptail_network_notice whiptail_dhcp_warn - whiptail_set_hostname + collect_hostname whiptail_management_nic ;; esac diff --git a/setup/so-setup b/setup/so-setup index 1a8a5d223..917273786 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -265,7 +265,7 @@ if ! [[ -f $install_opt_file ]]; then fi if [[ $is_minion ]]; then - whiptail_management_server + collect_mngr_hostname fi if [[ $is_minion ]] || [[ $reinit_networking ]] || ! [[ -f $net_init_file ]]; then @@ -367,6 +367,10 @@ fi # Start user prompts +if [[ $is_helix ]]; then + collect_helix_key +fi + if [[ $is_helix || $is_sensor ]]; then whiptail_sensor_nics fi @@ -376,10 +380,10 @@ if [[ $is_helix || $is_sensor || $is_import ]]; then fi if [[ ! $is_import ]]; then - whiptail_patch_schedule + collect_patch_schedule fi -whiptail_homenet_manager +collect_homenet_mngr if [[ $is_helix || $is_manager || $is_node || $is_import ]]; then set_base_heapsizes @@ -389,10 +393,10 @@ if [[ $is_manager && ! $is_eval ]]; then whiptail_manager_adv if [ "$MANAGERADV" = 'ADVANCED' ]; then if [ "$install_type" = 'MANAGER' ] || [ "$install_type" = 'MANAGERSEARCH' ]; then - whiptail_manager_adv_escluster + collect_es_cluster_name fi fi - + whiptail_metadata_tool [[ $MANAGERADV == "ADVANCED" ]] && [[ $ZEEKVERSION == "ZEEK" ]] && whiptail_manager_adv_service_zeeklogs @@ -403,9 +407,8 @@ if [[ $is_manager && ! $is_eval ]]; then whiptail_rule_setup if [ "$RULESETUP" != 'ETOPEN' ]; then - whiptail_oinkcode + collect_oinkcode fi - fi if [[ $is_manager ]]; then @@ -419,7 +422,7 @@ if [[ $is_manager ]]; then info "Disabling Strelka rules: STRELKA='$STRELKA'" fi - whiptail_dockernet_check + collect_dockernet fi if [[ $is_manager || $is_import ]]; then @@ -439,28 +442,28 @@ if [[ $is_distmanager ]]; then fi if [[ $is_sensor && ! $is_eval ]]; then - whiptail_homenet_sensor + collect_homenet_snsr whiptail_sensor_config if [ $NSMSETUP == 'ADVANCED' ]; then [[ $ZEEKVERSION == "ZEEK" ]] && whiptail_zeek_pins whiptail_suricata_pins - whiptail_bond_nics_mtu + collect_mtu else - [[ $ZEEKVERSION == "ZEEK" ]] && whiptail_basic_zeek - whiptail_basic_suri + [[ $ZEEKVERSION == "ZEEK" ]] && collect_zeek + collect_suri fi fi if [[ $is_node && ! $is_eval ]]; then whiptail_node_advanced if [ "$NODESETUP" == 'NODEADVANCED' ]; then - whiptail_node_es_heap - whiptail_node_ls_heap - whiptail_node_ls_pipeline_worker - whiptail_node_ls_pipline_batchsize - whiptail_node_ls_input_threads - whiptail_cur_close_days - whiptail_log_size_limit + collect_node_es_heap + collect_node_ls_heap + collect_node_ls_pipeline_worker_count + collect_node_ls_pipeline_batch_size + collect_node_ls_input + collect_cur_close_days + collect_es_space_limit else NODE_ES_HEAP_SIZE=$ES_HEAP_SIZE NODE_LS_HEAP_SIZE=$LS_HEAP_SIZE @@ -479,7 +482,7 @@ else FLEETNODEPASSWD1=$WEBPASSWD1 fi -if [[ $is_manager || $is_import ]]; then whiptail_so_allow; fi +if [[ $is_manager || $is_import ]]; then collect_so_allow; fi whiptail_make_changes diff --git a/setup/so-whiptail b/setup/so-whiptail index 0a72f1e3b..dd8083d55 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -75,7 +75,6 @@ whiptail_bond_nics_mtu() { local exitstatus=$? whiptail_check_exitstatus $exitstatus - } whiptail_cancel() { @@ -120,6 +119,8 @@ whiptail_create_admin_user() { ADMINUSER=$(whiptail --title "Security Onion Install" --inputbox \ "Please enter a username for a new system admin user: \nThe local onion account will be disabled during this install" 10 60 3>&1 1>&2 2>&3) + local exitstatus=$? + whiptail_check_exitstatus $exitstatus } whiptail_create_admin_user_password1() { @@ -300,20 +301,6 @@ whiptail_storage_requirements() { whiptail_check_exitstatus $exitstatus } -whiptail_invalid_pass_warning() { - - [ -n "$TESTING" ] && return - - whiptail --title "Security Onion Setup" --msgbox "Please choose a more secure password." 8 75 -} - -whiptail_invalid_pass_characters_warning() { - - [ -n "$TESTING" ] && return - - whiptail --title "Security Onion Setup" --msgbox "Password is invalid. Please exclude single quotes, double quotes, dollar signs, and backslashes from the password." 8 75 -} - whiptail_cur_close_days() { [ -n "$TESTING" ] && return @@ -391,11 +378,6 @@ whiptail_dockernet_check(){ whiptail --title "Security Onion Setup" --yesno \ "Do you want to keep the default Docker IP range? \n \n(Choose yes if you don't know what this means)" 10 75 - local exitstatus=$? - - if [[ $exitstatus == 1 ]]; then - whiptail_dockernet_net - fi } whiptail_dockernet_net() { @@ -495,32 +477,29 @@ whiptail_homenet_manager() { [ -n "$TESTING" ] && return HNMANAGER=$(whiptail --title "Security Onion Setup" --inputbox \ - "Enter your HOME_NET, separating CIDR blocks with a comma (,):" 10 75 10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 3>&1 1>&2 2>&3) + "Enter your HOME_NET, separating CIDR blocks with a comma (,):" 10 75 10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus export HNMANAGER } -whiptail_homenet_sensor() { - +whiptail_homenet_sensor_inherit() { [ -n "$TESTING" ] && return # Ask to inherit from manager whiptail --title "Security Onion Setup" --yesno "Do you want to inherit the HOME_NET from the Manager?" 8 75 +} - local exitstatus=$? +whiptail_homenet_sensor() { + [ -n "$TESTING" ] && return - if [ $exitstatus == 0 ]; then - export HNSENSOR=inherit - else - HNSENSOR=$(whiptail --title "Security Onion Setup" --inputbox \ + HNSENSOR=$(whiptail --title "Security Onion Setup" --inputbox \ "Enter your HOME_NET, separating CIDR blocks with a comma (,):" 10 75 10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 3>&1 1>&2 2>&3) - local exitstatus=$? - whiptail_check_exitstatus $exitstatus + local exitstatus=$? + whiptail_check_exitstatus $exitstatus - export HNSENSOR - fi + export HNSENSOR } whiptail_install_type() { @@ -602,6 +581,13 @@ whiptail_install_type_other() { export install_type } +whiptail_invalid_input() { # TODO: This should accept a list of arguments to specify what general pattern the input should follow + [ -n "$TESTING" ] && return + + whiptail --title "Security Onion Setup" --msgbox "Invalid input, please try again." 8 75 + +} + whiptail_invalid_pass_characters_warning() { [ -n "$TESTING" ] && return @@ -623,6 +609,18 @@ whiptail_invalid_user_warning() { whiptail --title "Security Onion Setup" --msgbox "Please enter a valid email address." 8 75 } +whiptail_invalid_hostname() { + [ -n "$TESTING" ] && return + + local error_message + error_message=$(echo "Please choose a valid hostname. It cannot be localhost; and must contain only \ + the ASCII letters 'A-Z' and 'a-z' (case-sensitive), the digits '0' through '9', \ + and hyphen ('-')" | tr -d '\t') + + whiptail --title "Security Onion Setup" \ + --msgbox "$error_message" 10 75 +} + whiptail_log_size_limit() { [ -n "$TESTING" ] && return @@ -692,6 +690,16 @@ whiptail_management_interface_gateway() { whiptail_check_exitstatus $exitstatus } +whiptail_management_interface_ip_mask() { + [ -n "$TESTING" ] && return + + manager_ip_mask=$(whiptail --title "Security Onion Setup" --inputbox \ + "Enter your IP address (with CIDR mask):" 10 60 3>&1 1>&2 2>&3) + + local exitstatus=$? + whiptail_check_exitstatus $exitstatus +} + whiptail_management_interface_ip() { [ -n "$TESTING" ] && return @@ -781,7 +789,6 @@ whiptail_net_setup_complete() { exit 0 } - whiptail_management_server() { [ -n "$TESTING" ] && return @@ -792,28 +799,6 @@ whiptail_management_server() { local exitstatus=$? whiptail_check_exitstatus $exitstatus - while [[ $MSRV == *'localhost'* || ! ( $MSRV =~ ^[a-zA-Z0-9\-]*$ ) ]] ; do - local error_message - error_message=$(echo "Please choose a valid hostname. It cannot contain localhost; and must contain only \ - the ASCII letters 'A-Z' and 'a-z' (case-sensitive), the digits '0' through '9', \ - and hyphen ('-')" | tr -d '\t') - - whiptail --title "Security Onion Setup" \ - --msgbox "$error_message" 10 75 - - MSRV=$(whiptail --title "Security Onion Setup" --inputbox \ - "Enter your Manager Server hostname. It is CASE SENSITIVE!" 10 75 XXXX 3>&1 1>&2 2>&3) - - local exitstatus=$? - whiptail_check_exitstatus $exitstatus - done - - if ! getent hosts "$MSRV"; then - add_manager_hostfile - else - MSRVIP=$(getent hosts "$MSRV" | awk 'NR==1{print $1}') - fi - } # Ask if you want to do advanced setup of the Manager @@ -839,11 +824,6 @@ whiptail_manager_adv_escluster(){ whiptail --title "Security Onion Setup" --yesno \ "Do you want to set up a traditional ES cluster for using replicas and/or Hot-Warm indices? Recommended only for those who have experience with ES clustering! " 12 75 - local exitstatus=$? - - if [[ $exitstatus == 0 ]]; then - whiptail_manager_adv_escluster_name - fi } # Get a cluster name @@ -1012,24 +992,24 @@ whiptail_node_ls_heap() { } -whiptail_node_ls_pipeline_worker() { +whiptail_node_ls_pipline_batchsize() { [ -n "$TESTING" ] && return - LSPIPELINEWORKERS=$(whiptail --title "Security Onion Setup" --inputbox \ - "\nEnter LogStash Pipeline Workers: \n \n(Recommended value is pre-populated)" 10 75 "$num_cpu_cores" 3>&1 1>&2 2>&3) + LSPIPELINEBATCH=$(whiptail --title "Security Onion Setup" --inputbox \ + "\nEnter Logstash pipeline batch size: \n \n(Default value is pre-populated)" 10 75 125 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus } -whiptail_node_ls_pipline_batchsize() { +whiptail_node_ls_pipeline_worker() { [ -n "$TESTING" ] && return - LSPIPELINEBATCH=$(whiptail --title "Security Onion Setup" --inputbox \ - "\nEnter LogStash Pipeline Batch Size: \n \n(Default value is pre-populated)" 10 75 125 3>&1 1>&2 2>&3) + LSPIPELINEWORKERS=$(whiptail --title "Security Onion Setup" --inputbox \ + "\nEnter number of Logstash pipeline workers: \n \n(Recommended value is pre-populated)" 10 75 "$num_cpu_cores" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -1041,7 +1021,7 @@ whiptail_node_ls_input_threads() { [ -n "$TESTING" ] && return LSINPUTTHREADS=$(whiptail --title "Security Onion Setup" --inputbox \ - "\nEnter LogStash Input Threads: \n \n(Default value is pre-populated)" 10 75 1 3>&1 1>&2 2>&3) + "\nEnter number of Logstash input threads: \n \n(Default value is pre-populated)" 10 75 1 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -1078,16 +1058,6 @@ whiptail_patch_name_new_schedule() { local exitstatus=$? whiptail_check_exitstatus $exitstatus - - while [[ -z "$PATCHSCHEDULENAME" ]]; do - whiptail --title "Security Onion Setup" --msgbox "Please enter a name for this OS patch schedule." 8 75 - PATCHSCHEDULENAME=$(whiptail --title "Security Onion Setup" --inputbox \ - "What name do you want to give this OS patch schedule? This schedule needs to be named uniquely. Available schedules can be found on the manager under /opt/so/salt/patch/os/schedules/.yml" 10 75 3>&1 1>&2 2>&3) - local exitstatus=$? - whiptail_check_exitstatus $exitstatus - done - - } whiptail_patch_schedule() { @@ -1104,27 +1074,6 @@ whiptail_patch_schedule() { local exitstatus=$? whiptail_check_exitstatus $exitstatus - - - case $patch_schedule in - 'New Schedule') - whiptail_patch_schedule_select_days - whiptail_patch_schedule_select_hours - whiptail_patch_name_new_schedule - patch_schedule_os_new - ;; - 'Import Schedule') - whiptail_patch_schedule_import - ;; - 'Automatic') - PATCHSCHEDULENAME='auto' - ;; - 'Manual') - PATCHSCHEDULENAME='manual' - ;; - esac - - } whiptail_patch_schedule_import() { @@ -1304,32 +1253,11 @@ whiptail_set_hostname() { [ -n "$TESTING" ] && return - HOSTNAME=$(cat /etc/hostname) - - if [[ "$HOSTNAME" == *'localhost'* ]]; then HOSTNAME=securityonion; fi - HOSTNAME=$(whiptail --title "Security Onion Setup" --inputbox \ - "Enter the hostname (not FQDN) you would like to set:" 10 75 "$HOSTNAME" 3>&1 1>&2 2>&3) + "Enter the hostname (not FQDN) you would like to set:" 10 75 "$HOSTNAME" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus - - while [[ $HOSTNAME == *'localhost'* || ! ( $HOSTNAME =~ ^[a-zA-Z0-9\-]*$ ) ]] ; do - local error_message - error_message=$(echo "Please choose a valid hostname. It cannot contain localhost; and must contain only \ - the ASCII letters 'a' through 'z' (case-insensitive), the digits '0' through '9', \ - and hyphen ('-')" | tr -d '\t') - - whiptail --title "Security Onion Setup" \ - --msgbox "$error_message" 10 75 - - HOSTNAME=$(whiptail --title "Security Onion Setup" --inputbox \ - "Enter the hostname (not FQDN) you would like to set:" 10 75 "$HOSTNAME" 3>&1 1>&2 2>&3) - - local exitstatus=$? - whiptail_check_exitstatus $exitstatus - done - } whiptail_set_redirect() { @@ -1350,7 +1278,7 @@ whiptail_set_redirect_host() { [ -n "$TESTING" ] && return REDIRECTHOST=$(whiptail --title "Security Onion Setup" --inputbox \ - "Enter the Hostname or IP you would like to use for the web interface:" 10 75 "$HOSTNAME" 3>&1 1>&2 2>&3) + "Enter the Hostname, IP, or FQDN you would like to use for the web interface:" 10 75 "$HOSTNAME" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus } @@ -1412,25 +1340,25 @@ whiptail_shard_count() { } -whiptail_so_allow() { - +whiptail_so_allow_yesno() { [ -n "$TESTING" ] && return whiptail --title "Security Onion Setup" \ - --yesno "Do you want to run so-allow to allow access to the web tools?" \ - 8 75 + --yesno "Do you want to run so-allow to allow access to the web tools?" \ + 8 75 +} - local exitstatus=$? +whiptail_so_allow() { - if [[ $exitstatus == 0 ]]; then - ALLOW_CIDR=$(whiptail --title "Security Onion Setup" \ + [ -n "$TESTING" ] && return + + ALLOW_CIDR=$(whiptail --title "Security Onion Setup" \ --inputbox "Enter a single IP address or an IP range, in CIDR notation, to allow:" \ 10 75 3>&1 1>&2 2>&3) - local exitstatus=$? - - export ALLOW_ROLE='a' - export ALLOW_CIDR - fi + local exitstatus=$? + + export ALLOW_ROLE='a' + export ALLOW_CIDR } whiptail_storage_requirements() { From 38e37a0385c43c83ba9793f7fc0261471bf72614 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 12 Jan 2021 11:04:40 -0500 Subject: [PATCH 31/56] [refactor] Remove whiptail shard count prompt --- setup/so-functions | 9 --------- setup/so-whiptail | 12 ------------ 2 files changed, 21 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 65e9a3a83..22bd514ac 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -425,15 +425,6 @@ collect_es_cluster_name() { fi } -collect_es_shard_count() { - whiptail_shard_count - - while ! valid_int "$SHARDCOUNT"; do - whiptail_invalid_input - whiptail_shard_count - done -} - collect_es_space_limit() { whiptail_log_size_limit diff --git a/setup/so-whiptail b/setup/so-whiptail index dd8083d55..78fd7b9c2 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1328,18 +1328,6 @@ whiptail_setup_failed() { whiptail --title "Security Onion Setup" --msgbox "$message" $height 75 } -whiptail_shard_count() { - - [ -n "$TESTING" ] && return - - SHARDCOUNT=$(whiptail --title "Security Onion Setup" --inputbox \ - "\nEnter ES Shard Count: \n \n(Default value is pre-populated)" 10 75 125 3>&1 1>&2 2>&3) - - local exitstatus=$? - whiptail_check_exitstatus $exitstatus - -} - whiptail_so_allow_yesno() { [ -n "$TESTING" ] && return From fb31b56c8b25bf9dc512f02117844865946c6972 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 12 Jan 2021 11:22:52 -0500 Subject: [PATCH 32/56] [fix] Only check for network init file if iso --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 917273786..5707212c6 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -268,7 +268,7 @@ if ! [[ -f $install_opt_file ]]; then collect_mngr_hostname fi - if [[ $is_minion ]] || [[ $reinit_networking ]] || ! [[ -f $net_init_file ]]; then + if [[ $is_minion ]] || [[ $reinit_networking ]] || [[ $is_iso ]] && ! [[ -f $net_init_file ]]; then whiptail_management_interface_setup fi From ff69d022b3f465ef721681ed9bf1460a31882155 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 12 Jan 2021 11:26:20 -0500 Subject: [PATCH 33/56] [fix] Correct function call --- setup/so-functions | 4 ++-- setup/so-whiptail | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 22bd514ac..3eeaee674 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -483,7 +483,7 @@ collect_helix_key() { collect_homenet_mngr() { whiptail_homenet_manager - while ! __validate_cidr_arr "$HNMANAGER"; do + while ! valid_cidr_lit "$HNMANAGER"; do whiptail_invalid_input whiptail_homenet_manager done @@ -495,7 +495,7 @@ collect_homenet_snsr() { else whiptail_homenet_sensor - while ! __validate_cidr_arr "$HNSENSOR"; do + while ! valid_cidr_list "$HNSENSOR"; do whiptail_invalid_input whiptail_homenet_sensor done diff --git a/setup/so-whiptail b/setup/so-whiptail index 78fd7b9c2..edb784b31 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -477,7 +477,7 @@ whiptail_homenet_manager() { [ -n "$TESTING" ] && return HNMANAGER=$(whiptail --title "Security Onion Setup" --inputbox \ - "Enter your HOME_NET, separating CIDR blocks with a comma (,):" 10 75 10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 3>&1 1>&2 2>&3) + "Enter your home network(s), separating CIDR blocks with a comma (,):" 10 75 "10.0.0.0/8,192.168.0.0/16,172.16.0.0/12" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -495,7 +495,7 @@ whiptail_homenet_sensor() { [ -n "$TESTING" ] && return HNSENSOR=$(whiptail --title "Security Onion Setup" --inputbox \ - "Enter your HOME_NET, separating CIDR blocks with a comma (,):" 10 75 10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 3>&1 1>&2 2>&3) + "Enter your home network(s), separating CIDR blocks with a comma (,):" 10 75 "10.0.0.0/8,192.168.0.0/16,172.16.0.0/12" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus From 5c6f8f9d4778b32d097eff5e0982266eaae9c976 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 12 Jan 2021 11:27:03 -0500 Subject: [PATCH 34/56] [fix] Correct function call (pt 2) --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 3eeaee674..da935518d 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -483,7 +483,7 @@ collect_helix_key() { collect_homenet_mngr() { whiptail_homenet_manager - while ! valid_cidr_lit "$HNMANAGER"; do + while ! valid_cidr_list "$HNMANAGER"; do whiptail_invalid_input whiptail_homenet_manager done From dd20002fd5999bd354bbefbc7c0486caee9cd2e8 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 12 Jan 2021 11:28:24 -0500 Subject: [PATCH 35/56] [fix] Dockernet prompt is negative, continue on "no" --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index da935518d..3f3b068d4 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -404,7 +404,7 @@ collect_dns_domain() { } collect_dockernet() { - if whiptail_dockernet_check; then + if ! whiptail_dockernet_check; then whiptail_dockernet_net while ! valid_ip4 "$DOCKERNET"; do From ebac17ce38f85585da85913a87f19f793b276d6d Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 13 Jan 2021 10:29:36 -0500 Subject: [PATCH 36/56] [wip] Attempting to fix missing patch schedule prompts --- setup/so-functions | 2 +- setup/so-whiptail | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 3f3b068d4..b132ba64d 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -605,7 +605,7 @@ collect_oinkcode() { collect_patch_schedule() { whiptail_patch_schedule - case $patch_schedule in + case "$patch_schedule" in 'New Schedule') whiptail_patch_schedule_select_days whiptail_patch_schedule_select_hours diff --git a/setup/so-whiptail b/setup/so-whiptail index edb784b31..e66cd3ccd 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1074,6 +1074,8 @@ whiptail_patch_schedule() { local exitstatus=$? whiptail_check_exitstatus $exitstatus + + export patch_schedule } whiptail_patch_schedule_import() { From 07349983158dede9c3b61e66c944d2279480d617 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 13 Jan 2021 10:39:24 -0500 Subject: [PATCH 37/56] [fix] patch_schedule should not be local --- setup/so-whiptail | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index e66cd3ccd..c0f7dba17 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1064,18 +1064,15 @@ whiptail_patch_schedule() { [ -n "$TESTING" ] && return - local patch_schedule patch_schedule=$(whiptail --title "Security Onion Setup" --radiolist \ - "Choose OS patch schedule: \nThis will NOT update Security Onion related tools such as Zeek, Elasticsearch, Kibana, SaltStack, etc." 15 75 5 \ - "Automatic" "Updates installed every 8 hours if available" ON \ - "Manual" "Updates will be installed manually" OFF \ - "Import Schedule" "Import named schedule on following screen" OFF \ - "New Schedule" "Configure and name new schedule on next screen" OFF 3>&1 1>&2 2>&3 ) + "Choose OS patch schedule: \nThis will NOT update Security Onion related tools such as Zeek, Elasticsearch, Kibana, SaltStack, etc." 15 75 5 \ + "Automatic" "Updates installed every 8 hours if available" ON \ + "Manual" "Updates will be installed manually" OFF \ + "Import Schedule" "Import named schedule on following screen" OFF \ + "New Schedule" "Configure and name new schedule on next screen" OFF 3>&1 1>&2 2>&3 ) local exitstatus=$? whiptail_check_exitstatus $exitstatus - - export patch_schedule } whiptail_patch_schedule_import() { From d254fd960abe167adbf5caf3492e6c4fe5bdb839 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 13 Jan 2021 11:04:35 -0500 Subject: [PATCH 38/56] [feat] Add message explaining strings cannot contain spaces --- setup/so-functions | 6 +++--- setup/so-whiptail | 9 ++++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index b132ba64d..d31305c5d 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -419,7 +419,7 @@ collect_es_cluster_name() { whiptail_manager_adv_escluster_name while ! valid_string "$ESCLUSTERNAME"; do - whiptail_invalid_input + whiptail_invalid_string "ES cluster name" whiptail_manager_adv_escluster_name done fi @@ -628,7 +628,7 @@ collect_patch_schedule_name_new() { whiptail_patch_name_new_schedule while ! valid_string "$PATCHSCHEDULENAME"; do - whiptail_invalid_input + whiptail_invalid_string "schedule name" whiptail_patch_name_new_schedule done } @@ -637,7 +637,7 @@ collect_patch_schedule_name_import() { whiptail_patch_schedule_import while ! valid_string "$PATCHSCHEDULENAME"; do - whiptail_invalid_input + whiptail_invalid_string "schedule name" whiptail_patch_schedule_import done } diff --git a/setup/so-whiptail b/setup/so-whiptail index c0f7dba17..a182fec5a 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -584,7 +584,14 @@ whiptail_install_type_other() { whiptail_invalid_input() { # TODO: This should accept a list of arguments to specify what general pattern the input should follow [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --msgbox "Invalid input, please try again." 8 75 + whiptail --title "Security Onion Setup" --msgbox " Invalid input, please try again." 7 40 + +} + +whiptail_invalid_string() { + [ -n "$TESTING" ] && return + + whiptail --title "Security Onion Setup" --msgbox "Invalid input, please try again.\n\nThe $1 cannot contain spaces." 9 45 } From 4dc3a6aa35280f84978ca2e4251841e44776f175 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 13 Jan 2021 11:36:20 -0500 Subject: [PATCH 39/56] [refactor] Standardize list inputs to comma separated --- salt/common/tools/sbin/so-common | 2 +- setup/so-functions | 2 ++ setup/so-whiptail | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 9fd4ef7dc..1f8a36d23 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -264,7 +264,7 @@ valid_cidr_list() { valid_dns_list() { local all_valid=0 - read -r -a dns_arr <<< "$1" + IFS="," read -r -a dns_arr <<< "$1" for addr in "${dns_arr[@]}"; do valid_ip4 "$addr" || all_valid=1 diff --git a/setup/so-functions b/setup/so-functions index d31305c5d..25407c8fa 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -392,6 +392,8 @@ collect_dns() { whiptail_invalid_input whiptail_management_interface_dns done + + MDNS=$(echo "$MDNS" | tr -s "," " ") # MDNS needs to be space separated, we prompt for comma separated for consistency } collect_dns_domain() { diff --git a/setup/so-whiptail b/setup/so-whiptail index a182fec5a..ad9e42f28 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -669,10 +669,11 @@ whiptail_management_interface_dns() { [ -n "$TESTING" ] && return MDNS=$(whiptail --title "Security Onion Setup" --inputbox \ - "Enter your DNS servers separated by a space:" 10 60 8.8.8.8 8.8.4.4 3>&1 1>&2 2>&3) + "Enter your DNS servers separated by commas:" 10 60 "8.8.8.8,8.8.4.4" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus + } whiptail_management_interface_dns_search() { From 6ea3a651a4881e20b1008d91941e7e75768bb5e5 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 13 Jan 2021 11:37:48 -0500 Subject: [PATCH 40/56] [fix] Fix unit tests for dns list --- tests/validation.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/validation.sh b/tests/validation.sh index 04cdaa7c7..d16c8bbb9 100644 --- a/tests/validation.sh +++ b/tests/validation.sh @@ -94,13 +94,13 @@ sleep 0.15s header "DNS" -test_fun 0 valid_dns_list "8.8.8.8 8.8.4.4" +test_fun 0 valid_dns_list "8.8.8.8,8.8.4.4" test_fun 0 valid_dns_list "8.8.8.8" -test_fun 1 valid_dns_list "8.8.8.8,8.8.4.4" +test_fun 1 valid_dns_list "8.8.8.8 8.8.4.4" -test_fun 1 valid_dns_list "8.8.8. 8.8.4.4" +test_fun 1 valid_dns_list "8.8.8.,8.8.4.4" test_fun 1 valid_dns_list "192.168.9." From 90f085b2d78467a75b34c2513e28f6edaf495a70 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 13 Jan 2021 15:57:21 -0500 Subject: [PATCH 41/56] [fix] Fail setup early if we can't determine version of manager --- setup/so-functions | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 25407c8fa..f1728dd71 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1213,12 +1213,21 @@ docker_seed_registry() { } download_repo_tarball() { - mkdir -p /root/manager_setup/securityonion - { - local manager_ver - manager_ver=$(ssh -i /root/.ssh/so.key soremote@"$MSRV" cat /etc/soversion) - scp -i /root/.ssh/so.key soremote@"$MSRV":/opt/so/repo/"$manager_ver".tar.gz /root/manager_setup - } >> "$setup_log" 2>&1 + + mkdir -p /root/manager_setup + + local manager_ver + manager_ver=$($sshcmd -i /root/.ssh/so.key soremote@"$MSRV" cat /etc/soversion) >> "$setup_log" 2>&1 + + # Fail if we can't determine the version + if [[ $manager_ver == '' ]]; then + rm /root/install_opt + local message="Could not determine the version of Security Onion running on the manager, please check your network settings." + echo "$message" | tee -a "$setup_log" + kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit + fi + + $scpcmd -i /root/.ssh/so.key soremote@"$MSRV":/opt/so/repo/"$manager_ver".tar.gz /root/manager_setup >> "$setup_log" 2>&1 # Fail if the file doesn't download if ! [ -f /root/manager_setup/"$manager_ver".tar.gz ]; then @@ -1227,7 +1236,8 @@ download_repo_tarball() { echo "$message" | tee -a "$setup_log" kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 fi - + + mkdir -p /root/manager_setup/securityonion { tar -xzf /root/manager_setup/"$manager_ver".tar.gz -C /root/manager_setup/securityonion rm -rf /root/manager_setup/"$manager_ver".tar.gz From 8245b258350773b8aad2f53c7652a1fa9be77a48 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 13 Jan 2021 17:28:19 -0500 Subject: [PATCH 42/56] [fix] Move metadata function --- setup/so-whiptail | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index ad9e42f28..365cb5c0d 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -931,6 +931,19 @@ whiptail_manager_updates_warning() { whiptail_check_exitstatus $exitstatus } +whiptail_metadata_tool() { + + [ -n "$TESTING" ] && return + + # Legacy variable naming + ZEEKVERSION=$(whiptail --title "Security Onion Setup" --radiolist "What tool would you like to use to generate metadata?" 20 75 4 \ + "ZEEK" "Zeek (formerly known as Bro)" ON \ + "SURICATA" "Suricata" OFF 3>&1 1>&2 2>&3) + + local exitstatus=$? + whiptail_check_exitstatus $exitstatus +} + whiptail_nids() { [ -n "$TESTING" ] && return @@ -1484,17 +1497,3 @@ whiptail_zeek_pins() { IFS=' ' read -ra ZEEKPINS <<< "$ZEEKPINS" } - -whiptail_metadata_tool() { - - [ -n "$TESTING" ] && return - - # Legacy variable naming - ZEEKVERSION=$(whiptail --title "Security Onion Setup" --radiolist "What tool would you like to use to generate metadata?" 20 75 4 \ - "ZEEK" "Zeek (formerly known as Bro)" ON \ - "SURICATA" "Suricata" OFF 3>&1 1>&2 2>&3) - - local exitstatus=$? - whiptail_check_exitstatus $exitstatus -} - From 3c22738ae11358a3d5e2f4c1fd3bea1b1c43cb74 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 14 Jan 2021 10:38:47 -0500 Subject: [PATCH 43/56] [fix] Add example CIDR notation, remove placeholder X.X.X.X --- setup/so-whiptail | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 365cb5c0d..b3976a7c9 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -692,7 +692,7 @@ whiptail_management_interface_gateway() { [ -n "$TESTING" ] && return MGATEWAY=$(whiptail --title "Security Onion Setup" --inputbox \ - "Enter your gateway:" 10 60 X.X.X.X 3>&1 1>&2 2>&3) + "Enter your gateway's IPv4 address:" 10 60 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -702,7 +702,7 @@ whiptail_management_interface_ip_mask() { [ -n "$TESTING" ] && return manager_ip_mask=$(whiptail --title "Security Onion Setup" --inputbox \ - "Enter your IP address (with CIDR mask):" 10 60 3>&1 1>&2 2>&3) + "Enter your IPv4 address with CIDR mask (e.g. 192.168.1.2/24):" 10 60 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus From 82c7832d60beea6cfeaea61609bfe9cbb160298e Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 14 Jan 2021 12:58:21 -0500 Subject: [PATCH 44/56] [fix] Fix indent in valid_hostname --- salt/common/tools/sbin/so-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 1f8a36d23..e3b01886b 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -284,7 +284,7 @@ valid_fqdn() { valid_hostname() { local hostname=$1 - [[ $hostname =~ ^[a-zA-Z0-9\-]+$ ]] && [[ $hostname != 'localhost' ]] && return 0 || return 1 + [[ $hostname =~ ^[a-zA-Z0-9\-]+$ ]] && [[ $hostname != 'localhost' ]] && return 0 || return 1 } valid_ip4() { From ddcd487edc612edfec4afe0ded8c48c1848b3bcb Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 14 Jan 2021 13:08:11 -0500 Subject: [PATCH 45/56] [fix] Remove files not in dev --- setup/automation/distributed_forwardnode_ami | 78 ------------------ setup/automation/distributed_manager_ami | 77 ------------------ setup/automation/distributed_searchnode_ami | 78 ------------------ setup/automation/eval_ami | 81 ------------------- setup/automation/eval_iso | 77 ------------------ setup/automation/standalone_ami | 83 -------------------- setup/automation/standalone_iso | 77 ------------------ 7 files changed, 551 deletions(-) delete mode 100644 setup/automation/distributed_forwardnode_ami delete mode 100644 setup/automation/distributed_manager_ami delete mode 100644 setup/automation/distributed_searchnode_ami delete mode 100644 setup/automation/eval_ami delete mode 100644 setup/automation/eval_iso delete mode 100644 setup/automation/standalone_ami delete mode 100644 setup/automation/standalone_iso diff --git a/setup/automation/distributed_forwardnode_ami b/setup/automation/distributed_forwardnode_ami deleted file mode 100644 index a3cd2cccb..000000000 --- a/setup/automation/distributed_forwardnode_ami +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash - -# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -TESTING=true - -address_type=DHCP -ADMINUSER=onionuser -ADMINPASS1=onionuser -ADMINPASS2=onionuser -#ALLOW_CIDR=0.0.0.0/0 -#ALLOW_ROLE=a -BASICZEEK=2 -BASICSURI=2 -# BLOGS= -BNICS=ens6 -ZEEKVERSION=ZEEK -# CURCLOSEDAYS= -# EVALADVANCED=BASIC -#GRAFANA=1 -# HELIXAPIKEY= -HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 -HNSENSOR=inherit -HOSTNAME=forwardnode-aws -install_type=SENSOR -# LSINPUTBATCHCOUNT= -# LSINPUTTHREADS= -# LSPIPELINEBATCH= -# LSPIPELINEWORKERS= -#MANAGERADV=BASIC -MANAGERUPDATES=1 -# MDNS= -# MGATEWAY= -# MIP= -# MMASK= -MNIC=ens5 -# MSEARCH= -MSRV=manager-aws -MSRVIP=172.16.163.10 -# MTU= -#NIDS=Suricata -# NODE_ES_HEAP_SIZE= -# NODE_LS_HEAP_SIZE= -#NODESETUP=NODEBASIC -NSMSETUP=BASIC -NODEUPDATES=MANAGER -# OINKCODE= -#OSQUERY=1 -# PATCHSCHEDULEDAYS= -# PATCHSCHEDULEHOURS= -PATCHSCHEDULENAME=auto -#PLAYBOOK=1 -# REDIRECTHOST= -#REDIRECTINFO=HOSTNAME -#RULESETUP=ETOPEN -# SHARDCOUNT= -SKIP_REBOOT=0 -SOREMOTEPASS1=onionuser -SOREMOTEPASS2=onionuser -#STRELKA=1 -#THEHIVE=1 -#WAZUH=1 -# WEBUSER=onionuser@somewhere.invalid -# WEBPASSWD1=0n10nus3r -# WEBPASSWD2=0n10nus3r diff --git a/setup/automation/distributed_manager_ami b/setup/automation/distributed_manager_ami deleted file mode 100644 index b1effcf7a..000000000 --- a/setup/automation/distributed_manager_ami +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash - -# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -TESTING=true - -address_type=DHCP -ADMINUSER=onionuser -ADMINPASS1=onionuser -ADMINPASS2=onionuser -ALLOW_CIDR=0.0.0.0/0 -ALLOW_ROLE=a -BASICZEEK=2 -BASICSURI=2 -# BLOGS= -BNICS=ens6 -ZEEKVERSION=ZEEK -# CURCLOSEDAYS= -# EVALADVANCED=BASIC -GRAFANA=1 -# HELIXAPIKEY= -HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 -HNSENSOR=inherit -HOSTNAME=manager-aws -install_type=MANAGER -# LSINPUTBATCHCOUNT= -# LSINPUTTHREADS= -# LSPIPELINEBATCH= -# LSPIPELINEWORKERS= -MANAGERADV=BASIC -MANAGERUPDATES=1 -# MDNS= -# MGATEWAY= -# MIP= -# MMASK= -MNIC=ens5 -# MSEARCH= -# MSRV= -# MTU= -NIDS=Suricata -# NODE_ES_HEAP_SIZE= -# NODE_LS_HEAP_SIZE= -NODESETUP=NODEBASIC -NSMSETUP=BASIC -NODEUPDATES=MANAGER -# OINKCODE= -OSQUERY=1 -# PATCHSCHEDULEDAYS= -# PATCHSCHEDULEHOURS= -PATCHSCHEDULENAME=auto -PLAYBOOK=1 -# REDIRECTHOST= -REDIRECTINFO=HOSTNAME -RULESETUP=ETOPEN -# SHARDCOUNT= -SKIP_REBOOT=0 -SOREMOTEPASS1=onionuser -SOREMOTEPASS2=onionuser -STRELKA=1 -THEHIVE=1 -WAZUH=1 -WEBUSER=onionuser@somewhere.invalid -WEBPASSWD1=0n10nus3r -WEBPASSWD2=0n10nus3r diff --git a/setup/automation/distributed_searchnode_ami b/setup/automation/distributed_searchnode_ami deleted file mode 100644 index e50e18475..000000000 --- a/setup/automation/distributed_searchnode_ami +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash - -# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -TESTING=true - -address_type=DHCP -ADMINUSER=onionuser -ADMINPASS1=onionuser -ADMINPASS2=onionuser -#ALLOW_CIDR=0.0.0.0/0 -#ALLOW_ROLE=a -#BASICZEEK=7 -#BASICSURI=7 -# BLOGS= -#BNICS=ens6 -#ZEEKVERSION=ZEEK -# CURCLOSEDAYS= -# EVALADVANCED=BASIC -#GRAFANA=1 -# HELIXAPIKEY= -HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 -HNSENSOR=inherit -HOSTNAME=searchnode-aws -install_type=SEARCHNODE -# LSINPUTBATCHCOUNT= -# LSINPUTTHREADS= -# LSPIPELINEBATCH= -# LSPIPELINEWORKERS= -#MANAGERADV=BASIC -MANAGERUPDATES=1 -# MDNS= -# MGATEWAY= -# MIP= -# MMASK= -MNIC=ens5 -# MSEARCH= -MSRV=manager-aws -MSRVIP=172.16.163.10 -# MTU= -#NIDS=Suricata -# NODE_ES_HEAP_SIZE= -# NODE_LS_HEAP_SIZE= -NODESETUP=NODEBASIC -NSMSETUP=BASIC -NODEUPDATES=MANAGER -# OINKCODE= -#OSQUERY=1 -# PATCHSCHEDULEDAYS= -# PATCHSCHEDULEHOURS= -PATCHSCHEDULENAME=auto -#PLAYBOOK=1 -# REDIRECTHOST= -#REDIRECTINFO=HOSTNAME -#RULESETUP=ETOPEN -# SHARDCOUNT= -SKIP_REBOOT=0 -SOREMOTEPASS1=onionuser -SOREMOTEPASS2=onionuser -#STRELKA=1 -#THEHIVE=1 -#WAZUH=1 -# WEBUSER=onionuser@somewhere.invalid -# WEBPASSWD1=0n10nus3r -# WEBPASSWD2=0n10nus3r diff --git a/setup/automation/eval_ami b/setup/automation/eval_ami deleted file mode 100644 index 91f418b14..000000000 --- a/setup/automation/eval_ami +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/bash - -# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -TESTING=true - -address_type=DHCP -ADMINUSER=onionuser -ADMINPASS1=onionuser -ADMINPASS2=onionuser -ALLOW_CIDR=0.0.0.0/0 -ALLOW_ROLE=a -BASICZEEK=2 -BASICSURI=2 -# BLOGS= -BNICS=eth1 -ZEEKVERSION=ZEEK -# CURCLOSEDAYS= -# EVALADVANCED=BASIC -GRAFANA=1 -# HELIXAPIKEY= -HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 -HNSENSOR=inherit -HOSTNAME=eval -install_type=EVAL -# LSINPUTBATCHCOUNT= -# LSINPUTTHREADS= -# LSPIPELINEBATCH= -# LSPIPELINEWORKERS= -MANAGERADV=BASIC -MANAGERUPDATES=1 -# MDNS= -# MGATEWAY= -# MIP= -# MMASK= -MNIC=eth0 -# MSEARCH= -# MSRV= -# MTU= -NIDS=Suricata -# NODE_ES_HEAP_SIZE= -# NODE_LS_HEAP_SIZE= -NODESETUP=NODEBASIC -NSMSETUP=BASIC -NODEUPDATES=MANAGER -# OINKCODE= -OSQUERY=1 -# PATCHSCHEDULEDAYS= -# PATCHSCHEDULEHOURS= -PATCHSCHEDULENAME=auto -PLAYBOOK=1 -# REDIRECTHOST= -REDIRECTINFO=IP -RULESETUP=ETOPEN -# SHARDCOUNT= -<<<<<<< HEAD:setup/automation/eval_ami -# SKIP_REBOOT=0 -======= -# SKIP_REBOOT= ->>>>>>> dev:setup/automation/eval-iso -SOREMOTEPASS1=onionuser -SOREMOTEPASS2=onionuser -STRELKA=1 -THEHIVE=1 -WAZUH=1 -WEBUSER=onionuser@somewhere.invalid -WEBPASSWD1=0n10nus3r -WEBPASSWD2=0n10nus3r diff --git a/setup/automation/eval_iso b/setup/automation/eval_iso deleted file mode 100644 index 6e5560028..000000000 --- a/setup/automation/eval_iso +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash - -# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -TESTING=true - -address_type=DHCP -ADMINUSER=onionuser -ADMINPASS1=onionuser -ADMINPASS2=onionuser -ALLOW_CIDR=0.0.0.0/0 -ALLOW_ROLE=a -BASICZEEK=7 -BASICSURI=7 -# BLOGS= -BNICS=eth1 -ZEEKVERSION=ZEEK -# CURCLOSEDAYS= -# EVALADVANCED=BASIC -GRAFANA=1 -# HELIXAPIKEY= -HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 -HNSENSOR=inherit -HOSTNAME=eval -install_type=EVAL -# LSINPUTBATCHCOUNT= -# LSINPUTTHREADS= -# LSPIPELINEBATCH= -# LSPIPELINEWORKERS= -MANAGERADV=BASIC -MANAGERUPDATES=1 -# MDNS= -# MGATEWAY= -# MIP= -# MMASK= -MNIC=eth0 -# MSEARCH= -# MSRV= -# MTU= -NIDS=Suricata -# NODE_ES_HEAP_SIZE= -# NODE_LS_HEAP_SIZE= -NODESETUP=NODEBASIC -NSMSETUP=BASIC -NODEUPDATES=MANAGER -# OINKCODE= -OSQUERY=1 -# PATCHSCHEDULEDAYS= -# PATCHSCHEDULEHOURS= -PATCHSCHEDULENAME=auto -PLAYBOOK=1 -# REDIRECTHOST= -REDIRECTINFO=IP -RULESETUP=ETOPEN -# SHARDCOUNT= -# SKIP_REBOOT= -SOREMOTEPASS1=onionuser -SOREMOTEPASS2=onionuser -STRELKA=1 -THEHIVE=1 -WAZUH=1 -WEBUSER=onionuser@somewhere.invalid -WEBPASSWD1=0n10nus3r -WEBPASSWD2=0n10nus3r diff --git a/setup/automation/standalone_ami b/setup/automation/standalone_ami deleted file mode 100644 index ae1101574..000000000 --- a/setup/automation/standalone_ami +++ /dev/null @@ -1,83 +0,0 @@ -#!/bin/bash - -# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -TESTING=true - -address_type=DHCP -ADMINUSER=onionuser -ADMINPASS1=onionuser -ADMINPASS2=onionuser -ALLOW_CIDR=0.0.0.0/0 -ALLOW_ROLE=a -BASICZEEK=2 -BASICSURI=2 -# BLOGS= -BNICS=eth1 -ZEEKVERSION=ZEEK -# CURCLOSEDAYS= -# EVALADVANCED=BASIC -GRAFANA=1 -# HELIXAPIKEY= -HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 -HNSENSOR=inherit -HOSTNAME=standalone -install_type=STANDALONE -INTERWEBS=AIRGAP -# LSINPUTBATCHCOUNT= -# LSINPUTTHREADS= -# LSPIPELINEBATCH= -# LSPIPELINEWORKERS= -MANAGERADV=BASIC -MANAGERUPDATES=1 -# MDNS= -# MGATEWAY= -# MIP= -# MMASK= -MNIC=eth0 -# MSEARCH= -# MSRV= -# MTU= -NIDS=Suricata -# NODE_ES_HEAP_SIZE= -# NODE_LS_HEAP_SIZE= -NODESETUP=NODEBASIC -NSMSETUP=BASIC -NODEUPDATES=MANAGER -# OINKCODE= -OSQUERY=1 -# PATCHSCHEDULEDAYS= -# PATCHSCHEDULEHOURS= -PATCHSCHEDULENAME=auto -PLAYBOOK=1 -<<<<<<< HEAD:setup/automation/standalone_ami -REDIRECTHOST=$(curl http://169.254.169.254/latest/meta-data/public-ipv4) -REDIRECTINFO=OTHER -======= -# REDIRECTHOST= -REDIRECTINFO=IP ->>>>>>> dev:setup/automation/standalone-airgap -RULESETUP=ETOPEN -# SHARDCOUNT= -# SKIP_REBOOT= -SOREMOTEPASS1=onionuser -SOREMOTEPASS2=onionuser -STRELKA=1 -THEHIVE=1 -WAZUH=1 -WEBUSER=onionuser@somewhere.invalid -WEBPASSWD1=0n10nus3r -WEBPASSWD2=0n10nus3r diff --git a/setup/automation/standalone_iso b/setup/automation/standalone_iso deleted file mode 100644 index 15b21e2df..000000000 --- a/setup/automation/standalone_iso +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash - -# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -TESTING=true - -address_type=DHCP -ADMINUSER=onionuser -ADMINPASS1=onionuser -ADMINPASS2=onionuser -ALLOW_CIDR=0.0.0.0/0 -ALLOW_ROLE=a -BASICZEEK=2 -BASICSURI=2 -# BLOGS= -BNICS=eth1 -ZEEKVERSION=ZEEK -# CURCLOSEDAYS= -# EVALADVANCED=BASIC -GRAFANA=1 -# HELIXAPIKEY= -HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 -HNSENSOR=inherit -HOSTNAME=standalone -install_type=STANDALONE -# LSINPUTBATCHCOUNT= -# LSINPUTTHREADS= -# LSPIPELINEBATCH= -# LSPIPELINEWORKERS= -MANAGERADV=BASIC -MANAGERUPDATES=1 -# MDNS= -# MGATEWAY= -# MIP= -# MMASK= -MNIC=eth0 -# MSEARCH= -# MSRV= -# MTU= -NIDS=Suricata -# NODE_ES_HEAP_SIZE= -# NODE_LS_HEAP_SIZE= -NODESETUP=NODEBASIC -NSMSETUP=BASIC -NODEUPDATES=MANAGER -# OINKCODE= -OSQUERY=1 -# PATCHSCHEDULEDAYS= -# PATCHSCHEDULEHOURS= -PATCHSCHEDULENAME=auto -PLAYBOOK=1 -# REDIRECTHOST= -REDIRECTINFO=IP -RULESETUP=ETOPEN -# SHARDCOUNT= -# SKIP_REBOOT= -SOREMOTEPASS1=onionuser -SOREMOTEPASS2=onionuser -STRELKA=1 -THEHIVE=1 -WAZUH=1 -WEBUSER=onionuser@somewhere.invalid -WEBPASSWD1=0n10nus3r -WEBPASSWD2=0n10nus3r From 8793965f4ae7a52665557d1a5ac308ec91cce6bd Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 14 Jan 2021 13:12:12 -0500 Subject: [PATCH 46/56] [fix] Capitalization --- setup/so-whiptail | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index b3976a7c9..390cd70a5 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -994,7 +994,7 @@ whiptail_node_es_heap() { [ -n "$TESTING" ] && return NODE_ES_HEAP_SIZE=$(whiptail --title "Security Onion Setup" --inputbox \ - "\nEnter ES Heap Size: \n \n(Recommended value is pre-populated)" 10 75 $ES_HEAP_SIZE 3>&1 1>&2 2>&3) + "\nEnter ES heap size: \n \n(Recommended value is pre-populated)" 10 75 $ES_HEAP_SIZE 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -1006,7 +1006,7 @@ whiptail_node_ls_heap() { [ -n "$TESTING" ] && return NODE_LS_HEAP_SIZE=$(whiptail --title "Security Onion Setup" --inputbox \ - "\nEnter LogStash Heap Size: \n \n(Recommended value is pre-populated)" 10 75 $LS_HEAP_SIZE 3>&1 1>&2 2>&3) + "\nEnter Logstash heap size: \n \n(Recommended value is pre-populated)" 10 75 $LS_HEAP_SIZE 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus From a7b9b565fdbedbfcac9cbc118366f1b805a2dd29 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 14 Jan 2021 13:19:29 -0500 Subject: [PATCH 47/56] [fix] Only return after all interfaces added to bond0 --- setup/so-functions | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index b7605506a..bcaad4a8e 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -885,10 +885,12 @@ configure_network_sensor() { connection.autoconnect "yes" >> "$setup_log" 2>&1 fi + local err for BNIC in "${BNICS[@]}"; do add_interface_bond0 "$BNIC" >> "$setup_log" 2>&1 - return $? + err=$? done + return $err } copy_salt_master_config() { From 2e23e0d690387ad907c2e43ee7fb5eda472c7746 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 14 Jan 2021 13:20:29 -0500 Subject: [PATCH 48/56] [fix] Only update err if return code is non-zero --- setup/so-functions | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index bcaad4a8e..c41ae42bd 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -888,7 +888,8 @@ configure_network_sensor() { local err for BNIC in "${BNICS[@]}"; do add_interface_bond0 "$BNIC" >> "$setup_log" 2>&1 - err=$? + local ret=$? + [[ $ret ]] || err=$ret done return $err } From df07cc578c3a7e08c9cea181c39765158650c0d2 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 14 Jan 2021 13:20:56 -0500 Subject: [PATCH 49/56] [fix] Only update err if return code is non-zero --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index c41ae42bd..87cbdb68e 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -889,7 +889,7 @@ configure_network_sensor() { for BNIC in "${BNICS[@]}"; do add_interface_bond0 "$BNIC" >> "$setup_log" 2>&1 local ret=$? - [[ $ret ]] || err=$ret + [[ $ret -eq 0 ]] || err=$ret done return $err } From ebc5a4314a9b9a71c13ad14e5e46696d775742b7 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 14 Jan 2021 13:43:00 -0500 Subject: [PATCH 50/56] [feat] Add salt logs to log rotation config --- salt/common/files/log-rotate.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/common/files/log-rotate.conf b/salt/common/files/log-rotate.conf index 8f1df0307..f8a16a038 100644 --- a/salt/common/files/log-rotate.conf +++ b/salt/common/files/log-rotate.conf @@ -19,6 +19,8 @@ /opt/so/log/telegraf/*.log /opt/so/log/redis/*.log /opt/so/log/salt/so-salt-minion-check +/opt/so/log/salt/minion +/opt/so/log/salt/master { {{ logrotate_conf | indent(width=4) }} } From dbe22f901da617d37de9e9b065334ee1f77137d4 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 14 Jan 2021 14:54:37 -0500 Subject: [PATCH 51/56] [fix] Add jinja raw block to so-common --- salt/common/tools/sbin/so-common | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 791443340..427721a49 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -347,6 +347,8 @@ valid_int() { [[ $num =~ ^[0-9]*$ ]] && [[ $num -ge $min ]] && [[ $num -le $max ]] && return 0 || return 1 } +# {% raw %} + valid_string() { local str=$1 local min_length=${2:-1} @@ -355,6 +357,8 @@ valid_string() { echo "$str" | grep -qP '^\S+$' && [[ ${#str} -ge $min_length ]] && [[ ${#str} -le $max_length ]] && return 0 || return 1 } +# {% endraw %} + valid_username() { local user=$1 From 0f6805823e1867689409d5e2be7557fd069c441b Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 15 Jan 2021 08:35:37 -0500 Subject: [PATCH 52/56] [fix] Add spacing to whiptail menu + preset err --- setup/so-functions | 2 +- setup/so-whiptail | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 87cbdb68e..31f381d0d 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -885,7 +885,7 @@ configure_network_sensor() { connection.autoconnect "yes" >> "$setup_log" 2>&1 fi - local err + local err=0 for BNIC in "${BNICS[@]}"; do add_interface_bond0 "$BNIC" >> "$setup_log" 2>&1 local ret=$? diff --git a/setup/so-whiptail b/setup/so-whiptail index 390cd70a5..f4e3a36dd 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -646,8 +646,8 @@ whiptail_first_menu_iso() { [ -n "$TESTING" ] && return option=$(whiptail --title "Security Onion Setup" --menu "Select an option" 10 75 2 \ - "Security Onion Installer" "Run the standard Security Onion installation " \ - "Configure Network" "Configure networking only " \ + "Install " "Run the standard Security Onion installation " \ + "Configure Network " "Configure networking only " \ 3>&1 1>&2 2>&3 ) local exitstatus=$? From 07b5f1d23e2907f490e64dc341f9a4465e69eebb Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 15 Jan 2021 08:55:30 -0500 Subject: [PATCH 53/56] Rename functions to avoid naming conflict with setup vars --- salt/common/tools/sbin/so-common | 6 +++--- salt/common/tools/sbin/so-tcpreplay | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 427721a49..4b722f57e 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -157,7 +157,7 @@ set_version() { } require_manager() { - if is_manager; then + if is_manager_node; then echo "This is a manager, We can proceed." else echo "Please run this command on the manager; the manager controls the grid." @@ -165,7 +165,7 @@ require_manager() { fi } -is_manager() { +is_manager_node() { # Check to see if this is a manager node role=$(lookup_role) is_single_node_grid && return 0 @@ -175,7 +175,7 @@ is_manager() { return 1 } -is_sensor() { +is_sensor_node() { # Check to see if this is a sensor (forward) node role=$(lookup_role) is_single_node_grid && return 0 diff --git a/salt/common/tools/sbin/so-tcpreplay b/salt/common/tools/sbin/so-tcpreplay index e8e24a474..22722ac9b 100755 --- a/salt/common/tools/sbin/so-tcpreplay +++ b/salt/common/tools/sbin/so-tcpreplay @@ -47,25 +47,25 @@ if ! docker ps | grep -q so-tcpreplay; then echo "Replay functionality not enabled; attempting to enable now (may require Internet access)..." echo - if is_manager; then + if is_manager_node; then TRUSTED_CONTAINERS=("so-tcpreplay") mkdir -p /opt/so/log/tcpreplay update_docker_containers "tcpreplay" "" "" "/opt/so/log/tcpreplay/init.log" fi - if is_sensor; then - if ! is_manager; then + if is_sensor_node; then + if ! is_manager_node; then echo "Attempting to start replay container. If this fails then you may need to run this command on the manager first." fi so-tcpreplay-start || fail "Unable to initialize tcpreplay" fi fi -if is_sensor; then +if is_sensor_node; then echo "Replaying PCAP(s) at ${REPLAYSPEED} Mbps on interface ${REPLAYIFACE}..." docker exec so-tcpreplay /usr/bin/bash -c "/usr/local/bin/tcpreplay -i ${REPLAYIFACE} -M${REPLAYSPEED} $@" echo "Replay completed. Warnings shown above are typically expected." -elif is_manager; then +elif is_manager_node; then echo "The sensor nodes in this grid can now replay traffic." else echo "Unable to replay traffic since this node is not a sensor node." From f4de5e28bf0144963cdfb43198e44e933a576897 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 15 Jan 2021 08:57:05 -0500 Subject: [PATCH 54/56] [fix] Padding 3->4 spaces, don't use lookup_pillar before salt is installed --- setup/so-functions | 4 +--- setup/so-whiptail | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 31f381d0d..f06bbd9e0 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -861,9 +861,6 @@ configure_network_sensor() { local nmcli_con_args=( "type" "bond" "mode" "0" ) fi - local MTU - MTU=$(lookup_pillar "mtu" "sensor") - # Create the bond interface only if it doesn't already exist nmcli -f name,uuid -p con | grep -q "$INTERFACE" >> "$setup_log" 2>&1 local found_int=$? @@ -2250,6 +2247,7 @@ sensor_pillar() { if [[ $NSMSETUP != 'ADVANCED' ]]; then if [[ $is_cloud ]]; then MTU=1575; else MTU=1500; fi fi + export MTU # Create the sensor pillar printf '%s\n'\ diff --git a/setup/so-whiptail b/setup/so-whiptail index f4e3a36dd..20627f13f 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -646,8 +646,8 @@ whiptail_first_menu_iso() { [ -n "$TESTING" ] && return option=$(whiptail --title "Security Onion Setup" --menu "Select an option" 10 75 2 \ - "Install " "Run the standard Security Onion installation " \ - "Configure Network " "Configure networking only " \ + "Install " "Run the standard Security Onion installation " \ + "Configure Network " "Configure networking only " \ 3>&1 1>&2 2>&3 ) local exitstatus=$? From ed129bcf1fb836ff89302d2a4b45a8c078304d8b Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 15 Jan 2021 09:25:04 -0500 Subject: [PATCH 55/56] [fix] Add verbose flag so that so-monitor-add only sees necessary information --- salt/common/tools/sbin/so-common | 21 ++++++++++++++++++--- setup/so-functions | 2 +- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 4b722f57e..d73ec18f2 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -41,8 +41,18 @@ add_interface_bond0() { fi done + case "$2" in + -v|--verbose) + local verbose=true + ;; + esac + for i in rx tx sg tso ufo gso gro lro; do - ethtool -K "$BNIC" $i off + if [[ $verbose != true ]]; then + ethtool -K "$BNIC" $i off + else + ethtool -K "$BNIC" $i off &>/dev/null + fi done # Check if the bond slave connection has already been created nmcli -f name,uuid -p con | grep -q "bond0-slave-$BNIC" @@ -64,8 +74,13 @@ add_interface_bond0() { ip link set dev "$BNIC" arp off multicast off allmulticast off promisc on - nmcli con up "bond0-slave-$BNIC" # Bring the slave interface up - + # Bring the slave interface up + if [[ $verbose != true ]]; then + nmcli con up "bond0-slave-$BNIC" + else + nmcli con up "bond0-slave-$BNIC" &>/dev/null + fi + if [ "$nic_error" != 0 ]; then return "$nic_error" fi diff --git a/setup/so-functions b/setup/so-functions index f06bbd9e0..132d6d202 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -884,7 +884,7 @@ configure_network_sensor() { local err=0 for BNIC in "${BNICS[@]}"; do - add_interface_bond0 "$BNIC" >> "$setup_log" 2>&1 + add_interface_bond0 "$BNIC" --verbose >> "$setup_log" 2>&1 local ret=$? [[ $ret -eq 0 ]] || err=$ret done From e440f6c44a1581010afb8fa8a1cc34d07b447c5e Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 15 Jan 2021 10:29:51 -0500 Subject: [PATCH 56/56] [fix] Set variables used by sensor pillar before generating the pillar --- setup/so-functions | 24 ++++++++++++++++-------- setup/so-setup | 1 + 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 132d6d202..83a3ec7fd 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -854,10 +854,8 @@ configure_network_sensor() { echo "Setting up sensor interface" >> "$setup_log" 2>&1 if [[ $is_cloud ]]; then - INTERFACE=${BNICS[0]} local nmcli_con_args=( "type" "ethernet" ) else - INTERFACE='bond0' local nmcli_con_args=( "type" "bond" "mode" "0" ) fi @@ -1307,6 +1305,22 @@ generate_repo_tarball() { tar -czf /opt/so/repo/"$SOVERSION".tar.gz ../. } +generate_sensor_vars() { + # Set the MTU + if [[ $NSMSETUP != 'ADVANCED' ]]; then + if [[ $is_cloud ]]; then MTU=1575; else MTU=1500; fi + fi + export MTU + + # Set interface variable + if [[ $is_cloud ]]; then + INTERFACE=${BNICS[0]} + else + INTERFACE='bond0' + fi + export INTERFACE +} + get_redirect() { whiptail_set_redirect if [ "$REDIRECTINFO" = "OTHER" ]; then @@ -2243,12 +2257,6 @@ sensor_pillar() { local pillar_file=$temp_install_dir/pillar/minions/$MINION_ID.sls - # Set the MTU - if [[ $NSMSETUP != 'ADVANCED' ]]; then - if [[ $is_cloud ]]; then MTU=1575; else MTU=1500; fi - fi - export MTU - # Create the sensor pillar printf '%s\n'\ "sensor:"\ diff --git a/setup/so-setup b/setup/so-setup index a4bf034cc..91103d21a 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -575,6 +575,7 @@ set_redirect >> $setup_log 2>&1 if [[ $is_sensor || $is_helix || $is_import ]]; then set_progress_str 3 'Generating sensor pillar' + generate_sensor_vars sensor_pillar >> $setup_log 2>&1 if [[ $is_sensor || $is_helix ]]; then steno_pillar >> $setup_log