Merge pull request #913 from Security-Onion-Solutions/feature/setup-changes

Feature/setup changes
This commit is contained in:
William Wernert
2020-07-02 14:01:46 -04:00
committed by GitHub
5 changed files with 206 additions and 92 deletions

View File

@@ -4,5 +4,5 @@ if [[ "$DEVICE_IFACE" != "$MNIC" && "$DEVICE_IFACE" != *"docker"* ]]; then
for i in rx tx sg tso ufo gso gro lro; do
ethtool -K "$DEVICE_IFACE" "$i" off;
done
ip link set dev "$DEVICE_IFACE" arp off multicast off allmulticast off
ip link set dev "$DEVICE_IFACE" arp off multicast off allmulticast off promisc on
fi

View File

@@ -388,8 +388,6 @@ configure_minion() {
printf '%s\n' '----';
cat "$minion_config";
} >> "$setup_log" 2>&1
}
checkin_at_boot() {
@@ -399,27 +397,33 @@ checkin_at_boot() {
echo "startup_states: highstate" >> "$minion_config"
}
check_requirements() {
local eval_or_dist=$1
local node_type=$2 # optional
local req_mem
local req_cores
local req_storage
local nic_list
readarray -t nic_list <<< "$(ip link| awk -F: '$0 !~ "lo|vir|veth|br|docker|wl|^[^0-9]"{print $2}' | grep -vwe "bond0" | sed 's/ //g')"
local num_nics=${#nic_list[@]}
if [[ "$eval_or_dist" == 'eval' ]]; then
req_mem=12
req_cores=4
req_nics=2
req_storage=100
elif [[ "$eval_or_dist" == 'dist' ]]; then
req_mem=8
req_cores=4
req_storage=40
if [[ "$node_type" == 'sensor' ]]; then req_nics=2; else req_nics=1; fi
if [[ "$node_type" == 'fleet' ]]; then req_mem=4; fi
fi
if (( $(echo "$free_space_root < $req_storage" | bc -l) )); then
whiptail_requirements_error "disk space" "${free_space_root} GB" "${req_storage} GB"
fi
if [[ $num_nics -lt $req_nics ]]; then
whiptail_requirements_error "NICs" "$num_nics" "$req_nics"
fi
@@ -429,7 +433,7 @@ check_requirements() {
fi
if [[ $total_mem_hr -lt $req_mem ]]; then
whiptail_requirements_error "memory" "${total_mem_hr}GB" "${req_mem}GB"
whiptail_requirements_error "memory" "${total_mem_hr} GB" "${req_mem} GB"
fi
}
@@ -481,42 +485,59 @@ copy_ssh_key() {
}
create_local_directories() {
echo "Creating local pillar and salt directories"
PILLARSALTDIR=${SCRIPTDIR::-5}
for i in "pillar" "salt"; do
for d in `find $PILLARSALTDIR/$i -type d`; do
suffixdir=${d//$PILLARSALTDIR/}
if [ ! -d "$local_salt_dir/$suffixdir" ]; then
mkdir -v "$local_salt_dir$suffixdir" >> "$setup_log" 2>&1
fi
done
chown -R socore:socore "$local_salt_dir/$i"
done
echo "Creating local pillar and salt directories"
PILLARSALTDIR=${SCRIPTDIR::-5}
for i in "pillar" "salt"; do
for d in $(find $PILLARSALTDIR/$i -type d); do
suffixdir=${d//$PILLARSALTDIR/}
if [ ! -d "$local_salt_dir/$suffixdir" ]; then
mkdir -v "$local_salt_dir$suffixdir" >> "$setup_log" 2>&1
fi
done
chown -R socore:socore "$local_salt_dir/$i"
done
}
create_sensor_bond() {
echo "Setting up sensor bond" >> "$setup_log" 2>&1
configure_network_sensor() {
echo "Setting up sensor interface" >> "$setup_log" 2>&1
local nic_error=0
check_network_manager_conf >> "$setup_log" 2>&1
# Set the MTU
if [[ $NSMSETUP != 'ADVANCED' ]]; then
MTU=1500
if [[ $is_ec2 ]]; then MTU=1575; else MTU=1500; fi
fi
if [[ $is_ec2 ]]; then
INTERFACE=${BNICS[0]}
local nmcli_con_arg="type ethernet"
else
INTERFACE='bond0'
local nmcli_con_arg="type bond mode 0"
fi
# Create the bond interface only if it doesn't already exist
if ! [[ $(nmcli -f name,uuid -p con | sed -n 's/bond0 //p' | tr -d ' ') ]]; then
nmcli con add ifname bond0 con-name "bond0" type bond mode 0 -- \
nmcli -f name,uuid -p con | grep -q "$INTERFACE"
local found_int=$?
if [[ $found_int != 0 ]]; then
nmcli con add ifname "$INTERFACE" con-name "$INTERFACE" $nmcli_con_arg -- \
ipv4.method disabled \
ipv6.method ignore \
ethernet.mtu $MTU \
connection.autoconnect "yes" >> "$setup_log" 2>&1
else
local int_uuid
int_uuid=$(nmcli -f name,uuid -p con | sed -n "s/$INTERFACE //p" | tr -d ' ')
nmcli con mod "$int_uuid" \
ipv4.method disabled \
ipv6.method ignore \
ethernet.mtu $MTU \
connection.autoconnect "yes" >> "$setup_log" 2>&1
fi
for BNIC in "${BNICS[@]}"; do
# Check if specific offload features are able to be disabled
for string in "generic-segmentation-offload" "generic-receive-offload" "tcp-segmentation-offload"; do
@@ -532,15 +553,29 @@ create_sensor_bond() {
ethtool -K "$BNIC" $i off >> "$setup_log" 2>&1
done
# Check if the bond slave connection has already been created
if ! [[ $(nmcli -f name,uuid -p con | sed -n "s/bond0-slave-$BNIC //p" | tr -d ' ') ]]; then
# Create the slave interface and assign it to the bond
nmcli con add type ethernet ifname "$BNIC" con-name "bond0-slave-$BNIC" master bond0 -- \
ethernet.mtu $MTU \
connection.autoconnect "yes" >> "$setup_log" 2>&1
fi
if [[ $is_ec2 ]]; then
nmcli con up "$BNIC" >> "$setup_log" 2>&1
else
# Check if the bond slave connection has already been created
nmcli -f name,uuid -p con | grep -q "bond0-slave-$BNIC"
local found_int=$?
if [[ $found_int != 0 ]]; then
# Create the slave interface and assign it to the bond
nmcli con add type ethernet ifname "$BNIC" con-name "bond0-slave-$BNIC" master bond0 -- \
ethernet.mtu $MTU \
connection.autoconnect "yes" >> "$setup_log" 2>&1
else
local int_uuid
int_uuid=$(nmcli -f name,uuid -p con | sed -n "s/bond0-slave-$BNIC //p" | tr -d ' ')
nmcli con up "bond0-slave-$BNIC" >> "$setup_log" 2>&1 # Bring the slave interface up
nmcli con mod "$int_uuid" \
ethernet.mtu $MTU \
connection.autoconnect "yes" >> "$setup_log" 2>&1
fi
nmcli con up "bond0-slave-$BNIC" >> "$setup_log" 2>&1 # Bring the slave interface up
fi
done
if [ $nic_error != 0 ]; then
@@ -565,11 +600,19 @@ detect_os() {
exit 1
fi
echo "Installing required packages to run installer"
# Install bind-utils so the host command exists
if ! command -v host > /dev/null 2>&1; then
echo "Installing required packages to run installer"
yum -y install bind-utils yum-plugin-versionlock >> "$setup_log" 2>&1
yum -y install bind-utils >> "$setup_log" 2>&1
fi
if ! command -v nmcli > /dev/null 2>&1; then
{
yum -y install NetworkManager;
systemctl enable NetworkManager;
systemctl start NetworkManager;
} >> "$setup_log" 2<&1
fi
yum -y install yum-plugin-versionlock bc >> "$setup_log" 2>&1
elif [ -f /etc/os-release ]; then
@@ -582,15 +625,17 @@ detect_os() {
echo "We do not support your current version of Ubuntu."
exit 1
fi
echo "Installing required packages to run installer"
# Install network manager so we can do interface stuff
if ! command -v nmcli > /dev/null 2>&1; then
echo "Installing required packages to run installer"
{
apt-get install -y network-manager;
systemctl enable NetworkManager;
systemctl start NetworkManager;
} >> "$setup_log" 2<&1
fi
apt-get install -y bc >> "$setup_log" 2>&1
else
echo "We were unable to determine if you are using a supported OS."
@@ -1317,7 +1362,7 @@ sensor_pillar() {
# Create the sensor pillar
printf '%s\n'\
"sensor:"\
" interface: bond0"\
" interface: $INTERFACE"\
" mainip: $MAINIP"\
" mainint: $MNIC" >> "$pillar_file"
@@ -1428,13 +1473,13 @@ set_initial_firewall_policy() {
$default_salt_dir/salt/common/tools/sbin/so-firewall --apply includehost search_node "$MAINIP"
case "$install_type" in
'EVAL')
$default_salt_dir/pillar/data/addtotab.sh evaltab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm" bond0 True
$default_salt_dir/pillar/data/addtotab.sh evaltab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm" $INTERFACE True
;;
'MASTERSEARCH')
$default_salt_dir/pillar/data/addtotab.sh mastersearchtab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm"
;;
'STANDALONE')
$default_salt_dir/pillar/data/addtotab.sh standalonetab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm" bond0
$default_salt_dir/pillar/data/addtotab.sh standalonetab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm" $INTERFACE
;;
esac
;;
@@ -1448,7 +1493,7 @@ set_initial_firewall_policy() {
case "$install_type" in
'SENSOR')
ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/salt/common/tools/sbin/so-firewall --apply includehost sensor "$MAINIP"
ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/pillar/data/addtotab.sh sensorstab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm" bond0
ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/pillar/data/addtotab.sh sensorstab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm" $INTERFACE
;;
'SEARCHNODE')
ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/salt/common/tools/sbin/so-firewall --apply includehost search_node "$MAINIP"
@@ -1457,7 +1502,7 @@ set_initial_firewall_policy() {
'HEAVYNODE')
ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/salt/common/tools/sbin/so-firewall includehost sensor "$MAINIP"
ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/salt/common/tools/sbin/so-firewall --apply includehost search_node "$MAINIP"
ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/pillar/data/addtotab.sh sensorstab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm" bond0
ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/pillar/data/addtotab.sh sensorstab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm" $INTERFACE
ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/pillar/data/addtotab.sh nodestab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm"
;;
'FLEET')
@@ -1620,3 +1665,8 @@ es_heapsize() {
export NODE_ES_HEAP_SIZE
fi
}
detect_ec2() {
# Check if EC2
if ( curl --fail -s -m 5 http://169.254.169.254/latest/meta-data/instance-id > /dev/null ); then export is_ec2="true"; fi
}

View File

@@ -51,10 +51,20 @@ echo "---- Starting setup at $(date -u) ----" >> $setup_log 2>&1
automated=no
function progress() {
if [ $automated == no ]; then
whiptail --title "Security Onion Install" --gauge 'Please wait while installing' 6 60 0
if grep -q "ERROR" $setup_log || [[ -s /var/spool/mail/root ]]; then
if [[ -s /var/spool/mail/root ]]; then
echo '[ ERROR ] /var/spool/mail/root grew unexpectedly' >> $setup_log 2>&1
fi
if [[ $automated == no ]]; then whiptail_setup_failed; else exit 1; fi
if [[ -z $SKIP_REBOOT ]]; then shutdown -r now; else exit 1; fi
else
cat >> $setup_log 2>&1
if [ $automated == no ]; then
whiptail --title "Security Onion Install" --gauge 'Please wait while installing' 6 60 0
else
cat >> $setup_log 2>&1
fi
fi
}
@@ -98,7 +108,7 @@ export PATH=$PATH:../salt/common/tools/sbin
got_root
detect_os
detect_os && detect_ec2
if [ "$OS" == ubuntu ]; then
update-alternatives --set newt-palette /etc/newt/palette.original >> $setup_log 2>&1
@@ -225,7 +235,7 @@ fi
# Start user prompts
if [[ $is_helix || $is_sensor ]]; then
whiptail_bond_nics
whiptail_sensor_nics
calculate_useable_cores
fi
@@ -334,14 +344,19 @@ if [[ "$setup_type" == 'iso' ]]; then
disable_onion_user
fi
set_hostname >> $setup_log 2>&1
set_version >> $setup_log 2>&1
clear_master >> $setup_log 2>&1
{
set_hostname;
set_version;
clear_master;
} >> $setup_log 2>&1
if [[ $is_master ]]; then
generate_passwords >> $setup_log 2>&1
secrets_pillar >> $setup_log 2>&1
add_socore_user_master >> $setup_log 2>&1
{
generate_passwords;
secrets_pillar;
add_socore_user_master;
} >> $setup_log 2>&1
fi
if [[ $is_master && ! $is_eval ]]; then
@@ -364,18 +379,17 @@ fi
# Set initial percentage to 0
export percentage=0
if [[ $is_minion ]]; then
set_progress_str 1 'Configuring firewall'
set_initial_firewall_policy >> $setup_log 2>&1
fi
if [[ $is_minion ]]; then
set_progress_str 1 'Configuring firewall'
set_initial_firewall_policy >> $setup_log 2>&1
fi
set_progress_str 2 'Updating packages'
update_packages >> $setup_log 2>&1
if [[ $is_sensor || $is_helix ]]; then
set_progress_str 3 'Creating bond interface'
create_sensor_bond >> $setup_log 2>&1
set_progress_str 3 'Configuring sensor interface'
configure_network_sensor >> $setup_log 2>&1
set_progress_str 4 'Generating sensor pillar'
sensor_pillar >> $setup_log 2>&1
fi
@@ -392,14 +406,15 @@ fi
set_progress_str 9 'Initializing Salt minion'
configure_minion "$minion_type" >> $setup_log 2>&1
if [[ $is_master || $is_helix ]]; then
set_progress_str 10 'Configuring Salt master'
create_local_directories >> $setup_log 2>&1
addtotab_generate_templates >> $setup_log 2>&1
copy_master_config >> $setup_log 2>&1
setup_salt_master_dirs >> $setup_log 2>&1
firewall_generate_templates >> $setup_log 2>&1
{
create_local_directories;
addtotab_generate_templates;
copy_master_config;
setup_salt_master_dirs;
firewall_generate_templates;
} >> $setup_log 2>&1
set_progress_str 11 'Updating sudoers file for soremote user'
update_sudoers >> $setup_log 2>&1
@@ -544,8 +559,8 @@ fi
salt-call state.apply -l info fleet.event_update-custom-hostname pillar="$pillar_override" >> $setup_log 2>&1
fi
set_progress_str 77 "$(print_salt_state_apply 'so-fleet-setup')"
so-fleet-setup $FLEETNODEUSER $FLEETNODEPASSWD1 >> $setup_log 2>&1
set_progress_str 74 "$(print_salt_state_apply 'so-fleet-setup')"
so-fleet-setup "$FLEETNODEUSER" "$FLEETNODEPASSWD1" >> $setup_log 2>&1
fi
@@ -600,10 +615,12 @@ fi
success=$(tail -10 $setup_log | grep Failed | awk '{ print $2}')
if [[ "$success" = 0 ]]; then
whiptail_setup_complete
if [[ -n $ALLOW_ROLE && -n $ALLOW_CIDR ]]; then
export IP=$ALLOW_CIDR
so-allow -$ALLOW_ROLE >> $setup_log 2>&1
fi
if [[ $THEHIVE == 1 ]]; then
check_hive_init
fi

View File

@@ -28,6 +28,9 @@ mkdir -p /nsm
filesystem_nsm=$(df /nsm | awk '$3 ~ /[0-9]+/ { print $2 * 1000 }')
export filesystem_nsm
free_space_root=$(df -Pk / | sed 1d | grep -v used | awk '{ print $4 / 1048576 }' | awk '{ printf("%.0f", $1) }')
export free_space_root
mkdir -p /root/installtmp/pillar/minions
export temp_install_dir=/root/installtmp
@@ -36,6 +39,11 @@ export percentage_str='Getting started'
export DEBIAN_FRONTEND=noninteractive
export default_salt_dir=/opt/so/saltstack/default
export local_salt_dir=/opt/so/saltstack/local
export SCRIPTDIR=$(cd `dirname $0` && pwd)
SCRIPTDIR=$(cd "$(dirname "$0")" && pwd)
export SCRIPTDIR
readarray -t nmcli_dev_status_list <<< "$(nmcli -t -f DEVICE,STATE -c no dev status)"
export nmcli_dev_status_list

View File

@@ -72,19 +72,27 @@ whiptail_bro_version() {
}
whiptail_bond_nics() {
whiptail_sensor_nics() {
[ -n "$TESTING" ] && return
filter_unused_nics
BNICS=$(whiptail --title "NIC Setup" --checklist "Please add NICs to the Monitor Interface" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3)
if [[ $is_ec2 ]]; then
local menu_text="Please select NIC for the Monitor Interface"
local list_type="radiolist"
else
local menu_text="Please add NICs to the Monitor Interface"
local list_type="checklist"
fi
BNICS=$(whiptail --title "NIC Setup" --$list_type "$menu_text" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3)
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
while [ -z "$BNICS" ]
do
BNICS=$(whiptail --title "NIC Setup" --checklist "Please add NICs to the Monitor Interface" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3 )
BNICS=$(whiptail --title "NIC Setup" --$list_type "$menu_text" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3 )
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
done
@@ -92,6 +100,16 @@ whiptail_bond_nics() {
BNICS=$(echo "$BNICS" | tr -d '"')
IFS=' ' read -ra BNICS <<< "$BNICS"
for bond_nic in "${BNICS[@]}"; do
if [[ "${nmcli_dev_status_list}" =~ $bond_nic\:unmanaged ]]; then
whiptail \
--title "Security Onion Setup" \
--msgbox "$bond_nic is unmanaged by Network Manager. Please remove it from other network management tools then re-run setup." \
8 75
exit
fi
done
}
whiptail_bond_nics_mtu() {
@@ -281,8 +299,6 @@ whiptail_fleet_custom_hostname() {
whiptail_check_exitstatus $exitstatus
}
whiptail_requirements_error() {
local requirement_needed=$1
@@ -292,7 +308,7 @@ whiptail_requirements_error() {
[ -n "$TESTING" ] && return
whiptail --title "Security Onion Setup" \
--yesno "This machine currently has $current_val $requirement_needed, but needs $needed_val to meet minimum requirements. Press YES to continue anyway, or press NO to cancel." 8 75
--yesno "This machine currently has $current_val $requirement_needed, but needs $needed_val to meet minimum requirements. Press YES to continue anyway, or press NO to cancel." 10 75
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
@@ -430,27 +446,37 @@ whiptail_install_type() {
# What kind of install are we doing?
install_type=$(whiptail --title "Security Onion Setup" --radiolist \
"Choose Install Type:" 20 75 13 \
"SENSOR" "Create a forward only sensor" ON \
"SEARCHNODE" "Add a Search Node with parsing" OFF \
"MASTER" "Start a new grid" OFF \
"EVAL" "Evaluate all the things" OFF \
"STANDALONE" "Standalone full install of everything" OFF \
"MASTERSEARCH" "Master + Search Node" OFF \
"HEAVYNODE" "Sensor + Search Node" OFF \
"HELIXSENSOR" "Connect this sensor to FireEye Helix" OFF \
"FLEET" "Dedicated Fleet Osquery Node" OFF \
"HOTNODE" "TODO Add Hot Node (Uses Elastic Clustering)" OFF \
"WARMNODE" "TODO Add Warm Node to existing Hot or Search node" OFF \
"WAZUH" "TODO Stand Alone Wazuh Server" OFF \
"STRELKA" "TODO Stand Alone Strelka Node" OFF 3>&1 1>&2 2>&3 )
"Choose install type:" 10 65 3 \
"EVAL" "Evaluation mode (not for production) " ON \
"STANDALONE" "Standalone production install " OFF \
"DISTRIBUTED" "Distributed install submenu " OFF \
3>&1 1>&2 2>&3
)
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
if [[ $install_type == "DISTRIBUTED" ]]; then
install_type=$(whiptail --title "Security Onion Setup" --radiolist \
"Choose distributed node type:" 13 60 6 \
"MASTER" "Start a new grid " ON \
"SENSOR" "Create a forward only sensor " OFF \
"SEARCHNODE" "Add a search node with parsing " OFF \
"MASTERSEARCH" "Master + search node " OFF \
"FLEET" "Dedicated Fleet Osquery Node " OFF \
"HEAVYNODE" "Sensor + Search Node " OFF \
3>&1 1>&2 2>&3
# "HOTNODE" "Add Hot Node (Uses Elastic Clustering)" OFF \ # TODO
# "WARMNODE" "Add Warm Node to existing Hot or Search node" OFF \ # TODO
# "WAZUH" "Stand Alone Wazuh Server" OFF \ # TODO
# "STRELKA" "Stand Alone Strelka Node" OFF \ # TODO
)
fi
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
export install_type
}
whiptail_log_size_limit() {
@@ -1128,7 +1154,20 @@ whiptail_you_sure() {
[ -n "$TESTING" ] && return
whiptail --title "Security Onion Setup" --yesno "Are you sure you want to continue a network install of Security Onion?" 8 75
read -r -d '' you_sure_text <<- EOM
Welcome to Security Onion Setup!
You can use Setup for lots of different use cases from a small standalone installation to a large distributed deployment for your enterprise.
Setup uses keyboard navigation and you can use arrow keys to move around. Certain screens may provide a list and ask you to select one or more items from that list. You can use [SPACE] to select items and [ENTER] to proceed to the next screen.
Would you like to continue?
EOM
whiptail \
--title "Security Onion Setup" \
--yesno "$you_sure_text" \
20 75
local exitstatus=$?
return $exitstatus