From 3ef3c157f379044ccbec27df6e71d4c3f2709667 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 16 Apr 2020 09:00:01 -0400 Subject: [PATCH] [refactor][WIP] Setup changes part 6 --- salt/strelka/init.sls | 1 + .../99-so-checksum-offload-disable | 2 +- setup/so-common-functions | 24 +- setup/so-constants | 25 ++ setup/so-functions | 213 ++++++++---- setup/so-setup | 305 ++++++++++-------- setup/so-variables | 23 +- setup/so-whiptail | 52 ++- 8 files changed, 389 insertions(+), 256 deletions(-) create mode 100644 setup/so-constants diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls index 19d07aca7..fa981ab34 100644 --- a/salt/strelka/init.sls +++ b/salt/strelka/init.sls @@ -14,6 +14,7 @@ # along with this program. If not, see . {%- set MASTER = grains['master'] %} {%- set MASTERIP = salt['pillar.get']('static:masterip', '') %} +{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.1') %} # Strelka config strelkaconfdir: diff --git a/setup/install_scripts/99-so-checksum-offload-disable b/setup/install_scripts/99-so-checksum-offload-disable index bcdd04029..b8c21a299 100755 --- a/setup/install_scripts/99-so-checksum-offload-disable +++ b/setup/install_scripts/99-so-checksum-offload-disable @@ -1,6 +1,6 @@ #!/bin/bash -if [[ "$DEVICE_IFACE" != "$MAININT" && "$DEVICE_IFACE" != *"docker"* ]]; then +if [[ "$DEVICE_IFACE" != "$MNIC" && "$DEVICE_IFACE" != *"docker"* ]]; then for i in rx tx sg tso ufo gso gro lro; do ethtool -K "$DEVICE_IFACE" "$i" off; done diff --git a/setup/so-common-functions b/setup/so-common-functions index 2419f0659..3e815fc63 100644 --- a/setup/so-common-functions +++ b/setup/so-common-functions @@ -16,8 +16,8 @@ filter_unused_nics() { fi # Finally, set filtered_nics to any NICs we aren't using (and ignore interfaces that aren't of use) - export filtered_nics filtered_nics=$(ip link | grep -vwe "$grep_string" | awk -F: '$0 !~ "lo|vir|veth|br|docker|wl|^[^0-9]"{print $2}') + export filtered_nics } calculate_useable_cores() { @@ -27,8 +27,8 @@ calculate_useable_cores() { local lb_procs_round lb_procs_round=$(printf "%.0f\n" $cores_for_bro) + if [ "$lb_procs_round" -lt 1 ]; then lb_procs=1; else lb_procs=$lb_procs_round; fi export lb_procs - if [ "$lb_procs_round" -lt 1 ]; then lb_procs=1; else lb_procs=$lb_procs_round; fi } set_defaul_log_size() { @@ -56,23 +56,3 @@ set_defaul_log_size() { export log_size_limit=$(( percentage_disk_space / 1000000000 )) } - -ls_heapsize() { - - if [ "$total_mem" -ge 32000 ]; then - LS_HEAP_SIZE='1000m' - return - fi - - case "$install_type" in - 'MASTERSEARCH' | 'HEAVYNODE' | 'HELIXSENSOR') - LS_HEAP_SIZE='1000m' - ;; - 'EVAL') - LS_HEAP_SIZE='700m' - ;; - *) - LS_HEAP_SIZE='500m' - ;; - esac -} \ No newline at end of file diff --git a/setup/so-constants b/setup/so-constants new file mode 100644 index 000000000..903d20a99 --- /dev/null +++ b/setup/so-constants @@ -0,0 +1,25 @@ +#!/bin/bash + +readonly create_bond_str='Creating bond interface' +export create_bond_str + +readonly salt_install_str='Installing Salt and dependencies' +export salt_install_str + +readonly docker_install_str='Installing Docker' +export docker_install_str + +readonly salt_minion_init_str='Initializing Salt minion' +export salt_minion_init_str + +readonly gen_patch_pillar_str='Generating the patch pillar' +export gen_patch_pillar_str + +readonly gen_static_pillar_str='Generating the static pillar' +export gen_static_pillar_str + +readonly gen_master_pillar_str='Generating the master pillar' +export gen_master_pillar_str + +readonly copy_temp_minion_str='Copying minion pillars to master' +export copy_temp_minion_str \ No newline at end of file diff --git a/setup/so-functions b/setup/so-functions index 7ecb8ec0d..e14340409 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -260,19 +260,38 @@ clear_master() { # This only happens if you re-install the master. if [ -f /etc/salt/pki/minion/minion_master.pub ]; then echo "Clearing old master key" >> "$setup_log" 2>&1 - rm /etc/salt/pki/minion/minion_master.pub - service salt-minion restart + rm -f /etc/salt/pki/minion/minion_master.pub + sytemctl -q restart salt-minion fi } +collect_soremote_inputs() { + SCMATCH=no + while [ $SCMATCH != yes ]; do + whiptail_create_soremote_user_password1 + whiptail_create_soremote_user_password2 + check_soremote_pass + done +} + +collect_adminuser_inputs() { + APMATCH=no + while [ $APMATCH != yes ]; do + whiptail_create_admin_user_password1 + whiptail_create_admin_user_password2 + check_admin_pass + done +} + + collect_webuser_inputs() { # Get a password for the web admin user - local VALIDUSER=no - while [ $VALIDUSER != yes ]; do + local valid_user=no + while [ $valid_user != yes ]; do whiptail_create_web_user if so-user valemail "$WEBUSER"; then - VALIDUSER=yes + valid_user=yes else whiptail_invalid_user_warning fi @@ -616,21 +635,6 @@ docker_seed_registry() { } -es_heapsize() { - - # Determine ES Heap Size - if [ "$total_mem" -lt 8000 ] ; then - ES_HEAP_SIZE="600m" - elif [ "$total_mem" -ge 100000 ]; then - # Set a max of 25GB for heap size - # https://www.elastic.co/guide/en/elasticsearch/guide/current/heap-sizing.html - ES_HEAP_SIZE="25000m" - else - # Set heap size to 25% of available memory - ES_HEAP_SIZE=$(( total_mem / 4 ))"m" - fi - -} fireeye_pillar() { @@ -669,22 +673,8 @@ generate_passwords(){ KRATOSKEY=$(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 20 | head -n 1) } -get_main_ip() { - - # Get the main IP address the box is using - - # FIXME: find a way to get the ip of MNIC instead - - # Add some logic because Bubntu 18.04 like to be different - if [ $OSVER = 'bionic' ]; then - MAINIP=$(ip route get 1 | awk '{print $7;exit}') - else - MAINIP=$(ip route get 1 | awk '{print $NF;exit}') - fi - - # FIXME: should MAININT be MNIC? - MAININT=$(ip route get 1 | awk '{print $5;exit}') - +set_main_ip() { + MAINIP=$(ip route get 1 | awk '{print $7;exit}') } get_redirect() { @@ -713,22 +703,25 @@ install_cleanup() { } -# TODO: figure out if this is necessary -install_master() { - - # Install the salt master package - if [ $OS != 'centos' ]; then - if [ $OSVER != "xenial" ]; then - apt-get install -y salt-common=2019.2.3+ds-1 salt-master=2019.2.3+ds-1 salt-minion=2019.2.3+ds-1 libssl-dev python-m2crypto - apt-mark hold salt-common salt-master salt-minion - else - apt-get install -y salt-common=2019.2.3+ds-1 salt-master=2019.2.3+ds-1 salt-minion=2019.2.3+ds-1 libssl-dev python-m2crypto - apt-mark hold salt-common salt-master salt-minion - fi - fi - - copy_master_config +get_minion_type() { + local minion_type + case "$install_type" in + 'EVAL' | 'MASTERSEARCH' | 'MASTER' | 'SENSOR' | 'HEAVYNODE' | 'FLEET') + minion_type=$(echo "$install_type" | tr '[:upper:]' '[:lower:]') + ;; + 'HELIXSENSOR') + minion_type='helix' + ;; + '*NODE') + minion_type='node' + ;; + esac + echo "$minion_type" +} +set_base_heapsizes() { + es_heapsize + ls_heapsize } master_pillar() { @@ -845,14 +838,14 @@ network_setup() { echo "... Setting ONBOOT for management interface"; if ! netplan > /dev/null 2>&1; then - nmcli con mod "$MAININT" connection.autoconnect "yes"; + nmcli con mod "$MNIC" connection.autoconnect "yes"; fi echo "... Copying 99-so-checksum-offload-disable"; cp "$SCRIPTDIR"/install_scripts/99-so-checksum-offload-disable /etc/NetworkManager/dispatcher.d/pre-up.d/99-so-checksum-offload-disable ; echo "... Modifying 99-so-checksum-offload-disable"; - sed -i "s/\$MAININT/${MAININT}/g" /etc/NetworkManager/dispatcher.d/pre-up.d/99-so-checksum-offload-disable; + sed -i "s/\$MNIC/${MNIC}/g" /etc/NetworkManager/dispatcher.d/pre-up.d/99-so-checksum-offload-disable; } >> "$setup_log" 2>&1 } @@ -864,7 +857,7 @@ node_pillar() { printf '%s\n'\ "node:"\ " mainip: $MAINIP"\ - " mainint: $MAININT"\ + " mainint: $MNIC"\ " esheap: $NODE_ES_HEAP_SIZE"\ " esclustername: {{ grains.host }}"\ " lsheap: $NODE_LS_HEAP_SIZE"\ @@ -947,13 +940,15 @@ saltify() { yum -y install wget https://repo.saltstack.com/py3/redhat/salt-py3-repo-latest-2.el7.noarch.rpm cp /etc/yum.repos.d/salt-py3-latest.repo /etc/yum.repos.d/salt-py3-2019-2.repo sed -i 's/latest/2019.2/g' /etc/yum.repos.d/salt-py3-2019-2.repo - yum -y install sqlite3 argon2 curl jq openssl + set_progress_str 6 'Installing various dependencies' + yum -y install sqlite3 argon2 curl jq openssl mariadb-devel # Download Ubuntu Keys in case master updates = 1 mkdir -p /opt/so/gpg wget --inet4-only -O /opt/so/gpg/SALTSTACK-GPG-KEY.pub https://repo.saltstack.com/apt/ubuntu/16.04/amd64/latest/SALTSTACK-GPG-KEY.pub wget --inet4-only -O /opt/so/gpg/docker.pub https://download.docker.com/linux/ubuntu/gpg wget --inet4-only -O /opt/so/gpg/GPG-KEY-WAZUH https://packages.wazuh.com/key/GPG-KEY-WAZUH cp "$./yum_repos/wazuh.repo" /etc/yum.repos.d/wazuh.repo + set_progress_str 7 'Installing salt-master' yum -y install salt-master-2019.2.3 systemctl enable salt-master ;; @@ -977,6 +972,7 @@ saltify() { esac cp "$./yum_repos/wazuh.repo" /etc/yum.repos.d/wazuh.repo yum clean expire-cache + set_progress_str 8 'Installing salt-minion & python modules' yum -y install epel-release\ salt-minion-2019.2.3\ python3\ @@ -1041,8 +1037,11 @@ saltify() { # Initialize the new repos apt-get update >> "$setup_log" 2>&1 # FIXME: Install salt-master on Ubuntu? - apt-get -y install sqlite3 argon2 openssl >> "$setup_log" 2>&1 - if [ "$OSVER" != 'xenial' ]; then apt-get -y install python3-mysqldb >> "$setup_log" 2>&1; else apt-get -y install python-mysqldb >> "$setup_log" 2>&1; fi + set_progress_str 6 'Installing various dependencies' + apt-get -y install sqlite3 argon2 openssl libssl-dev >> "$setup_log" 2>&1 + set_progress_str 7 'Installing salt-master' + apt-get -y salt-master=2019.2.3+ds-1 >> "$setup_log" 2>&1 + apt-mark hold salt-master ;; *) # Copy down the gpg keys and install them from the master @@ -1057,10 +1056,15 @@ saltify() { ;; esac apt-get update >> "$setup_log" 2>&1 + set_progress_str 8 'Installing salt-minion & python modules' apt-get -y install salt-minion=2019.2.3+ds-1\ salt-common=2019.2.3+ds-1 >> "$setup_log" 2>&1 apt-mark hold salt-minion salt-common - if [ "$OSVER" != 'xenial' ]; then apt-get -y install python3-dateutil python3-m2crypto >> "$setup_log" 2>&1; else apt-get -y install python-dateutil python-m2crypto >> "$setup_log" 2>&1; fi + if [ "$OSVER" != 'xenial' ]; then + apt-get -y install python3-dateutil python3-m2crypto python3-mysqldb >> "$setup_log" 2>&1 + else + apt-get -y install python-dateutil python-m2crypto python-mysqldb >> "$setup_log" 2>&1 + fi fi } @@ -1115,6 +1119,29 @@ setup_salt_master_dirs() { chown -R socore:socore /opt/so } +# $1 => percentage +# $2 => text for progress bar +set_progress_str() { + local percentage_input + percentage_input=$1 + local progress_bar_text + progress_bar_text=$2 + + if [ "$percentage_input" -lt "$percentage" ]; then + percentage="$percentage_input" + fi + + percentage_str="XXX\n${percentage_input}\n${progress_bar_text}\nXXX" + export percentage_str +} + +progress_str_printer() { + while [ "$percentage" -lt 100 ]; do + echo "$percentage_str" + done + sleep 0.5s +} + sensor_pillar() { local pillar_file=$temp_install_dir/pillar/minions/$MINION_ID.sls @@ -1124,7 +1151,7 @@ sensor_pillar() { "sensor"\ " interface: bond0"\ " mainip: $MAINIP"\ - " mainint: $MAININT" > "$pillar_file" + " mainint: $MNIC" > "$pillar_file" if [ "$NSMSETUP" = 'ADVANCED' ]; then echo " bro_pins:" >> "$pillar_file" @@ -1186,12 +1213,12 @@ set_hostname_iso() { set_initial_firewall_policy() { - get_main_ip + set_main_ip case "$install_type" in 'MASTER') printf " - %s\n" "$MAINIP" | tee /opt/so/saltstack/pillar/firewall/minions.sls /opt/so/saltstack/pillar/firewall/masterfw.sls - /opt/so/saltstack/pillar/data/addtotab.sh mastertab "$MINION_ID" "$MAINIP" "$CPUCORES" "$random_uid" "$MAININT" "$FSROOT" "$FSNSM" + /opt/so/saltstack/pillar/data/addtotab.sh mastertab "$MINION_ID" "$MAINIP" "$CPUCORES" "$random_uid" "$MNIC" "$FSROOT" "$FSNSM" ;; 'EVAL' | 'MASTERSEARCH') printf " - %s\n" "$MAINIP" | tee /opt/so/saltstack/pillar/firewall/minions.sls\ @@ -1200,10 +1227,10 @@ set_initial_firewall_policy() { /opt/so/saltstack/pillar/firewall/search_nodes.sls case "$install_type" in 'EVAL') - /opt/so/saltstack/pillar/data/addtotab.sh evaltab "$MINION_ID" "$MAINIP" "$CPUCORES" "$random_uid" "$MAININT" "$FSROOT" "$FSNSM" bond0 + /opt/so/saltstack/pillar/data/addtotab.sh evaltab "$MINION_ID" "$MAINIP" "$CPUCORES" "$random_uid" "$MNIC" "$FSROOT" "$FSNSM" bond0 ;; 'MASTERSEARCH') - /opt/so/saltstack/pillar/data/addtotab.sh nodestab "$MINION_ID" "$MAINIP" "$CPUCORES" "$random_uid" "$MAININT" "$FSROOT" "$FSNSM" + /opt/so/saltstack/pillar/data/addtotab.sh nodestab "$MINION_ID" "$MAINIP" "$CPUCORES" "$random_uid" "$MNIC" "$FSROOT" "$FSNSM" ;; esac ;; @@ -1217,17 +1244,17 @@ set_initial_firewall_policy() { case "$INSTALLERTYPE" in 'SENSOR') ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh forward_nodes "$MAINIP" - ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo /opt/so/saltstack/pillar/data/addtotab.sh sensorstab "$MINION_ID" "$MAINIP" "$CPUCORES" "$random_uid" "$MAININT" "$FSROOT" "$FSNSM" bond0 + ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo /opt/so/saltstack/pillar/data/addtotab.sh sensorstab "$MINION_ID" "$MAINIP" "$CPUCORES" "$random_uid" "$MNIC" "$FSROOT" "$FSNSM" bond0 ;; 'SEARCHNODE') ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh search_nodes "$MAINIP" - ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo /opt/so/saltstack/pillar/data/addtotab.sh nodestab "$MINION_ID" "$MAINIP" "$CPUCORES" "$random_uid" "$MAININT" "$FSROOT" "$FSNSM" + ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo /opt/so/saltstack/pillar/data/addtotab.sh nodestab "$MINION_ID" "$MAINIP" "$CPUCORES" "$random_uid" "$MNIC" "$FSROOT" "$FSNSM" ;; 'HEAVYNODE') ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh forward_nodes "$MAINIP" ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh search_nodes "$MAINIP" - ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo /opt/so/saltstack/pillar/data/addtotab.sh sensorstab "$MINION_ID" "$MAINIP" "$CPUCORES" "$random_uid" "$MAININT" "$FSROOT" "$FSNSM" bond0 - ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo /opt/so/saltstack/pillar/data/addtotab.sh nodestab "$MINION_ID" "$MAINIP" "$CPUCORES" "$random_uid" "$MAININT" "$FSROOT" "$FSNSM" + ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo /opt/so/saltstack/pillar/data/addtotab.sh sensorstab "$MINION_ID" "$MAINIP" "$CPUCORES" "$random_uid" "$MNIC" "$FSROOT" "$FSNSM" bond0 + ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo /opt/so/saltstack/pillar/data/addtotab.sh nodestab "$MINION_ID" "$MAINIP" "$CPUCORES" "$random_uid" "$MNIC" "$FSROOT" "$FSNSM" ;; esac ;; @@ -1246,7 +1273,7 @@ set_initial_firewall_policy() { # Set up the management interface on the ISO set_management_interface() { - if [ "$ADDRESSTYPE" = 'DHCP' ]; then + if [ "$address_type" = 'DHCP' ]; then nmcli con mod "$MNIC" connection.autoconnect yes nmcli con up "$MNIC" else @@ -1309,6 +1336,54 @@ update_sudoers() { echo "soremote ALL=(ALL) NOPASSWD:/opt/so/saltstack/pillar/data/addtotab.sh" | tee -a /etc/sudoers echo "soremote ALL=(ALL) NOPASSWD:/opt/so/saltstack/salt/master/files/add_minion.sh" | tee -a /etc/sudoers else - echo "User soremote already granted sudo privileges" + echo "User soremote already granted sudo privileges" >> "$setup_log" + fi +} + +ls_heapsize() { + + if [ "$total_mem" -ge 32000 ]; then + LS_HEAP_SIZE='1000m' + return + fi + + case "$install_type" in + 'MASTERSEARCH' | 'HEAVYNODE' | 'HELIXSENSOR') + LS_HEAP_SIZE='1000m' + ;; + 'EVAL') + LS_HEAP_SIZE='700m' + ;; + *) + LS_HEAP_SIZE='500m' + ;; + esac + export LS_HEAP_SIZE + + if [[ "$install_type" =~ ^(EVAL|MASTERSEARCH)$ ]]; then + NODE_LS_HEAP_SIZE=LS_HEAP_SIZE + export NODE_LS_HEAP_SIZE + fi +} + + +es_heapsize() { + + # Determine ES Heap Size + if [ "$total_mem" -lt 8000 ] ; then + ES_HEAP_SIZE="600m" + elif [ "$total_mem" -ge 100000 ]; then + # Set a max of 25GB for heap size + # https://www.elastic.co/guide/en/elasticsearch/guide/current/heap-sizing.html + ES_HEAP_SIZE="25000m" + else + # Set heap size to 25% of available memory + ES_HEAP_SIZE=$(( total_mem / 4 ))"m" + fi + export ES_HEAP_SIZE + + if [[ "$install_type" =~ ^(EVAL|MASTERSEARCH)$ ]]; then + NODE_ES_HEAP_SIZE=ES_HEAP_SIZE + export NODE_ES_HEAP_SIZE fi } \ No newline at end of file diff --git a/setup/so-setup b/setup/so-setup index b391a6e13..3fe9e5709 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -20,10 +20,12 @@ cd "$(dirname "$0")" || exit 255 source "./so-functions" source "./so-whiptail" source "./so-variables" +source "./so-constants" source "./so-common-functions" # See if this is an ISO install -install_type=$1 +setup_type=$1 +export setup_type case "$install_type" in iso | network) # Accepted values @@ -39,112 +41,151 @@ esac export PATH=$PATH:"$SCRIPTDIR/../salt/common/tools/sbin" -# Reset the Install Log -date -u > $setup_log 2>&1 -echo "stty size is: $(stty size)" >> $setup_log 2>&1 +init_common() { + date -u > $setup_log 2>&1 + echo "stty size is: $(stty size)" >> $setup_log 2>&1 -# Check for prerequisites -got_root -detect_os - -if [ "$OS" == ubuntu ]; then - # Override the horrible Ubuntu whiptail color pallete - update-alternatives --set newt-palette /etc/newt/palette.original >> $setup_log 2>&1 -fi - -# Question Time -echo "Asking user if they are sure they want to proceed" >> $setup_log 2>&1 -if (whiptail_you_sure) ; then - - # Create a temp dir to get started - setterm -blank 0 - - if [ $install_type == network ]; then - # Let folks know they need their management interface already set up. - whiptail_network_notice - - # Set the hostname to reduce errors - whiptail_set_hostname - - # Set management nic - whiptail_management_nic - - else - - # Set the hostname - whiptail_set_hostname - whiptail_management_nic - - # Ask if you want dhcp or static - whiptail_dhcp_or_static - - # Do this if it static is selected - if [ "$ADDRESSTYPE" != 'DHCP' ]; then - whiptail_management_interface_ip - whiptail_management_interface_mask - whiptail_management_interface_gateway - whiptail_management_interface_dns - whiptail_management_interface_dns_search - fi - - # Go ahead and bring up networking so other parts of the install work - set_hostname_iso - set_management_interface - - # Add an admin user - whiptail_create_admin_user - - # Get a password for the admin user - APMATCH=no - while [ $APMATCH != yes ]; do - whiptail_create_admin_user_password1 - whiptail_create_admin_user_password2 - check_admin_pass - done + got_root + detect_os + if [ "$OS" == ubuntu ]; then + update-alternatives --set newt-palette /etc/newt/palette.original >> $setup_log 2>&1 fi - # Go ahead and gen the keys so we can use them for any sensor type - Disabled for now - #minio_generate_keys + setterm -blank 0 + + echo "Asking user if they are sure they want to proceed" >> $setup_log 2>&1 + if whiptail_you_sure; then + true + else + echo "User not sure. Cancelling setup.">> $setup_log 2>&1 + whiptail_cancel + fi + + case "$setup_type" in + 'iso') + whiptail_set_hostname + whiptail_management_nic + whiptail_dhcp_or_static + + if [ "$address_type" != 'DHCP' ]; then + whiptail_management_interface_ip + whiptail_management_interface_mask + whiptail_management_interface_gateway + whiptail_management_interface_dns + whiptail_management_interface_dns_search + fi + + # Init networking so rest of install works + set_hostname_iso + set_management_interface + + whiptail_create_admin_user + add_admin_user + disable_onion_user + + collect_adminuser_inputs + ;; + 'network') + whiptail_network_notice + whiptail_set_hostname + whiptail_management_nic + ;; + esac # What kind of install are we doing? whiptail_install_type - SHORTNAME=$(echo "$hostname" | awk -F. '{print $1}') - MINION_ID=$(echo "$SHORTNAME'_'$install_type" | tr '[:upper:]' '[:lower:]') + + local short_name + short_name=$(echo "$hostname" | awk -F. '{print $1}') + + MINION_ID=$(echo "${short_name}_${install_type}" | tr '[:upper:]' '[:lower:]') + export MINION_ID echo "MINION_ID = $MINION_ID" >> $setup_log 2>&1 - # How do we want to handle OS patching? manual, auto or scheduled days and hours whiptail_patch_schedule - case $PATCHSCHEDULE in - 'New Schedule') - whiptail_patch_schedule_select_days - whiptail_patch_schedule_select_hours - whiptail_patch_name_new_schedule - patch_schedule_os_new - ;; - 'Import Schedule') - whiptail_patch_schedule_import - ;; - 'Automatic') - PATCHSCHEDULENAME=auto - ;; - 'Manual') - PATCHSCHEDULENAME=manual - ;; - esac + + # TODO: figure out if this works + filter_unused_nics + minion_type=$(get_minion_type) +} + +init_eval() { + whiptail_bond_nics + whiptail_homenet_master + whiptail_components_adv_warning + whiptail_enable_components +} + +init_helix() { + whiptail_bond_nics + whiptail_homenet_master + whiptail_helix_apikey + + set_base_heapsizes + calculate_usable_cores + set_hostname + set_version + clear_master + set_main_ip + + # Set various constants + RULESETUP=ETOPEN + NSMSETUP=BASIC + home_network_sensor=inherit + MASTERUPDATES=0 + + { + progress_str_printer & + + set_progress_str 1 create_bond_str + create_sensor_bond + + set_progress_str 2 salt_install_str + saltify # finishes at progress=8 + + set_progress_str 10 docker_install_str + docker_install + + set_progress_str 12 salt_minion_init_str + configure_minion "$minion_type" + + set_progress_str 13 'Configuring salt master' + copy_master_config + setup_salt_master_dirs + + set_progress_str 15 'Updating sudoers file for soremote user' + update_sudoers + + set_progress_str 16 gen_static_pillar_str + master_static + + set_progress_str 17 gen_master_pillar_str + master_pillar + + set_progress_str 18 gen_patch_pillar_str + patch_pillar + + set_progress_str 19 'Generating the FireEye pillar' + fireeye_pillar + + + + } | whiptail --title "Hybrid Hunter Install" --gauge "Please wait while installing" 6 60 0 +} #################### ## Helix ## #################### - if [ $"install_type" == 'HELIXSENSOR' ]; then - MASTERUPDATES=0 + if [ "$install_type" == 'HELIXSENSOR' ]; then filter_unused_nics - [[ "$SKIP_BOND" != 'yes' ]] && whiptail_bond_nics - whiptail_helix_apikey + whiptail_bond_nics whiptail_homenet_master + whiptail_helix_apikey RULESETUP=ETOPEN NSMSETUP=BASIC home_network_sensor=inherit + MASTERUPDATES=0 es_heapsize >> $setup_log 2>&1 ls_heapsize >> $setup_log 2>&1 calculate_useable_cores >> "$setup_log" 2>&1 @@ -154,12 +195,8 @@ if (whiptail_you_sure) ; then set_version; clear_master; } >> "$setup_log" 2>&1 - mkdir -p /nsm get_main_ip - if [ $install_type == iso ]; then - add_admin_user - disable_onion_user - fi + # Install salt and dependencies { @@ -177,7 +214,7 @@ if (whiptail_you_sure) ; then echo " ** Configuring Minion **" >> $setup_log configure_minion helix >> $setup_log 2>&1 echo " ** Installing Salt Master **" >> $setup_log - install_master >> $setup_log 2>&1 + copy_master_config >> $setup_log 2>&1 setup_salt_master_dirs >> $setup_log 2>&1 update_sudoers >> $setup_log 2>&1 echo -e "XXX\n25\nConfiguring Default Pillars... \nXXX" @@ -221,7 +258,7 @@ if (whiptail_you_sure) ; then salt-call state.apply filebeat >> $setup_log 2>&1 salt-call state.apply utility >> $setup_log 2>&1 salt-call state.apply schedule >> $setup_log 2>&1 - echo -e "XX\n97\nFinishing touches... \nXXX" + echo -e "XXX\n97\nFinishing touches... \nXXX" filter_unused_nics >> $setup_log 2>&1 network_setup >> $setup_log 2>&1 echo -e "XXX\n98\nVerifying Setup... \nXXX" @@ -301,10 +338,6 @@ if (whiptail_you_sure) ; then # Figure out the main IP address get_main_ip - if [ $install_type == iso ]; then - add_admin_user - disable_onion_user - fi # Add the user so we can sit back and relax add_socore_user_master @@ -322,7 +355,7 @@ if (whiptail_you_sure) ; then echo " ** Configuring Minion **" >> $setup_log configure_minion master >> $setup_log 2>&1 echo " ** Installing Salt Master **" >> $setup_log - install_master >> $setup_log 2>&1 + copy_master_config >> $setup_log 2>&1 setup_salt_master_dirs >> $setup_log 2>&1 update_sudoers >> $setup_log 2>&1 es_heapsize >> $setup_log 2>&1 @@ -397,7 +430,7 @@ if (whiptail_you_sure) ; then echo -e "XXX\n89\nInstalling Playbook... \nXXX" salt-call state.apply playbook >> $setup_log 2>&1 fi - echo -e "XX\n97\nFinishing touches... \nXXX" + echo -e "XXX\n97\nFinishing touches... \nXXX" filter_unused_nics >> $setup_log 2>&1 network_setup >> $setup_log 2>&1 echo -e "XXX\n98\nAdding user to SOC... \nXXX" @@ -426,7 +459,7 @@ if (whiptail_you_sure) ; then if [ $install_type == 'SENSOR' ]; then filter_unused_nics - [[ $SKIP_BOND != 'yes' ]] && whiptail_bond_nics + whiptail_bond_nics whiptail_management_server whiptail_master_updates whiptail_homenet_sensor @@ -445,10 +478,7 @@ if (whiptail_you_sure) ; then set_updates set_hostname clear_master - if [ $install_type == iso ]; then - add_admin_user - disable_onion_user - fi + copy_ssh_key >> $setup_log 2>&1 { sleep 0.5 @@ -485,7 +515,7 @@ if (whiptail_you_sure) ; then salt-call state.apply suricata >> $setup_log 2>&1 echo -e "XXX\n70\nInstalling Strelka... \nXXX" salt-call state.apply strelka >> $setup_log 2>&1 - echo -e "XX\n97\nFinishing touches... \nXXX" + echo -e "XXX\n97\nFinishing touches... \nXXX" filter_unused_nics >> $setup_log 2>&1 network_setup >> $setup_log 2>&1 echo -e "XXX\n98\nVerifying Setup... \nXXX" @@ -513,7 +543,7 @@ if (whiptail_you_sure) ; then if [ $install_type == 'EVAL' ]; then TYPE='eval' # Select which NICs are in the bond - [[ $SKIP_BOND != 'yes' ]] && whiptail_bond_nics + whiptail_bond_nics elif [ $install_type == 'MASTERSEARCH' ]; then TYPE='mastersearch' fi @@ -560,27 +590,36 @@ if (whiptail_you_sure) ; then clear_master set_defaul_log_size get_main_ip - if [ $install_type == iso ]; then - add_admin_user - disable_onion_user - fi # Add the user so we can sit back and relax add_socore_user_master add_soremote_user_master + + { + # Check progress in child process + progress_str_printer & + + if [ "$install_type" = 'EVAL' ]; then + set_progress_str 1 "Creating bond interface..." + create_sensor_bond >> $setup_log 2>&1 + fi + + set_progress_str 2 "Installing Salt..." + + } | whiptail --title "Hybrid Hunter Install" --gauge "Please wait while installing" 6 60 0 + { sleep 0.5 if [ $install_type == 'EVAL' ]; then echo -e "XXX\n0\nCreating Bond Interface... \nXXX" create_sensor_bond >> $setup_log 2>&1 fi - echo -e "XXX\n1\nInstalling Python 3... \nXXX" echo -e "XXX\n2\nInstalling saltstack... \nXXX" saltify >> $setup_log 2>&1 echo -e "XXX\n3\nInstalling docker... \nXXX" docker_install >> $setup_log 2>&1 echo -e "XXX\n5\nInstalling master code... \nXXX" - install_master >> $setup_log 2>&1 + copy_master_config >> $setup_log 2>&1 echo -e "XXX\n6\nCopying salt code... \nXXX" setup_salt_master_dirs >> $setup_log 2>&1 echo -e "XXX\n6\nupdating suduers... \nXXX" @@ -644,7 +683,7 @@ if (whiptail_you_sure) ; then echo -e "XXX\n52\nInstalling Suricata... \nXXX" salt-call state.apply suricata >> $setup_log 2>&1 echo -e "XXX\n54\nInstalling Zeek... \nXXX" - salt-call state.apply bro >> $setup_log 2>&1 + salt-call state.apply bro >> $setup_log 2>&1 fi echo -e "XXX\n56\nInstalling curator... \nXXX" @@ -674,7 +713,7 @@ if (whiptail_you_sure) ; then echo -e "XXX\n95\nInstalling Strelka... \nXXX" salt-call state.apply strelka >> $setup_log 2>&1 fi - echo -e "XX\n97\nFinishing touches... \nXXX" + echo -e "XXX\n97\nFinishing touches... \nXXX" filter_unused_nics >> $setup_log 2>&1 network_setup >> $setup_log 2>&1 echo -e "XXX\n98\nAdding user to SOC... \nXXX" @@ -729,10 +768,6 @@ if (whiptail_you_sure) ; then set_updates set_hostname clear_master - if [ $install_type == iso ]; then - add_admin_user - disable_onion_user - fi copy_ssh_key >> $setup_log 2>&1 { sleep 0.5 @@ -766,7 +801,7 @@ if (whiptail_you_sure) ; then salt-call state.apply elasticsearch >> $setup_log 2>&1 salt-call state.apply curator >> $setup_log 2>&1 salt-call state.apply filebeat >> $setup_log 2>&1 - echo -e "XX\n97\nFinishing touches... \nXXX" + echo -e "XXX\n97\nFinishing touches... \nXXX" filter_unused_nics >> $setup_log 2>&1 network_setup >> $setup_log 2>&1 echo -e "XXX\n98\nVerifying Setup... \nXXX" @@ -789,7 +824,7 @@ if (whiptail_you_sure) ; then if [ $install_type == 'HEAVYNODE' ]; then filter_unused_nics - [[ $SKIP_BOND != 'yes' ]] && whiptail_bond_nics + whiptail_bond_nics whiptail_management_server whiptail_master_updates whiptail_homenet_sensor @@ -831,10 +866,7 @@ if (whiptail_you_sure) ; then set_updates set_hostname clear_master - if [ $install_type == iso ]; then - add_admin_user - disable_onion_user - fi + copy_ssh_key >> $setup_log 2>&1 { sleep 0.5 @@ -879,7 +911,7 @@ if (whiptail_you_sure) ; then salt-call state.apply pcap >> $setup_log 2>&1 echo -e "XXX\n60\nInstalling IDS components... \nXXX" salt-call state.apply suricata >> $setup_log 2>&1 - echo -e "XX\n97\nFinishing touches... \nXXX" + echo -e "XXX\n97\nFinishing touches... \nXXX" filter_unused_nics >> $setup_log 2>&1 network_setup >> $setup_log 2>&1 echo -e "XXX\n98\nVerifying Setup... \nXXX" @@ -895,9 +927,9 @@ if (whiptail_you_sure) ; then fi -########################### -### Standalone Fleet ### -########################### + ########################### + ### Standalone Fleet ### + ########################### if [ $install_type == 'FLEET' ]; then whiptail_management_server @@ -906,10 +938,7 @@ if (whiptail_you_sure) ; then set_updates set_hostname clear_master - if [ $install_type == iso ]; then - add_admin_user - disable_onion_user - fi + copy_ssh_key >> $setup_log 2>&1 { sleep 0.5 @@ -945,7 +974,7 @@ if (whiptail_you_sure) ; then salt-call state.apply redis >> $setup_log 2>&1 echo -e "XXX\n60\nInstalling Fleet... \nXXX" salt-call state.apply fleet >> $setup_log 2>&1 - echo -e "XX\n97\nFinishing touches... \nXXX" + echo -e "XXX\n97\nFinishing touches... \nXXX" filter_unused_nics >> $setup_log 2>&1 network_setup >> $setup_log 2>&1 echo -e "XXX\n98\nVerifying Setup... \nXXX" @@ -960,9 +989,7 @@ if (whiptail_you_sure) ; then shutdown -r now fi fi - - else - echo "User not sure. Cancelling setup.">> $setup_log 2>&1 - whiptail_cancel + echo "User not sure. Cancelling setup.">> $setup_log 2>&1 + whiptail_cancel fi diff --git a/setup/so-variables b/setup/so-variables index 1044fbf33..16155001e 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -1,36 +1,39 @@ #!/bin/bash -export total_mem total_mem=$(grep MemTotal /proc/meminfo | awk '{print $2}' | sed -r 's/.{3}$//') +export total_mem -export hostname hostname=$(cat /etc/hostname) +export hostname -export all_nics all_nics=$(ip link | awk -F: '$0 !~ "lo|vir|veth|br|docker|wl|^[^0-9]"{print $2 " \"" "Interface" "\"" " OFF"}') +export all_nics -export cpu_cores cpu_cores=$(grep -c "processor" /proc/cpuinfo | grep -v "KVM") +export cpu_cores -export cpu_core_list cpu_core_list=$(grep "processor" /proc/cpuinfo | grep -v "KVM" | awk '{print $3 " \"" "core" "\""}') +export cpu_core_list -export random_uid random_uid=$(&1 1>&2 2>&3 ) + export address_type local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -302,7 +303,7 @@ whiptail_eval_adv() { whiptail_check_exitstatus $exitstatus } -whiptail_eval_adv_warning() { +whiptail_components_adv_warning() { [ -z "$QUIET" ] && return @@ -326,9 +327,9 @@ whiptail_homenet_master() { [ -z "$QUIET" ] && return # Ask for the HOME_NET on the master - export home_network_master home_network_master=$(whiptail --title "Security Onion Setup" --inputbox \ "Enter your HOME_NET separated by ," 10 75 10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 3>&1 1>&2 2>&3) + export home_network_master local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -346,9 +347,10 @@ whiptail_homenet_sensor() { if [ $exitstatus == 0 ]; then export home_network_sensor=inherit else - export home_network_sensor home_network_sensor=$(whiptail --title "Security Onion Setup" --inputbox \ "Enter your HOME_NET separated by ," 10 75 10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 3>&1 1>&2 2>&3) + export home_network_sensor + local exitstatus=$? whiptail_check_exitstatus $exitstatus fi @@ -360,7 +362,6 @@ whiptail_install_type() { [ -z "$QUIET" ] && return # What kind of install are we doing? - export install_type install_type=$(whiptail --title "Security Onion Setup" --radiolist \ "Choose Install Type:" 20 75 13 \ "SENSOR" "Create a forward only sensor" ON \ @@ -376,6 +377,7 @@ whiptail_install_type() { "WAZUH" "TODO Stand Alone Wazuh Node" OFF \ "STRELKA" "TODO Stand Alone Strelka Node" OFF \ "PARSINGNODE" "TODO Add a dedicated Parsing Node" OFF 3>&1 1>&2 2>&3 ) + export install_type local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -607,7 +609,6 @@ whiptail_node_es_heap() { [ -z "$QUIET" ] && return - es_heapsize NODE_ES_HEAP_SIZE=$(whiptail --title "Security Onion Setup" --inputbox \ "\nEnter ES Heap Size: \n \n(Recommended value is pre-populated)" 10 75 $ES_HEAP_SIZE 3>&1 1>&2 2>&3) @@ -620,7 +621,6 @@ whiptail_node_ls_heap() { [ -z "$QUIET" ] && return - ls_heapsize NODE_LS_HEAP_SIZE=$(whiptail --title "Security Onion Setup" --inputbox \ "\nEnter LogStash Heap Size: \n \n(Recommended value is pre-populated)" 10 75 $LS_HEAP_SIZE 3>&1 1>&2 2>&3) @@ -713,14 +713,32 @@ whiptail_patch_schedule() { [ -z "$QUIET" ] && return - # What kind of patch schedule are we doing? - PATCHSCHEDULE=$(whiptail --title "Security Onion Setup" --radiolist \ + local patch_schedule + patch_schedule=$(whiptail --title "Security Onion Setup" --radiolist \ "Choose OS patch schedule. This will NOT update Security Onion related tools such as Zeek, Elasticsearch, Kibana, SaltStack, etc." 15 75 5 \ "Automatic" "Updates installed every 8 hours if available" ON \ "Manual" "Updates will be installed manually" OFF \ "Import Schedule" "Import named schedule on following screen" OFF \ "New Schedule" "Configure and name new schedule on next screen" OFF 3>&1 1>&2 2>&3 ) + case $patch_schedule in + 'New Schedule') + whiptail_patch_schedule_select_days + whiptail_patch_schedule_select_hours + whiptail_patch_name_new_schedule + patch_schedule_os_new + ;; + 'Import Schedule') + whiptail_patch_schedule_import + ;; + 'Automatic') + PATCHSCHEDULENAME='auto' + ;; + 'Manual') + PATCHSCHEDULENAME='manual' + ;; + esac + local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -942,16 +960,20 @@ whiptail_master_updates() { [ -z "$QUIET" ] && return - MASTERUPDATES=$(whiptail --title "Security Onion Setup" --radiolist \ + local update_string + update_string=$(whiptail --title "Security Onion Setup" --radiolist \ "How would you like to download updates for your grid?:" 20 75 4 \ "MASTER" "Master node is proxy for OS/Docker updates." ON \ "OPEN" "Each node connect to the Internet for updates" OFF 3>&1 1>&2 2>&3 ) - if [ "$MASTERUPDATES" == "MASTER" ]; then - MASTERUPDATES=1 - else - MASTERUPDATES=0 - fi + case "$update_string" in + 'MASTER') + MASTERUPDATES=1 + ;; + *) + MASTERUPDATES=0 + ;; + esac local exitstatus=$? whiptail_check_exitstatus $exitstatus