diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index acad465d1..b5d0b0293 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -1,11 +1,8 @@ {%- set NODE_ROUTE_TYPE = salt['pillar.get']('elasticsearch:node_route_type', 'hot') %} -{%- if salt['pillar.get']('elasticsearch:hot_warm_enabled') or salt['pillar.get']('elasticsearch:true_cluster') %} -{%- set ESCLUSTERNAME = salt['pillar.get']('elasticsearch:true_cluster_name', '') %} -{%- else %} {%- set ESCLUSTERNAME = salt['pillar.get']('elasticsearch:esclustername', '') %} -{%- endif %} -{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} +{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') %} +{%- set FEATURES = salt['pillar.get']('elastic:features', False) %} +{%- set TRUECLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %} cluster.name: "{{ ESCLUSTERNAME }}" network.host: 0.0.0.0 @@ -42,5 +39,10 @@ cluster.routing.allocation.disk.watermark.flood_stage: 98% # authz_exception: true {%- endif %} node.attr.box_type: {{ NODE_ROUTE_TYPE }} -node.name: {{ ESCLUSTERNAME }} +node.name: {{ grains.host }} script.max_compilations_rate: 1000/1m +{%- if TRUECLUSTER is sameas true %} + {%- if grains.role == 'so-manager' %} +node.roles: [ master ] + {%- endif %} +{%- endif %} diff --git a/setup/so-functions b/setup/so-functions index 3cf268869..51d896acd 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1176,15 +1176,23 @@ manager_global() { "elastic:"\ " features: False"\ "elasticsearch:"\ - " replicas: 0"\ - " true_cluster: False"\ - " true_cluster_name: 'so'"\ + " replicas: 0" >> "$global_pillar" + if [ -n "$ESCLUSTERNAME" ]; then + printf '%s\n'\ + " true_cluster: True"\ + " true_cluster_name: '$ESCLUSTERNAME'" >> "$global_pillar" + else + printf '%s\n'\ + " true_cluster: False"\ + " true_cluster_name: 'so'" >> "$global_pillar" + fi + printf '%s\n'\ " discovery_nodes: 1"\ " hot_warm_enabled: False"\ " cluster_routing_allocation_disk.threshold_enabled: true"\ - " cluster_routing_allocation_disk_watermark_low: '95%'"\ - " cluster_routing_allocation_disk_watermark_high: '98%'"\ - " cluster_routing_allocation_disk_watermark_flood_stage: '98%'"\ + " cluster_routing_allocation_disk_watermark_low: '95%'"\ + " cluster_routing_allocation_disk_watermark_high: '98%'"\ + " cluster_routing_allocation_disk_watermark_flood_stage: '98%'"\ " index_settings:"\ " so-beats:"\ " shards: 1"\ @@ -1300,8 +1308,15 @@ elasticsearch_pillar() { "elasticsearch:"\ " mainip: '$MAINIP'"\ " mainint: '$MNIC'"\ - " esheap: '$NODE_ES_HEAP_SIZE'"\ - " esclustername: {{ grains.host }}"\ + " esheap: '$NODE_ES_HEAP_SIZE'" + if [ -n "$ESCLUSTERNAME" ]; then + printf '%s\n'\ + " esclustername: $ESCLUSTERNAME" + else + printf '%s\n'\ + " esclustername: {{ grains.host }}" + fi + printf '%s\n'\ " node_type: '$NODETYPE'"\ " es_port: $node_es_port"\ " log_size_limit: $log_size_limit"\ diff --git a/setup/so-setup b/setup/so-setup index 77c579cfc..4cfb1fa16 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -336,6 +336,11 @@ fi if [[ $is_manager && ! $is_eval ]]; then whiptail_manager_adv + if [ "$MANAGERADV" = 'ADVANCED' ]; then + if [ "$install_type" = 'MANAGER' ] || [ "$install_type" = 'MANAGERSEARCH' ]; then + whiptail_manager_adv_escluster + fi + fi whiptail_zeek_version # Don't run this function for now since Snort is not yet supported # whiptail_nids diff --git a/setup/so-whiptail b/setup/so-whiptail index 11d968910..ebc515782 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -31,24 +31,6 @@ whiptail_airgap() { whiptail_check_exitstatus $exitstatus } -whiptail_basic_zeek() { - - [ -n "$TESTING" ] && return - - if [[ $is_smooshed ]]; then - local PROCS=$(expr $lb_procs / 2) - if [ "$PROCS" -lt 1 ]; then PROCS=1; else PROCS=$PROCS; fi - else - local PROCS=$lb_procs - fi - - BASICZEEK=$(whiptail --title "Security Onion Setup" --inputbox \ - "Enter the number of zeek processes:" 10 75 "$PROCS" 3>&1 1>&2 2>&3) - - local exitstatus=$? - whiptail_check_exitstatus $exitstatus -} - whiptail_basic_suri() { [ -n "$TESTING" ] && return @@ -68,15 +50,10 @@ whiptail_basic_suri() { } -whiptail_zeek_pins() { +whiptail_basic_zeek() { [ -n "$TESTING" ] && return - local cpu_core_list_whiptail=() - for item in "${cpu_core_list[@]}"; do - cpu_core_list_whiptail+=("$item" "OFF") - done - if [[ $is_smooshed ]]; then local PROCS=$(expr $lb_procs / 2) if [ "$PROCS" -lt 1 ]; then PROCS=1; else PROCS=$PROCS; fi @@ -103,47 +80,6 @@ whiptail_zeek_version() { local exitstatus=$? whiptail_check_exitstatus $exitstatus - -} - -whiptail_sensor_nics() { - - [ -n "$TESTING" ] && return - - filter_unused_nics - - if [[ $is_ec2 ]]; then - local menu_text="Please select NIC for the Monitor Interface:" - local list_type="radiolist" - else - local menu_text="Please add NICs to the Monitor Interface:" - local list_type="checklist" - fi - - BNICS=$(whiptail --title "NIC Setup" --$list_type "$menu_text" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3) - local exitstatus=$? - whiptail_check_exitstatus $exitstatus - - while [ -z "$BNICS" ] - do - BNICS=$(whiptail --title "NIC Setup" --$list_type "$menu_text" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3 ) - local exitstatus=$? - whiptail_check_exitstatus $exitstatus - done - - BNICS=$(echo "$BNICS" | tr -d '"') - - IFS=' ' read -ra BNICS <<< "$BNICS" - - for bond_nic in "${BNICS[@]}"; do - if [[ "${nmcli_dev_status_list}" =~ $bond_nic\:unmanaged ]]; then - whiptail \ - --title "Security Onion Setup" \ - --msgbox "$bond_nic is unmanaged by Network Manager. Please remove it from other network management tools then re-run setup." \ - 8 75 - exit - fi - done } whiptail_bond_nics_mtu() { @@ -187,6 +123,13 @@ whiptail_check_exitstatus() { esac } +whiptail_components_adv_warning() { + + [ -n "$TESTING" ] && return + + whiptail --title "Security Onion Setup" --msgbox "Please keep in mind the more services that you enable the more RAM that is required." 8 75 +} + whiptail_create_admin_user() { [ -n "$TESTING" ] && return @@ -294,13 +237,6 @@ whiptail_create_web_user() { whiptail_check_exitstatus $exitstatus } -whiptail_invalid_user_warning() { - - [ -n "$TESTING" ] && return - - whiptail --title "Security Onion Setup" --msgbox "Please enter a valid email address." 8 75 -} - whiptail_create_web_user_password1() { [ -n "$TESTING" ] && return @@ -546,11 +482,26 @@ whiptail_eval_adv() { whiptail_check_exitstatus $exitstatus } -whiptail_components_adv_warning() { +whiptail_fleet_custom_hostname() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --msgbox "Please keep in mind the more services that you enable the more RAM that is required." 8 75 + FLEETCUSTOMHOSTNAME=$(whiptail --title "Security Onion Install" --inputbox \ + "What FQDN should osquery clients use for connections to this Fleet node? Leave blank if the local system hostname will be used." 10 60 3>&1 1>&2 2>&3) + + local exitstatus=$? + whiptail_check_exitstatus $exitstatus +} + +whiptail_gauge_post_setup() { + + if [ -n "$TESTING" ]; then + cat >> $setup_log 2>&1 + else + local msg=$1 + + whiptail --title "Security Onion Setup" --gauge "$msg" 6 60 96 + fi } whiptail_helix_apikey() { @@ -677,6 +628,27 @@ whiptail_install_type_other() { export install_type } +whiptail_invalid_pass_characters_warning() { + + [ -n "$TESTING" ] && return + + whiptail --title "Security Onion Setup" --msgbox "Password is invalid. Please exclude single quotes, double quotes and backslashes from the password." 8 75 +} + +whiptail_invalid_pass_warning() { + + [ -n "$TESTING" ] && return + + whiptail --title "Security Onion Setup" --msgbox "Please choose a more secure password." 8 75 +} + +whiptail_invalid_user_warning() { + + [ -n "$TESTING" ] && return + + whiptail --title "Security Onion Setup" --msgbox "Please enter a valid email address." 8 75 +} + whiptail_log_size_limit() { [ -n "$TESTING" ] && return @@ -691,6 +663,17 @@ whiptail_log_size_limit() { } +whiptail_make_changes() { + + [ -n "$TESTING" ] && return + + whiptail --title "Security Onion Setup" --yesno "We are going to set this machine up as a $install_type. Please press YES to make changes or NO to cancel." 8 75 + + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + +} + whiptail_management_interface_dns() { [ -n "$TESTING" ] && return @@ -765,43 +748,6 @@ whiptail_management_nic() { } -whiptail_nids() { - - [ -n "$TESTING" ] && return - - NIDS=$(whiptail --title "Security Onion Setup" --radiolist \ - "Choose which IDS to run: \n\n(Snort 3.0 support will be added once it is out of beta.)" 25 75 4 \ - "Suricata" "Suricata" ON \ - "Snort" "Placeholder for Snort 3.0 " OFF 3>&1 1>&2 2>&3 ) - - local exitstatus=$? - whiptail_check_exitstatus $exitstatus - -} - -whiptail_oinkcode() { - - [ -n "$TESTING" ] && return - - OINKCODE=$(whiptail --title "Security Onion Setup" --inputbox \ - "Enter your ET Pro or oinkcode:" 10 75 XXXXXXX 3>&1 1>&2 2>&3) - - local exitstatus=$? - whiptail_check_exitstatus $exitstatus - -} - -whiptail_make_changes() { - - [ -n "$TESTING" ] && return - - whiptail --title "Security Onion Setup" --yesno "We are going to set this machine up as a $install_type. Please press YES to make changes or NO to cancel." 8 75 - - local exitstatus=$? - whiptail_check_exitstatus $exitstatus - -} - whiptail_management_server() { [ -n "$TESTING" ] && return @@ -851,6 +797,30 @@ whiptail_manager_adv() { } +# Ask if you want to do true clustering +whiptail_manager_adv_escluster(){ + + [ -n "$TESTING" ] && return + + whiptail --title "Security Onion Setup" --yesno \ + "Do you want to set up a traditional ES cluster?" 8 75 + + local exitstatus=$? + + if [[ $exitstatus == 0 ]]; then + whiptail_manager_adv_escluster_name + fi +} + +# Get a cluster name +whiptail_manager_adv_escluster_name(){ + + [ -n "$TESTING" ] && return + + ESCLUSTERNAME=$(whiptail --title "Security Onion Setup" --inputbox \ + "Enter a name for your ES cluster!" 10 75 securityonion 3>&1 1>&2 2>&3) +} + # Ask which additional components to install whiptail_manager_adv_service_zeeklogs() { @@ -905,6 +875,54 @@ whiptail_manager_adv_service_zeeklogs() { } +whiptail_manager_updates() { + + [ -n "$TESTING" ] && return + + local update_string + update_string=$(whiptail --title "Security Onion Setup" --radiolist \ + "How would you like to download OS package updates for your grid?" 20 75 4 \ + "MANAGER" "Manager node is proxy for updates" ON \ + "OPEN" "Each node connects to the Internet for updates" OFF 3>&1 1>&2 2>&3 ) + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + + case "$update_string" in + 'MANAGER') + export MANAGERUPDATES='1' + ;; + *) + export MANAGERUPDATES='0' + ;; + esac + +} + +whiptail_manager_updates_warning() { + [ -n "$TESTING" ] && return + + whiptail --title "Security Onion Setup"\ + --msgbox "Updating through the manager node requires the manager to have internet access, press ENTER to continue."\ + 8 75 + + local exitstatus=$? + whiptail_check_exitstatus $exitstatus +} + +whiptail_nids() { + + [ -n "$TESTING" ] && return + + NIDS=$(whiptail --title "Security Onion Setup" --radiolist \ + "Choose which IDS to run: \n\n(Snort 3.0 support will be added once it is out of beta.)" 25 75 4 \ + "Suricata" "Suricata" ON \ + "Snort" "Placeholder for Snort 3.0 " OFF 3>&1 1>&2 2>&3 ) + + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + +} + whiptail_network_notice() { [ -n "$TESTING" ] && return @@ -990,6 +1008,18 @@ whiptail_node_ls_input_threads() { } +whiptail_oinkcode() { + + [ -n "$TESTING" ] && return + + OINKCODE=$(whiptail --title "Security Onion Setup" --inputbox \ + "Enter your ET Pro or oinkcode:" 10 75 XXXXXXX 3>&1 1>&2 2>&3) + + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + +} + #TODO: helper function to display error message or exit if batch mode # exit_if_batch <"Error string"> @@ -1144,6 +1174,21 @@ whiptail_patch_schedule_select_hours() { } +whiptail_requirements_error() { + + local requirement_needed=$1 + local current_val=$2 + local needed_val=$3 + + [ -n "$TESTING" ] && return + + whiptail --title "Security Onion Setup" \ + --yesno "This machine currently has $current_val $requirement_needed, but needs $needed_val to meet minimum requirements. Press YES to continue anyway, or press NO to cancel." 10 75 + + local exitstatus=$? + whiptail_check_exitstatus $exitstatus +} + whiptail_rule_setup() { [ -n "$TESTING" ] && return @@ -1175,6 +1220,46 @@ whiptail_sensor_config() { } +whiptail_sensor_nics() { + + [ -n "$TESTING" ] && return + + filter_unused_nics + + if [[ $is_ec2 ]]; then + local menu_text="Please select NIC for the Monitor Interface:" + local list_type="radiolist" + else + local menu_text="Please add NICs to the Monitor Interface:" + local list_type="checklist" + fi + + BNICS=$(whiptail --title "NIC Setup" --$list_type "$menu_text" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3) + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + + while [ -z "$BNICS" ] + do + BNICS=$(whiptail --title "NIC Setup" --$list_type "$menu_text" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3 ) + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + done + + BNICS=$(echo "$BNICS" | tr -d '"') + + IFS=' ' read -ra BNICS <<< "$BNICS" + + for bond_nic in "${BNICS[@]}"; do + if [[ "${nmcli_dev_status_list}" =~ $bond_nic\:unmanaged ]]; then + whiptail \ + --title "Security Onion Setup" \ + --msgbox "$bond_nic is unmanaged by Network Manager. Please remove it from other network management tools then re-run setup." \ + 8 75 + exit + fi + done +} + whiptail_set_hostname() { [ -n "$TESTING" ] && return @@ -1295,15 +1380,30 @@ whiptail_so_allow() { fi } -whiptail_gauge_post_setup() { +whiptail_storage_requirements() { + local mount=$1 + local current_val=$2 + local needed_val=$3 - if [ -n "$TESTING" ]; then - cat >> $setup_log 2>&1 - else - local msg=$1 + [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --gauge "$msg" 6 60 96 - fi + read -r -d '' message <<- EOM + Free space on mount point '${mount}' is currently ${current_val}. + + You need ${needed_val} to meet minimum requirements. + + Visit https://docs.securityonion.net/en/2.1/hardware.html for more information. + + Press YES to continue anyway, or press NO to cancel. + EOM + + whiptail \ + --title "Security Onion Setup" \ + --yesno "$message" \ + 14 75 + + local exitstatus=$? + whiptail_check_exitstatus $exitstatus } whiptail_strelka_rules() { @@ -1346,40 +1446,6 @@ whiptail_suricata_pins() { } -whiptail_manager_updates() { - - [ -n "$TESTING" ] && return - - local update_string - update_string=$(whiptail --title "Security Onion Setup" --radiolist \ - "How would you like to download OS package updates for your grid?" 20 75 4 \ - "MANAGER" "Manager node is proxy for updates" ON \ - "OPEN" "Each node connects to the Internet for updates" OFF 3>&1 1>&2 2>&3 ) - local exitstatus=$? - whiptail_check_exitstatus $exitstatus - - case "$update_string" in - 'MANAGER') - export MANAGERUPDATES='1' - ;; - *) - export MANAGERUPDATES='0' - ;; - esac - -} - -whiptail_manager_updates_warning() { - [ -n "$TESTING" ] && return - - whiptail --title "Security Onion Setup"\ - --msgbox "Updating through the manager node requires the manager to have internet access, press ENTER to continue."\ - 8 75 - - local exitstatus=$? - whiptail_check_exitstatus $exitstatus -} - whiptail_node_updates() { [ -n "$TESTING" ] && return @@ -1417,3 +1483,40 @@ whiptail_you_sure() { return $exitstatus } + +whiptail_zeek_pins() { + + [ -n "$TESTING" ] && return + + local cpu_core_list_whiptail=() + for item in "${cpu_core_list[@]}"; do + cpu_core_list_whiptail+=("$item" "OFF") + done + + if [[ $is_smooshed ]]; then + local PROCS=$(expr $lb_procs / 2) + if [ "$PROCS" -lt 1 ]; then PROCS=1; else PROCS=$PROCS; fi + else + local PROCS=$lb_procs + fi + + ZEEKPINS=$(whiptail --noitem --title "Pin Zeek CPUS" --checklist "Please select $PROCS cores to pin Zeek to:" 20 75 12 "${cpu_core_list_whiptail[@]}" 3>&1 1>&2 2>&3 ) + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + + ZEEKPINS=$(echo "$ZEEKPINS" | tr -d '"') + + IFS=' ' read -ra ZEEKPINS <<< "$ZEEKPINS" +} + +whiptail_zeek_version() { + + [ -n "$TESTING" ] && return + + ZEEKVERSION=$(whiptail --title "Security Onion Setup" --radiolist "What tool would you like to use to generate metadata?" 20 75 4 "ZEEK" "Zeek (formerly known as Bro)" ON \ + "SURICATA" "Suricata" OFF 3>&1 1>&2 2>&3) + + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + +}