Merge pull request #2174 from Security-Onion-Solutions/escluster

Escluster
This commit is contained in:
Mike Reeves
2020-12-02 13:23:08 -05:00
committed by GitHub
4 changed files with 292 additions and 167 deletions

View File

@@ -1,11 +1,8 @@
{%- set NODE_ROUTE_TYPE = salt['pillar.get']('elasticsearch:node_route_type', 'hot') %}
{%- if salt['pillar.get']('elasticsearch:hot_warm_enabled') or salt['pillar.get']('elasticsearch:true_cluster') %}
{%- set ESCLUSTERNAME = salt['pillar.get']('elasticsearch:true_cluster_name', '') %}
{%- else %}
{%- set ESCLUSTERNAME = salt['pillar.get']('elasticsearch:esclustername', '') %}
{%- endif %}
{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') %}
{%- set FEATURES = salt['pillar.get']('elastic:features', False) %}
{%- set TRUECLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %}
cluster.name: "{{ ESCLUSTERNAME }}"
network.host: 0.0.0.0
@@ -42,5 +39,10 @@ cluster.routing.allocation.disk.watermark.flood_stage: 98%
# authz_exception: true
{%- endif %}
node.attr.box_type: {{ NODE_ROUTE_TYPE }}
node.name: {{ ESCLUSTERNAME }}
node.name: {{ grains.host }}
script.max_compilations_rate: 1000/1m
{%- if TRUECLUSTER is sameas true %}
{%- if grains.role == 'so-manager' %}
node.roles: [ master ]
{%- endif %}
{%- endif %}

View File

@@ -1176,15 +1176,23 @@ manager_global() {
"elastic:"\
" features: False"\
"elasticsearch:"\
" replicas: 0"\
" true_cluster: False"\
" true_cluster_name: 'so'"\
" replicas: 0" >> "$global_pillar"
if [ -n "$ESCLUSTERNAME" ]; then
printf '%s\n'\
" true_cluster: True"\
" true_cluster_name: '$ESCLUSTERNAME'" >> "$global_pillar"
else
printf '%s\n'\
" true_cluster: False"\
" true_cluster_name: 'so'" >> "$global_pillar"
fi
printf '%s\n'\
" discovery_nodes: 1"\
" hot_warm_enabled: False"\
" cluster_routing_allocation_disk.threshold_enabled: true"\
" cluster_routing_allocation_disk_watermark_low: '95%'"\
" cluster_routing_allocation_disk_watermark_high: '98%'"\
" cluster_routing_allocation_disk_watermark_flood_stage: '98%'"\
" cluster_routing_allocation_disk_watermark_low: '95%'"\
" cluster_routing_allocation_disk_watermark_high: '98%'"\
" cluster_routing_allocation_disk_watermark_flood_stage: '98%'"\
" index_settings:"\
" so-beats:"\
" shards: 1"\
@@ -1300,8 +1308,15 @@ elasticsearch_pillar() {
"elasticsearch:"\
" mainip: '$MAINIP'"\
" mainint: '$MNIC'"\
" esheap: '$NODE_ES_HEAP_SIZE'"\
" esclustername: {{ grains.host }}"\
" esheap: '$NODE_ES_HEAP_SIZE'"
if [ -n "$ESCLUSTERNAME" ]; then
printf '%s\n'\
" esclustername: $ESCLUSTERNAME"
else
printf '%s\n'\
" esclustername: {{ grains.host }}"
fi
printf '%s\n'\
" node_type: '$NODETYPE'"\
" es_port: $node_es_port"\
" log_size_limit: $log_size_limit"\

View File

@@ -336,6 +336,11 @@ fi
if [[ $is_manager && ! $is_eval ]]; then
whiptail_manager_adv
if [ "$MANAGERADV" = 'ADVANCED' ]; then
if [ "$install_type" = 'MANAGER' ] || [ "$install_type" = 'MANAGERSEARCH' ]; then
whiptail_manager_adv_escluster
fi
fi
whiptail_zeek_version
# Don't run this function for now since Snort is not yet supported
# whiptail_nids

View File

@@ -31,24 +31,6 @@ whiptail_airgap() {
whiptail_check_exitstatus $exitstatus
}
whiptail_basic_zeek() {
[ -n "$TESTING" ] && return
if [[ $is_smooshed ]]; then
local PROCS=$(expr $lb_procs / 2)
if [ "$PROCS" -lt 1 ]; then PROCS=1; else PROCS=$PROCS; fi
else
local PROCS=$lb_procs
fi
BASICZEEK=$(whiptail --title "Security Onion Setup" --inputbox \
"Enter the number of zeek processes:" 10 75 "$PROCS" 3>&1 1>&2 2>&3)
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
}
whiptail_basic_suri() {
[ -n "$TESTING" ] && return
@@ -68,15 +50,10 @@ whiptail_basic_suri() {
}
whiptail_zeek_pins() {
whiptail_basic_zeek() {
[ -n "$TESTING" ] && return
local cpu_core_list_whiptail=()
for item in "${cpu_core_list[@]}"; do
cpu_core_list_whiptail+=("$item" "OFF")
done
if [[ $is_smooshed ]]; then
local PROCS=$(expr $lb_procs / 2)
if [ "$PROCS" -lt 1 ]; then PROCS=1; else PROCS=$PROCS; fi
@@ -103,47 +80,6 @@ whiptail_zeek_version() {
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
}
whiptail_sensor_nics() {
[ -n "$TESTING" ] && return
filter_unused_nics
if [[ $is_ec2 ]]; then
local menu_text="Please select NIC for the Monitor Interface:"
local list_type="radiolist"
else
local menu_text="Please add NICs to the Monitor Interface:"
local list_type="checklist"
fi
BNICS=$(whiptail --title "NIC Setup" --$list_type "$menu_text" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3)
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
while [ -z "$BNICS" ]
do
BNICS=$(whiptail --title "NIC Setup" --$list_type "$menu_text" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3 )
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
done
BNICS=$(echo "$BNICS" | tr -d '"')
IFS=' ' read -ra BNICS <<< "$BNICS"
for bond_nic in "${BNICS[@]}"; do
if [[ "${nmcli_dev_status_list}" =~ $bond_nic\:unmanaged ]]; then
whiptail \
--title "Security Onion Setup" \
--msgbox "$bond_nic is unmanaged by Network Manager. Please remove it from other network management tools then re-run setup." \
8 75
exit
fi
done
}
whiptail_bond_nics_mtu() {
@@ -187,6 +123,13 @@ whiptail_check_exitstatus() {
esac
}
whiptail_components_adv_warning() {
[ -n "$TESTING" ] && return
whiptail --title "Security Onion Setup" --msgbox "Please keep in mind the more services that you enable the more RAM that is required." 8 75
}
whiptail_create_admin_user() {
[ -n "$TESTING" ] && return
@@ -294,13 +237,6 @@ whiptail_create_web_user() {
whiptail_check_exitstatus $exitstatus
}
whiptail_invalid_user_warning() {
[ -n "$TESTING" ] && return
whiptail --title "Security Onion Setup" --msgbox "Please enter a valid email address." 8 75
}
whiptail_create_web_user_password1() {
[ -n "$TESTING" ] && return
@@ -546,11 +482,26 @@ whiptail_eval_adv() {
whiptail_check_exitstatus $exitstatus
}
whiptail_components_adv_warning() {
whiptail_fleet_custom_hostname() {
[ -n "$TESTING" ] && return
whiptail --title "Security Onion Setup" --msgbox "Please keep in mind the more services that you enable the more RAM that is required." 8 75
FLEETCUSTOMHOSTNAME=$(whiptail --title "Security Onion Install" --inputbox \
"What FQDN should osquery clients use for connections to this Fleet node? Leave blank if the local system hostname will be used." 10 60 3>&1 1>&2 2>&3)
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
}
whiptail_gauge_post_setup() {
if [ -n "$TESTING" ]; then
cat >> $setup_log 2>&1
else
local msg=$1
whiptail --title "Security Onion Setup" --gauge "$msg" 6 60 96
fi
}
whiptail_helix_apikey() {
@@ -677,6 +628,27 @@ whiptail_install_type_other() {
export install_type
}
whiptail_invalid_pass_characters_warning() {
[ -n "$TESTING" ] && return
whiptail --title "Security Onion Setup" --msgbox "Password is invalid. Please exclude single quotes, double quotes and backslashes from the password." 8 75
}
whiptail_invalid_pass_warning() {
[ -n "$TESTING" ] && return
whiptail --title "Security Onion Setup" --msgbox "Please choose a more secure password." 8 75
}
whiptail_invalid_user_warning() {
[ -n "$TESTING" ] && return
whiptail --title "Security Onion Setup" --msgbox "Please enter a valid email address." 8 75
}
whiptail_log_size_limit() {
[ -n "$TESTING" ] && return
@@ -691,6 +663,17 @@ whiptail_log_size_limit() {
}
whiptail_make_changes() {
[ -n "$TESTING" ] && return
whiptail --title "Security Onion Setup" --yesno "We are going to set this machine up as a $install_type. Please press YES to make changes or NO to cancel." 8 75
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
}
whiptail_management_interface_dns() {
[ -n "$TESTING" ] && return
@@ -765,43 +748,6 @@ whiptail_management_nic() {
}
whiptail_nids() {
[ -n "$TESTING" ] && return
NIDS=$(whiptail --title "Security Onion Setup" --radiolist \
"Choose which IDS to run: \n\n(Snort 3.0 support will be added once it is out of beta.)" 25 75 4 \
"Suricata" "Suricata" ON \
"Snort" "Placeholder for Snort 3.0 " OFF 3>&1 1>&2 2>&3 )
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
}
whiptail_oinkcode() {
[ -n "$TESTING" ] && return
OINKCODE=$(whiptail --title "Security Onion Setup" --inputbox \
"Enter your ET Pro or oinkcode:" 10 75 XXXXXXX 3>&1 1>&2 2>&3)
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
}
whiptail_make_changes() {
[ -n "$TESTING" ] && return
whiptail --title "Security Onion Setup" --yesno "We are going to set this machine up as a $install_type. Please press YES to make changes or NO to cancel." 8 75
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
}
whiptail_management_server() {
[ -n "$TESTING" ] && return
@@ -851,6 +797,30 @@ whiptail_manager_adv() {
}
# Ask if you want to do true clustering
whiptail_manager_adv_escluster(){
[ -n "$TESTING" ] && return
whiptail --title "Security Onion Setup" --yesno \
"Do you want to set up a traditional ES cluster?" 8 75
local exitstatus=$?
if [[ $exitstatus == 0 ]]; then
whiptail_manager_adv_escluster_name
fi
}
# Get a cluster name
whiptail_manager_adv_escluster_name(){
[ -n "$TESTING" ] && return
ESCLUSTERNAME=$(whiptail --title "Security Onion Setup" --inputbox \
"Enter a name for your ES cluster!" 10 75 securityonion 3>&1 1>&2 2>&3)
}
# Ask which additional components to install
whiptail_manager_adv_service_zeeklogs() {
@@ -905,6 +875,54 @@ whiptail_manager_adv_service_zeeklogs() {
}
whiptail_manager_updates() {
[ -n "$TESTING" ] && return
local update_string
update_string=$(whiptail --title "Security Onion Setup" --radiolist \
"How would you like to download OS package updates for your grid?" 20 75 4 \
"MANAGER" "Manager node is proxy for updates" ON \
"OPEN" "Each node connects to the Internet for updates" OFF 3>&1 1>&2 2>&3 )
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
case "$update_string" in
'MANAGER')
export MANAGERUPDATES='1'
;;
*)
export MANAGERUPDATES='0'
;;
esac
}
whiptail_manager_updates_warning() {
[ -n "$TESTING" ] && return
whiptail --title "Security Onion Setup"\
--msgbox "Updating through the manager node requires the manager to have internet access, press ENTER to continue."\
8 75
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
}
whiptail_nids() {
[ -n "$TESTING" ] && return
NIDS=$(whiptail --title "Security Onion Setup" --radiolist \
"Choose which IDS to run: \n\n(Snort 3.0 support will be added once it is out of beta.)" 25 75 4 \
"Suricata" "Suricata" ON \
"Snort" "Placeholder for Snort 3.0 " OFF 3>&1 1>&2 2>&3 )
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
}
whiptail_network_notice() {
[ -n "$TESTING" ] && return
@@ -990,6 +1008,18 @@ whiptail_node_ls_input_threads() {
}
whiptail_oinkcode() {
[ -n "$TESTING" ] && return
OINKCODE=$(whiptail --title "Security Onion Setup" --inputbox \
"Enter your ET Pro or oinkcode:" 10 75 XXXXXXX 3>&1 1>&2 2>&3)
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
}
#TODO: helper function to display error message or exit if batch mode
# exit_if_batch <"Error string"> <Error code (int)>
@@ -1144,6 +1174,21 @@ whiptail_patch_schedule_select_hours() {
}
whiptail_requirements_error() {
local requirement_needed=$1
local current_val=$2
local needed_val=$3
[ -n "$TESTING" ] && return
whiptail --title "Security Onion Setup" \
--yesno "This machine currently has $current_val $requirement_needed, but needs $needed_val to meet minimum requirements. Press YES to continue anyway, or press NO to cancel." 10 75
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
}
whiptail_rule_setup() {
[ -n "$TESTING" ] && return
@@ -1175,6 +1220,46 @@ whiptail_sensor_config() {
}
whiptail_sensor_nics() {
[ -n "$TESTING" ] && return
filter_unused_nics
if [[ $is_ec2 ]]; then
local menu_text="Please select NIC for the Monitor Interface:"
local list_type="radiolist"
else
local menu_text="Please add NICs to the Monitor Interface:"
local list_type="checklist"
fi
BNICS=$(whiptail --title "NIC Setup" --$list_type "$menu_text" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3)
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
while [ -z "$BNICS" ]
do
BNICS=$(whiptail --title "NIC Setup" --$list_type "$menu_text" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3 )
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
done
BNICS=$(echo "$BNICS" | tr -d '"')
IFS=' ' read -ra BNICS <<< "$BNICS"
for bond_nic in "${BNICS[@]}"; do
if [[ "${nmcli_dev_status_list}" =~ $bond_nic\:unmanaged ]]; then
whiptail \
--title "Security Onion Setup" \
--msgbox "$bond_nic is unmanaged by Network Manager. Please remove it from other network management tools then re-run setup." \
8 75
exit
fi
done
}
whiptail_set_hostname() {
[ -n "$TESTING" ] && return
@@ -1295,15 +1380,30 @@ whiptail_so_allow() {
fi
}
whiptail_gauge_post_setup() {
whiptail_storage_requirements() {
local mount=$1
local current_val=$2
local needed_val=$3
if [ -n "$TESTING" ]; then
cat >> $setup_log 2>&1
else
local msg=$1
[ -n "$TESTING" ] && return
whiptail --title "Security Onion Setup" --gauge "$msg" 6 60 96
fi
read -r -d '' message <<- EOM
Free space on mount point '${mount}' is currently ${current_val}.
You need ${needed_val} to meet minimum requirements.
Visit https://docs.securityonion.net/en/2.1/hardware.html for more information.
Press YES to continue anyway, or press NO to cancel.
EOM
whiptail \
--title "Security Onion Setup" \
--yesno "$message" \
14 75
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
}
whiptail_strelka_rules() {
@@ -1346,40 +1446,6 @@ whiptail_suricata_pins() {
}
whiptail_manager_updates() {
[ -n "$TESTING" ] && return
local update_string
update_string=$(whiptail --title "Security Onion Setup" --radiolist \
"How would you like to download OS package updates for your grid?" 20 75 4 \
"MANAGER" "Manager node is proxy for updates" ON \
"OPEN" "Each node connects to the Internet for updates" OFF 3>&1 1>&2 2>&3 )
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
case "$update_string" in
'MANAGER')
export MANAGERUPDATES='1'
;;
*)
export MANAGERUPDATES='0'
;;
esac
}
whiptail_manager_updates_warning() {
[ -n "$TESTING" ] && return
whiptail --title "Security Onion Setup"\
--msgbox "Updating through the manager node requires the manager to have internet access, press ENTER to continue."\
8 75
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
}
whiptail_node_updates() {
[ -n "$TESTING" ] && return
@@ -1417,3 +1483,40 @@ whiptail_you_sure() {
return $exitstatus
}
whiptail_zeek_pins() {
[ -n "$TESTING" ] && return
local cpu_core_list_whiptail=()
for item in "${cpu_core_list[@]}"; do
cpu_core_list_whiptail+=("$item" "OFF")
done
if [[ $is_smooshed ]]; then
local PROCS=$(expr $lb_procs / 2)
if [ "$PROCS" -lt 1 ]; then PROCS=1; else PROCS=$PROCS; fi
else
local PROCS=$lb_procs
fi
ZEEKPINS=$(whiptail --noitem --title "Pin Zeek CPUS" --checklist "Please select $PROCS cores to pin Zeek to:" 20 75 12 "${cpu_core_list_whiptail[@]}" 3>&1 1>&2 2>&3 )
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
ZEEKPINS=$(echo "$ZEEKPINS" | tr -d '"')
IFS=' ' read -ra ZEEKPINS <<< "$ZEEKPINS"
}
whiptail_zeek_version() {
[ -n "$TESTING" ] && return
ZEEKVERSION=$(whiptail --title "Security Onion Setup" --radiolist "What tool would you like to use to generate metadata?" 20 75 4 "ZEEK" "Zeek (formerly known as Bro)" ON \
"SURICATA" "Suricata" OFF 3>&1 1>&2 2>&3)
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
}