From f07e583013f01686fab37e049e3698d64dc56d83 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 8 Jan 2021 16:33:38 -0500 Subject: [PATCH 1/4] increase salt logging to info --- files/salt/master/master | 2 ++ setup/so-functions | 2 ++ 2 files changed, 4 insertions(+) diff --git a/files/salt/master/master b/files/salt/master/master index 42e7866d9..93e8ff938 100644 --- a/files/salt/master/master +++ b/files/salt/master/master @@ -13,6 +13,8 @@ # user: socore log_file: /opt/so/log/salt/master +log_level_logfile: info +log_level: info ##### File Server settings ##### ########################################## diff --git a/setup/so-functions b/setup/so-functions index 315334a18..83b5aef3c 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -467,6 +467,8 @@ configure_minion() { printf '%s\n'\ "use_superseded:"\ " - module.run"\ + "log_level: info"\ + "log_level_logfile: info"\ "log_file: /opt/so/log/salt/minion" >> "$minion_config" { From 63047b4b853b82b3bd14288b0807cb59d93f7cc8 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sun, 10 Jan 2021 00:56:10 -0500 Subject: [PATCH 2/4] Add retry logic around salt key acceptance during setup --- salt/common/tools/sbin/so-common | 19 +++++++++++++++++++ setup/so-functions | 2 +- setup/so-setup | 4 ++-- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index c71e9150c..caa19cd37 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -141,6 +141,25 @@ get_random_value() { head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1 } +retry() { + maxAttempts=$1 + sleepDelay=$2 + cmd=$3 + attempt=0 + while [[ $attempt -lt $maxAttempts ]]; do + attempt=$((attempt+1)) + logCmd "$cmd" + exitcode=$? + if [[ $exitcode -eq 0 ]]; then + return $exitCode + fi + info "Command failed with exit code $exitcode; will retry in $sleepDelay seconds ($attempt / $maxAttempts)..." + sleep $sleepDelay + done + error "Command continues to fail; giving up." + return 1 +} + wait_for_apt() { local progress_callback=$1 diff --git a/setup/so-functions b/setup/so-functions index 83b5aef3c..b2b7b688c 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1972,7 +1972,7 @@ set_progress_str() { printf '%s\n' \ '----'\ - "$percentage% - ${progress_bar_text^^}"\ + info "$percentage% - ${progress_bar_text^^}"\ "----" >> "$setup_log" 2>&1 } diff --git a/setup/so-setup b/setup/so-setup index 15189746f..28f0bc0bb 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -637,12 +637,12 @@ set_redirect >> $setup_log 2>&1 if [[ $is_minion ]]; then set_progress_str 20 'Accepting Salt key on manager' - accept_salt_key_remote >> $setup_log 2>&1 + retry 20 10 accept_salt_key_remote fi if [[ $is_manager || $is_import || $is_helix ]]; then set_progress_str 20 'Accepting Salt key' - salt-key -ya "$MINION_ID" >> $setup_log 2>&1 + retry 20 10 "salt-key -ya $MINION_ID" fi set_progress_str 21 'Copying minion pillars to manager' From bc8e200919f31936fd6e875916d8caccefb91ff6 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sun, 10 Jan 2021 02:34:46 -0500 Subject: [PATCH 3/4] Continued retry implementation for salt-key acceptance; improve timestamp coverage in setup --- salt/common/tools/sbin/so-common | 13 +++++++++++-- setup/so-functions | 4 +++- setup/so-setup | 4 ++-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index caa19cd37..0c18c4482 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -145,12 +145,21 @@ retry() { maxAttempts=$1 sleepDelay=$2 cmd=$3 + expectedOutput=$4 attempt=0 while [[ $attempt -lt $maxAttempts ]]; do attempt=$((attempt+1)) - logCmd "$cmd" + info "Executing command with retry support: $cmd" + output=$($cmd) + info "Results: $output" exitcode=$? - if [[ $exitcode -eq 0 ]]; then + if [ -n "$expectedOutput" ]; then + if [[ "$output" =~ "$expectedOutput" ]]; then + return $exitCode + else + info "Expected '$expectedOutput' but got '$output'" + fi + elif [[ $exitcode -eq 0 ]]; then return $exitCode fi info "Command failed with exit code $exitcode; will retry in $sleepDelay seconds ($attempt / $maxAttempts)..." diff --git a/setup/so-functions b/setup/so-functions index b2b7b688c..e9de76233 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1970,9 +1970,11 @@ set_progress_str() { echo -e "$percentage_str" + info "Progressing ($percentage%): $progress_bar_text" + printf '%s\n' \ '----'\ - info "$percentage% - ${progress_bar_text^^}"\ + "$percentage% - ${progress_bar_text^^}"\ "----" >> "$setup_log" 2>&1 } diff --git a/setup/so-setup b/setup/so-setup index 28f0bc0bb..e62dc434f 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -637,12 +637,12 @@ set_redirect >> $setup_log 2>&1 if [[ $is_minion ]]; then set_progress_str 20 'Accepting Salt key on manager' - retry 20 10 accept_salt_key_remote + retry 20 10 accept_salt_key_remote "going to be accepted" fi if [[ $is_manager || $is_import || $is_helix ]]; then set_progress_str 20 'Accepting Salt key' - retry 20 10 "salt-key -ya $MINION_ID" + retry 20 10 "salt-key -ya $MINION_ID" "going to be accepted" fi set_progress_str 21 'Copying minion pillars to manager' From 8b49876e26d49f5fd9570ade5e7664447bd0eb04 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 11 Jan 2021 12:04:57 -0500 Subject: [PATCH 4/4] First pass at distribute ISO automation files --- setup/automation/distributed-iso-manager | 77 +++++++++++++++++++++++ setup/automation/distributed-iso-search | 78 ++++++++++++++++++++++++ setup/automation/distributed-iso-sensor | 78 ++++++++++++++++++++++++ 3 files changed, 233 insertions(+) create mode 100644 setup/automation/distributed-iso-manager create mode 100644 setup/automation/distributed-iso-search create mode 100644 setup/automation/distributed-iso-sensor diff --git a/setup/automation/distributed-iso-manager b/setup/automation/distributed-iso-manager new file mode 100644 index 000000000..cbf803dd2 --- /dev/null +++ b/setup/automation/distributed-iso-manager @@ -0,0 +1,77 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +TESTING=true + +address_type=DHCP +ADMINUSER=onionuser +ADMINPASS1=onionuser +ADMINPASS2=onionuser +ALLOW_CIDR=0.0.0.0/0 +ALLOW_ROLE=a +BASICZEEK=7 +BASICSURI=7 +# BLOGS= +#BNICS=eth1 +ZEEKVERSION=ZEEK +# CURCLOSEDAYS= +# EVALADVANCED=BASIC +GRAFANA=1 +# HELIXAPIKEY= +HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 +HNSENSOR=inherit +HOSTNAME=distributed-manager +install_type=MANAGER +# LSINPUTBATCHCOUNT= +# LSINPUTTHREADS= +# LSPIPELINEBATCH= +# LSPIPELINEWORKERS= +MANAGERADV=BASIC +MANAGERUPDATES=1 +# MDNS= +# MGATEWAY= +# MIP= +# MMASK= +MNIC=eth0 +# MSEARCH= +# MSRV= +# MTU= +NIDS=Suricata +# NODE_ES_HEAP_SIZE= +# NODE_LS_HEAP_SIZE= +NODESETUP=NODEBASIC +NSMSETUP=BASIC +NODEUPDATES=MANAGER +# OINKCODE= +OSQUERY=1 +# PATCHSCHEDULEDAYS= +# PATCHSCHEDULEHOURS= +PATCHSCHEDULENAME=auto +PLAYBOOK=1 +# REDIRECTHOST= +REDIRECTINFO=IP +RULESETUP=ETOPEN +# SHARDCOUNT= +# SKIP_REBOOT= +SOREMOTEPASS1=onionuser +SOREMOTEPASS2=onionuser +STRELKA=1 +THEHIVE=1 +WAZUH=1 +WEBUSER=onionuser@somewhere.invalid +WEBPASSWD1=0n10nus3r +WEBPASSWD2=0n10nus3r diff --git a/setup/automation/distributed-iso-search b/setup/automation/distributed-iso-search new file mode 100644 index 000000000..d37a7d935 --- /dev/null +++ b/setup/automation/distributed-iso-search @@ -0,0 +1,78 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +TESTING=true + +address_type=DHCP +ADMINUSER=onionuser +ADMINPASS1=onionuser +ADMINPASS2=onionuser +# ALLOW_CIDR=0.0.0.0/0 +# ALLOW_ROLE=a +# BASICZEEK=7 +# BASICSURI=7 +# BLOGS= +# BNICS=eth1 +# ZEEKVERSION=ZEEK +# CURCLOSEDAYS= +# EVALADVANCED=BASIC +# GRAFANA=1 +# HELIXAPIKEY= +HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 +HNSENSOR=inherit +HOSTNAME=distributed-search +install_type=SEARCHNODE +# LSINPUTBATCHCOUNT= +# LSINPUTTHREADS= +# LSPIPELINEBATCH= +# LSPIPELINEWORKERS= +# MANAGERADV=BASIC +MANAGERUPDATES=1 +# MDNS= +# MGATEWAY= +# MIP= +# MMASK= +MNIC=eth0 +# MSEARCH= +MSRV=distributed-manager +MSRVIP=10.66.166.42 +# MTU= +# NIDS=Suricata +# NODE_ES_HEAP_SIZE= +# NODE_LS_HEAP_SIZE= +NODESETUP=NODEBASIC +NSMSETUP=BASIC +NODEUPDATES=MANAGER +# OINKCODE= +# OSQUERY=1 +# PATCHSCHEDULEDAYS= +# PATCHSCHEDULEHOURS= +PATCHSCHEDULENAME=auto +# PLAYBOOK=1 +# REDIRECTHOST= +# REDIRECTINFO=IP +# RULESETUP=ETOPEN +# SHARDCOUNT= +# SKIP_REBOOT= +SOREMOTEPASS1=onionuser +SOREMOTEPASS2=onionuser +# STRELKA=1 +# THEHIVE=1 +WAZUH=1 +WEBUSER=onionuser@somewhere.invalid +WEBPASSWD1=0n10nus3r +WEBPASSWD2=0n10nus3r diff --git a/setup/automation/distributed-iso-sensor b/setup/automation/distributed-iso-sensor new file mode 100644 index 000000000..402049be9 --- /dev/null +++ b/setup/automation/distributed-iso-sensor @@ -0,0 +1,78 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +TESTING=true + +address_type=DHCP +ADMINUSER=onionuser +ADMINPASS1=onionuser +ADMINPASS2=onionuser +# ALLOW_CIDR=0.0.0.0/0 +# ALLOW_ROLE=a +BASICZEEK=7 +BASICSURI=7 +# BLOGS= +BNICS=eth1 +ZEEKVERSION=ZEEK +# CURCLOSEDAYS= +# EVALADVANCED=BASIC +# GRAFANA=1 +# HELIXAPIKEY= +HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 +HNSENSOR=inherit +HOSTNAME=distributed-sensor +install_type=SENSOR +# LSINPUTBATCHCOUNT= +# LSINPUTTHREADS= +# LSPIPELINEBATCH= +# LSPIPELINEWORKERS= +# MANAGERADV=BASIC +MANAGERUPDATES=1 +# MDNS= +# MGATEWAY= +# MIP= +# MMASK= +MNIC=eth0 +# MSEARCH= +MSRV=distributed-manager +MSRVIP=10.66.166.42 +# MTU= +# NIDS=Suricata +# NODE_ES_HEAP_SIZE= +# NODE_LS_HEAP_SIZE= +# NODESETUP=NODEBASIC +NSMSETUP=BASIC +NODEUPDATES=MANAGER +# OINKCODE= +# OSQUERY=1 +# PATCHSCHEDULEDAYS= +# PATCHSCHEDULEHOURS= +PATCHSCHEDULENAME=auto +# PLAYBOOK=1 +# REDIRECTHOST= +# REDIRECTINFO=IP +# RULESETUP=ETOPEN +# SHARDCOUNT= +# SKIP_REBOOT= +SOREMOTEPASS1=onionuser +SOREMOTEPASS2=onionuser +STRELKA=1 +# THEHIVE=1 +WAZUH=1 +WEBUSER=onionuser@somewhere.invalid +WEBPASSWD1=0n10nus3r +WEBPASSWD2=0n10nus3r