From 9cf150f988aef5ad820acf4a0bd8e1b7a5c1dd09 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 23 Dec 2020 15:11:43 -0500 Subject: [PATCH 01/39] Switch from Jinja syntax to bash --- salt/thehive/scripts/cortex_init | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/thehive/scripts/cortex_init b/salt/thehive/scripts/cortex_init index 0ec8785f2..e7f78400e 100644 --- a/salt/thehive/scripts/cortex_init +++ b/salt/thehive/scripts/cortex_init @@ -20,7 +20,7 @@ cortex_clean(){ cortex_init(){ CORTEX_URL="{{URLBASE}}/cortex" - CORTEX_API_URL="{{CORTEX_URL}}/api" + CORTEX_API_URL="$CORTEX_URL/api" CORTEX_USER="{{CORTEXUSER}}" CORTEX_PASSWORD="{{CORTEXPASSWORD}}" CORTEX_KEY="{{CORTEXKEY}}" From baf5be1a3a6cffc37d3a99d63c6cd75beb12cc45 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 23 Dec 2020 20:14:46 -0500 Subject: [PATCH 02/39] Return adequate exit code when init fails; Logs output of init scripts for troubleshooting failed installations --- salt/thehive/init.sls | 4 ++-- salt/thehive/scripts/cortex_init | 6 ++++-- salt/thehive/scripts/hive_init | 6 ++++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/salt/thehive/init.sls b/salt/thehive/init.sls index c89017dda..6c3d4d5ac 100644 --- a/salt/thehive/init.sls +++ b/salt/thehive/init.sls @@ -123,7 +123,7 @@ cortexscript: - source: salt://thehive/scripts/cortex_init - cwd: /opt/so - template: jinja - - hide_output: True + - hide_output: False so-thehive: docker_container.running: @@ -148,7 +148,7 @@ thehivescript: - source: salt://thehive/scripts/hive_init - cwd: /opt/so - template: jinja - - hide_output: True + - hide_output: False {% else %} diff --git a/salt/thehive/scripts/cortex_init b/salt/thehive/scripts/cortex_init index e7f78400e..a7f7f2727 100644 --- a/salt/thehive/scripts/cortex_init +++ b/salt/thehive/scripts/cortex_init @@ -58,6 +58,7 @@ cortex_init(){ touch /opt/so/state/cortex.txt else echo "We experienced an issue connecting to Cortex!" + exit 1 fi } @@ -65,10 +66,11 @@ if [ -f /opt/so/state/cortex.txt ]; then cortex_clean exit 0 else - if wait_for_web_response http://{{MANAGERIP}}:9400 '"status":"green"'; then + if wait_for_web_response http://{{MANAGERIP}}:9400/_cluster/health '"status":"green"'; then cortex_init cortex_clean else - echo "TheHive Elasticsearch server is not ready; unable to proceed with cortex init." + echo "TheHive Elasticsearch server is not ready; unable to proceed with Cortex init." + exit 1 fi fi diff --git a/salt/thehive/scripts/hive_init b/salt/thehive/scripts/hive_init index 70b770a1c..855d0d4f6 100755 --- a/salt/thehive/scripts/hive_init +++ b/salt/thehive/scripts/hive_init @@ -36,6 +36,7 @@ thehive_init(){ touch /opt/so/state/thehive.txt else echo "We experienced an issue connecting to TheHive!" + exit 1 fi } @@ -43,10 +44,11 @@ if [ -f /opt/so/state/thehive.txt ]; then thehive_clean exit 0 else - if wait_for_web_response http://{{MANAGERIP}}:9400 '"status":"green"'; then + if wait_for_web_response http://{{MANAGERIP}}:9400/_cluster/health '"status":"green"'; then thehive_init thehive_clean else - echo "TheHive Elasticsearch server is not ready; unable to proceed with hive init." + echo "TheHive Elasticsearch server is not ready; unable to proceed with TheHive init." + exit 1 fi fi From 8f15d794bc8d1fee2fefc73143a8ee478dc5e75f Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 24 Dec 2020 08:44:28 -0500 Subject: [PATCH 03/39] Silence curl progress output during hive/cortex init --- salt/thehive/scripts/cortex_init | 14 +++++++------- salt/thehive/scripts/hive_init | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/salt/thehive/scripts/cortex_init b/salt/thehive/scripts/cortex_init index a7f7f2727..084957885 100644 --- a/salt/thehive/scripts/cortex_init +++ b/salt/thehive/scripts/cortex_init @@ -32,25 +32,25 @@ cortex_init(){ if wait_for_web_response https://$CORTEX_URL "Cortex"; then # Migrate DB - curl -v -k -XPOST -L "https://$CORTEX_API_URL/maintenance/migrate" + curl -sk -XPOST -L "https://$CORTEX_API_URL/maintenance/migrate" # Create intial Cortex superadmin - curl -v -k -L "https://$CORTEX_API_URL/user" -H "Content-Type: application/json" -d "{\"login\" : \"$CORTEX_USER\",\"name\" : \"$CORTEX_USER\",\"roles\" : [\"superadmin\"],\"preferences\" : \"{}\",\"password\" : \"$CORTEX_PASSWORD\", \"key\": \"$CORTEX_KEY\"}" + curl -sk -L "https://$CORTEX_API_URL/user" -H "Content-Type: application/json" -d "{\"login\" : \"$CORTEX_USER\",\"name\" : \"$CORTEX_USER\",\"roles\" : [\"superadmin\"],\"preferences\" : \"{}\",\"password\" : \"$CORTEX_PASSWORD\", \"key\": \"$CORTEX_KEY\"}" # Create user-supplied org - curl -k -XPOST -H "Authorization: Bearer $CORTEX_KEY" -H "Content-Type: application/json" -L "https://$CORTEX_API_URL/organization" -d "{ \"name\": \"$CORTEX_ORG_NAME\",\"description\": \"$CORTEX_ORG_DESC\",\"status\": \"Active\"}" + curl -sk -XPOST -H "Authorization: Bearer $CORTEX_KEY" -H "Content-Type: application/json" -L "https://$CORTEX_API_URL/organization" -d "{ \"name\": \"$CORTEX_ORG_NAME\",\"description\": \"$CORTEX_ORG_DESC\",\"status\": \"Active\"}" # Create user-supplied org user - curl -k -XPOST -H "Authorization: Bearer $CORTEX_KEY" -H "Content-Type: application/json" -L "https://$CORTEX_API_URL/user" -d "{\"name\": \"$CORTEX_ORG_USER\",\"roles\": [\"read\",\"analyze\",\"orgadmin\"],\"organization\": \"$CORTEX_ORG_NAME\",\"login\": \"$CORTEX_ORG_USER\",\"key\": \"$CORTEX_ORG_USER_KEY\" }" + curl -sk -XPOST -H "Authorization: Bearer $CORTEX_KEY" -H "Content-Type: application/json" -L "https://$CORTEX_API_URL/user" -d "{\"name\": \"$CORTEX_ORG_USER\",\"roles\": [\"read\",\"analyze\",\"orgadmin\"],\"organization\": \"$CORTEX_ORG_NAME\",\"login\": \"$CORTEX_ORG_USER\",\"key\": \"$CORTEX_ORG_USER_KEY\" }" # Enable URLScan.io Analyzer - curl -v -k -XPOST -H "Authorization: Bearer $CORTEX_ORG_USER_KEY" -H "Content-Type: application/json" -L "https://$CORTEX_API_URL/organization/analyzer/Urlscan_io_Search_0_1_0" -d '{"name":"Urlscan_io_Search_0_1_0","configuration":{"auto_extract_artifacts":false,"check_tlp":true,"max_tlp":2}}' + curl -sv -k -XPOST -H "Authorization: Bearer $CORTEX_ORG_USER_KEY" -H "Content-Type: application/json" -L "https://$CORTEX_API_URL/organization/analyzer/Urlscan_io_Search_0_1_0" -d '{"name":"Urlscan_io_Search_0_1_0","configuration":{"auto_extract_artifacts":false,"check_tlp":true,"max_tlp":2}}' # Enable Cert PassiveDNS Analyzer - curl -v -k -XPOST -H "Authorization: Bearer $CORTEX_ORG_USER_KEY" -H "Content-Type: application/json" -L "https://$CORTEX_API_URL/organization/analyzer/CERTatPassiveDNS_2_0" -d '{"name":"CERTatPassiveDNS_2_0","configuration":{"auto_extract_artifacts":false,"check_tlp":true,"max_tlp":2, "limit": 100}}' + curl -sv -k -XPOST -H "Authorization: Bearer $CORTEX_ORG_USER_KEY" -H "Content-Type: application/json" -L "https://$CORTEX_API_URL/organization/analyzer/CERTatPassiveDNS_2_0" -d '{"name":"CERTatPassiveDNS_2_0","configuration":{"auto_extract_artifacts":false,"check_tlp":true,"max_tlp":2, "limit": 100}}' # Revoke $CORTEX_USER key - curl -k -XDELETE -H "Authorization: Bearer $CORTEX_KEY" -L "https://$CORTEX_API_URL/user/$CORTEX_USER/key" + curl -sk -XDELETE -H "Authorization: Bearer $CORTEX_KEY" -L "https://$CORTEX_API_URL/user/$CORTEX_USER/key" # Update SOCtopus config with apikey value #sed -i "s/cortex_key = .*/cortex_key = $CORTEX_KEY/" $SOCTOPUS_CONFIG diff --git a/salt/thehive/scripts/hive_init b/salt/thehive/scripts/hive_init index 855d0d4f6..f0f1a5f52 100755 --- a/salt/thehive/scripts/hive_init +++ b/salt/thehive/scripts/hive_init @@ -23,15 +23,15 @@ thehive_init(){ echo -n "Waiting for TheHive..." if wait_for_web_response https://$THEHIVE_URL "TheHive"; then # Migrate DB - curl -v -k -XPOST -L "https://$THEHIVE_API_URL/maintenance/migrate" + curl -sk -XPOST -L "https://$THEHIVE_API_URL/maintenance/migrate" # Create intial TheHive user - curl -v -k -L "https://$THEHIVE_API_URL/user" -H "Content-Type: application/json" -d "{\"login\" : \"$THEHIVE_USER\",\"name\" : \"$THEHIVE_USER\",\"roles\" : [\"read\",\"alert\",\"write\",\"admin\"],\"preferences\" : \"{}\",\"password\" : \"$THEHIVE_PASSWORD\", \"key\": \"$THEHIVE_KEY\"}" + curl -sk -L "https://$THEHIVE_API_URL/user" -H "Content-Type: application/json" -d "{\"login\" : \"$THEHIVE_USER\",\"name\" : \"$THEHIVE_USER\",\"roles\" : [\"read\",\"alert\",\"write\",\"admin\"],\"preferences\" : \"{}\",\"password\" : \"$THEHIVE_PASSWORD\", \"key\": \"$THEHIVE_KEY\"}" # Pre-load custom fields # # reputation - curl -v -k -L "https://$THEHIVE_API_URL/list/custom_fields" -H "Authorization: Bearer $THEHIVE_KEY" -H "Content-Type: application/json" -d "{\"value\":{\"name\": \"reputation\", \"reference\": \"reputation\", \"description\": \"This field provides an overall reputation status for an address/domain.\", \"type\": \"string\", \"options\": []}}" + curl -sk -L "https://$THEHIVE_API_URL/list/custom_fields" -H "Authorization: Bearer $THEHIVE_KEY" -H "Content-Type: application/json" -d "{\"value\":{\"name\": \"reputation\", \"reference\": \"reputation\", \"description\": \"This field provides an overall reputation status for an address/domain.\", \"type\": \"string\", \"options\": []}}" touch /opt/so/state/thehive.txt else From 4d1cb374684e6ec0fe8641bc189c4e1289970b82 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 28 Dec 2020 09:35:51 -0500 Subject: [PATCH 04/39] [feat] Add function to wait for dpkg lock --- salt/common/tools/sbin/so-common | 235 +++++++++++++++++++------------ setup/so-functions | 148 +++++++++++-------- 2 files changed, 233 insertions(+), 150 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 86763dda8..29e065ac0 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -17,8 +17,8 @@ # Check for prerequisites if [ "$(id -u)" -ne 0 ]; then - echo "This script must be run using sudo!" - exit 1 + echo "This script must be run using sudo!" + exit 1 fi # Define a banner to separate sections @@ -26,46 +26,46 @@ banner="======================================================================== header() { echo - printf '%s\n' "$banner" "$*" "$banner" + printf '%s\n' "$banner" "$*" "$banner" } lookup_salt_value() { - key=$1 - group=$2 - kind=$3 + key=$1 + group=$2 + kind=$3 - if [ -z "$kind" ]; then - kind=pillar - fi + if [ -z "$kind" ]; then + kind=pillar + fi - if [ -n "$group" ]; then - group=${group}: - fi + if [ -n "$group" ]; then + group=${group}: + fi - salt-call --no-color ${kind}.get ${group}${key} --out=newline_values_only + salt-call --no-color ${kind}.get ${group}${key} --out=newline_values_only } lookup_pillar() { - key=$1 - pillar=$2 - if [ -z "$pillar" ]; then - pillar=global - fi - lookup_salt_value "$key" "$pillar" "pillar" + key=$1 + pillar=$2 + if [ -z "$pillar" ]; then + pillar=global + fi + lookup_salt_value "$key" "$pillar" "pillar" } lookup_pillar_secret() { - lookup_pillar "$1" "secrets" + lookup_pillar "$1" "secrets" } lookup_grain() { - lookup_salt_value "$1" "" "grains" + lookup_salt_value "$1" "" "grains" } lookup_role() { - id=$(lookup_grain id) - pieces=($(echo $id | tr '_' ' ')) - echo ${pieces[1]} + id=$(lookup_grain id) + pieces=($(echo $id | tr '_' ' ')) + echo ${pieces[1]} } check_container() { @@ -74,98 +74,147 @@ check_container() { } check_password() { - local password=$1 - echo "$password" | egrep -v "'|\"|\\$|\\\\" > /dev/null 2>&1 - return $? + local password=$1 + echo "$password" | egrep -v "'|\"|\\$|\\\\" > /dev/null 2>&1 + return $? } set_os() { - if [ -f /etc/redhat-release ]; then - OS=centos - else - OS=ubuntu - fi + if [ -f /etc/redhat-release ]; then + OS=centos + else + OS=ubuntu + fi } set_minionid() { - MINIONID=$(lookup_grain id) + MINIONID=$(lookup_grain id) } set_version() { - CURRENTVERSION=0.0.0 - if [ -f /etc/soversion ]; then - CURRENTVERSION=$(cat /etc/soversion) - fi - if [ -z "$VERSION" ]; then - if [ -z "$NEWVERSION" ]; then - if [ "$CURRENTVERSION" == "0.0.0" ]; then - echo "ERROR: Unable to detect Security Onion version; terminating script." - exit 1 - else - VERSION=$CURRENTVERSION - fi - else - VERSION="$NEWVERSION" - fi - fi + CURRENTVERSION=0.0.0 + if [ -f /etc/soversion ]; then + CURRENTVERSION=$(cat /etc/soversion) + fi + if [ -z "$VERSION" ]; then + if [ -z "$NEWVERSION" ]; then + if [ "$CURRENTVERSION" == "0.0.0" ]; then + echo "ERROR: Unable to detect Security Onion version; terminating script." + exit 1 + else + VERSION=$CURRENTVERSION + fi + else + VERSION="$NEWVERSION" + fi + fi } require_manager() { - # Check to see if this is a manager - MANAGERCHECK=$(cat /etc/salt/grains | grep role | awk '{print $2}') - if [ $MANAGERCHECK == 'so-eval' ] || [ $MANAGERCHECK == 'so-manager' ] || [ $MANAGERCHECK == 'so-managersearch' ] || [ $MANAGERCHECK == 'so-standalone' ] || [ $MANAGERCHECK == 'so-helix' ] || [ $MANAGERCHECK == 'so-import' ]; then - echo "This is a manager, We can proceed." - else - echo "Please run this command on the manager; the manager controls the grid." - exit 1 - fi + # Check to see if this is a manager + MANAGERCHECK=$(cat /etc/salt/grains | grep role | awk '{print $2}') + if [ $MANAGERCHECK == 'so-eval' ] || [ $MANAGERCHECK == 'so-manager' ] || [ $MANAGERCHECK == 'so-managersearch' ] || [ $MANAGERCHECK == 'so-standalone' ] || [ $MANAGERCHECK == 'so-helix' ] || [ $MANAGERCHECK == 'so-import' ]; then + echo "This is a manager, We can proceed." + else + echo "Please run this command on the manager; the manager controls the grid." + exit 1 + fi } is_single_node_grid() { - role=$(lookup_role) - if [ "$role" != "eval" ] && [ "$role" != "standalone" ] && [ "$role" != "import" ]; then - return 1 - fi - return 0 + role=$(lookup_role) + if [ "$role" != "eval" ] && [ "$role" != "standalone" ] && [ "$role" != "import" ]; then + return 1 + fi + return 0 } fail() { - msg=$1 - echo "ERROR: $msg" - echo "Exiting." - exit 1 + msg=$1 + echo "ERROR: $msg" + echo "Exiting." + exit 1 } get_random_value() { - length=${1:-20} - head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1 + length=${1:-20} + head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1 +} + +wait_for_apt() { + local progress_callback=$1 + + local retry_count=2 + local retry_timeout='30s' + local lock_msg='Could not acquire dpkg lock, is another process using it?' + local retry_msg="Waiting $retry_timeout for dpkg lock to release." + if [ -z "$progress_callback" ]; then + if [ -z "$progress_bar_text" ]; then + local old_text="Installing..." + else + local old_text="$progress_bar_text" + fi + fi + + while [[ "$retry_count" -lt "$retry_count" ]]; do + if __check_apt_lock; then + if [ -z "$progress_callback" ]; then + echo "$lock_msg" + echo "$retry_msg" + else + $progress_callback "$retry_msg" + fi + else + $progress_callback "$old_text" + return 0 + fi + ((retry_count++)) + sleep "$retry_timeout" + done + + if __check_apt_lock; then + if [[ -z $progress_callback ]]; then + echo "$lock_msg" + echo "Could not acquire lock after $retry_count attempts, aborting." + fi + return 1 + else + return 0 + fi +} + +__check_apt_lock() { + lsof /var/lib/dpkg/lock &> /dev/null + local lock=$? + + return $lock } wait_for_web_response() { - url=$1 - expected=$2 - maxAttempts=${3:-300} - logfile=/root/wait_for_web_response.log - attempt=0 - while [[ $attempt -lt $maxAttempts ]]; do - attempt=$((attempt+1)) - echo "Waiting for value '$expected' at '$url' ($attempt/$maxAttempts)" - result=$(curl -ks -L $url) - exitcode=$? + url=$1 + expected=$2 + maxAttempts=${3:-300} + logfile=/root/wait_for_web_response.log + attempt=0 + while [[ $attempt -lt $maxAttempts ]]; do + attempt=$((attempt+1)) + echo "Waiting for value '$expected' at '$url' ($attempt/$maxAttempts)" + result=$(curl -ks -L $url) + exitcode=$? - echo "--------------------------------------------------" >> $logfile - echo "$(date) - Checking web URL: $url ($attempt/$maxAttempts)" >> $logfile - echo "$result" >> $logfile - echo "exit code=$exitcode" >> $logfile - echo "" >> $logfile + echo "--------------------------------------------------" >> $logfile + echo "$(date) - Checking web URL: $url ($attempt/$maxAttempts)" >> $logfile + echo "$result" >> $logfile + echo "exit code=$exitcode" >> $logfile + echo "" >> $logfile - if [[ $exitcode -eq 0 && "$result" =~ $expected ]]; then - echo "Received expected response; proceeding." - return 0 - fi - echo "Server is not ready" - sleep 1 - done - echo "Server still not ready after $maxAttempts attempts; giving up." - return 1 -} \ No newline at end of file + if [[ $exitcode -eq 0 && "$result" =~ $expected ]]; then + echo "Received expected response; proceeding." + return 0 + fi + echo "Server is not ready" + sleep 1 + done + echo "Server still not ready after $maxAttempts attempts; giving up." + return 1 +} diff --git a/setup/so-functions b/setup/so-functions index e723a89c8..a8a7517a5 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -84,7 +84,7 @@ calculate_useable_cores() { lb_procs_round=$(printf "%.0f\n" $cores_for_zeek) if [ "$lb_procs_round" -lt 1 ]; then lb_procs=1; else lb_procs=$lb_procs_round; fi - export lb_procs + export lb_procs } airgap_rules() { @@ -799,13 +799,13 @@ detect_os() { echo "Installing required packages to run installer..." # Install network manager so we can do interface stuff if ! command -v nmcli > /dev/null 2>&1; then + if wait_for_apt; then apt-get install -y network-manager >> "$setup_log" 2<&1; else exit 1; fi { - apt-get install -y network-manager; - systemctl enable NetworkManager; - systemctl start NetworkManager; + systemctl enable NetworkManager + systemctl start NetworkManager } >> "$setup_log" 2<&1 fi - apt-get install -y bc curl >> "$setup_log" 2>&1 + if wait_for_apt; then apt-get install -y bc curl >> "$setup_log" 2>&1; else exit 1; fi else echo "We were unable to determine if you are using a supported OS." @@ -882,21 +882,28 @@ docker_install() { else case "$install_type" in 'MANAGER' | 'EVAL' | 'STANDALONE' | 'MANAGERSEARCH' | 'IMPORT') - apt-get update >> "$setup_log" 2>&1 + if wait_for_apt 'whiptail_prog_new_message'; then apt-get update >> "$setup_log" 2>&1; else kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1; fi ;; *) - { - apt-key add "$temp_install_dir"/gpg/docker.pub; - add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"; - apt-get update; - } >> "$setup_log" 2>&1 + if wait_for_apt 'whiptail_prog_new_message'; then + { + apt-key add "$temp_install_dir"/gpg/docker.pub; + add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"; + apt-get update; + } >> "$setup_log" 2>&1 + else + kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 + fi ;; esac - - if [ $OSVER != "xenial" ]; then - apt-get -y install docker-ce python3-docker >> "$setup_log" 2>&1 + if wait_for_apt 'whiptail_prog_new_message'; then + if [ $OSVER != "xenial" ]; then + apt-get -y install docker-ce python3-docker >> "$setup_log" 2>&1 + else + apt-get -y install docker-ce python-docker >> "$setup_log" 2>&1 + fi else - apt-get -y install docker-ce python-docker >> "$setup_log" 2>&1 + kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 fi fi docker_registry @@ -939,7 +946,7 @@ docker_seed_update() { if [ "$install_type" == 'HELIXSENSOR' ]; then percent_delta=6 fi - ((docker_seed_update_percent=docker_seed_update_percent+percent_delta)) + ((docker_seed_update_percent+=percent_delta)) set_progress_str "$docker_seed_update_percent" "Downloading $name" } @@ -1572,7 +1579,11 @@ remove_package() { fi else if dpkg -l | grep -q "$package_name"; then - apt purge -y "$package_name" + if wait_for_apt 'whiptail_prog_new_message'; then + apt purge -y "$package_name" + else + exit 1 + fi fi fi } @@ -1644,36 +1655,43 @@ saltify() { yum -y install epel-release yum -y install salt-minion-3002.2\ python3\ - python36-docker\ - python36-dateutil\ - python36-m2crypto\ - python36-mysql\ - yum-utils\ - device-mapper-persistent-data\ - lvm2\ - openssl\ - jq; - yum -y update --exclude=salt*; + python36-docker\ + python36-dateutil\ + python36-m2crypto\ + python36-mysql\ + yum-utils\ + device-mapper-persistent-data\ + lvm2\ + openssl\ + jq; + yum -y update --exclude=salt*; fi systemctl enable salt-minion; } >> "$setup_log" 2>&1 yum versionlock salt* else - DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" upgrade >> "$setup_log" 2>&1 + if wait_for_apt 'whiptail_prog_new_message'; then + DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" upgrade >> "$setup_log" 2>&1 + else + kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 + fi if [ $OSVER != "xenial" ]; then # Switch to Python 3 as default if this is not xenial update-alternatives --install /usr/bin/python python /usr/bin/python3.6 10 >> "$setup_log" 2>&1 fi - # Add the pre-requisites for installing docker-ce - apt-get -y install ca-certificates\ - curl\ - software-properties-common\ - apt-transport-https\ - openssl\ - netcat\ - jq >> "$setup_log" 2>&1 - + if wait_for_apt 'whiptail_prog_new_message'; then + # Add the pre-requisites for installing docker-ce + apt-get -y install ca-certificates\ + curl\ + software-properties-common\ + apt-transport-https\ + openssl\ + netcat\ + jq >> "$setup_log" 2>&1 + else + kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 + fi # Grab the version from the os-release file local ubuntu_version ubuntu_version=$(grep VERSION_ID /etc/os-release | awk -F '[ "]' '{print $2}') @@ -1681,7 +1699,11 @@ saltify() { case "$install_type" in 'FLEET') - if [ "$OSVER" != 'xenial' ]; then apt-get -y install python3-mysqldb >> "$setup_log" 2>&1; else apt-get -y install python-mysqldb >> "$setup_log" 2>&1; fi + if wait_for_apt 'whiptail_prog_new_message'; then + if [ "$OSVER" != 'xenial' ]; then apt-get -y install python3-mysqldb >> "$setup_log" 2>&1; else apt-get -y install python-mysqldb >> "$setup_log" 2>&1; fi + else + kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 + fi ;; 'MANAGER' | 'EVAL' | 'MANAGERSEARCH' | 'STANDALONE' | 'IMPORT' | 'HELIXSENSOR') @@ -1703,13 +1725,18 @@ saltify() { curl -s https://packages.wazuh.com/key/GPG-KEY-WAZUH | apt-key add - >> "$setup_log" 2>&1 # Add repo echo "deb https://packages.wazuh.com/3.x/apt/ stable main" > /etc/apt/sources.list.d/wazuh.list 2>> "$setup_log" - # Initialize the new repos - apt-get update >> "$setup_log" 2>&1 - set_progress_str 6 'Installing various dependencies' - apt-get -y install sqlite3 argon2 libssl-dev >> "$setup_log" 2>&1 - set_progress_str 7 'Installing salt-master' - apt-get -y install salt-master=3002.2+ds-1 >> "$setup_log" 2>&1 - apt-mark hold salt-master >> "$setup_log" 2>&1 + + if wait_for_apt 'whiptail_prog_new_message'; then + # Initialize the new repos + apt-get update >> "$setup_log" 2>&1 + set_progress_str 6 'Installing various dependencies' + apt-get -y install sqlite3 argon2 libssl-dev >> "$setup_log" 2>&1 + set_progress_str 7 'Installing salt-master' + apt-get -y install salt-master=3002.2+ds-1 >> "$setup_log" 2>&1 + apt-mark hold salt-master >> "$setup_log" 2>&1 + else + kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 + fi ;; *) # Copy down the gpg keys and install them from the manager @@ -1723,18 +1750,21 @@ saltify() { echo "deb https://packages.wazuh.com/3.x/apt/ stable main" > /etc/apt/sources.list.d/wazuh.list 2>> "$setup_log" ;; esac - apt-get update >> "$setup_log" 2>&1 - set_progress_str 8 'Installing salt-minion & python modules' - apt-get -y install salt-minion=3002.2+ds-1\ - salt-common=3002.2+ds-1 >> "$setup_log" 2>&1 - apt-mark hold salt-minion salt-common >> "$setup_log" 2>&1 - if [ "$OSVER" != 'xenial' ]; then - apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb >> "$setup_log" 2>&1 - else - apt-get -y install python-pip python-dateutil python-m2crypto python-mysqldb >> "$setup_log" 2>&1 + if wait_for_apt 'whiptail_prog_new_message'; then + apt-get update >> "$setup_log" 2>&1 + set_progress_str 8 'Installing salt-minion & python modules' + apt-get -y install salt-minion=3002.2+ds-1\ + salt-common=3002.2+ds-1 >> "$setup_log" 2>&1 + apt-mark hold salt-minion salt-common >> "$setup_log" 2>&1 + if [ "$OSVER" != 'xenial' ]; then + apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb >> "$setup_log" 2>&1 + else + apt-get -y install python-pip python-dateutil python-m2crypto python-mysqldb >> "$setup_log" 2>&1 + fi + else + kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 fi fi - } salt_checkin() { @@ -2154,8 +2184,12 @@ update_packages() { if [ "$OS" = 'centos' ]; then yum -y update >> "$setup_log" else - apt-get -y update >> "$setup_log" - apt-get -y upgrade >> "$setup_log" + if wait_for_apt 'whiptail_prog_new_message'; then + apt-get -y update >> "$setup_log" + apt-get -y upgrade >> "$setup_log" + else + kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1 + fi fi } From 7fe0182ede5df2378cc07eada6b93d4c956ae790 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 28 Dec 2020 11:26:56 -0500 Subject: [PATCH 05/39] Refactor so-test and so-tcpreplay to be compatible with SO 2.3.20+; Change hive_init and cortex_init to initialize the cortex and fleet services directly on the manager IP instead of attempting to use the public URL --- salt/common/tools/sbin/so-tcpreplay | 46 +++++++++++++++++++++++------ salt/common/tools/sbin/so-test | 29 +----------------- salt/tcpreplay/init.sls | 15 ++-------- salt/thehive/scripts/cortex_init | 19 ++++++------ salt/thehive/scripts/hive_init | 11 ++++--- 5 files changed, 55 insertions(+), 65 deletions(-) diff --git a/salt/common/tools/sbin/so-tcpreplay b/salt/common/tools/sbin/so-tcpreplay index 99ed5665c..4cd473c0c 100755 --- a/salt/common/tools/sbin/so-tcpreplay +++ b/salt/common/tools/sbin/so-tcpreplay @@ -17,14 +17,42 @@ # Usage: so-tcpreplay "/opt/samples/*" -REPLAY_ENABLED=$(docker images | grep so-tcpreplay) -REPLAY_RUNNING=$(docker ps | grep so-tcpreplay) +. /usr/sbin/so-common +. /usr/sbin/so-image-common -if [ "$REPLAY_ENABLED" != "" ] && [ "$REPLAY_RUNNING" != "" ]; then - docker cp so-tcpreplay:/opt/samples /opt/samples - docker exec -it so-tcpreplay /usr/local/bin/tcpreplay -i bond0 -M10 $1 -else - echo "Replay functionality not enabled! To enable, run `so-tcpreplay-start`" - echo - echo "Note that you will need internet access to download the appropriate components" +REPLAYIFACE=${REPLAYIFACE:-bond0} +REPLAYSPEED=${REPLAYSPEED:-10} + +if [[ $# -lt 1 ]]; then + echo "Replays one or more PCAP sample files to the Security Onion monitoring interface." + echo + echo "Usage: $0 " + echo + echo "All PCAPs must be placed in the /opt/so/samples directory unless replaying" + echo "a sample pcap that is included in the so-tcpreplay image. Those PCAP sampes" + echo "are located in the /opt/samples directory inside of the image." + echo + echo "Customer provided PCAP example:" + echo " $0 /opt/so/samples/some_event.pcap" + echo + echo "Security Onion-provided PCAP example:" + echo " $0 /opt/samples/4in6.pcap" + + exit 1 fi + +if ! docker ps | grep -q so-tcpreplay; then + echo "Replay functionality not enabled; attempting to enable now (may require Internet access)..." + echo + + TRUSTED_CONTAINERS=("so-tcpreplay") + update_docker_containers "tcpreplay" + so-tcpreplay-start || fail "Unable to initialize tcpreplay" + mkdir -p /opt/so/samples + docker cp so-tcpreplay:/opt/samples/* /opt/so/samples +fi + +echo "Replaying PCAP(s) at ${REPLAYSPEED} Mbps on interface ${REPLAYIFACE}..." +docker exec -it so-tcpreplay /usr/bin/bash -c "/usr/local/bin/tcpreplay -i ${REPLAYIFACE} -M${REPLAYSPEED} $@" + +echo "Replay completed. Warnings shown above are typically expected." diff --git a/salt/common/tools/sbin/so-test b/salt/common/tools/sbin/so-test index 4270c9e76..8e981aecf 100755 --- a/salt/common/tools/sbin/so-test +++ b/salt/common/tools/sbin/so-test @@ -15,31 +15,4 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -# Usage: so-test - -. /usr/sbin/so-common - -REPLAY_ENABLED=$(docker images | grep so-tcpreplay) -REPLAY_RUNNING=$(docker ps | grep so-tcpreplay) - -if [ "$REPLAY_ENABLED" != "" ] && [ "$REPLAY_RUNNING" != "" ]; then - echo - echo "Preparing to replay PCAPs..." - docker cp so-tcpreplay:/opt/samples /opt/samples - docker exec -it so-tcpreplay /usr/local/bin/tcpreplay -i bond0 -M10 /opt/samples/* - echo - echo "PCAP's have been replayed - it is normal to see some warnings." - echo -else - echo "Replay functionality not enabled! Enabling Now...." - echo - echo "Note that you will need internet access to download the appropriate components" - /usr/sbin/so-start tcpreplay - echo "Replay functionality enabled. Replaying PCAPs Now...." - docker cp so-tcpreplay:/opt/samples /opt/samples - docker exec -it so-tcpreplay /usr/local/bin/tcpreplay -i bond0 -M10 /opt/samples/* - echo - echo "PCAP's have been replayed - it is normal to see some warnings." - echo -fi - +so-tcpreplay /opt/samples/* diff --git a/salt/tcpreplay/init.sls b/salt/tcpreplay/init.sls index d9050ffbb..a684d2fef 100644 --- a/salt/tcpreplay/init.sls +++ b/salt/tcpreplay/init.sls @@ -3,18 +3,6 @@ {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} -so-tcpreplayimage: - cmd.run: - - name: docker pull --disable-content-trust=false docker.io/{{ IMAGEREPO }}/so-tcpreplay:{{ VERSION }} - -so-tcpreplaytag: - cmd.run: - - name: docker tag {{ IMAGEREPO }}/so-tcpreplay:{{ VERSION }} {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-tcpreplay:{{ VERSION }} - -so-tcpreplaypush: - cmd.run: - - name: docker push {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-tcpreplay:{{ VERSION }} - so-tcpreplay: docker_container.running: - network_mode: "host" @@ -23,6 +11,9 @@ so-tcpreplay: - user: root - interactive: True - tty: True + - binds: + - /opt/so/samples:/opt/so/samples:ro + {% else %} diff --git a/salt/thehive/scripts/cortex_init b/salt/thehive/scripts/cortex_init index 084957885..64a228f16 100644 --- a/salt/thehive/scripts/cortex_init +++ b/salt/thehive/scripts/cortex_init @@ -1,6 +1,5 @@ #!/bin/bash # {%- set MANAGERIP = salt['pillar.get']('global:managerip', '') %} -# {%- set URLBASE = salt['pillar.get']('global:url_base', '') %} # {%- set CORTEXUSER = salt['pillar.get']('global:cortexuser', 'cortexadmin') %} # {%- set CORTEXPASSWORD = salt['pillar.get']('global:cortexpassword', 'cortexchangeme') %} # {%- set CORTEXKEY = salt['pillar.get']('global:cortexkey', '') %} @@ -19,7 +18,7 @@ cortex_clean(){ } cortex_init(){ - CORTEX_URL="{{URLBASE}}/cortex" + CORTEX_URL="http://{{MANAGERIP}}:9001" CORTEX_API_URL="$CORTEX_URL/api" CORTEX_USER="{{CORTEXUSER}}" CORTEX_PASSWORD="{{CORTEXPASSWORD}}" @@ -30,27 +29,27 @@ cortex_init(){ CORTEX_ORG_USER_KEY="{{CORTEXORGUSERKEY}}" SOCTOPUS_CONFIG="$default_salt_dir/salt/soctopus/files/SOCtopus.conf" - if wait_for_web_response https://$CORTEX_URL "Cortex"; then + if wait_for_web_response $CORTEX_URL "Cortex"; then # Migrate DB - curl -sk -XPOST -L "https://$CORTEX_API_URL/maintenance/migrate" + curl -sk -XPOST -L "$CORTEX_API_URL/maintenance/migrate" # Create intial Cortex superadmin - curl -sk -L "https://$CORTEX_API_URL/user" -H "Content-Type: application/json" -d "{\"login\" : \"$CORTEX_USER\",\"name\" : \"$CORTEX_USER\",\"roles\" : [\"superadmin\"],\"preferences\" : \"{}\",\"password\" : \"$CORTEX_PASSWORD\", \"key\": \"$CORTEX_KEY\"}" + curl -sk -L "$CORTEX_API_URL/user" -H "Content-Type: application/json" -d "{\"login\" : \"$CORTEX_USER\",\"name\" : \"$CORTEX_USER\",\"roles\" : [\"superadmin\"],\"preferences\" : \"{}\",\"password\" : \"$CORTEX_PASSWORD\", \"key\": \"$CORTEX_KEY\"}" # Create user-supplied org - curl -sk -XPOST -H "Authorization: Bearer $CORTEX_KEY" -H "Content-Type: application/json" -L "https://$CORTEX_API_URL/organization" -d "{ \"name\": \"$CORTEX_ORG_NAME\",\"description\": \"$CORTEX_ORG_DESC\",\"status\": \"Active\"}" + curl -sk -XPOST -H "Authorization: Bearer $CORTEX_KEY" -H "Content-Type: application/json" -L "$CORTEX_API_URL/organization" -d "{ \"name\": \"$CORTEX_ORG_NAME\",\"description\": \"$CORTEX_ORG_DESC\",\"status\": \"Active\"}" # Create user-supplied org user - curl -sk -XPOST -H "Authorization: Bearer $CORTEX_KEY" -H "Content-Type: application/json" -L "https://$CORTEX_API_URL/user" -d "{\"name\": \"$CORTEX_ORG_USER\",\"roles\": [\"read\",\"analyze\",\"orgadmin\"],\"organization\": \"$CORTEX_ORG_NAME\",\"login\": \"$CORTEX_ORG_USER\",\"key\": \"$CORTEX_ORG_USER_KEY\" }" + curl -sk -XPOST -H "Authorization: Bearer $CORTEX_KEY" -H "Content-Type: application/json" -L "$CORTEX_API_URL/user" -d "{\"name\": \"$CORTEX_ORG_USER\",\"roles\": [\"read\",\"analyze\",\"orgadmin\"],\"organization\": \"$CORTEX_ORG_NAME\",\"login\": \"$CORTEX_ORG_USER\",\"key\": \"$CORTEX_ORG_USER_KEY\" }" # Enable URLScan.io Analyzer - curl -sv -k -XPOST -H "Authorization: Bearer $CORTEX_ORG_USER_KEY" -H "Content-Type: application/json" -L "https://$CORTEX_API_URL/organization/analyzer/Urlscan_io_Search_0_1_0" -d '{"name":"Urlscan_io_Search_0_1_0","configuration":{"auto_extract_artifacts":false,"check_tlp":true,"max_tlp":2}}' + curl -sv -k -XPOST -H "Authorization: Bearer $CORTEX_ORG_USER_KEY" -H "Content-Type: application/json" -L "$CORTEX_API_URL/organization/analyzer/Urlscan_io_Search_0_1_0" -d '{"name":"Urlscan_io_Search_0_1_0","configuration":{"auto_extract_artifacts":false,"check_tlp":true,"max_tlp":2}}' # Enable Cert PassiveDNS Analyzer - curl -sv -k -XPOST -H "Authorization: Bearer $CORTEX_ORG_USER_KEY" -H "Content-Type: application/json" -L "https://$CORTEX_API_URL/organization/analyzer/CERTatPassiveDNS_2_0" -d '{"name":"CERTatPassiveDNS_2_0","configuration":{"auto_extract_artifacts":false,"check_tlp":true,"max_tlp":2, "limit": 100}}' + curl -sv -k -XPOST -H "Authorization: Bearer $CORTEX_ORG_USER_KEY" -H "Content-Type: application/json" -L "$CORTEX_API_URL/organization/analyzer/CERTatPassiveDNS_2_0" -d '{"name":"CERTatPassiveDNS_2_0","configuration":{"auto_extract_artifacts":false,"check_tlp":true,"max_tlp":2, "limit": 100}}' # Revoke $CORTEX_USER key - curl -sk -XDELETE -H "Authorization: Bearer $CORTEX_KEY" -L "https://$CORTEX_API_URL/user/$CORTEX_USER/key" + curl -sk -XDELETE -H "Authorization: Bearer $CORTEX_KEY" -L "$CORTEX_API_URL/user/$CORTEX_USER/key" # Update SOCtopus config with apikey value #sed -i "s/cortex_key = .*/cortex_key = $CORTEX_KEY/" $SOCTOPUS_CONFIG diff --git a/salt/thehive/scripts/hive_init b/salt/thehive/scripts/hive_init index f0f1a5f52..55a408f60 100755 --- a/salt/thehive/scripts/hive_init +++ b/salt/thehive/scripts/hive_init @@ -1,6 +1,5 @@ #!/bin/bash # {%- set MANAGERIP = salt['pillar.get']('global:managerip', '') %} -# {%- set URLBASE = salt['pillar.get']('global:url_base', '') %} # {%- set THEHIVEUSER = salt['pillar.get']('global:hiveuser', 'hiveadmin') %} # {%- set THEHIVEPASSWORD = salt['pillar.get']('global:hivepassword', 'hivechangeme') %} # {%- set THEHIVEKEY = salt['pillar.get']('global:hivekey', '') %} @@ -13,7 +12,7 @@ thehive_clean(){ } thehive_init(){ - THEHIVE_URL="{{URLBASE}}/thehive" + THEHIVE_URL="http://{{MANAGERIP}}:9000/thehive" THEHIVE_API_URL="$THEHIVE_URL/api" THEHIVE_USER="{{THEHIVEUSER}}" THEHIVE_PASSWORD="{{THEHIVEPASSWORD}}" @@ -21,17 +20,17 @@ thehive_init(){ SOCTOPUS_CONFIG="/opt/so/saltstack/salt/soctopus/files/SOCtopus.conf" echo -n "Waiting for TheHive..." - if wait_for_web_response https://$THEHIVE_URL "TheHive"; then + if wait_for_web_response $THEHIVE_URL "TheHive"; then # Migrate DB - curl -sk -XPOST -L "https://$THEHIVE_API_URL/maintenance/migrate" + curl -sk -XPOST -L "$THEHIVE_API_URL/maintenance/migrate" # Create intial TheHive user - curl -sk -L "https://$THEHIVE_API_URL/user" -H "Content-Type: application/json" -d "{\"login\" : \"$THEHIVE_USER\",\"name\" : \"$THEHIVE_USER\",\"roles\" : [\"read\",\"alert\",\"write\",\"admin\"],\"preferences\" : \"{}\",\"password\" : \"$THEHIVE_PASSWORD\", \"key\": \"$THEHIVE_KEY\"}" + curl -sk -L "$THEHIVE_API_URL/user" -H "Content-Type: application/json" -d "{\"login\" : \"$THEHIVE_USER\",\"name\" : \"$THEHIVE_USER\",\"roles\" : [\"read\",\"alert\",\"write\",\"admin\"],\"preferences\" : \"{}\",\"password\" : \"$THEHIVE_PASSWORD\", \"key\": \"$THEHIVE_KEY\"}" # Pre-load custom fields # # reputation - curl -sk -L "https://$THEHIVE_API_URL/list/custom_fields" -H "Authorization: Bearer $THEHIVE_KEY" -H "Content-Type: application/json" -d "{\"value\":{\"name\": \"reputation\", \"reference\": \"reputation\", \"description\": \"This field provides an overall reputation status for an address/domain.\", \"type\": \"string\", \"options\": []}}" + curl -sk -L "$THEHIVE_API_URL/list/custom_fields" -H "Authorization: Bearer $THEHIVE_KEY" -H "Content-Type: application/json" -d "{\"value\":{\"name\": \"reputation\", \"reference\": \"reputation\", \"description\": \"This field provides an overall reputation status for an address/domain.\", \"type\": \"string\", \"options\": []}}" touch /opt/so/state/thehive.txt else From 7050b1fce5deeab1642882809bd0be377f7cf9e1 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 28 Dec 2020 13:55:03 -0500 Subject: [PATCH 06/39] [fix] Don't use same variable for increment and limit --- salt/common/tools/sbin/so-common | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 29e065ac0..4909bc5e1 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -155,8 +155,8 @@ wait_for_apt() { local old_text="$progress_bar_text" fi fi - - while [[ "$retry_count" -lt "$retry_count" ]]; do + local count=0 + while [[ "$count" -lt "$retry_count" ]]; do if __check_apt_lock; then if [ -z "$progress_callback" ]; then echo "$lock_msg" @@ -168,8 +168,8 @@ wait_for_apt() { $progress_callback "$old_text" return 0 fi - ((retry_count++)) sleep "$retry_timeout" + ((count++)) done if __check_apt_lock; then From 1e0525b1ad7f17fd292d804d2e2ef87f2b64a4ab Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 28 Dec 2020 13:57:44 -0500 Subject: [PATCH 07/39] [fix] Only call progress callback if arg passed --- salt/common/tools/sbin/so-common | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 4909bc5e1..9382fe472 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -158,14 +158,14 @@ wait_for_apt() { local count=0 while [[ "$count" -lt "$retry_count" ]]; do if __check_apt_lock; then - if [ -z "$progress_callback" ]; then + if [[ -z $progress_callback ]]; then echo "$lock_msg" echo "$retry_msg" else $progress_callback "$retry_msg" fi else - $progress_callback "$old_text" + [[ -z $progress_callback ]] && $progress_callback "$old_text" return 0 fi sleep "$retry_timeout" From 0dd80a664ffba784e5c90a1ddd6c538668084539 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 28 Dec 2020 14:00:09 -0500 Subject: [PATCH 08/39] [fix] Only call progress callback if arg passed --- salt/common/tools/sbin/so-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 9382fe472..f124e4f64 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -165,7 +165,7 @@ wait_for_apt() { $progress_callback "$retry_msg" fi else - [[ -z $progress_callback ]] && $progress_callback "$old_text" + [[ -z $progress_callback ]] || $progress_callback "$old_text" return 0 fi sleep "$retry_timeout" From 17160dcdbeecb55994f91b4107506c3185460637 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 28 Dec 2020 14:02:46 -0500 Subject: [PATCH 09/39] [fix] Don't repeat fail message on last attempt --- salt/common/tools/sbin/so-common | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index f124e4f64..fa1dec2f8 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -174,7 +174,6 @@ wait_for_apt() { if __check_apt_lock; then if [[ -z $progress_callback ]]; then - echo "$lock_msg" echo "Could not acquire lock after $retry_count attempts, aborting." fi return 1 From f7a60a011bf5e82693ff090fac329d97522daef9 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 28 Dec 2020 14:06:33 -0500 Subject: [PATCH 10/39] [fix] Message formatting --- salt/common/tools/sbin/so-common | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index fa1dec2f8..af254790a 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -157,10 +157,12 @@ wait_for_apt() { fi local count=0 while [[ "$count" -lt "$retry_count" ]]; do + ((count++)) + echo "Attempting to acquire dpkg lock... (Attempt $count/$retry_count)" if __check_apt_lock; then if [[ -z $progress_callback ]]; then - echo "$lock_msg" - echo "$retry_msg" + echo " $lock_msg" + echo " $retry_msg" else $progress_callback "$retry_msg" fi @@ -169,7 +171,6 @@ wait_for_apt() { return 0 fi sleep "$retry_timeout" - ((count++)) done if __check_apt_lock; then From a0533dd6b55c5a11a4a653937c136c6052a396d1 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 28 Dec 2020 14:17:27 -0500 Subject: [PATCH 11/39] [feat] Increase retry_count, decrease wait time, change wording --- salt/common/tools/sbin/so-common | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index af254790a..7dd2e89ce 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -144,10 +144,10 @@ get_random_value() { wait_for_apt() { local progress_callback=$1 - local retry_count=2 - local retry_timeout='30s' - local lock_msg='Could not acquire dpkg lock, is another process using it?' - local retry_msg="Waiting $retry_timeout for dpkg lock to release." + local retry_count=30 + local retry_timeout='10s' + local lock_msg='Could not acquire lock.' + local retry_msg="Waiting $retry_timeout for lock to release." if [ -z "$progress_callback" ]; then if [ -z "$progress_bar_text" ]; then local old_text="Installing..." From 5bd15b91eab5aaf52b17a89e8e096d17e1c9ed1c Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 28 Dec 2020 14:18:43 -0500 Subject: [PATCH 12/39] [fix] Message formatting changes --- salt/common/tools/sbin/so-common | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 7dd2e89ce..79d5f7499 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -146,8 +146,8 @@ wait_for_apt() { local retry_count=30 local retry_timeout='10s' - local lock_msg='Could not acquire lock.' - local retry_msg="Waiting $retry_timeout for lock to release." + local lock_msg='Could not acquire lock' + local retry_msg="waiting $retry_timeout for lock to release." if [ -z "$progress_callback" ]; then if [ -z "$progress_bar_text" ]; then local old_text="Installing..." @@ -161,8 +161,7 @@ wait_for_apt() { echo "Attempting to acquire dpkg lock... (Attempt $count/$retry_count)" if __check_apt_lock; then if [[ -z $progress_callback ]]; then - echo " $lock_msg" - echo " $retry_msg" + echo " $lock_msg, $retry_msg" else $progress_callback "$retry_msg" fi From a4239d7fe4912247eb4cb594bcc83f6c7c4a162d Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 28 Dec 2020 14:20:37 -0500 Subject: [PATCH 13/39] [fix] Clarify why dpkg lock is needed --- salt/common/tools/sbin/so-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 79d5f7499..96414f2ed 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -158,7 +158,7 @@ wait_for_apt() { local count=0 while [[ "$count" -lt "$retry_count" ]]; do ((count++)) - echo "Attempting to acquire dpkg lock... (Attempt $count/$retry_count)" + echo "Attempting to acquire dpkg lock to run apt command... (Attempt $count/$retry_count)" if __check_apt_lock; then if [[ -z $progress_callback ]]; then echo " $lock_msg, $retry_msg" From 5836d22525b0176b5c450682d4df6bb78bfdcf8a Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 28 Dec 2020 14:29:03 -0500 Subject: [PATCH 14/39] [fix] Change text printed to whiptail progress bar --- salt/common/tools/sbin/so-common | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 96414f2ed..270d538b9 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -146,8 +146,7 @@ wait_for_apt() { local retry_count=30 local retry_timeout='10s' - local lock_msg='Could not acquire lock' - local retry_msg="waiting $retry_timeout for lock to release." + local lock_msg="Could not acquire lock, waiting $retry_timeout for lock to release." if [ -z "$progress_callback" ]; then if [ -z "$progress_bar_text" ]; then local old_text="Installing..." @@ -161,9 +160,9 @@ wait_for_apt() { echo "Attempting to acquire dpkg lock to run apt command... (Attempt $count/$retry_count)" if __check_apt_lock; then if [[ -z $progress_callback ]]; then - echo " $lock_msg, $retry_msg" + echo " $lock_msg" else - $progress_callback "$retry_msg" + $progress_callback "$lock_msg" fi else [[ -z $progress_callback ]] || $progress_callback "$old_text" From b03408df6bf8ea1134e9f547ec828837b72737dc Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 28 Dec 2020 14:30:34 -0500 Subject: [PATCH 15/39] [fix] Add missing function --- setup/so-functions | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/setup/so-functions b/setup/so-functions index a8a7517a5..642d386fe 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2260,6 +2260,12 @@ es_heapsize() { fi } + +whiptail_prog_new_message() { + local message=$1 + set_progress_str "$percentage" "$message" +} + # Enable Zeek Logs zeek_logs_enabled() { echo "Enabling Zeek Logs" >> "$setup_log" 2>&1 From af62e648529e63ce72e95dde9ba1f74573aac769 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 28 Dec 2020 14:40:17 -0500 Subject: [PATCH 16/39] [fix] Message changes --- salt/common/tools/sbin/so-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 270d538b9..8c13a8d2d 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -146,7 +146,7 @@ wait_for_apt() { local retry_count=30 local retry_timeout='10s' - local lock_msg="Could not acquire lock, waiting $retry_timeout for lock to release." + local lock_msg="Could not acquire dpkg lock, waiting $retry_timeout for lock to release." if [ -z "$progress_callback" ]; then if [ -z "$progress_bar_text" ]; then local old_text="Installing..." From 5059373485c9771a91e2995d07b891ccc232d839 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 28 Dec 2020 14:43:33 -0500 Subject: [PATCH 17/39] [fix] Change text printed to whiptail progress bar --- salt/common/tools/sbin/so-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 8c13a8d2d..4543ad9fb 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -162,7 +162,7 @@ wait_for_apt() { if [[ -z $progress_callback ]]; then echo " $lock_msg" else - $progress_callback "$lock_msg" + $progress_callback "$lock_msg (Attempt $count/$retry_count)" fi else [[ -z $progress_callback ]] || $progress_callback "$old_text" From cdb6dfcea0c677b3e23f527bda7b84c11f0b7436 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 28 Dec 2020 14:55:15 -0500 Subject: [PATCH 18/39] [fix][wip] Fix whiptail output --- salt/common/tools/sbin/so-common | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 4543ad9fb..6cefb11dc 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -157,10 +157,10 @@ wait_for_apt() { local count=0 while [[ "$count" -lt "$retry_count" ]]; do ((count++)) - echo "Attempting to acquire dpkg lock to run apt command... (Attempt $count/$retry_count)" + [[ -z $progress_callback ]] && echo "Attempting to acquire dpkg lock to run apt command... (Attempt $count/$retry_count)" if __check_apt_lock; then if [[ -z $progress_callback ]]; then - echo " $lock_msg" + echo " $lock_msg" | tee -a "$setup_log" else $progress_callback "$lock_msg (Attempt $count/$retry_count)" fi @@ -172,9 +172,7 @@ wait_for_apt() { done if __check_apt_lock; then - if [[ -z $progress_callback ]]; then - echo "Could not acquire lock after $retry_count attempts, aborting." - fi + [[ -z $progress_callback ]] && echo "Could not acquire lock after $retry_count attempts, aborting." return 1 else return 0 From 97466957a7d447d9fe2125e488d61e508359818d Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 28 Dec 2020 15:06:03 -0500 Subject: [PATCH 19/39] [fix] Fix text printed to whiptail progress bar --- salt/common/tools/sbin/so-common | 6 +++--- setup/so-functions | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 6cefb11dc..c71e9150c 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -147,8 +147,8 @@ wait_for_apt() { local retry_count=30 local retry_timeout='10s' local lock_msg="Could not acquire dpkg lock, waiting $retry_timeout for lock to release." - if [ -z "$progress_callback" ]; then - if [ -z "$progress_bar_text" ]; then + if [[ -z $progress_callback ]]; then + if [[ -z $progress_bar_text ]]; then local old_text="Installing..." else local old_text="$progress_bar_text" @@ -162,7 +162,7 @@ wait_for_apt() { if [[ -z $progress_callback ]]; then echo " $lock_msg" | tee -a "$setup_log" else - $progress_callback "$lock_msg (Attempt $count/$retry_count)" + $progress_callback "Could not acquire dpkg lock, waiting $retry_timeout ($count/$retry_count)" fi else [[ -z $progress_callback ]] || $progress_callback "$old_text" diff --git a/setup/so-functions b/setup/so-functions index 642d386fe..06dcd10de 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1927,7 +1927,8 @@ setup_salt_master_dirs() { set_progress_str() { local percentage_input=$1 - local progress_bar_text=$2 + progress_bar_text=$2 + export progress_bar_text if (( "$percentage_input" >= "$percentage" )); then percentage="$percentage_input" From deb38844bac91ab8ae7f5e2ac09168f96564be7b Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 28 Dec 2020 16:20:33 -0500 Subject: [PATCH 20/39] Correct hive init urls --- salt/thehive/scripts/cortex_init | 4 ++-- salt/thehive/scripts/hive_init | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/thehive/scripts/cortex_init b/salt/thehive/scripts/cortex_init index 64a228f16..c2f00390d 100644 --- a/salt/thehive/scripts/cortex_init +++ b/salt/thehive/scripts/cortex_init @@ -18,8 +18,8 @@ cortex_clean(){ } cortex_init(){ - CORTEX_URL="http://{{MANAGERIP}}:9001" - CORTEX_API_URL="$CORTEX_URL/api" + CORTEX_URL="http://{{MANAGERIP}}:9001/cortex/" + CORTEX_API_URL="${CORTEX_URL}api" CORTEX_USER="{{CORTEXUSER}}" CORTEX_PASSWORD="{{CORTEXPASSWORD}}" CORTEX_KEY="{{CORTEXKEY}}" diff --git a/salt/thehive/scripts/hive_init b/salt/thehive/scripts/hive_init index 55a408f60..ca6f7aa2b 100755 --- a/salt/thehive/scripts/hive_init +++ b/salt/thehive/scripts/hive_init @@ -12,8 +12,8 @@ thehive_clean(){ } thehive_init(){ - THEHIVE_URL="http://{{MANAGERIP}}:9000/thehive" - THEHIVE_API_URL="$THEHIVE_URL/api" + THEHIVE_URL="http://{{MANAGERIP}}:9000/thehive/" + THEHIVE_API_URL="${THEHIVE_URL}api" THEHIVE_USER="{{THEHIVEUSER}}" THEHIVE_PASSWORD="{{THEHIVEPASSWORD}}" THEHIVE_KEY="{{THEHIVEKEY}}" From ea5e25c4a5d9bbc7671a729329d6f4b6418fb49f Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 29 Dec 2020 10:34:27 -0500 Subject: [PATCH 21/39] Monitor interface will not always be bond0 - pull correct value from pillar; Replay test data after automated test installations complete. --- salt/common/tools/sbin/so-tcpreplay | 6 +++--- setup/so-setup | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/so-tcpreplay b/salt/common/tools/sbin/so-tcpreplay index 4cd473c0c..a5c75c267 100755 --- a/salt/common/tools/sbin/so-tcpreplay +++ b/salt/common/tools/sbin/so-tcpreplay @@ -20,9 +20,11 @@ . /usr/sbin/so-common . /usr/sbin/so-image-common -REPLAYIFACE=${REPLAYIFACE:-bond0} +REPLAYIFACE=${REPLAYIFACE:-$(lookup_pillar interface sensor)} REPLAYSPEED=${REPLAYSPEED:-10} +mkdir -p /opt/so/samples + if [[ $# -lt 1 ]]; then echo "Replays one or more PCAP sample files to the Security Onion monitoring interface." echo @@ -48,8 +50,6 @@ if ! docker ps | grep -q so-tcpreplay; then TRUSTED_CONTAINERS=("so-tcpreplay") update_docker_containers "tcpreplay" so-tcpreplay-start || fail "Unable to initialize tcpreplay" - mkdir -p /opt/so/samples - docker cp so-tcpreplay:/opt/samples/* /opt/so/samples fi echo "Replaying PCAP(s) at ${REPLAYSPEED} Mbps on interface ${REPLAYIFACE}..." diff --git a/setup/so-setup b/setup/so-setup index 8300fe6ae..fc5996c04 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -842,6 +842,8 @@ if [[ -n $SO_ERROR ]]; then else echo "Successfully completed setup! Continuing with post-installation steps" >> $setup_log 2>&1 { + [ -n "$TESTING" ] && so-test + export percentage=95 # set to last percentage used in previous subshell if [[ -n $ALLOW_ROLE && -n $ALLOW_CIDR ]]; then set_progress_str 97 "Running so-allow -${ALLOW_ROLE} for ${ALLOW_CIDR}" From 74dd2187fb13b79975e77a06e6e91077d6ba61db Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 29 Dec 2020 11:16:57 -0500 Subject: [PATCH 22/39] Use AMI's public IP for external access --- setup/automation/aws_standalone_defaults | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/automation/aws_standalone_defaults b/setup/automation/aws_standalone_defaults index db199986b..6c00cc789 100644 --- a/setup/automation/aws_standalone_defaults +++ b/setup/automation/aws_standalone_defaults @@ -62,7 +62,7 @@ OSQUERY=1 # PATCHSCHEDULEHOURS= PATCHSCHEDULENAME=auto PLAYBOOK=1 -REDIRECTHOST=securityonion +REDIRECTHOST=$(curl http://169.254.169.254/latest/meta-data/public-ipv4) REDIRECTINFO=OTHER RULESETUP=ETOPEN # SHARDCOUNT= From 0a57b789007a807eb93ef467be6d35f80976a3ba Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 29 Dec 2020 12:52:21 -0500 Subject: [PATCH 23/39] [feat] Add option to set up only network on an iso --- setup/so-functions | 39 ++++++++++++++++++++++ setup/so-setup | 81 +++++++++++++++++++--------------------------- setup/so-variables | 6 ++++ setup/so-whiptail | 11 +++++++ 4 files changed, 89 insertions(+), 48 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 06dcd10de..b3ea2b476 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1377,6 +1377,45 @@ minio_generate_keys() { } +network_init() { + case "$setup_type" in + 'iso') + whiptail_set_hostname + whiptail_management_nic + whiptail_dhcp_or_static + + if [ "$address_type" != 'DHCP' ]; then + whiptail_management_interface_ip + whiptail_management_interface_mask + whiptail_management_interface_gateway + whiptail_management_interface_dns + whiptail_management_interface_dns_search + fi + ;; + 'network') + whiptail_network_notice + whiptail_dhcp_warn + whiptail_set_hostname + whiptail_management_nic + ;; + esac + + if [[ $is_minion ]]; then + whiptail_management_server + fi + + if [[ $is_minion || $is_iso ]]; then + whiptail_management_interface_setup + fi + + # Init networking so rest of install works + disable_ipv6 + set_hostname + if [[ "$setup_type" == 'iso' ]]; then + set_management_interface + fi +} + network_setup() { { echo "Finishing up network setup"; diff --git a/setup/so-setup b/setup/so-setup index 8300fe6ae..f0954fa3a 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -67,7 +67,7 @@ while [[ $# -gt 0 ]]; do esac done -if ! [ -f /root/install_opt ] && [ -d /root/manager_setup/securityonion ] && [[ $(pwd) != /root/manager_setup/securityonion/setup ]]; then +if ! [ -f $install_opt_file ] && [ -d /root/manager_setup/securityonion ] && [[ $(pwd) != /root/manager_setup/securityonion/setup ]]; then exec bash /root/manager_setup/securityonion/setup/so-setup "${original_args[@]}" fi @@ -81,7 +81,7 @@ fi parse_install_username -if ! [ -f /root/install_opt ]; then +if ! [ -f $install_opt_file ]; then # Begin Installation pre-processing title "Initializing Setup" info "Installing as the $INSTALLUSERNAME user" @@ -164,17 +164,33 @@ if [ "$automated" == no ]; then fi fi -if ! [ -f /root/install_opt ]; then +if ! [[ -f $install_opt_file ]]; then if (whiptail_you_sure); then true else echo "User cancelled setup." | tee -a "$setup_log" whiptail_cancel fi - - whiptail_install_type + if [[ $setup_type == 'iso' ]]; then + whiptail_first_menu_iso + if [[ $option == "Configure Network" ]] && ! [[ -f $net_init_file ]]; then + network_init + printf '%s\n' \ + "MNIC=$MNIC" \ + "HOSTNAME=$HOSTNAME" \ + "MSRV=$MSRV" \ + "MSRVIP=$MSRVIP" > "$net_init_file" + whiptail --title "Security Onion Setup" \ + --msgbox "Successfully set up networking, setup will now exit." 7 75 + exit 0 + else + whiptail_install_type + fi + else + whiptail_install_type + fi else - source /root/install_opt + source $install_opt_file fi if [ "$install_type" = 'EVAL' ]; then @@ -230,7 +246,7 @@ if [[ ( $is_manager || $is_import ) && $is_iso ]]; then fi fi -if ! [ -f /root/install_opt ]; then +if ! [ -f $install_opt_file ]; then if [[ $is_manager && $is_sensor ]]; then check_requirements "standalone" elif [[ $is_fleet_standalone ]]; then @@ -243,41 +259,10 @@ if ! [ -f /root/install_opt ]; then check_requirements "import" fi - case "$setup_type" in - 'iso') - whiptail_set_hostname - whiptail_management_nic - whiptail_dhcp_or_static - - if [ "$address_type" != 'DHCP' ]; then - whiptail_management_interface_ip - whiptail_management_interface_mask - whiptail_management_interface_gateway - whiptail_management_interface_dns - whiptail_management_interface_dns_search - fi - ;; - 'network') - whiptail_network_notice - whiptail_dhcp_warn - whiptail_set_hostname - whiptail_management_nic - ;; - esac - - if [[ $is_minion ]]; then - whiptail_management_server - fi - - if [[ $is_minion || $is_iso ]]; then - whiptail_management_interface_setup - fi - - # Init networking so rest of install works - disable_ipv6 - set_hostname - if [[ "$setup_type" == 'iso' ]]; then - set_management_interface + if [[ -f $net_init_file ]]; then + network_init + else + source $net_init_file fi if [[ -n "$TURBO" ]]; then @@ -298,8 +283,8 @@ if ! [ -f /root/install_opt ]; then "install_type=$install_type" \ "MNIC=$MNIC" \ "HOSTNAME=$HOSTNAME" \ - "MSRV=$MSRV"\ - "MSRVIP=$MSRVIP" > /root/install_opt + "MSRV=$MSRV" \ + "MSRVIP=$MSRVIP" > "$install_opt_file" download_repo_tarball exec bash /root/manager_setup/securityonion/setup/so-setup "${original_args[@]}" fi @@ -396,7 +381,10 @@ if [[ $is_manager && ! $is_eval ]]; then whiptail_manager_adv_escluster fi fi - whiptail_zeek_version + whiptail_metadata_tool + if [ "$MANAGERADV" = 'ADVANCED' ] && [ "$ZEEKVERSION" != 'SURICATA' ]; then + whiptail_manager_adv_service_zeeklogs + fi # Don't run this function for now since Snort is not yet supported # whiptail_nids NIDS=Suricata @@ -406,9 +394,6 @@ if [[ $is_manager && ! $is_eval ]]; then whiptail_oinkcode fi - if [ "$MANAGERADV" = 'ADVANCED' ] && [ "$ZEEKVERSION" != 'SURICATA' ]; then - whiptail_manager_adv_service_zeeklogs - fi fi if [[ $is_manager ]]; then diff --git a/setup/so-variables b/setup/so-variables index 1f154a5c0..e3ae8cbc8 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -66,3 +66,9 @@ mkdir -p "$local_salt_dir" SCRIPTDIR=$(pwd) export SCRIPTDIR + +install_opt_file=/root/install_opt +export install_opt_file + +net_init_file=/root/net_int +export net_init_file diff --git a/setup/so-whiptail b/setup/so-whiptail index d60352bdd..2cee5f7f1 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -658,6 +658,17 @@ whiptail_log_size_limit() { } +whiptail_first_menu_iso() { + [ -n "$TESTING" ] && return + + option=$(whiptail --title "Security Onion Setup" --menu "Select an option" 10 75 2 \ + "Configure Network" "Configure networking only " \ + "Security Onion Installer" "Run the standard Security Onion installation " \ + 3>&1 1>&2 2>&3 + ) + local exitstatus=$? + whiptail_check_exitstatus $exitstatus +} whiptail_make_changes() { [ -n "$TESTING" ] && return From 989e2b8b780411b266c7ee1ad6a9ad141b7a823c Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 29 Dec 2020 16:15:10 -0500 Subject: [PATCH 24/39] Add eval automation --- setup/automation/pm_eval_defaults | 77 +++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 setup/automation/pm_eval_defaults diff --git a/setup/automation/pm_eval_defaults b/setup/automation/pm_eval_defaults new file mode 100644 index 000000000..a2acf0457 --- /dev/null +++ b/setup/automation/pm_eval_defaults @@ -0,0 +1,77 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +TESTING=true + +address_type=DHCP +ADMINUSER=onionuser +ADMINPASS1=onionuser +ADMINPASS2=onionuser +ALLOW_CIDR=0.0.0.0/0 +ALLOW_ROLE=a +BASICZEEK=7 +BASICSURI=7 +# BLOGS= +BNICS=eth1 +ZEEKVERSION=ZEEK +# CURCLOSEDAYS= +# EVALADVANCED=BASIC +GRAFANA=1 +# HELIXAPIKEY= +HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 +HNSENSOR=inherit +HOSTNAME=eval +install_type=EVAL +# LSINPUTBATCHCOUNT= +# LSINPUTTHREADS= +# LSPIPELINEBATCH= +# LSPIPELINEWORKERS= +MANAGERADV=BASIC +MANAGERUPDATES=1 +# MDNS= +# MGATEWAY= +# MIP= +# MMASK= +MNIC=eth0 +# MSEARCH= +# MSRV= +# MTU= +NIDS=Suricata +# NODE_ES_HEAP_SIZE= +# NODE_LS_HEAP_SIZE= +NODESETUP=NODEBASIC +NSMSETUP=BASIC +NODEUPDATES=MANAGER +# OINKCODE= +OSQUERY=1 +# PATCHSCHEDULEDAYS= +# PATCHSCHEDULEHOURS= +PATCHSCHEDULENAME=auto +PLAYBOOK=1 +# REDIRECTHOST= +REDIRECTINFO=IP +RULESETUP=ETOPEN +# SHARDCOUNT= +SKIP_REBOOT=1 +SOREMOTEPASS1=onionuser +SOREMOTEPASS2=onionuser +STRELKA=1 +THEHIVE=1 +WAZUH=1 +WEBUSER=onionuser@somewhere.invalid +WEBPASSWD1=0n10nus3r +WEBPASSWD2=0n10nus3r From 827a571db8c754919f4363307d2cb5ad797d2697 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 29 Dec 2020 17:25:53 -0500 Subject: [PATCH 25/39] Ensure so-test is logged --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index fc5996c04..752afb9a9 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -842,7 +842,7 @@ if [[ -n $SO_ERROR ]]; then else echo "Successfully completed setup! Continuing with post-installation steps" >> $setup_log 2>&1 { - [ -n "$TESTING" ] && so-test + [ -n "$TESTING" ] && logCmd so-setup export percentage=95 # set to last percentage used in previous subshell if [[ -n $ALLOW_ROLE && -n $ALLOW_CIDR ]]; then From a49ddfb887c42def14e29128d2c8028ec6d84bac Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 29 Dec 2020 20:42:50 -0500 Subject: [PATCH 26/39] Reboot to ensure thehive falls in line before kicking off the test --- setup/automation/aws_eval_defaults | 2 +- setup/automation/aws_standalone_defaults | 2 +- setup/automation/pm_eval_defaults | 2 +- setup/automation/pm_standalone_defaults | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/setup/automation/aws_eval_defaults b/setup/automation/aws_eval_defaults index e038bf29d..2c5a8a52d 100644 --- a/setup/automation/aws_eval_defaults +++ b/setup/automation/aws_eval_defaults @@ -66,7 +66,7 @@ PLAYBOOK=1 REDIRECTINFO=HOSTNAME RULESETUP=ETOPEN # SHARDCOUNT= -SKIP_REBOOT=0 +# SKIP_REBOOT=0 SOREMOTEPASS1=onionuser SOREMOTEPASS2=onionuser STRELKA=1 diff --git a/setup/automation/aws_standalone_defaults b/setup/automation/aws_standalone_defaults index 6c00cc789..d32e1fad7 100644 --- a/setup/automation/aws_standalone_defaults +++ b/setup/automation/aws_standalone_defaults @@ -66,7 +66,7 @@ REDIRECTHOST=$(curl http://169.254.169.254/latest/meta-data/public-ipv4) REDIRECTINFO=OTHER RULESETUP=ETOPEN # SHARDCOUNT= -SKIP_REBOOT=0 +# SKIP_REBOOT= SOREMOTEPASS1=onionuser SOREMOTEPASS2=onionuser STRELKA=1 diff --git a/setup/automation/pm_eval_defaults b/setup/automation/pm_eval_defaults index a2acf0457..6e5560028 100644 --- a/setup/automation/pm_eval_defaults +++ b/setup/automation/pm_eval_defaults @@ -66,7 +66,7 @@ PLAYBOOK=1 REDIRECTINFO=IP RULESETUP=ETOPEN # SHARDCOUNT= -SKIP_REBOOT=1 +# SKIP_REBOOT= SOREMOTEPASS1=onionuser SOREMOTEPASS2=onionuser STRELKA=1 diff --git a/setup/automation/pm_standalone_defaults b/setup/automation/pm_standalone_defaults index d7bc1ea1f..0561a2883 100644 --- a/setup/automation/pm_standalone_defaults +++ b/setup/automation/pm_standalone_defaults @@ -66,7 +66,7 @@ PLAYBOOK=1 REDIRECTINFO=IP RULESETUP=ETOPEN # SHARDCOUNT= -SKIP_REBOOT=1 +# SKIP_REBOOT= SOREMOTEPASS1=onionuser SOREMOTEPASS2=onionuser STRELKA=1 From 19d14cf277ca815d272b41232ef873b17c7c29b9 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 30 Dec 2020 10:31:04 -0500 Subject: [PATCH 27/39] Fix script typo to correctly run the so-test --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 752afb9a9..299727eca 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -842,7 +842,7 @@ if [[ -n $SO_ERROR ]]; then else echo "Successfully completed setup! Continuing with post-installation steps" >> $setup_log 2>&1 { - [ -n "$TESTING" ] && logCmd so-setup + [ -n "$TESTING" ] && logCmd so-test export percentage=95 # set to last percentage used in previous subshell if [[ -n $ALLOW_ROLE && -n $ALLOW_CIDR ]]; then From 13f0ddabfc407b6f988bec2d293e529d8bc7454b Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 30 Dec 2020 12:02:42 -0500 Subject: [PATCH 28/39] Use manager internal IP for intra-service comms --- salt/soctopus/files/SOCtopus.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soctopus/files/SOCtopus.conf b/salt/soctopus/files/SOCtopus.conf index 29f31f95f..4f58ecf83 100644 --- a/salt/soctopus/files/SOCtopus.conf +++ b/salt/soctopus/files/SOCtopus.conf @@ -1,4 +1,4 @@ -{%- set MANAGER = salt['pillar.get']('global:url_base', '') %} +{%- set MANAGER = salt['pillar.get']('manager:mainip', '') %} {%- set URLBASE = salt['pillar.get']('global:url_base', '') %} {%- set HIVEKEY = salt['pillar.get']('global:hivekey', '') %} {%- set CORTEXKEY = salt['pillar.get']('global:cortexorguserkey', '') %} From 3f3fe78322ffa2a6938c4d0d5810eaad90bf6227 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 30 Dec 2020 14:01:20 -0500 Subject: [PATCH 29/39] [fix] Correct reversed logic --- setup/so-setup | 6 +++--- setup/so-variables | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index f0954fa3a..4fb100caa 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -246,7 +246,7 @@ if [[ ( $is_manager || $is_import ) && $is_iso ]]; then fi fi -if ! [ -f $install_opt_file ]; then +if ! [[ -f $install_opt_file ]]; then if [[ $is_manager && $is_sensor ]]; then check_requirements "standalone" elif [[ $is_fleet_standalone ]]; then @@ -260,9 +260,9 @@ if ! [ -f $install_opt_file ]; then fi if [[ -f $net_init_file ]]; then - network_init + source "$net_init_file" else - source $net_init_file + network_init fi if [[ -n "$TURBO" ]]; then diff --git a/setup/so-variables b/setup/so-variables index e3ae8cbc8..a2fdf03c6 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -70,5 +70,5 @@ export SCRIPTDIR install_opt_file=/root/install_opt export install_opt_file -net_init_file=/root/net_int +net_init_file=/root/net_init export net_init_file From df305c49a66b360ef5f6aa5a61f308b7f7870756 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 30 Dec 2020 16:33:46 -0500 Subject: [PATCH 30/39] Stop SOC prior to opening the firewall for analysts, this ensures no outside requests can be processed prior to the server rebooting --- setup/so-setup | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-setup b/setup/so-setup index 299727eca..3ee0326c0 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -845,6 +845,7 @@ else [ -n "$TESTING" ] && logCmd so-test export percentage=95 # set to last percentage used in previous subshell + so-soc-stop # Stop SOC so it doesn't accept external requests prior to the reboot if [[ -n $ALLOW_ROLE && -n $ALLOW_CIDR ]]; then set_progress_str 97 "Running so-allow -${ALLOW_ROLE} for ${ALLOW_CIDR}" IP=$ALLOW_CIDR so-allow -$ALLOW_ROLE >> $setup_log 2>&1 From e167bfed20e706a9ca42589fc3fb38874c00aee3 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 30 Dec 2020 18:48:56 -0500 Subject: [PATCH 31/39] Redirect tcpreplay init output to file --- salt/common/tools/sbin/so-tcpreplay | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-tcpreplay b/salt/common/tools/sbin/so-tcpreplay index a5c75c267..3f5c0aead 100755 --- a/salt/common/tools/sbin/so-tcpreplay +++ b/salt/common/tools/sbin/so-tcpreplay @@ -48,7 +48,8 @@ if ! docker ps | grep -q so-tcpreplay; then echo TRUSTED_CONTAINERS=("so-tcpreplay") - update_docker_containers "tcpreplay" + mkdir -p /opt/so/log/tcpreplay + update_docker_containers "tcpreplay" "" "" "/opt/so/log/tcpreplay/init.log" so-tcpreplay-start || fail "Unable to initialize tcpreplay" fi From 6b81419d3870f2755291146612745fbe02d97a98 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 30 Dec 2020 22:02:19 -0500 Subject: [PATCH 32/39] tcpreplay doesn't need an interactive terminal to run, remove 'it' --- salt/common/tools/sbin/so-tcpreplay | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-tcpreplay b/salt/common/tools/sbin/so-tcpreplay index 3f5c0aead..fa992bdd8 100755 --- a/salt/common/tools/sbin/so-tcpreplay +++ b/salt/common/tools/sbin/so-tcpreplay @@ -54,6 +54,6 @@ if ! docker ps | grep -q so-tcpreplay; then fi echo "Replaying PCAP(s) at ${REPLAYSPEED} Mbps on interface ${REPLAYIFACE}..." -docker exec -it so-tcpreplay /usr/bin/bash -c "/usr/local/bin/tcpreplay -i ${REPLAYIFACE} -M${REPLAYSPEED} $@" +docker exec so-tcpreplay /usr/bin/bash -c "/usr/local/bin/tcpreplay -i ${REPLAYIFACE} -M${REPLAYSPEED} $@" echo "Replay completed. Warnings shown above are typically expected." From 4b244645ba907cff7401b890b00a40ab263410ad Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 31 Dec 2020 10:52:59 -0500 Subject: [PATCH 33/39] so-fleet-setup doesn't need an interactive terminal to run, remove 'it' --- salt/common/tools/sbin/so-fleet-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-fleet-setup b/salt/common/tools/sbin/so-fleet-setup index 8de83b118..a3aa013a2 100755 --- a/salt/common/tools/sbin/so-fleet-setup +++ b/salt/common/tools/sbin/so-fleet-setup @@ -16,7 +16,7 @@ if [ ! "$(docker ps -q -f name=so-fleet)" ]; then fi docker exec so-fleet fleetctl config set --address https://127.0.0.1:8080 --tls-skip-verify --url-prefix /fleet -docker exec -it so-fleet bash -c 'while [[ "$(curl -s -o /dev/null --insecure -w ''%{http_code}'' https://127.0.0.1:8080/fleet)" != "301" ]]; do sleep 5; done' +docker exec so-fleet bash -c 'while [[ "$(curl -s -o /dev/null --insecure -w ''%{http_code}'' https://127.0.0.1:8080/fleet)" != "301" ]]; do sleep 5; done' docker exec so-fleet fleetctl setup --email $1 --password $2 docker exec so-fleet fleetctl apply -f /packs/palantir/Fleet/Endpoints/MacOS/osquery.yaml From 455da7ec5db9f1eb73852690f2ba2e42e4b28ce5 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 31 Dec 2020 15:09:22 -0500 Subject: [PATCH 34/39] Only stop SOC if is_manager or is_import --- ...{aws_forwardnode_defaults => distributed_forwardnode_ami} | 0 .../{aws_manager_defaults => distributed_manager_ami} | 0 .../{aws_searchnode_defaults => distributed_searchnode_ami} | 0 setup/automation/{aws_eval_defaults => eval_ami} | 0 setup/automation/{pm_eval_defaults => eval_iso} | 0 setup/automation/{aws_standalone_defaults => standalone_ami} | 0 setup/automation/{pm_standalone_defaults => standalone_iso} | 0 setup/so-setup | 5 ++++- 8 files changed, 4 insertions(+), 1 deletion(-) rename setup/automation/{aws_forwardnode_defaults => distributed_forwardnode_ami} (100%) rename setup/automation/{aws_manager_defaults => distributed_manager_ami} (100%) rename setup/automation/{aws_searchnode_defaults => distributed_searchnode_ami} (100%) rename setup/automation/{aws_eval_defaults => eval_ami} (100%) rename setup/automation/{pm_eval_defaults => eval_iso} (100%) rename setup/automation/{aws_standalone_defaults => standalone_ami} (100%) rename setup/automation/{pm_standalone_defaults => standalone_iso} (100%) diff --git a/setup/automation/aws_forwardnode_defaults b/setup/automation/distributed_forwardnode_ami similarity index 100% rename from setup/automation/aws_forwardnode_defaults rename to setup/automation/distributed_forwardnode_ami diff --git a/setup/automation/aws_manager_defaults b/setup/automation/distributed_manager_ami similarity index 100% rename from setup/automation/aws_manager_defaults rename to setup/automation/distributed_manager_ami diff --git a/setup/automation/aws_searchnode_defaults b/setup/automation/distributed_searchnode_ami similarity index 100% rename from setup/automation/aws_searchnode_defaults rename to setup/automation/distributed_searchnode_ami diff --git a/setup/automation/aws_eval_defaults b/setup/automation/eval_ami similarity index 100% rename from setup/automation/aws_eval_defaults rename to setup/automation/eval_ami diff --git a/setup/automation/pm_eval_defaults b/setup/automation/eval_iso similarity index 100% rename from setup/automation/pm_eval_defaults rename to setup/automation/eval_iso diff --git a/setup/automation/aws_standalone_defaults b/setup/automation/standalone_ami similarity index 100% rename from setup/automation/aws_standalone_defaults rename to setup/automation/standalone_ami diff --git a/setup/automation/pm_standalone_defaults b/setup/automation/standalone_iso similarity index 100% rename from setup/automation/pm_standalone_defaults rename to setup/automation/standalone_iso diff --git a/setup/so-setup b/setup/so-setup index 3ee0326c0..435ccf502 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -845,8 +845,11 @@ else [ -n "$TESTING" ] && logCmd so-test export percentage=95 # set to last percentage used in previous subshell - so-soc-stop # Stop SOC so it doesn't accept external requests prior to the reboot if [[ -n $ALLOW_ROLE && -n $ALLOW_CIDR ]]; then + if [[ $is_manager || $is_import ]]; then + set_progress_str 96 "Stopping SOC prior to adjusting firewall rules" + so-soc-stop # Stop SOC so it doesn't accept external requests prior to the reboot + fi set_progress_str 97 "Running so-allow -${ALLOW_ROLE} for ${ALLOW_CIDR}" IP=$ALLOW_CIDR so-allow -$ALLOW_ROLE >> $setup_log 2>&1 fi From a714d36b99b078a6a5f0891409af4e53a215540c Mon Sep 17 00:00:00 2001 From: William Wernert Date: Sat, 2 Jan 2021 21:03:15 -0500 Subject: [PATCH 35/39] [fix] Remove condition for stopping SOC, since the parent condition covers what's tested --- setup/so-setup | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 435ccf502..808ee3825 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -846,10 +846,9 @@ else export percentage=95 # set to last percentage used in previous subshell if [[ -n $ALLOW_ROLE && -n $ALLOW_CIDR ]]; then - if [[ $is_manager || $is_import ]]; then - set_progress_str 96 "Stopping SOC prior to adjusting firewall rules" - so-soc-stop # Stop SOC so it doesn't accept external requests prior to the reboot - fi + set_progress_str 96 "Stopping SOC prior to adjusting firewall rules" + so-soc-stop # Stop SOC so it doesn't accept external requests prior to the reboot + set_progress_str 97 "Running so-allow -${ALLOW_ROLE} for ${ALLOW_CIDR}" IP=$ALLOW_CIDR so-allow -$ALLOW_ROLE >> $setup_log 2>&1 fi From 0fa001ed92ceb05eb14fe0cb21e1709a9b2717f5 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 4 Jan 2021 09:27:22 -0500 Subject: [PATCH 36/39] [fix] Add more logic to network-only process --- setup/so-functions | 23 ++++++++--------------- setup/so-setup | 28 +++++++++++++++++++++------- setup/so-whiptail | 12 +++++++++++- 3 files changed, 40 insertions(+), 23 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index b3ea2b476..d0e502941 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1378,6 +1378,14 @@ minio_generate_keys() { } network_init() { + disable_ipv6 + set_hostname + if [[ "$setup_type" == 'iso' ]]; then + set_management_interface + fi +} + +network_init_whiptail() { case "$setup_type" in 'iso') whiptail_set_hostname @@ -1399,21 +1407,6 @@ network_init() { whiptail_management_nic ;; esac - - if [[ $is_minion ]]; then - whiptail_management_server - fi - - if [[ $is_minion || $is_iso ]]; then - whiptail_management_interface_setup - fi - - # Init networking so rest of install works - disable_ipv6 - set_hostname - if [[ "$setup_type" == 'iso' ]]; then - set_management_interface - fi } network_setup() { diff --git a/setup/so-setup b/setup/so-setup index 4fb100caa..677e46478 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -23,7 +23,7 @@ if [ "$uid" -ne 0 ]; then fi # Save the original argument array since we modify it -readarray -t original_args <<< "$@" +original_args=("$@") cd "$(dirname "$0")" || exit 255 @@ -174,12 +174,12 @@ if ! [[ -f $install_opt_file ]]; then if [[ $setup_type == 'iso' ]]; then whiptail_first_menu_iso if [[ $option == "Configure Network" ]] && ! [[ -f $net_init_file ]]; then + network_init_whiptail + whiptail_management_interface_setup network_init printf '%s\n' \ "MNIC=$MNIC" \ - "HOSTNAME=$HOSTNAME" \ - "MSRV=$MSRV" \ - "MSRVIP=$MSRVIP" > "$net_init_file" + "HOSTNAME=$HOSTNAME" > "$net_init_file" whiptail --title "Security Onion Setup" \ --msgbox "Successfully set up networking, setup will now exit." 7 75 exit 0 @@ -259,12 +259,26 @@ if ! [[ -f $install_opt_file ]]; then check_requirements "import" fi - if [[ -f $net_init_file ]]; then - source "$net_init_file" + [[ -f $net_init_file ]] && whiptail_net_reinit && reinit_networking=true + + if [[ $reinit_networking ]] || ! [[ -f $net_init_file ]]; then + network_init_whiptail else - network_init + source "$net_init_file" fi + if [[ $is_minion ]]; then + whiptail_management_server + fi + + if [[ $is_minion ]] || [[ $reinit_networking ]] || ! [[ -f $net_init_file ]]; then + whiptail_management_interface_setup + fi + + if [[ $reinit_networking ]] || ! [[ -f $net_init_file ]]; then + network_init + fi + if [[ -n "$TURBO" ]]; then use_turbo_proxy fi diff --git a/setup/so-whiptail b/setup/so-whiptail index 2cee5f7f1..b034ab679 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -771,7 +771,11 @@ whiptail_management_interface_setup() { if [[ $is_iso ]]; then if [[ $minion_msg != "" ]]; then - msg="initialize networking and $minion_msg" + if [[ -f $net_init_file ]]; then + msg=$minion_msg + else + msg="initialize networking and $minion_msg" + fi else msg="initialize networking" fi @@ -978,6 +982,12 @@ whiptail_network_notice() { } +whiptail_net_reinit() { + [ -n "$TESTING" ] && return + + whiptail --title "Security Onion Setup" --yesno "The management interface has already been configured. Do you want to reconfigure it?" 8 75 +} + whiptail_node_advanced() { [ -n "$TESTING" ] && return From 535820bfa7a5caae1a77f7e37094a3b5ce7938a3 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 4 Jan 2021 10:18:32 -0500 Subject: [PATCH 37/39] Remove old Strelka cron job --- salt/manager/init.sls | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 4136b276d..502c89579 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -88,6 +88,13 @@ append_so-aptcacherng_so-status.conf: {% endif %} +strelka_yara_update_old: + cron.absent: + - user: root + - name: '/usr/sbin/so-yara-update > /dev/null 2>&1' + - hour: '7' + - minute: '1' + strelka_yara_update: cron.present: - user: root From f8c7413b1527cb148240650a2562c1086e3fb12c Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 4 Jan 2021 10:37:07 -0500 Subject: [PATCH 38/39] [fix] Move is_iso variable assignment up --- setup/so-setup | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 677e46478..686ae52e6 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -67,6 +67,10 @@ while [[ $# -gt 0 ]]; do esac done +if [[ "$setup_type" == 'iso' ]]; then + is_iso=true +fi + if ! [ -f $install_opt_file ] && [ -d /root/manager_setup/securityonion ] && [[ $(pwd) != /root/manager_setup/securityonion/setup ]]; then exec bash /root/manager_setup/securityonion/setup/so-setup "${original_args[@]}" fi @@ -233,11 +237,6 @@ elif [ "$install_type" = 'ANALYST' ]; then is_analyst=true fi -# Say yes to the dress if its an ISO install -if [[ "$setup_type" == 'iso' ]]; then - is_iso=true -fi - # Check if this is an airgap install if [[ ( $is_manager || $is_import ) && $is_iso ]]; then whiptail_airgap From 7bfac1e8df318fa4e590cfa8855afa72305e1e27 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 4 Jan 2021 11:58:25 -0500 Subject: [PATCH 39/39] [fix] Don't prompt to only set up network and then skip if network was previously configured --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 3addaf208..bede7990d 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -177,7 +177,7 @@ if ! [[ -f $install_opt_file ]]; then fi if [[ $setup_type == 'iso' ]]; then whiptail_first_menu_iso - if [[ $option == "Configure Network" ]] && ! [[ -f $net_init_file ]]; then + if [[ $option == "Configure Network" ]]; then network_init_whiptail whiptail_management_interface_setup network_init