diff --git a/pillar/data/addtotab.sh b/pillar/data/addtotab.sh deleted file mode 100644 index 65f9446dd..000000000 --- a/pillar/data/addtotab.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env bash - -# This script adds sensors/nodes/etc to the nodes tab -default_salt_dir=/opt/so/saltstack/default -local_salt_dir=/opt/so/saltstack/local -TYPE=$1 -NAME=$2 -IPADDRESS=$3 -CPUS=$4 -GUID=$5 -MANINT=$6 -ROOTFS=$7 -NSM=$8 -MONINT=$9 -#NODETYPE=$10 -#HOTNAME=$11 - -echo "Seeing if this host is already in here. If so delete it" -if grep -q $NAME "$local_salt_dir/pillar/data/$TYPE.sls"; then - echo "Node Already Present - Let's re-add it" - awk -v blah=" $NAME:" 'BEGIN{ print_flag=1 } -{ - if( $0 ~ blah ) - { - print_flag=0; - next - } - if( $0 ~ /^ [a-zA-Z0-9]+:$/ ) - { - print_flag=1; - } - if ( print_flag == 1 ) - print $0 - -} ' $local_salt_dir/pillar/data/$TYPE.sls > $local_salt_dir/pillar/data/tmp.$TYPE.sls -mv $local_salt_dir/pillar/data/tmp.$TYPE.sls $local_salt_dir/pillar/data/$TYPE.sls -echo "Deleted $NAME from the tab. Now adding it in again with updated info" -fi -echo " $NAME:" >> $local_salt_dir/pillar/data/$TYPE.sls -echo " ip: $IPADDRESS" >> $local_salt_dir/pillar/data/$TYPE.sls -echo " manint: $MANINT" >> $local_salt_dir/pillar/data/$TYPE.sls -echo " totalcpus: $CPUS" >> $local_salt_dir/pillar/data/$TYPE.sls -echo " guid: $GUID" >> $local_salt_dir/pillar/data/$TYPE.sls -echo " rootfs: $ROOTFS" >> $local_salt_dir/pillar/data/$TYPE.sls -echo " nsmfs: $NSM" >> $local_salt_dir/pillar/data/$TYPE.sls -if [ $TYPE == 'sensorstab' ]; then - echo " monint: bond0" >> $local_salt_dir/pillar/data/$TYPE.sls -fi -if [ $TYPE == 'evaltab' ] || [ $TYPE == 'standalonetab' ]; then - echo " monint: bond0" >> $local_salt_dir/pillar/data/$TYPE.sls - if [ ! $10 ]; then - salt-call state.apply utility queue=True - fi -fi -if [ $TYPE == 'nodestab' ]; then - salt-call state.apply elasticsearch queue=True -# echo " nodetype: $NODETYPE" >> $local_salt_dir/pillar/data/$TYPE.sls -# echo " hotname: $HOTNAME" >> $local_salt_dir/pillar/data/$TYPE.sls -fi diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index c831b45fe..72e4bbe82 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -37,8 +37,7 @@ 'elasticfleet', 'elasticfleet.manager', 'elasticsearch.cluster', - 'elastic-fleet-package-registry', - 'utility' + 'elastic-fleet-package-registry' ] %} {% set sensor_states = [ diff --git a/salt/kibana/enabled.sls b/salt/kibana/enabled.sls index ea2482798..0e986dbc2 100644 --- a/salt/kibana/enabled.sls +++ b/salt/kibana/enabled.sls @@ -70,7 +70,7 @@ wait_for_so-kibana: - ssl: True - verify_ssl: False - status: 200 - - wait_for: 300 + - wait_for: 600 - request_interval: 15 - require: - docker_container: so-kibana diff --git a/salt/manager/files/repodownload.conf b/salt/manager/files/repodownload.conf index 67ae4b121..9c9cb5109 100644 --- a/salt/manager/files/repodownload.conf +++ b/salt/manager/files/repodownload.conf @@ -11,8 +11,8 @@ name=Security Onion Repo repo mirrorlist=file:///opt/so/conf/reposync/mirror.txt enabled=1 gpgcheck=1 -[securityonionkernel] -name=Security Onion Repo repo +[securityonionkernelsync] +name=Security Onion Kernel Repo repo mirrorlist=file:///opt/so/conf/reposync/mirror-kernel.txt enabled=1 gpgcheck=1 diff --git a/salt/manager/tools/sbin/so-repo-sync b/salt/manager/tools/sbin/so-repo-sync index 6c1b9d509..d6a290c25 100755 --- a/salt/manager/tools/sbin/so-repo-sync +++ b/salt/manager/tools/sbin/so-repo-sync @@ -17,9 +17,9 @@ createrepo /nsm/repo # The kernel repo section is deployed to repodownload.conf by the manager highstate, which # runs AFTER this script during soup. On the first upgrade to a kernel-aware version the # on-disk config still predates the section, so guard on its presence to avoid dnf's -# "Unknown repo: 'securityonionkernel'" aborting the sync (set -e). The next sync after the +# "Unknown repo: 'securityonionkernelsync'" aborting the sync (set -e). The next sync after the # highstate deploys the section will pick it up. -if grep -q '^\[securityonionkernel\]' /opt/so/conf/reposync/repodownload.conf; then - dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionkernel --download-metadata -p /nsm/kernelrepo/ +if grep -q '^\[securityonionkernelsync\]' /opt/so/conf/reposync/repodownload.conf; then + dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionkernelsync --download-metadata -p /nsm/kernelrepo/ createrepo /nsm/kernelrepo fi diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index f7ca96212..566d1482e 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -245,6 +245,7 @@ check_airgap() { UPDATE_DIR=/tmp/soagupdate/SecurityOnion AGDOCKER=/tmp/soagupdate/docker AGREPO=/tmp/soagupdate/minimal/Packages + AGUEKREPO=/tmp/soagupdate/uek/Packages else is_airgap=1 fi @@ -1021,13 +1022,19 @@ update_airgap_rules() { rsync -a $UPDATE_DIR/agrules/securityonion-resources/* /nsm/securityonion-resources/ } -update_airgap_repo() { +update_airgap_repos() { # Update the files in the repo - echo "Syncing new updates to /nsm/repo" - rsync -a $AGREPO/* /nsm/repo/ - echo "Creating repo" + echo "Syncing new updates to /nsm/repo & /nsm/kernelrepo" + # Airgap soup copies new files into the local repo, but doesn't remove old packages. Retaining the ability to rollback package updates + rsync -a "$AGREPO"/ /nsm/repo/ + rsync -a "$AGUEKREPO"/ /nsm/kernelrepo/ + dnf -y install yum-utils createrepo_c + + echo "Running createrepo for /nsm/repo" createrepo /nsm/repo + echo "Running createrepo for /nsm/kernelrepo" + createrepo /nsm/kernelrepo } update_salt_mine() { @@ -1783,7 +1790,7 @@ main() { set -e if [[ $is_airgap -eq 0 ]]; then - update_airgap_repo + update_airgap_repos dnf clean all check_os_updates elif [[ $OS == 'oracle' ]]; then diff --git a/salt/suricata/enabled.sls b/salt/suricata/enabled.sls index 10c04e5b9..53f367971 100644 --- a/salt/suricata/enabled.sls +++ b/salt/suricata/enabled.sls @@ -66,10 +66,11 @@ so-suricata: - file: suriclassifications surirulereload: - cmd.run: + cmd.run: - name: /usr/sbin/so-suricata-reload-rules >> /opt/so/log/suricata/reload.log 2>&1 - - onchanges: + - onchanges: - file: surirulesync + - onlyif: test -f /opt/so/rules/suricata/all-rulesets.rules - require: - docker_container: so-suricata diff --git a/salt/suricata/tools/sbin/so-suricata-reload-rules b/salt/suricata/tools/sbin/so-suricata-reload-rules index b966e4bc0..6db519413 100644 --- a/salt/suricata/tools/sbin/so-suricata-reload-rules +++ b/salt/suricata/tools/sbin/so-suricata-reload-rules @@ -11,13 +11,12 @@ RULES_FILE="/opt/so/rules/suricata/all-rulesets.rules" SOCKET="/var/run/suricata/suricata-command.socket" SURICATASC="docker exec so-suricata /opt/suricata/bin/suricatasc" -# Epoch mtime of the ruleset we need Suricata to have loaded. Captured once so a -# file update mid-reload does not move the goalpost. -target_mtime=$(stat -c %Y "$RULES_FILE") || fail "Could not stat the Suricata rules file: $RULES_FILE" - # Format an epoch as a human-readable local timestamp for log messages. fmt_time() { date -d "@$1" '+%Y-%m-%d %H:%M:%S %Z' 2>/dev/null; } +# Prefix each input line with the current timestamp. +timestamp_lines() { while IFS= read -r line; do printf '%s %s\n' "$(date '+%Y-%m-%d %H:%M:%S %Z')" "$line"; done; } + # Epoch of Suricata's last *completed* ruleset reload; non-zero return on failure. suricata_reload_epoch() { local out ts @@ -53,4 +52,14 @@ reload_and_verify() { return 1 } -retry 60 3 'reload_and_verify' || fail "Suricata did not load the current ruleset in time." +# Run the reload/verify, timestamping every line of output (ours and the +# retry/fail helpers') so reload.log shows when each step ran. The pipeline is +# synchronous, so the log is fully flushed and ordered before we exit; the +# script's real exit code is preserved via PIPESTATUS. +{ + # Epoch mtime of the ruleset we need Suricata to have loaded. Captured once so + # a file update mid-reload does not move the goalpost. + target_mtime=$(stat -c %Y "$RULES_FILE") || fail "Could not stat the Suricata rules file: $RULES_FILE" + retry 60 3 'reload_and_verify' || fail "Suricata did not load the current ruleset in time." +} 2>&1 | timestamp_lines +exit "${PIPESTATUS[0]}" diff --git a/salt/top.sls b/salt/top.sls index 5e194b046..27a3481a2 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -84,7 +84,6 @@ base: - zeek - strelka - elastalert - - utility - elasticfleet - pcap.cleanup @@ -114,7 +113,6 @@ base: - zeek - strelka - elastalert - - utility - elasticfleet - stig - kafka @@ -142,7 +140,6 @@ base: - elastic-fleet-package-registry - kibana - elastalert - - utility - elasticfleet - stig - kafka @@ -169,7 +166,6 @@ base: - elastic-fleet-package-registry - kibana - elastalert - - utility - elasticfleet - kafka @@ -199,7 +195,6 @@ base: - elastic-fleet-package-registry - kibana - elastalert - - utility - elasticfleet - stig - kafka @@ -223,7 +218,6 @@ base: - elasticsearch - elastic-fleet-package-registry - kibana - - utility - suricata - zeek - elasticfleet diff --git a/salt/utility/bin/eval b/salt/utility/bin/eval deleted file mode 100644 index f30f0f421..000000000 --- a/salt/utility/bin/eval +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# Wait for ElasticSearch to come up, so that we can query for version infromation -echo -n "Waiting for ElasticSearch..." -COUNT=0 -ELASTICSEARCH_CONNECTED="no" -while [[ "$COUNT" -le 30 ]]; do - curl -K /opt/so/conf/elasticsearch/curl.config -k --output /dev/null --silent --head --fail -L https://{{ GLOBALS.manager_ip }}:9200 - if [ $? -eq 0 ]; then - ELASTICSEARCH_CONNECTED="yes" - echo "connected!" - break - else - ((COUNT+=1)) - sleep 1 - echo -n "." - fi -done -if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then - echo - echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'docker ps' \n -running 'sudo so-elastic-restart'" - echo - - exit -fi - -echo "Applying cross cluster search config..." - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -XPUT -L https://{{ GLOBALS.manager_ip }}:9200/_cluster/settings \ - -H 'Content-Type: application/json' \ - -d "{\"persistent\": {\"search\": {\"remote\": {\"{{ grains.host }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}" diff --git a/salt/utility/init.sls b/salt/utility/init.sls deleted file mode 100644 index 49bb2cb0c..000000000 --- a/salt/utility/init.sls +++ /dev/null @@ -1,22 +0,0 @@ -{% from 'allowed_states.map.jinja' import allowed_states %} -{% from 'vars/globals.map.jinja' import GLOBALS %} - -{% if sls in allowed_states %} - {% if grains['role'] in ['so-eval', 'so-import'] %} -fixsearch: - cmd.script: - - shell: /bin/bash - - cwd: /opt/so - - source: salt://utility/bin/eval - - template: jinja - - defaults: - GLOBALS: {{ GLOBALS }} - {% endif %} - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/setup/so-functions b/setup/so-functions index 8ac53fd95..d607b9092 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -29,8 +29,12 @@ title() { } fail_setup() { + local err_msg=$1 + if [[ -n "$err_msg" ]]; then + error "$err_msg" + fi error "Setup encountered an unrecoverable failure, exiting" - touch /root/failure + echo "setup incomplete: $err_msg" > /root/failure exit 1 } @@ -697,7 +701,7 @@ compare_main_nic_ip() { EOM [[ -n $TESTING ]] || whiptail --title "$whiptail_title" --msgbox "$message" 11 75 - kill -SIGINT "$(ps --pid $$ -oppid=)"; fail_setup + kill -SIGINT "$(ps --pid $$ -oppid=)"; fail_setup "Main IP mismatch" fi else # Setup uses MAINIP, but since we ignore the equality condition when using a VPN @@ -755,8 +759,7 @@ configure_management_bond() { info "Setting up $bond_name management interface with mode $bond_mode" if [[ ${#MBNICS[@]} -eq 0 ]]; then - error "[ERROR] No management bond NICs were selected." - fail_setup + fail_setup "No management bond NICs selected" fi nmcli -t -f NAME con show | grep -Fxq "$bond_name" @@ -914,8 +917,7 @@ detect_os() { is_rpm=true is_supported=true else - info "This OS is not supported. Security Onion requires Oracle Linux 9." - fail_setup + fail_setup "This OS is not supported. Security Onion requires Oracle Linux 9." fi info "Found OS: $OS $OSVER" @@ -923,7 +925,7 @@ detect_os() { download_elastic_agent_artifacts() { if ! update_elastic_agent 2>&1 | tee -a "$setup_log"; then - fail_setup + fail_setup "Failed to update Elastic Agent" fi } @@ -1567,7 +1569,7 @@ proxy_validate() { error "Received error: $proxy_test_err" if [[ -n $TESTING ]]; then error "Exiting setup" - kill -SIGINT "$(ps --pid $$ -oppid=)"; fail_setup + kill -SIGINT "$(ps --pid $$ -oppid=)"; fail_setup "Proxy validation failed" fi fi return $ret @@ -1774,8 +1776,7 @@ ensure_pyyaml() { local result=$? set +o pipefail if [[ $result -ne 0 ]] || ! rpm -q python3-pyyaml >/dev/null 2>&1; then - error "Failed to install python3-pyyaml (exit=$result)" - fail_setup + fail_setup "Failed to install python3-pyyaml (exit=$result)" fi info "python3-pyyaml installed successfully" } @@ -1910,8 +1911,8 @@ repo_sync_local() { if [[ ! $is_airgap ]]; then curl --retry 5 --retry-delay 60 -A "netinstall/$SOVERSION/$OS/$(uname -r)/1" https://sigs.securityonion.net/checkup --output /tmp/install - retry 5 60 "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/" >> "$setup_log" 2>&1 || fail_setup - retry 5 60 "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionkernel --download-metadata -p /nsm/kernelrepo/" >> "$setup_log" 2>&1 || fail_setup + retry 5 60 "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/" >> "$setup_log" 2>&1 || fail_setup "Failed to sync repos" + retry 5 60 "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionkernel --download-metadata -p /nsm/kernelrepo/" >> "$setup_log" 2>&1 || fail_setup "Failed to sync kernel repos" # After the download is complete run createrepo create_repo fi @@ -1924,10 +1925,10 @@ saltify() { if [[ $waitforstate ]]; then # install all for a manager - retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -M -X stable $SALTVERSION" || fail_setup + retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -M -X stable $SALTVERSION" || fail_setup "Failed to install salt master" else # just a minion - retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -X stable $SALTVERSION" || fail_setup + retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -X stable $SALTVERSION" || fail_setup "Failed to install salt minion" fi salt_install_module_deps @@ -1999,7 +2000,7 @@ set_main_ip() { info "MAINIP=$MAINIP" info "MNIC_IP=$MNIC_IP" whiptail_error_message "The management IP could not be determined. Please check the log at /root/sosetup.log and verify the network configuration. Select OK to exit." - fail_setup + fail_setup "Could not determine MAINIP or MNIC_IP" fi sleep 1 done @@ -2203,7 +2204,7 @@ set_initial_firewall_access() { set_management_interface() { title "Setting up the main interface" if [[ $MNIC == "bond1" ]]; then - configure_management_bond || fail_setup + configure_management_bond || fail_setup "Failed to configure management bond" fi if [ "$address_type" = 'DHCP' ]; then diff --git a/setup/so-setup b/setup/so-setup index 4285e8e89..a44934088 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -90,8 +90,7 @@ if [[ "$setup_type" == 'iso' ]]; then if [[ $is_rpm ]]; then is_iso=true else - echo "Only use 'so-setup iso' for an ISO install on Security Onion ISO images. Please run 'so-setup network' instead." - fail_setup + fail_setup "Only use 'so-setup iso' for an ISO install on Security Onion ISO images. Please run 'so-setup network' instead." fi fi @@ -130,7 +129,7 @@ catch() { info "Fatal error occurred at $1 in so-setup, failing setup." grep --color=never "ERROR" "$setup_log" > "$error_log" whiptail_setup_failed - fail_setup + fail_setup "Fatal error occurred at $1 in so-setup" } # Add the progress function for manager node type installs @@ -238,8 +237,7 @@ case "$setup_type" in info "Beginning Security Onion $setup_type install" ;; *) - error "Invalid install type, must be 'iso', 'network' or 'desktop'." - fail_setup + fail_setup "Invalid install type, must be 'iso', 'network' or 'desktop'." ;; esac @@ -773,8 +771,7 @@ if ! [[ -f $install_opt_file ]]; then logCmd "salt-call state.apply -l info registry" title "Seeding the docker registry" if ! docker_seed_registry; then - error "Failed to seed the docker registry" - fail_setup + fail_setup "Failed to seed the docker registry" fi title "Applying the manager state" logCmd "salt-call state.apply -l info manager" @@ -797,8 +794,7 @@ if ! [[ -f $install_opt_file ]]; then title "Setting up Elastic Fleet" logCmd "salt-call state.apply elasticfleet.config" if ! logCmd so-elastic-fleet-setup; then - error "Failed to run so-elastic-fleet-setup" - fail_setup + fail_setup "Failed to run so-elastic-fleet-setup" fi mark_setup_complete set_initial_firewall_access diff --git a/setup/so-verify b/setup/so-verify index 672ed70cc..660424c72 100755 --- a/setup/so-verify +++ b/setup/so-verify @@ -143,15 +143,15 @@ main() { cat $error_log echo "--------------------------" exit_code=1 - touch /root/failure + echo "Found setup errors. Check $error_log for details" > /root/failure elif using_iso && cron_error_in_mail_spool; then echo "WARNING: Unexpected cron job output in mail spool" exit_code=1 - touch /root/failure + echo "Unexpected cron job output found in /var/spool/mail/" > /root/failure elif is_manager_node && status_failed; then echo "WARNING: Containers are not in a healthy state" exit_code=1 - touch /root/failure + echo "Containers are not in a healthy state. Check so-status for details" > /root/failure else echo "Successfully completed setup!" touch /root/success