Merge remote-tracking branch 'origin/3/dev' into saltthangs

This commit is contained in:
Josh Patterson
2026-07-02 10:27:54 -04:00
14 changed files with 61 additions and 164 deletions
-59
View File
@@ -1,59 +0,0 @@
#!/usr/bin/env bash
# This script adds sensors/nodes/etc to the nodes tab
default_salt_dir=/opt/so/saltstack/default
local_salt_dir=/opt/so/saltstack/local
TYPE=$1
NAME=$2
IPADDRESS=$3
CPUS=$4
GUID=$5
MANINT=$6
ROOTFS=$7
NSM=$8
MONINT=$9
#NODETYPE=$10
#HOTNAME=$11
echo "Seeing if this host is already in here. If so delete it"
if grep -q $NAME "$local_salt_dir/pillar/data/$TYPE.sls"; then
echo "Node Already Present - Let's re-add it"
awk -v blah=" $NAME:" 'BEGIN{ print_flag=1 }
{
if( $0 ~ blah )
{
print_flag=0;
next
}
if( $0 ~ /^ [a-zA-Z0-9]+:$/ )
{
print_flag=1;
}
if ( print_flag == 1 )
print $0
} ' $local_salt_dir/pillar/data/$TYPE.sls > $local_salt_dir/pillar/data/tmp.$TYPE.sls
mv $local_salt_dir/pillar/data/tmp.$TYPE.sls $local_salt_dir/pillar/data/$TYPE.sls
echo "Deleted $NAME from the tab. Now adding it in again with updated info"
fi
echo " $NAME:" >> $local_salt_dir/pillar/data/$TYPE.sls
echo " ip: $IPADDRESS" >> $local_salt_dir/pillar/data/$TYPE.sls
echo " manint: $MANINT" >> $local_salt_dir/pillar/data/$TYPE.sls
echo " totalcpus: $CPUS" >> $local_salt_dir/pillar/data/$TYPE.sls
echo " guid: $GUID" >> $local_salt_dir/pillar/data/$TYPE.sls
echo " rootfs: $ROOTFS" >> $local_salt_dir/pillar/data/$TYPE.sls
echo " nsmfs: $NSM" >> $local_salt_dir/pillar/data/$TYPE.sls
if [ $TYPE == 'sensorstab' ]; then
echo " monint: bond0" >> $local_salt_dir/pillar/data/$TYPE.sls
fi
if [ $TYPE == 'evaltab' ] || [ $TYPE == 'standalonetab' ]; then
echo " monint: bond0" >> $local_salt_dir/pillar/data/$TYPE.sls
if [ ! $10 ]; then
salt-call state.apply utility queue=True
fi
fi
if [ $TYPE == 'nodestab' ]; then
salt-call state.apply elasticsearch queue=True
# echo " nodetype: $NODETYPE" >> $local_salt_dir/pillar/data/$TYPE.sls
# echo " hotname: $HOTNAME" >> $local_salt_dir/pillar/data/$TYPE.sls
fi
+1 -2
View File
@@ -37,8 +37,7 @@
'elasticfleet',
'elasticfleet.manager',
'elasticsearch.cluster',
'elastic-fleet-package-registry',
'utility'
'elastic-fleet-package-registry'
] %}
{% set sensor_states = [
+1 -1
View File
@@ -70,7 +70,7 @@ wait_for_so-kibana:
- ssl: True
- verify_ssl: False
- status: 200
- wait_for: 300
- wait_for: 600
- request_interval: 15
- require:
- docker_container: so-kibana
+2 -2
View File
@@ -11,8 +11,8 @@ name=Security Onion Repo repo
mirrorlist=file:///opt/so/conf/reposync/mirror.txt
enabled=1
gpgcheck=1
[securityonionkernel]
name=Security Onion Repo repo
[securityonionkernelsync]
name=Security Onion Kernel Repo repo
mirrorlist=file:///opt/so/conf/reposync/mirror-kernel.txt
enabled=1
gpgcheck=1
+3 -3
View File
@@ -17,9 +17,9 @@ createrepo /nsm/repo
# The kernel repo section is deployed to repodownload.conf by the manager highstate, which
# runs AFTER this script during soup. On the first upgrade to a kernel-aware version the
# on-disk config still predates the section, so guard on its presence to avoid dnf's
# "Unknown repo: 'securityonionkernel'" aborting the sync (set -e). The next sync after the
# "Unknown repo: 'securityonionkernelsync'" aborting the sync (set -e). The next sync after the
# highstate deploys the section will pick it up.
if grep -q '^\[securityonionkernel\]' /opt/so/conf/reposync/repodownload.conf; then
dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionkernel --download-metadata -p /nsm/kernelrepo/
if grep -q '^\[securityonionkernelsync\]' /opt/so/conf/reposync/repodownload.conf; then
dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionkernelsync --download-metadata -p /nsm/kernelrepo/
createrepo /nsm/kernelrepo
fi
+12 -5
View File
@@ -245,6 +245,7 @@ check_airgap() {
UPDATE_DIR=/tmp/soagupdate/SecurityOnion
AGDOCKER=/tmp/soagupdate/docker
AGREPO=/tmp/soagupdate/minimal/Packages
AGUEKREPO=/tmp/soagupdate/uek/Packages
else
is_airgap=1
fi
@@ -1021,13 +1022,19 @@ update_airgap_rules() {
rsync -a $UPDATE_DIR/agrules/securityonion-resources/* /nsm/securityonion-resources/
}
update_airgap_repo() {
update_airgap_repos() {
# Update the files in the repo
echo "Syncing new updates to /nsm/repo"
rsync -a $AGREPO/* /nsm/repo/
echo "Creating repo"
echo "Syncing new updates to /nsm/repo & /nsm/kernelrepo"
# Airgap soup copies new files into the local repo, but doesn't remove old packages. Retaining the ability to rollback package updates
rsync -a "$AGREPO"/ /nsm/repo/
rsync -a "$AGUEKREPO"/ /nsm/kernelrepo/
dnf -y install yum-utils createrepo_c
echo "Running createrepo for /nsm/repo"
createrepo /nsm/repo
echo "Running createrepo for /nsm/kernelrepo"
createrepo /nsm/kernelrepo
}
update_salt_mine() {
@@ -1783,7 +1790,7 @@ main() {
set -e
if [[ $is_airgap -eq 0 ]]; then
update_airgap_repo
update_airgap_repos
dnf clean all
check_os_updates
elif [[ $OS == 'oracle' ]]; then
+3 -2
View File
@@ -66,10 +66,11 @@ so-suricata:
- file: suriclassifications
surirulereload:
cmd.run:
cmd.run:
- name: /usr/sbin/so-suricata-reload-rules >> /opt/so/log/suricata/reload.log 2>&1
- onchanges:
- onchanges:
- file: surirulesync
- onlyif: test -f /opt/so/rules/suricata/all-rulesets.rules
- require:
- docker_container: so-suricata
@@ -11,13 +11,12 @@ RULES_FILE="/opt/so/rules/suricata/all-rulesets.rules"
SOCKET="/var/run/suricata/suricata-command.socket"
SURICATASC="docker exec so-suricata /opt/suricata/bin/suricatasc"
# Epoch mtime of the ruleset we need Suricata to have loaded. Captured once so a
# file update mid-reload does not move the goalpost.
target_mtime=$(stat -c %Y "$RULES_FILE") || fail "Could not stat the Suricata rules file: $RULES_FILE"
# Format an epoch as a human-readable local timestamp for log messages.
fmt_time() { date -d "@$1" '+%Y-%m-%d %H:%M:%S %Z' 2>/dev/null; }
# Prefix each input line with the current timestamp.
timestamp_lines() { while IFS= read -r line; do printf '%s %s\n' "$(date '+%Y-%m-%d %H:%M:%S %Z')" "$line"; done; }
# Epoch of Suricata's last *completed* ruleset reload; non-zero return on failure.
suricata_reload_epoch() {
local out ts
@@ -53,4 +52,14 @@ reload_and_verify() {
return 1
}
retry 60 3 'reload_and_verify' || fail "Suricata did not load the current ruleset in time."
# Run the reload/verify, timestamping every line of output (ours and the
# retry/fail helpers') so reload.log shows when each step ran. The pipeline is
# synchronous, so the log is fully flushed and ordered before we exit; the
# script's real exit code is preserved via PIPESTATUS.
{
# Epoch mtime of the ruleset we need Suricata to have loaded. Captured once so
# a file update mid-reload does not move the goalpost.
target_mtime=$(stat -c %Y "$RULES_FILE") || fail "Could not stat the Suricata rules file: $RULES_FILE"
retry 60 3 'reload_and_verify' || fail "Suricata did not load the current ruleset in time."
} 2>&1 | timestamp_lines
exit "${PIPESTATUS[0]}"
-6
View File
@@ -84,7 +84,6 @@ base:
- zeek
- strelka
- elastalert
- utility
- elasticfleet
- pcap.cleanup
@@ -114,7 +113,6 @@ base:
- zeek
- strelka
- elastalert
- utility
- elasticfleet
- stig
- kafka
@@ -142,7 +140,6 @@ base:
- elastic-fleet-package-registry
- kibana
- elastalert
- utility
- elasticfleet
- stig
- kafka
@@ -169,7 +166,6 @@ base:
- elastic-fleet-package-registry
- kibana
- elastalert
- utility
- elasticfleet
- kafka
@@ -199,7 +195,6 @@ base:
- elastic-fleet-package-registry
- kibana
- elastalert
- utility
- elasticfleet
- stig
- kafka
@@ -223,7 +218,6 @@ base:
- elasticsearch
- elastic-fleet-package-registry
- kibana
- utility
- suricata
- zeek
- elasticfleet
-29
View File
@@ -1,29 +0,0 @@
#!/bin/bash
# Wait for ElasticSearch to come up, so that we can query for version infromation
echo -n "Waiting for ElasticSearch..."
COUNT=0
ELASTICSEARCH_CONNECTED="no"
while [[ "$COUNT" -le 30 ]]; do
curl -K /opt/so/conf/elasticsearch/curl.config -k --output /dev/null --silent --head --fail -L https://{{ GLOBALS.manager_ip }}:9200
if [ $? -eq 0 ]; then
ELASTICSEARCH_CONNECTED="yes"
echo "connected!"
break
else
((COUNT+=1))
sleep 1
echo -n "."
fi
done
if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
echo
echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'docker ps' \n -running 'sudo so-elastic-restart'"
echo
exit
fi
echo "Applying cross cluster search config..."
curl -K /opt/so/conf/elasticsearch/curl.config -s -k -XPUT -L https://{{ GLOBALS.manager_ip }}:9200/_cluster/settings \
-H 'Content-Type: application/json' \
-d "{\"persistent\": {\"search\": {\"remote\": {\"{{ grains.host }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}"
-22
View File
@@ -1,22 +0,0 @@
{% from 'allowed_states.map.jinja' import allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% if sls in allowed_states %}
{% if grains['role'] in ['so-eval', 'so-import'] %}
fixsearch:
cmd.script:
- shell: /bin/bash
- cwd: /opt/so
- source: salt://utility/bin/eval
- template: jinja
- defaults:
GLOBALS: {{ GLOBALS }}
{% endif %}
{% else %}
{{sls}}_state_not_allowed:
test.fail_without_changes:
- name: {{sls}}_state_not_allowed
{% endif %}
+17 -16
View File
@@ -29,8 +29,12 @@ title() {
}
fail_setup() {
local err_msg=$1
if [[ -n "$err_msg" ]]; then
error "$err_msg"
fi
error "Setup encountered an unrecoverable failure, exiting"
touch /root/failure
echo "setup incomplete: $err_msg" > /root/failure
exit 1
}
@@ -697,7 +701,7 @@ compare_main_nic_ip() {
EOM
[[ -n $TESTING ]] || whiptail --title "$whiptail_title" --msgbox "$message" 11 75
kill -SIGINT "$(ps --pid $$ -oppid=)"; fail_setup
kill -SIGINT "$(ps --pid $$ -oppid=)"; fail_setup "Main IP mismatch"
fi
else
# Setup uses MAINIP, but since we ignore the equality condition when using a VPN
@@ -755,8 +759,7 @@ configure_management_bond() {
info "Setting up $bond_name management interface with mode $bond_mode"
if [[ ${#MBNICS[@]} -eq 0 ]]; then
error "[ERROR] No management bond NICs were selected."
fail_setup
fail_setup "No management bond NICs selected"
fi
nmcli -t -f NAME con show | grep -Fxq "$bond_name"
@@ -914,8 +917,7 @@ detect_os() {
is_rpm=true
is_supported=true
else
info "This OS is not supported. Security Onion requires Oracle Linux 9."
fail_setup
fail_setup "This OS is not supported. Security Onion requires Oracle Linux 9."
fi
info "Found OS: $OS $OSVER"
@@ -923,7 +925,7 @@ detect_os() {
download_elastic_agent_artifacts() {
if ! update_elastic_agent 2>&1 | tee -a "$setup_log"; then
fail_setup
fail_setup "Failed to update Elastic Agent"
fi
}
@@ -1567,7 +1569,7 @@ proxy_validate() {
error "Received error: $proxy_test_err"
if [[ -n $TESTING ]]; then
error "Exiting setup"
kill -SIGINT "$(ps --pid $$ -oppid=)"; fail_setup
kill -SIGINT "$(ps --pid $$ -oppid=)"; fail_setup "Proxy validation failed"
fi
fi
return $ret
@@ -1774,8 +1776,7 @@ ensure_pyyaml() {
local result=$?
set +o pipefail
if [[ $result -ne 0 ]] || ! rpm -q python3-pyyaml >/dev/null 2>&1; then
error "Failed to install python3-pyyaml (exit=$result)"
fail_setup
fail_setup "Failed to install python3-pyyaml (exit=$result)"
fi
info "python3-pyyaml installed successfully"
}
@@ -1910,8 +1911,8 @@ repo_sync_local() {
if [[ ! $is_airgap ]]; then
curl --retry 5 --retry-delay 60 -A "netinstall/$SOVERSION/$OS/$(uname -r)/1" https://sigs.securityonion.net/checkup --output /tmp/install
retry 5 60 "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/" >> "$setup_log" 2>&1 || fail_setup
retry 5 60 "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionkernel --download-metadata -p /nsm/kernelrepo/" >> "$setup_log" 2>&1 || fail_setup
retry 5 60 "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/" >> "$setup_log" 2>&1 || fail_setup "Failed to sync repos"
retry 5 60 "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionkernel --download-metadata -p /nsm/kernelrepo/" >> "$setup_log" 2>&1 || fail_setup "Failed to sync kernel repos"
# After the download is complete run createrepo
create_repo
fi
@@ -1924,10 +1925,10 @@ saltify() {
if [[ $waitforstate ]]; then
# install all for a manager
retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -M -X stable $SALTVERSION" || fail_setup
retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -M -X stable $SALTVERSION" || fail_setup "Failed to install salt master"
else
# just a minion
retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -X stable $SALTVERSION" || fail_setup
retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -X stable $SALTVERSION" || fail_setup "Failed to install salt minion"
fi
salt_install_module_deps
@@ -1999,7 +2000,7 @@ set_main_ip() {
info "MAINIP=$MAINIP"
info "MNIC_IP=$MNIC_IP"
whiptail_error_message "The management IP could not be determined. Please check the log at /root/sosetup.log and verify the network configuration. Select OK to exit."
fail_setup
fail_setup "Could not determine MAINIP or MNIC_IP"
fi
sleep 1
done
@@ -2203,7 +2204,7 @@ set_initial_firewall_access() {
set_management_interface() {
title "Setting up the main interface"
if [[ $MNIC == "bond1" ]]; then
configure_management_bond || fail_setup
configure_management_bond || fail_setup "Failed to configure management bond"
fi
if [ "$address_type" = 'DHCP' ]; then
+5 -9
View File
@@ -90,8 +90,7 @@ if [[ "$setup_type" == 'iso' ]]; then
if [[ $is_rpm ]]; then
is_iso=true
else
echo "Only use 'so-setup iso' for an ISO install on Security Onion ISO images. Please run 'so-setup network' instead."
fail_setup
fail_setup "Only use 'so-setup iso' for an ISO install on Security Onion ISO images. Please run 'so-setup network' instead."
fi
fi
@@ -130,7 +129,7 @@ catch() {
info "Fatal error occurred at $1 in so-setup, failing setup."
grep --color=never "ERROR" "$setup_log" > "$error_log"
whiptail_setup_failed
fail_setup
fail_setup "Fatal error occurred at $1 in so-setup"
}
# Add the progress function for manager node type installs
@@ -238,8 +237,7 @@ case "$setup_type" in
info "Beginning Security Onion $setup_type install"
;;
*)
error "Invalid install type, must be 'iso', 'network' or 'desktop'."
fail_setup
fail_setup "Invalid install type, must be 'iso', 'network' or 'desktop'."
;;
esac
@@ -773,8 +771,7 @@ if ! [[ -f $install_opt_file ]]; then
logCmd "salt-call state.apply -l info registry"
title "Seeding the docker registry"
if ! docker_seed_registry; then
error "Failed to seed the docker registry"
fail_setup
fail_setup "Failed to seed the docker registry"
fi
title "Applying the manager state"
logCmd "salt-call state.apply -l info manager"
@@ -797,8 +794,7 @@ if ! [[ -f $install_opt_file ]]; then
title "Setting up Elastic Fleet"
logCmd "salt-call state.apply elasticfleet.config"
if ! logCmd so-elastic-fleet-setup; then
error "Failed to run so-elastic-fleet-setup"
fail_setup
fail_setup "Failed to run so-elastic-fleet-setup"
fi
mark_setup_complete
set_initial_firewall_access
+3 -3
View File
@@ -143,15 +143,15 @@ main() {
cat $error_log
echo "--------------------------"
exit_code=1
touch /root/failure
echo "Found setup errors. Check $error_log for details" > /root/failure
elif using_iso && cron_error_in_mail_spool; then
echo "WARNING: Unexpected cron job output in mail spool"
exit_code=1
touch /root/failure
echo "Unexpected cron job output found in /var/spool/mail/" > /root/failure
elif is_manager_node && status_failed; then
echo "WARNING: Containers are not in a healthy state"
exit_code=1
touch /root/failure
echo "Containers are not in a healthy state. Check so-status for details" > /root/failure
else
echo "Successfully completed setup!"
touch /root/success