CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2025-12-06 17:22:49 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'remotes/origin/dev' into feature/so-pcap-pull

Browse Source
This commit is contained in:
Josh Brower
2021-05-26 09:52:58 -04:00
parent 0187c9d6df 67828a86c1
commit ed249600d3
18 changed files with 438 additions and 487 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
HOTFIX
View File

@@ -1 +0,0 @@
GRIDFIX

4
README.md
View File

@@ -1,6 +1,6 @@
## Security Onion 2.3.50
## Security Onion 2.3.51
Security Onion 2.3.50 is here!
Security Onion 2.3.51 is here!
## Screenshots

22
VERIFY_ISO.md
View File

@@ -1,17 +1,17 @@
### 2.3.50 ISO image built on 2021/04/27
### 2.3.51 ISO image built on 2021/04/27
### Download and Verify
2.3.50 ISO image:
https://download.securityonion.net/file/securityonion/securityonion-2.3.50.iso
2.3.51 ISO image:
https://download.securityonion.net/file/securityonion/securityonion-2.3.51.iso
MD5: C39CEA68B5A8AFC5CFFB2481797C0374
SHA1: 00AD9F29ABE3AB495136989E62EBB8FA00DA82C6
SHA256: D77AE370D7863837A989F6735413D1DD46B866D8D135A4C363B0633E3990387E
MD5: 7CFB525BEFC0A9F2ED148F5831E387FA
SHA1: 8CC34FCCC36822B309B8168AA706B3D1EC7F3BFD
SHA256: 9892C2546C9AE5A48015160F379B070F0BE30C89693B97F3F1E1592DDCE1DEE0
Signature for ISO image:
https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.50.iso.sig
https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.51.iso.sig
Signing key:
https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS
@@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/ma
Download the signature file for the ISO:
```
wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.50.iso.sig
wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.51.iso.sig
```
Download the ISO image:
```
wget https://download.securityonion.net/file/securityonion/securityonion-2.3.50.iso
wget https://download.securityonion.net/file/securityonion/securityonion-2.3.51.iso
```
Verify the downloaded ISO image using the signature file:
```
gpg --verify securityonion-2.3.50.iso.sig securityonion-2.3.50.iso
gpg --verify securityonion-2.3.51.iso.sig securityonion-2.3.51.iso
```
The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
```
gpg: Signature made Tue 27 Apr 2021 02:17:25 PM EDT using RSA key ID FE507013
gpg: Signature made Thu 20 May 2021 07:49:57 AM EDT using RSA key ID FE507013
gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
gpg: WARNING: This key is not certified with a trusted signature!
gpg: There is no indication that the signature belongs to the owner.

208
pillar/docker/config.sls
View File

@@ -1,208 +0,0 @@
{%- set FLEETMANAGER = salt['pillar.get']('global:fleet_manager', False) -%}
{%- set FLEETNODE = salt['pillar.get']('global:fleet_node', False) -%}
{% set WAZUH = salt['pillar.get']('manager:wazuh', '0') %}
{% set THEHIVE = salt['pillar.get']('manager:thehive', '0') %}
{% set PLAYBOOK = salt['pillar.get']('manager:playbook', '0') %}
{% set FREQSERVER = salt['pillar.get']('manager:freq', '0') %}
{% set DOMAINSTATS = salt['pillar.get']('manager:domainstats', '0') %}
{% set ZEEKVER = salt['pillar.get']('global:mdengine', 'COMMUNITY') %}
{% set GRAFANA = salt['pillar.get']('manager:grafana', '0') %}
eval:
containers:
- so-nginx
- so-telegraf
{% if GRAFANA == '1' %}
- so-influxdb
- so-grafana
{% endif %}
- so-dockerregistry
- so-soc
- so-kratos
- so-idstools
{% if FLEETMANAGER %}
- so-mysql
- so-fleet
- so-redis
{% endif %}
- so-elasticsearch
- so-logstash
- so-kibana
- so-steno
- so-suricata
- so-zeek
- so-curator
- so-elastalert
{% if WAZUH != '0' %}
- so-wazuh
{% endif %}
- so-soctopus
{% if THEHIVE != '0' %}
- so-thehive
- so-thehive-es
- so-cortex
{% endif %}
{% if PLAYBOOK != '0' %}
- so-playbook
{% endif %}
{% if FREQSERVER != '0' %}
- so-freqserver
{% endif %}
{% if DOMAINSTATS != '0' %}
- so-domainstats
{% endif %}
heavy_node:
containers:
- so-nginx
- so-telegraf
- so-redis
- so-logstash
- so-elasticsearch
- so-curator
- so-steno
- so-suricata
- so-wazuh
- so-filebeat
{% if ZEEKVER != 'SURICATA' %}
- so-zeek
{% endif %}
helix:
containers:
- so-nginx
- so-telegraf
- so-idstools
- so-steno
- so-zeek
- so-redis
- so-logstash
- so-filebeat
hot_node:
containers:
- so-nginx
- so-telegraf
- so-logstash
- so-elasticsearch
- so-curator
manager_search:
containers:
- so-nginx
- so-telegraf
- so-soc
- so-kratos
- so-acng
- so-idstools
- so-redis
- so-logstash
- so-elasticsearch
- so-curator
- so-kibana
- so-elastalert
- so-filebeat
- so-soctopus
{% if FLEETMANAGER %}
- so-mysql
- so-fleet
- so-redis
{% endif %}
{% if WAZUH != '0' %}
- so-wazuh
{% endif %}
- so-soctopus
{% if THEHIVE != '0' %}
- so-thehive
- so-thehive-es
- so-cortex
{% endif %}
{% if PLAYBOOK != '0' %}
- so-playbook
{% endif %}
{% if FREQSERVER != '0' %}
- so-freqserver
{% endif %}
{% if DOMAINSTATS != '0' %}
- so-domainstats
{% endif %}
manager:
containers:
- so-dockerregistry
- so-nginx
- so-telegraf
{% if GRAFANA == '1' %}
- so-influxdb
- so-grafana
{% endif %}
- so-soc
- so-kratos
- so-acng
- so-idstools
- so-redis
- so-elasticsearch
- so-logstash
- so-kibana
- so-elastalert
- so-filebeat
{% if FLEETMANAGER %}
- so-mysql
- so-fleet
- so-redis
{% endif %}
{% if WAZUH != '0' %}
- so-wazuh
{% endif %}
- so-soctopus
{% if THEHIVE != '0' %}
- so-thehive
- so-thehive-es
- so-cortex
{% endif %}
{% if PLAYBOOK != '0' %}
- so-playbook
{% endif %}
{% if FREQSERVER != '0' %}
- so-freqserver
{% endif %}
{% if DOMAINSTATS != '0' %}
- so-domainstats
{% endif %}
parser_node:
containers:
- so-nginx
- so-telegraf
- so-logstash
search_node:
containers:
- so-nginx
- so-telegraf
- so-logstash
- so-elasticsearch
- so-curator
- so-filebeat
{% if WAZUH != '0' %}
- so-wazuh
{% endif %}
sensor:
containers:
- so-nginx
- so-telegraf
- so-steno
- so-suricata
{% if ZEEKVER != 'SURICATA' %}
- so-zeek
{% endif %}
- so-wazuh
- so-filebeat
warm_node:
containers:
- so-nginx
- so-telegraf
- so-elasticsearch
fleet:
containers:
{% if FLEETNODE %}
- so-mysql
- so-fleet
- so-redis
- so-filebeat
- so-nginx
- so-telegraf
{% endif %}

26
salt/common/tools/sbin/so-common
View File

@@ -302,6 +302,7 @@ retry() {
cmd=$3
expectedOutput=$4
attempt=0
local exitcode=0
while [[ $attempt -lt $maxAttempts ]]; do
attempt=$((attempt+1))
echo "Executing command with retry support: $cmd"
@@ -321,7 +322,29 @@ retry() {
sleep $sleepDelay
done
echo "Command continues to fail; giving up."
return 1
return $exitcode
}
run_check_net_err() {
local cmd=$1
local err_msg=${2:-"Unknown error occured, please check /root/$WHATWOULDYOUSAYYAHDOHERE.log for details."} # Really need to rename that variable
local no_retry=$3
local exit_code
if [[ -z $no_retry ]]; then
retry 5 60 "$cmd"
exit_code=$?
else
eval "$cmd"
exit_code=$?
fi
if [[ $exit_code -ne 0 ]]; then
ERR_HANDLED=true
[[ -z $no_retry ]] || echo "Command failed with error $exit_code"
echo "$err_msg"
exit $exit_code
fi
}
set_os() {
@@ -487,6 +510,7 @@ wait_for_web_response() {
expected=$2
maxAttempts=${3:-300}
logfile=/root/wait_for_web_response.log
truncate -s 0 "$logfile"
attempt=0
while [[ $attempt -lt $maxAttempts ]]; do
attempt=$((attempt+1))

18
salt/common/tools/sbin/so-docker-prune
View File

@@ -32,13 +32,15 @@ def get_image_version(string) -> str:
ver = string.split(':')[-1]
if ver == 'latest':
# Version doesn't like "latest", so use a high semver
return '999999.9.9'
return '99999.9.9'
else:
try:
Version(ver)
except InvalidVersion:
# Strip the last substring following a hyphen for automated branches
ver = '-'.join(ver.split('-')[:-1])
# Also return a very high semver for any version
# with a dash in it since it will likely be a dev version of some kind
if '-' in ver:
return '999999.9.9'
return ver
@@ -73,8 +75,12 @@ def main(quiet):
for tag in group:
if not quiet: print(f'Removing image {tag}')
client.images.remove(tag)
except InvalidVersion as e:
print(f'so-{get_so_image_basename(t_list[0])}: {e.args[0]}', file=sys.stderr)
except (docker.errors.APIError, InvalidVersion) as e:
print(f'so-{get_so_image_basename(t_list[0])}: {e}', file=sys.stderr)
exit(1)
except Exception as e:
print('Unhandled exception occurred:')
print(f'so-{get_so_image_basename(t_list[0])}: {e}', file=sys.stderr)
exit(1)
if no_prunable and not quiet:
@@ -86,4 +92,4 @@ if __name__ == "__main__":
main_parser.add_argument('-q', '--quiet', action='store_const', const=True, required=False)
args = main_parser.parse_args(sys.argv[1:])
main(args.quiet)
main(args.quiet)

22
salt/common/tools/sbin/so-image-common
View File

@@ -18,6 +18,7 @@
# NOTE: This script depends on so-common
IMAGEREPO=security-onion-solutions
# shellcheck disable=SC2120
container_list() {
MANAGERCHECK=$1
@@ -128,13 +129,13 @@ update_docker_containers() {
mkdir -p $SIGNPATH >> "$LOG_FILE" 2>&1
# Let's make sure we have the public key
retry 50 10 "curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" >> "$LOG_FILE" 2>&1
run_check_net_err \
"curl --retry 5 --retry-delay 60 -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" \
"Could not pull signature key file, please ensure connectivity to https://raw.gihubusercontent.com" \
noretry >> "$LOG_FILE" 2>&1
result=$?
if [[ $result -eq 0 ]]; then
cat $SIGNPATH/KEYS | gpg --import - >> "$LOG_FILE" 2>&1
else
echo "Failed to pull signature key file: $result"
exit 1
fi
# Download the containers from the interwebs
@@ -148,14 +149,15 @@ update_docker_containers() {
# Pull down the trusted docker image
local image=$i:$VERSION$IMAGE_TAG_SUFFIX
retry 50 10 "docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image" >> "$LOG_FILE" 2>&1
run_check_net_err \
"docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image" \
"Could not pull $image, please ensure connectivity to $CONTAINER_REGISTRY" >> "$LOG_FILE" 2>&1
# Get signature
retry 50 10 "curl -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" >> "$LOG_FILE" 2>&1
if [[ $? -ne 0 ]]; then
echo "Unable to pull signature file for $image" >> "$LOG_FILE" 2>&1
exit 1
fi
run_check_net_err \
"curl --retry 5 --retry-delay 60 -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" \
"Could not pull signature file for $image, please ensure connectivity to https://sigs.securityonion.net " \
noretry >> "$LOG_FILE" 2>&1
# Dump our hash values
DOCKERINSPECT=$(docker inspect $CONTAINER_REGISTRY/$IMAGEREPO/$image)

566
salt/common/tools/sbin/soup
View File

@@ -20,12 +20,79 @@
UPDATE_DIR=/tmp/sogh/securityonion
INSTALLEDVERSION=$(cat /etc/soversion)
POSTVERSION=$INSTALLEDVERSION
INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk {'print $2'})
INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk '{print $2}')
BATCHSIZE=5
SOUP_LOG=/root/soup.log
INFLUXDB_MIGRATION_LOG=/opt/so/log/influxdb/soup_migration.log
WHATWOULDYOUSAYYAHDOHERE=soup
check_err() {
local exit_code=$1
local err_msg="Unhandled error occured, please check $SOUP_LOG for details."
[[ $ERR_HANDLED == true ]] && exit $exit_code
if [[ $exit_code -ne 0 ]]; then
printf '%s' "Soup failed with error $exit_code: "
case $exit_code in
2)
echo 'No such file or directory'
;;
5)
echo 'Interrupted system call'
;;
12)
echo 'Out of memory'
;;
28)
echo 'No space left on device'
echo 'Likely ran out of space on disk, please review hardware requirements for Security Onion: https://docs.securityonion.net/en/2.3/hardware.html'
;;
30)
echo 'Read-only file system'
;;
35)
echo 'Resource temporarily unavailable'
;;
64)
echo 'Machine is not on the network'
;;
67)
echo 'Link has been severed'
;;
100)
echo 'Network is down'
;;
101)
echo 'Network is unreachable'
;;
102)
echo 'Network reset'
;;
110)
echo 'Connection timed out'
;;
111)
echo 'Connection refused'
;;
112)
echo 'Host is down'
;;
113)
echo 'No route to host'
;;
*)
echo 'Unhandled error'
echo "$err_msg"
;;
esac
if [[ $exit_code -ge 64 && $exit_code -le 113 ]]; then
echo "$err_msg"
fi
exit $exit_code
fi
}
add_common() {
cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
@@ -47,8 +114,8 @@ airgap_mounted() {
echo "Example: /home/user/securityonion-2.X.0.iso"
echo "Example: /dev/sdx1"
echo ""
read -p 'Enter the location of the iso: ' ISOLOC
if [ -f $ISOLOC ]; then
read -rp 'Enter the location of the iso: ' ISOLOC
if [[ -f $ISOLOC ]]; then
# Mounting the ISO image
mkdir -p /tmp/soagupdate
mount -t iso9660 -o loop $ISOLOC /tmp/soagupdate
@@ -60,7 +127,7 @@ airgap_mounted() {
else
echo "ISO has been mounted!"
fi
elif [ -f $ISOLOC/SecurityOnion/VERSION ]; then
elif [[ -f $ISOLOC/SecurityOnion/VERSION ]]; then
ln -s $ISOLOC /tmp/soagupdate
echo "Found the update content"
else
@@ -78,9 +145,9 @@ airgap_mounted() {
}
airgap_update_dockers() {
if [ $is_airgap -eq 0 ]; then
if [[ $is_airgap -eq 0 ]]; then
# Let's copy the tarball
if [ ! -f $AGDOCKER/registry.tar ]; then
if [[ ! -f $AGDOCKER/registry.tar ]]; then
echo "Unable to locate registry. Exiting"
exit 1
else
@@ -88,9 +155,9 @@ airgap_update_dockers() {
docker stop so-dockerregistry
docker rm so-dockerregistry
echo "Copying the new dockers over"
tar xvf $AGDOCKER/registry.tar -C /nsm/docker-registry/docker
tar xvf "$AGDOCKER/registry.tar" -C /nsm/docker-registry/docker
echo "Add Registry back"
docker load -i $AGDOCKER/registry_image.tar
docker load -i "$AGDOCKER/registry_image.tar"
fi
fi
}
@@ -102,9 +169,9 @@ update_registry() {
}
check_sudoers() {
if grep -q "so-setup" /etc/sudoers; then
echo "There is an entry for so-setup in the sudoers file, this can be safely deleted using \"visudo\"."
fi
if grep -q "so-setup" /etc/sudoers; then
echo "There is an entry for so-setup in the sudoers file, this can be safely deleted using \"visudo\"."
fi
}
check_log_size_limit() {
@@ -178,7 +245,9 @@ check_os_updates() {
echo "Continuing without updating packages"
elif [[ "$confirm" == [uU] ]]; then
echo "Applying Grid Updates"
salt \* -b 5 state.apply patch.os queue=True
set +e
run_check_net_err "salt '*' -b 5 state.apply patch.os queue=True" 'Could not apply OS updates, please check your network connection.'
set -e
else
echo "Exiting soup"
exit 0
@@ -206,7 +275,9 @@ clone_to_tmp() {
if [ -n "$BRANCH" ]; then
SOUP_BRANCH="-b $BRANCH"
fi
git clone $SOUP_BRANCH https://github.com/Security-Onion-Solutions/securityonion.git
set +e
run_check_net_err "git clone $SOUP_BRANCH https://github.com/Security-Onion-Solutions/securityonion.git" "Could not clone repo, please ensure network access to https://github.com"
set -e
cd /tmp
if [ ! -f $UPDATE_DIR/VERSION ]; then
echo "Update was unable to pull from github. Please check your internet."
@@ -253,6 +324,7 @@ preupgrade_changes_2.3.50_repo() {
echo "Checking to see if 2.3.50 repo changes are needed."
[[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]] && up_2.3.3X_to_2.3.50_repo
true
}
preupgrade_changes() {
@@ -265,6 +337,7 @@ preupgrade_changes() {
[[ "$INSTALLEDVERSION" == 2.3.0 || "$INSTALLEDVERSION" == 2.3.1 || "$INSTALLEDVERSION" == 2.3.2 || "$INSTALLEDVERSION" == 2.3.10 ]] && up_2.3.0_to_2.3.20
[[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30
[[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]] && up_2.3.3X_to_2.3.50
true
}
postupgrade_changes() {
@@ -275,6 +348,7 @@ postupgrade_changes() {
[[ "$POSTVERSION" == 2.3.20 || "$POSTVERSION" == 2.3.21 ]] && post_2.3.2X_to_2.3.30
[[ "$POSTVERSION" == 2.3.30 ]] && post_2.3.30_to_2.3.40
[[ "$POSTVERSION" == 2.3.50 ]] && post_2.3.5X_to_2.3.60
true
}
post_rc1_to_2.3.21() {
@@ -432,7 +506,7 @@ up_2.3.2X_to_2.3.30() {
sed -i "/ imagerepo: securityonion/c\ imagerepo: 'security-onion-solutions'" /opt/so/saltstack/local/pillar/global.sls
# Strelka rule repo pillar addition
if [ $is_airgap -eq 0 ]; then
if [[ $is_airgap -eq 0 ]]; then
# Add manager as default Strelka YARA rule repo
sed -i "/^strelka:/a \\ repos: \n - https://$HOSTNAME/repo/rules/strelka" /opt/so/saltstack/local/pillar/global.sls;
else
@@ -459,7 +533,7 @@ up_2.3.3X_to_2.3.50_repo() {
rm -f "/etc/yum.repos.d/$DELREPO.repo"
fi
done
if [ $is_airgap -eq 1 ]; then
if [[ $is_airgap -eq 1 ]]; then
# Copy the new repo file if not airgap
cp $UPDATE_DIR/salt/repo/client/files/centos/securityonion.repo /etc/yum.repos.d/
yum clean all
@@ -575,7 +649,7 @@ upgrade_check() {
# Let's make sure we actually need to update.
NEWVERSION=$(cat $UPDATE_DIR/VERSION)
HOTFIXVERSION=$(cat $UPDATE_DIR/HOTFIX)
CURRENTHOTFIX=$(cat /etc/sohotfix 2>/dev/null)
[[ -f /etc/sohotfix ]] && CURRENTHOTFIX=$(cat /etc/sohotfix)
if [ "$INSTALLEDVERSION" == "$NEWVERSION" ]; then
echo "Checking to see if there are hotfixes needed"
if [ "$HOTFIXVERSION" == "$CURRENTHOTFIX" ]; then
@@ -592,13 +666,14 @@ upgrade_check() {
}
upgrade_check_salt() {
NEWSALTVERSION=$(grep version: $UPDATE_DIR/salt/salt/master.defaults.yaml | awk {'print $2'})
NEWSALTVERSION=$(grep version: $UPDATE_DIR/salt/salt/master.defaults.yaml | awk '{print $2}')
if [ "$INSTALLEDSALTVERSION" == "$NEWSALTVERSION" ]; then
echo "You are already running the correct version of Salt for Security Onion."
else
UPGRADESALT=1
fi
}
upgrade_salt() {
SALTUPGRADED=True
echo "Performing upgrade of Salt from $INSTALLEDSALTVERSION to $NEWSALTVERSION."
@@ -610,7 +685,11 @@ upgrade_salt() {
yum versionlock delete "salt-*"
echo "Updating Salt packages and restarting services."
echo ""
sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable "$NEWSALTVERSION"
set +e
run_check_net_err \
"sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \
"Could not update salt, please check $SOUP_LOG for details."
set -e
echo "Applying yum versionlock for Salt."
echo ""
yum versionlock add "salt-*"
@@ -623,7 +702,11 @@ upgrade_salt() {
apt-mark unhold "salt-minion"
echo "Updating Salt packages and restarting services."
echo ""
sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable "$NEWSALTVERSION"
set +e
run_check_net_err \
"sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable \"$NEWSALTVERSION\"" \
"Could not update salt, please check $SOUP_LOG for details."
set -e
echo "Applying apt hold for Salt."
echo ""
apt-mark hold "salt-common"
@@ -648,234 +731,244 @@ verify_latest_update_script() {
cp $UPDATE_DIR/salt/common/tools/sbin/soup $DEFAULT_SALT_DIR/salt/common/tools/sbin/
cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
salt-call state.apply common queue=True
salt-call state.apply -l info common queue=True
echo ""
echo "soup has been updated. Please run soup again."
exit 0
fi
}
main () {
echo "### Preparing soup at `date` ###"
while getopts ":b" opt; do
case "$opt" in
b ) # process option b
shift
BATCHSIZE=$1
if ! [[ "$BATCHSIZE" =~ ^[0-9]+$ ]]; then
echo "Batch size must be a number greater than 0."
exit 1
fi
;;
\? )
echo "Usage: cmd [-b]"
;;
esac
done
echo "Checking to see if this is a manager."
echo ""
require_manager
set_minionid
echo "Checking to see if this is an airgap install"
echo ""
check_airgap
echo "Found that Security Onion $INSTALLEDVERSION is currently installed."
echo ""
set_os
set_palette
check_elastic_license
echo ""
if [ $is_airgap -eq 0 ]; then
# Let's mount the ISO since this is airgap
airgap_mounted
else
echo "Cloning Security Onion github repo into $UPDATE_DIR."
echo "Removing previous upgrade sources."
rm -rf $UPDATE_DIR
clone_to_tmp
fi
check_os_updates
echo ""
echo "Verifying we have the latest soup script."
verify_latest_update_script
echo ""
echo "Generating new repo archive"
generate_and_clean_tarballs
if [ -f /usr/sbin/so-image-common ]; then
. /usr/sbin/so-image-common
else
add_common
fi
echo "Let's see if we need to update Security Onion."
upgrade_check
upgrade_space
echo "Checking for Salt Master and Minion updates."
upgrade_check_salt
if [ "$is_hotfix" == "true" ]; then
echo "Applying $HOTFIXVERSION"
copy_new_files
echo ""
update_version
salt-call state.highstate -l info queue=True
main() {
set -e
trap 'check_err $?' EXIT
else
echo ""
echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION."
echo ""
echo "Updating dockers to $NEWVERSION."
if [ $is_airgap -eq 0 ]; then
airgap_update_dockers
update_centos_repo
yum clean all
check_os_updates
else
update_registry
update_docker_containers "soup"
fi
echo ""
echo "Stopping Salt Minion service."
systemctl stop salt-minion
echo "Killing any remaining Salt Minion processes."
pkill -9 -ef /usr/bin/salt-minion
echo ""
echo "Stopping Salt Master service."
systemctl stop salt-master
echo ""
preupgrade_changes_2.3.50_repo
# Does salt need upgraded. If so update it.
if [ "$UPGRADESALT" == "1" ]; then
echo "Upgrading Salt"
# Update the repo files so it can actually upgrade
upgrade_salt
fi
echo "Checking if Salt was upgraded."
echo ""
# Check that Salt was upgraded
SALTVERSIONPOSTUPGRADE=$(salt --versions-report | grep Salt: | awk {'print $2'})
if [[ "$SALTVERSIONPOSTUPGRADE" != "$NEWSALTVERSION" ]]; then
echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG."
echo "Once the issue is resolved, run soup again."
echo "Exiting."
echo ""
exit 1
else
echo "Salt upgrade success."
echo ""
fi
preupgrade_changes
echo ""
if [ $is_airgap -eq 0 ]; then
echo "Updating Rule Files to the Latest."
update_airgap_rules
fi
# Only update the repo if its airgap
if [[ $is_airgap -eq 0 ]] && [[ "$UPGRADESALT" != "1" ]]; then
update_centos_repo
fi
echo ""
echo "Copying new Security Onion code from $UPDATE_DIR to $DEFAULT_SALT_DIR."
copy_new_files
echo ""
update_version
echo ""
echo "Locking down Salt Master for upgrade"
masterlock
echo ""
echo "Starting Salt Master service."
systemctl start salt-master
# Testing that that salt-master is up by checking that is it connected to itself
retry 50 10 "salt-call state.show_top -l error" || exit 1
echo ""
echo "Ensuring python modules for Salt are installed and patched."
salt-call state.apply salt.python3-influxdb -l info queue=True
echo ""
# Only regenerate osquery packages if Fleet is enabled
FLEET_MANAGER=$(lookup_pillar fleet_manager)
FLEET_NODE=$(lookup_pillar fleet_node)
if [[ "$FLEET_MANAGER" == "True" || "$FLEET_NODE" == "True" ]]; then
echo ""
echo "Regenerating Osquery Packages.... This will take several minutes."
salt-call state.apply fleet.event_gen-packages -l info queue=True
echo ""
fi
echo ""
echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes."
salt-call state.highstate -l info queue=True
echo ""
echo "Upgrade from $INSTALLEDVERSION to $NEWVERSION complete."
echo ""
echo "Stopping Salt Master to remove ACL"
systemctl stop salt-master
masterunlock
echo ""
echo "Starting Salt Master service."
systemctl start salt-master
# Testing that that salt-master is up by checking that is it connected to itself
retry 50 10 "salt-call state.show_top -l error" || exit 1
echo "Running a highstate. This could take several minutes."
salt-call state.highstate -l info queue=True
postupgrade_changes
unmount_update
thehive_maint
if [ "$UPGRADESALT" == "1" ]; then
if [ $is_airgap -eq 0 ]; then
echo ""
echo "Cleaning repos on remote Security Onion nodes."
salt -C 'not *_eval and not *_helixsensor and not *_manager and not *_managersearch and not *_standalone and G@os:CentOS' cmd.run "yum clean all"
echo ""
fi
fi
check_sudoers
if [[ -n $lsl_msg ]]; then
case $lsl_msg in
'distributed')
echo "[INFO] The value of log_size_limit in any heavy node minion pillars may be incorrect."
echo " -> We recommend checking and adjusting the values as necessary."
echo " -> Minion pillar directory: /opt/so/saltstack/local/pillar/minions/"
echo "### Preparing soup at $(date) ###"
while getopts ":b" opt; do
case "$opt" in
b ) # process option b
shift
BATCHSIZE=$1
if ! [[ "$BATCHSIZE" =~ ^[0-9]+$ ]]; then
echo "Batch size must be a number greater than 0."
exit 1
fi
;;
'single-node')
# We can assume the lsl_details array has been set if lsl_msg has this value
echo "[WARNING] The value of log_size_limit (${lsl_details[0]}) does not match the recommended value of ${lsl_details[1]}."
echo " -> We recommend checking and adjusting the value as necessary."
echo " -> File: /opt/so/saltstack/local/pillar/minions/${lsl_details[2]}.sls"
\? )
echo "Usage: cmd [-b]"
;;
esac
done
echo "Checking to see if this is a manager."
echo ""
require_manager
set_minionid
echo "Checking to see if this is an airgap install"
echo ""
check_airgap
echo "Found that Security Onion $INSTALLEDVERSION is currently installed."
echo ""
set_os
set_palette
check_elastic_license
echo ""
if [[ $is_airgap -eq 0 ]]; then
# Let's mount the ISO since this is airgap
airgap_mounted
else
echo "Cloning Security Onion github repo into $UPDATE_DIR."
echo "Removing previous upgrade sources."
rm -rf $UPDATE_DIR
clone_to_tmp
fi
check_os_updates
echo ""
echo "Verifying we have the latest soup script."
verify_latest_update_script
echo ""
echo "Generating new repo archive"
generate_and_clean_tarballs
if [ -f /usr/sbin/so-image-common ]; then
. /usr/sbin/so-image-common
else
add_common
fi
NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l)
echo "Let's see if we need to update Security Onion."
upgrade_check
upgrade_space
if [ $NUM_MINIONS -gt 1 ]; then
echo "Checking for Salt Master and Minion updates."
upgrade_check_salt
cat << EOF
if [ "$is_hotfix" == "true" ]; then
echo "Applying $HOTFIXVERSION"
copy_new_files
echo ""
update_version
salt-call state.highstate -l info queue=True
else
echo ""
echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION."
echo ""
echo "Updating dockers to $NEWVERSION."
if [[ $is_airgap -eq 0 ]]; then
airgap_update_dockers
update_centos_repo
yum clean all
check_os_updates
else
update_registry
set +e
update_docker_containers "soup"
set -e
fi
echo ""
echo "Stopping Salt Minion service."
systemctl stop salt-minion
echo "Killing any remaining Salt Minion processes."
set +e
pkill -9 -ef /usr/bin/salt-minion
set -e
echo ""
echo "Stopping Salt Master service."
systemctl stop salt-master
echo ""
preupgrade_changes_2.3.50_repo
# Does salt need upgraded. If so update it.
if [[ $UPGRADESALT -eq 1 ]]; then
echo "Upgrading Salt"
# Update the repo files so it can actually upgrade
upgrade_salt
fi
echo "Checking if Salt was upgraded."
echo ""
# Check that Salt was upgraded
SALTVERSIONPOSTUPGRADE=$(salt --versions-report | grep Salt: | awk '{print $2}')
if [[ "$SALTVERSIONPOSTUPGRADE" != "$NEWSALTVERSION" ]]; then
echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG."
echo "Once the issue is resolved, run soup again."
echo "Exiting."
echo ""
exit 1
else
echo "Salt upgrade success."
echo ""
fi
preupgrade_changes
echo ""
if [[ $is_airgap -eq 0 ]]; then
echo "Updating Rule Files to the Latest."
update_airgap_rules
fi
# Only update the repo if its airgap
if [[ $is_airgap -eq 0 && $UPGRADESALT -ne 1 ]]; then
update_centos_repo
fi
echo ""
echo "Copying new Security Onion code from $UPDATE_DIR to $DEFAULT_SALT_DIR."
copy_new_files
echo ""
update_version
echo ""
echo "Locking down Salt Master for upgrade"
masterlock
echo ""
echo "Starting Salt Master service."
systemctl start salt-master
# Testing that salt-master is up by checking that is it connected to itself
set +e
retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check $SOUP_LOG for details."
set -e
echo ""
echo "Ensuring python modules for Salt are installed and patched."
salt-call state.apply salt.python3-influxdb -l info queue=True
echo ""
# Only regenerate osquery packages if Fleet is enabled
FLEET_MANAGER=$(lookup_pillar fleet_manager)
FLEET_NODE=$(lookup_pillar fleet_node)
if [[ "$FLEET_MANAGER" == "True" || "$FLEET_NODE" == "True" ]]; then
echo ""
echo "Regenerating Osquery Packages.... This will take several minutes."
salt-call state.apply fleet.event_gen-packages -l info queue=True
echo ""
fi
echo ""
echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes."
salt-call state.highstate -l info queue=True
echo ""
echo "Upgrade from $INSTALLEDVERSION to $NEWVERSION complete."
echo ""
echo "Stopping Salt Master to remove ACL"
systemctl stop salt-master
masterunlock
echo ""
echo "Starting Salt Master service."
systemctl start salt-master
# Testing that salt-master is up by checking that is it connected to itself
set +e
retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check $SOUP_LOG for details."
set -e
echo "Running a highstate. This could take several minutes."
salt-call state.highstate -l info queue=True
postupgrade_changes
[[ $is_airgap -eq 0 ]] && unmount_update
thehive_maint
if [[ $UPGRADESALT -eq 1 ]]; then
if [[ $is_airgap -eq 0 ]]; then
echo ""
echo "Cleaning repos on remote Security Onion nodes."
salt -C 'not *_eval and not *_helixsensor and not *_manager and not *_managersearch and not *_standalone and G@os:CentOS' cmd.run "yum clean all"
echo ""
fi
fi
check_sudoers
if [[ -n $lsl_msg ]]; then
case $lsl_msg in
'distributed')
echo "[INFO] The value of log_size_limit in any heavy node minion pillars may be incorrect."
echo " -> We recommend checking and adjusting the values as necessary."
echo " -> Minion pillar directory: /opt/so/saltstack/local/pillar/minions/"
;;
'single-node')
# We can assume the lsl_details array has been set if lsl_msg has this value
echo "[WARNING] The value of log_size_limit (${lsl_details[0]}) does not match the recommended value of ${lsl_details[1]}."
echo " -> We recommend checking and adjusting the value as necessary."
echo " -> File: /opt/so/saltstack/local/pillar/minions/${lsl_details[2]}.sls"
;;
esac
fi
NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l)
if [[ $NUM_MINIONS -gt 1 ]]; then
cat << EOF
@@ -889,10 +982,10 @@ For more information, please see https://docs.securityonion.net/en/2.3/soup.html
EOF
fi
fi
fi
echo "### soup has been served at `date` ###"
echo "### soup has been served at $(date) ###"
}
cat << EOF
@@ -907,6 +1000,7 @@ Press Enter to continue or Ctrl-C to cancel.
EOF
read input
read -r input
main "$@" | tee -a $SOUP_LOG

2
salt/grafana/init.sls
View File

@@ -11,7 +11,7 @@
{% set GRAFANA_SETTINGS = salt['grains.filter_by'](default_settings, default='grafana', merge=salt['pillar.get']('grafana', {})) %}
{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone'] and GRAFANA == 1 %}
{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone'] or (grains.role == 'so-eval' and GRAFANA == 1) %}
# Grafana all the things
grafanadir:

5
salt/influxdb/init.sls
View File

@@ -3,7 +3,7 @@
{% set GRAFANA = salt['pillar.get']('manager:grafana', '0') %}
{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone'] and GRAFANA == 1 %}
{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone'] or (grains.role == 'so-eval' and GRAFANA == 1) %}
{% set MANAGER = salt['grains.get']('master') %}
{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
@@ -17,7 +17,6 @@
include:
- salt.minion
- salt.python3-influxdb
# Influx DB
influxconfdir:
file.directory:
@@ -139,4 +138,4 @@ so_downsample_cq:
test.fail_without_changes:
- name: {{sls}}_state_not_allowed
{% endif %}
{% endif %}

16
salt/salt/minion.sls
View File

@@ -44,12 +44,24 @@ hold_salt_packages:
{% endfor %}
{% endif %}
remove_info_log_level_logfile:
file.line:
- name: /etc/salt/minion
- match: "log_level_logfile: info"
- mode: delete
remove_info_log_level:
file.line:
- name: /etc/salt/minion
- match: "log_level: info"
- mode: delete
set_log_levels:
file.append:
- name: /etc/salt/minion
- text:
- "log_level: info"
- "log_level_logfile: info"
- "log_level: error"
- "log_level_logfile: error"
- listen_in:
- service: salt_minion_service

2
salt/soc/files/soc/soc.json
View File

@@ -54,7 +54,7 @@
"verifyCert": false
},
"influxdb": {
{%- if grains['role'] in ['so-import'] %}
{%- if grains['role'] in ['so-import'] or (grains['role'] == 'so-eval' and GRAFANA == 0) %}
"hostUrl": "",
{%- else %}
"hostUrl": "https://{{ MANAGERIP }}:8086",

11
salt/top.sls
View File

@@ -38,7 +38,6 @@ base:
- patch.os.schedule
- motd
- salt.minion-check
- sensoroni
- salt.lasthighstate
'*_helixsensor and G@saltversion:{{saltversion}}':
@@ -47,6 +46,7 @@ base:
- ca
- ssl
- registry
- sensoroni
- telegraf
- firewall
- idstools
@@ -66,6 +66,7 @@ base:
- match: compound
- ca
- ssl
- sensoroni
- telegraf
- firewall
- nginx
@@ -92,6 +93,7 @@ base:
- ca
- ssl
- registry
- sensoroni
- manager
- nginx
- telegraf
@@ -160,6 +162,7 @@ base:
- ca
- ssl
- registry
- sensoroni
- nginx
- telegraf
- influxdb
@@ -220,6 +223,7 @@ base:
- ca
- ssl
- registry
- sensoroni
- manager
- nginx
- telegraf
@@ -290,6 +294,7 @@ base:
- match: compound
- ca
- ssl
- sensoroni
- nginx
- telegraf
- firewall
@@ -320,6 +325,7 @@ base:
- ca
- ssl
- registry
- sensoroni
- nginx
- telegraf
- influxdb
@@ -382,6 +388,7 @@ base:
- match: compound
- ca
- ssl
- sensoroni
- nginx
- telegraf
- firewall
@@ -424,6 +431,7 @@ base:
- match: compound
- ca
- ssl
- sensoroni
- nginx
- telegraf
- firewall
@@ -441,6 +449,7 @@ base:
- ca
- ssl
- registry
- sensoroni
- manager
- nginx
- soc

1
salt/zeek/init.sls
View File

@@ -78,6 +78,7 @@ zeekspoolownership:
file.directory:
- name: /nsm/zeek/spool
- user: 937
- max_depth: 0
- recurse:
- user

2
setup/automation/distributed-iso-sensor
View File

@@ -34,7 +34,7 @@ ZEEKVERSION=ZEEK
# HELIXAPIKEY=
HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12
HNSENSOR=inherit
HOSTNAME=distributed-sensor
HOSTNAME=Distributed-Sensor
install_type=SENSOR
# LSINPUTBATCHCOUNT=
# LSINPUTTHREADS=

7
setup/so-functions
View File

@@ -1348,15 +1348,16 @@ filter_unused_nics() {
nic_list=()
for nic in "${filtered_nics[@]}"; do
local nic_mac=$(cat "/sys/class/net/${nic}/address" 2>/dev/null)
case $(cat "/sys/class/net/${nic}/carrier" 2>/dev/null) in
1)
nic_list+=("$nic" "Link UP " "OFF")
nic_list+=("$nic" "$nic_mac Link UP " "OFF")
;;
0)
nic_list+=("$nic" "Link DOWN " "OFF")
nic_list+=("$nic" "$nic_mac Link DOWN " "OFF")
;;
*)
nic_list+=("$nic" "Link UNKNOWN " "OFF")
nic_list+=("$nic" "$nic_mac Link UNKNOWN " "OFF")
;;
esac
done

12
setup/so-whiptail
View File

@@ -408,6 +408,7 @@ whiptail_enable_components() {
PLAYBOOK=0
STRELKA=0
if [[ $is_eval ]]; then
COMPONENTS=$(whiptail --title "Security Onion Setup" --checklist \
"Select Components to install:" 20 75 8 \
GRAFANA "Enable Grafana for system monitoring" ON \
@@ -416,6 +417,17 @@ whiptail_enable_components() {
THEHIVE "Enable TheHive" ON \
PLAYBOOK "Enable Playbook" ON \
STRELKA "Enable Strelka" ON 3>&1 1>&2 2>&3)
else
COMPONENTS=$(whiptail --title "Security Onion Setup" --checklist \
"Select Components to install:" 20 75 7 \
OSQUERY "Enable Fleet with osquery" ON \
WAZUH "Enable Wazuh" ON \
THEHIVE "Enable TheHive" ON \
PLAYBOOK "Enable Playbook" ON \
STRELKA "Enable Strelka" ON 3>&1 1>&2 2>&3)
export "GRAFANA=1"
fi
local exitstatus=$?
whiptail_check_exitstatus $exitstatus

BIN
sigs/securityonion-2.3.51.iso.sig Normal file
View File

Binary file not shown.