Merge pull request #3032 from Security-Onion-Solutions/dev

Update kilo to latest dev
This commit is contained in:
Doug Burks
2021-02-19 04:53:23 -08:00
committed by GitHub
12 changed files with 197 additions and 90 deletions

View File

@@ -6,5 +6,17 @@
nocompress
create
sharedscripts
endscript
}
/nsm/strelka/log/strelka.log
{
daily
rotate 14
missingok
copytruncate
compress
create
extension .log
dateext
dateyesterday
}

View File

@@ -268,7 +268,7 @@ docker:
# Reserve OS ports for Docker proxy in case boot settings are not already applied/present
dockerapplyports:
cmd.run:
- name: if [ ! -f /etc/sysctl.d/99-reserved-ports.conf ]; then sysctl -w net.ipv4.ip_local_reserved_ports="55000,57314"; fi
- name: if [ ! -s /etc/sysctl.d/99-reserved-ports.conf ]; then sysctl -w net.ipv4.ip_local_reserved_ports="55000,57314"; fi
# Reserve OS ports for Docker proxy
dockerreserveports:

View File

@@ -38,9 +38,10 @@ if [ ! -f $BACKUPFILE ]; then
fi
# Find oldest backup file and remove it
# Find oldest backup files and remove them
NUMBACKUPS=$(find /nsm/backup/ -type f -name "so-config-backup*" | wc -l)
OLDESTBACKUP=$(find /nsm/backup/ -type f -name "so-config-backup*" | ls -1t | tail -1)
if [ "$NUMBACKUPS" -gt "$MAXBACKUPS" ]; then
rm -f /nsm/backup/$OLDESTBACKUP
fi
while [ "$NUMBACKUPS" -gt "$MAXBACKUPS" ]; do
OLDESTBACKUP=$(find /nsm/backup/ -type f -name "so-config-backup*" -type f -printf '%T+ %p\n' | sort | head -n 1 | awk -F" " '{print $2}')
rm -f $OLDESTBACKUP
NUMBACKUPS=$(find /nsm/backup/ -type f -name "so-config-backup*" | wc -l)
done

View File

@@ -55,12 +55,14 @@ rulesdir:
- group: 939
- makedirs: True
# Don't show changes because all.rules can be large
synclocalnidsrules:
file.recurse:
- name: /opt/so/rules/nids/
- source: salt://idstools/
- user: 939
- group: 939
- show_changes: False
- include_pat: 'E@.rules'
so-idstools:

View File

@@ -0,0 +1,32 @@
#!/bin/bash
#
# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.. /usr/sbin/so-common
APP=so-suricata-eve-clean
lf=/tmp/$APP-pidLockFile
# create empty lock file if none exists
cat /dev/null >> $lf
read lastPID < $lf
# if lastPID is not null and a process with that pid exists , exit
[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
echo $$ > $lf
MAXDAYS={{ salt['pillar.get']('suricata:cleanup:eve_json', 7) }}
if [ "$MAXDAYS" -lt 1 ]; then
MAXDAYS=1
fi
find /nsm/suricata/ -type f -mtime +$MAXDAYS -name 'eve-*.json*' -execdir rm -- '{}' \;

View File

@@ -1,6 +1,6 @@
#!/bin/bash
# Gzip the eve logs
find /nsm/suricata/eve*.json -type f -printf '%T@\t%p\n' | sort -t $'\t' -g | head -n -1 | cut -d $'\t' -f 2- | xargs nice gzip
find /nsm/suricata/eve*.json -type f -printf '%T@\t%p\n' | sort -t $'\t' -g | head -n -1 | cut -d $'\t' -f 2 | xargs nice gzip
# TODO Add stats log

View File

@@ -74,6 +74,7 @@ surirulesync:
- source: salt://suricata/rules/
- user: 940
- group: 940
- show_changes: False
surilogscript:
file.managed:
@@ -177,6 +178,27 @@ disable_so-suricata_so-status.conf:
- month: '*'
- dayweek: '*'
so-suricata-eve-clean:
file.managed:
- name: /usr/sbin/so-suricata-eve-clean
- user: root
- group: root
- mode: 755
- template: jinja
- source: salt://suricata/cron/so-suricata-eve-clean
# Add eve clean cron
clean_suricata_eve_files:
cron.present:
- name: /usr/sbin/so-suricata-eve-clean > /dev/null 2>&1
- identifier: clean_suricata_eve_files
- user: root
- minute: '10'
- hour: '0'
- daymonth: '*'
- month: '*'
- dayweek: '*'
{% else %}
{{sls}}_state_not_allowed:

View File

@@ -34,7 +34,7 @@ GRAFANA=1
# HELIXAPIKEY=
HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12
HNSENSOR=inherit
HOSTNAME=standalone
HOSTNAME=eval
install_type=EVAL
# LSINPUTBATCHCOUNT=
# LSINPUTTHREADS=

View File

@@ -34,8 +34,8 @@ GRAFANA=1
# HELIXAPIKEY=
HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12
HNSENSOR=inherit
HOSTNAME=standalone
install_type=STANDALONE
HOSTNAME=eval
install_type=EVAL
# LSINPUTBATCHCOUNT=
# LSINPUTTHREADS=
# LSPIPELINEBATCH=

View File

@@ -398,6 +398,13 @@ collect_hostname() {
whiptail_set_hostname "$HOSTNAME"
if [[ $HOSTNAME == 'securityonion' ]]; then # Will only check HOSTNAME=securityonion once
if ! (whiptail_avoid_default_hostname); then
whiptail_set_hostname
fi
fi
while ! valid_hostname "$HOSTNAME"; do
whiptail_invalid_hostname
whiptail_set_hostname "$HOSTNAME"
@@ -963,15 +970,16 @@ installer_prereq_packages() {
elif [ "$OS" == ubuntu ]; then
# Print message to stdout so the user knows setup is doing something
echo "Installing required packages to run installer..."
retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1
# Install network manager so we can do interface stuff
if ! command -v nmcli > /dev/null 2>&1; then
retry 50 10 "apt-get install -y network-manager" >> "$setup_log" 2>&1 || exit 1
retry 50 10 "apt-get -y install network-manager" >> "$setup_log" 2>&1 || exit 1
{
systemctl enable NetworkManager
systemctl start NetworkManager
} >> "$setup_log" 2<&1
fi
retry 50 10 "apt-get install -y bc curl" >> "$setup_log" 2>&1 || exit 1
retry 50 10 "apt-get -y install bc curl" >> "$setup_log" 2>&1 || exit 1
fi
}
@@ -1487,50 +1495,50 @@ manager_global() {
fi
if [ -z "$DOCKERNET" ]; then
DOCKERNET=172.17.0.0
DOCKERBIP=$(echo $DOCKERNET | awk -F'.' '{print $1,$2,$3,1}' OFS='.')/24
DOCKERNET=172.17.0.0
DOCKERBIP=$(echo $DOCKERNET | awk -F'.' '{print $1,$2,$3,1}' OFS='.')/24
else
DOCKERBIP=$(echo $DOCKERNET | awk -F'.' '{print $1,$2,$3,1}' OFS='.')/24
DOCKERBIP=$(echo $DOCKERNET | awk -F'.' '{print $1,$2,$3,1}' OFS='.')/24
fi
# Create a global file for global values
printf '%s\n'\
"global:"\
" soversion: '$SOVERSION'"\
" hnmanager: '$HNMANAGER'"\
" ntpserver: '$NTPSERVER'"\
" dockernet: '$DOCKERNET'"\
" proxy: '$PROXY'"\
" mdengine: '$ZEEKVERSION'"\
" ids: '$NIDS'"\
" url_base: '$REDIRECTIT'"\
" managerip: '$MAINIP'" > "$global_pillar"
if [[ $is_airgap ]]; then
printf '%s\n'\
" airgap: True"\ >> "$global_pillar"
else
printf '%s\n'\
" airgap: False"\ >> "$global_pillar"
fi
printf '%s\n'\
"global:"\
" soversion: '$SOVERSION'"\
" hnmanager: '$HNMANAGER'"\
" ntpserver: '$NTPSERVER'"\
" dockernet: '$DOCKERNET'"\
" proxy: '$PROXY'"\
" mdengine: '$ZEEKVERSION'"\
" ids: '$NIDS'"\
" url_base: '$REDIRECTIT'"\
" managerip: '$MAINIP'" > "$global_pillar"
if [[ $is_airgap ]]; then
printf '%s\n'\
" airgap: True"\ >> "$global_pillar"
else
printf '%s\n'\
" airgap: False"\ >> "$global_pillar"
fi
# Check if TheHive is enabled. If so, add creds and other details
if [[ "$THEHIVE" == "1" ]]; then
printf '%s\n'\
" hiveuser: '$WEBUSER'"\
" hivepassword: '$WEBPASSWD1'"\
" hivekey: '$HIVEKEY'"\
" hiveplaysecret: '$HIVEPLAYSECRET'"\
" cortexuser: '$WEBUSER'"\
" cortexpassword: '$WEBPASSWD1'"\
" cortexkey: '$CORTEXKEY'"\
" cortexorgname: 'SecurityOnion'"\
" cortexorguser: 'soadmin'"\
" cortexorguserkey: '$CORTEXORGUSERKEY'"\
" cortexplaysecret: '$CORTEXPLAYSECRET'" >> "$global_pillar"
fi
# Check if TheHive is enabled. If so, add creds and other details
if [[ "$THEHIVE" == "1" ]]; then
printf '%s\n'\
" hiveuser: '$WEBUSER'"\
" hivepassword: '$WEBPASSWD1'"\
" hivekey: '$HIVEKEY'"\
" hiveplaysecret: '$HIVEPLAYSECRET'"\
" cortexuser: '$WEBUSER'"\
" cortexpassword: '$WEBPASSWD1'"\
" cortexkey: '$CORTEXKEY'"\
" cortexorgname: 'SecurityOnion'"\
" cortexorguser: 'soadmin'"\
" cortexorguserkey: '$CORTEXORGUSERKEY'"\
" cortexplaysecret: '$CORTEXPLAYSECRET'" >> "$global_pillar"
fi
# Continue adding other details
# Continue adding other details
printf '%s\n'\
" fleet_custom_hostname: "\
" fleet_manager: False"\
@@ -1541,7 +1549,7 @@ manager_global() {
" fleet_ip: 'N/A'"\
" sensoronikey: '$SENSORONIKEY'"\
" wazuh: $WAZUH"\
" managerupdate: $MANAGERUPDATES"\
" managerupdate: $MANAGERUPDATES"\
" imagerepo: '$IMAGEREPO'"\
" pipeline: 'redis'"\
"sensoroni:"\
@@ -1557,22 +1565,22 @@ manager_global() {
" features: False"\
"elasticsearch:"\
" replicas: 0" >> "$global_pillar"
if [ -n "$ESCLUSTERNAME" ]; then
printf '%s\n'\
" true_cluster: True"\
" true_cluster_name: '$ESCLUSTERNAME'" >> "$global_pillar"
else
printf '%s\n'\
" true_cluster: False"\
" true_cluster_name: 'so'" >> "$global_pillar"
fi
if [ -n "$ESCLUSTERNAME" ]; then
printf '%s\n'\
" true_cluster: True"\
" true_cluster_name: '$ESCLUSTERNAME'" >> "$global_pillar"
else
printf '%s\n'\
" true_cluster: False"\
" true_cluster_name: 'so'" >> "$global_pillar"
fi
printf '%s\n'\
" discovery_nodes: 1"\
" hot_warm_enabled: False"\
" cluster_routing_allocation_disk.threshold_enabled: true"\
" cluster_routing_allocation_disk_watermark_low: '95%'"\
" cluster_routing_allocation_disk_watermark_high: '98%'"\
" cluster_routing_allocation_disk_watermark_flood_stage: '98%'"\
" cluster_routing_allocation_disk_watermark_low: '95%'"\
" cluster_routing_allocation_disk_watermark_high: '98%'"\
" cluster_routing_allocation_disk_watermark_flood_stage: '98%'"\
" index_settings:"\
" so-beats:"\
" shards: 1"\
@@ -1633,10 +1641,10 @@ manager_global() {
" upload_queue_size: 4"\
" encoding: 'gzip'"\
" interval: 5"\
"backup:"\
" locations:"\
" - /opt/so/saltstack/local"\
"soctopus:"\
"backup:"\
" locations:"\
" - /opt/so/saltstack/local"\
"soctopus:"\
" playbook:"\
" rulesets:"\
" - windows"\
@@ -1790,7 +1798,7 @@ reinstall_init() {
local service_retry_count=20
{
if command -v salt-call &> /dev/null; then
if command -v salt-call &> /dev/null && grep -q "master:" /etc/salt/minion 2> /dev/null; then
# Disable schedule so highstate doesn't start running during the install
salt-call -l info schedule.disable
@@ -1950,9 +1958,7 @@ saltify() {
} >> "$setup_log" 2>&1
yum versionlock salt*
else
if ! (DEBIAN_FRONTEND=noninteractive retry 50 10 "apt-get -y -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" upgrade" >> "$setup_log" 2>&1); then
exit 1
fi
DEBIAN_FRONTEND=noninteractive retry 50 10 "apt-get -y -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" upgrade" >> "$setup_log" 2>&1 || exit 1
if [ $OSVER != "xenial" ]; then
# Switch to Python 3 as default if this is not xenial
@@ -2277,7 +2283,7 @@ sensor_pillar() {
set_default_log_size() {
local percentage
case $INSTALLTYPE in
case $install_type in
STANDALONE | EVAL | HEAVYNODE)
percentage=50
;;

View File

@@ -379,6 +379,11 @@ if [[ $is_import ]]; then
PLAYBOOK=0
fi
if [[ $is_airgap ]]; then
PATCHSCHEDULENAME=manual
MANAGERUPDATES=0
fi
# Start user prompts
if [[ $is_helix ]]; then
@@ -393,7 +398,7 @@ if [[ $is_helix || $is_sensor || $is_import ]]; then
calculate_useable_cores
fi
if [[ ! $is_import ]]; then
if [[ ! $is_airgap && ! $is_import ]]; then
collect_patch_schedule
fi
@@ -446,7 +451,7 @@ if [[ $is_manager || $is_import ]]; then
get_redirect
fi
if [[ $is_distmanager || ( $is_sensor || $is_node || $is_fleet_standalone ) && ! $is_eval ]]; then
if [[ ! $is_airgap && ( $is_distmanager || ( $is_sensor || $is_node || $is_fleet_standalone ) && ! $is_eval ) ]]; then
whiptail_manager_updates
if [[ $setup_type == 'network' && $MANAGERUPDATES == 1 ]]; then
whiptail_manager_updates_warning
@@ -703,32 +708,33 @@ set_redirect >> $setup_log 2>&1
set_progress_str 60 "$(print_salt_state_apply 'manager')"
salt-call state.apply -l info manager >> $setup_log 2>&1
set_progress_str 61 "$(print_salt_state_apply 'idstools')"
create_local_nids_rules >> $setup_log 2>&1
salt-call state.apply -l info idstools >> $setup_log 2>&1
set_progress_str 61 "$(print_salt_state_apply 'suricata.manager')"
salt-call state.apply -l info suricata.manager >> $setup_log 2>&1
fi
set_progress_str 62 "$(print_salt_state_apply 'firewall')"
set_progress_str 61 "$(print_salt_state_apply 'firewall')"
salt-call state.apply -l info firewall >> $setup_log 2>&1
if [ $OS = 'centos' ]; then
set_progress_str 63 'Installing Yum utilities'
set_progress_str 62 'Installing Yum utilities'
salt-call state.apply -l info yum.packages >> $setup_log 2>&1
fi
set_progress_str 63 "$(print_salt_state_apply 'common')"
set_progress_str 62 "$(print_salt_state_apply 'common')"
salt-call state.apply -l info common >> $setup_log 2>&1
if [[ ! $is_helix ]]; then
set_progress_str 64 "$(print_salt_state_apply 'nginx')"
set_progress_str 62 "$(print_salt_state_apply 'nginx')"
salt-call state.apply -l info nginx >> $setup_log 2>&1
fi
if [[ $is_manager || $is_helix || $is_import ]]; then
set_progress_str 63 "$(print_salt_state_apply 'idstools')"
create_local_nids_rules >> $setup_log 2>&1
salt-call state.apply -l info idstools >> $setup_log 2>&1
set_progress_str 63 "$(print_salt_state_apply 'suricata.manager')"
salt-call state.apply -l info suricata.manager >> $setup_log 2>&1
fi
if [[ $is_manager || $is_node || $is_import || $is_helix ]]; then
set_progress_str 64 "$(print_salt_state_apply 'elasticsearch')"
salt-call state.apply -l info elasticsearch >> $setup_log 2>&1

View File

@@ -28,6 +28,20 @@ whiptail_airgap() {
whiptail_check_exitstatus $exitstatus
}
whiptail_avoid_default_hostname() {
[ -n "$TESTING" ] && return
read -r -d '' message <<- EOM
To prevent hostname conflicts, avoid using the default 'securityonion' hostname in a distributed environment.
You can choose to use this default hostname anyway, or change it to a new hostname.
EOM
whiptail --title "Security Onion Setup" \
--yesno "$message" 11 75 \
--yes-button "Use Anyway" --no-button "Change" --defaultno
}
whiptail_basic_suri() {
[ -n "$TESTING" ] && return
@@ -937,13 +951,25 @@ whiptail_metadata_tool() {
[ -n "$TESTING" ] && return
read -r -d '' message <<- EOM
What tool would you like to use to generate metadata?
This question is asking specifically about metadata, which would be things like the connection log, DNS log, HTTP log, etc. This does not include NIDS alerts.
If you choose Zeek for metadata, Suricata will still run to generate NIDS alerts.
If you choose Suricata for metadata, it will generate NIDS alerts and metadata, and Zeek will not run at all.
EOM
# Legacy variable naming
ZEEKVERSION=$(whiptail --title "Security Onion Setup" --radiolist "What tool would you like to use to generate metadata?" 20 75 4 \
"ZEEK" "Zeek (formerly known as Bro)" ON \
"SURICATA" "Suricata" OFF 3>&1 1>&2 2>&3)
ZEEKVERSION=$(whiptail --title "Security Onion Setup" --menu "$message" 20 75 2 \
"Zeek " "Use Zeek (Bro) for metadata and Suricata for NIDS alerts" \
"Suricata " "Use Suricata for both metadata and NIDS alerts" 3>&1 1>&2 2>&3)
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
ZEEKVERSION=$(echo "${ZEEKVERSION^^}" | tr -d ' ')
}
whiptail_nids() {