diff --git a/salt/common/tools/sbin/so-zeek-stats b/salt/common/tools/sbin/so-zeek-stats index ff89c6506..d3e9b40a6 100755 --- a/salt/common/tools/sbin/so-zeek-stats +++ b/salt/common/tools/sbin/so-zeek-stats @@ -24,11 +24,11 @@ show_stats() { echo echo "Average throughput:" echo - docker exec -it so-zeek /opt/zeek/bin/zeekctl capstats + docker exec so-zeek env -i PATH=/bin:/usr/bin:/sbin:/usr/sbin:/opt/bin:/usr/local/bin:/usr/local/sbin runuser -l zeek -c '/opt/zeek/bin/zeekctl capstats' echo echo "Average packet loss:" echo - docker exec -it so-zeek /opt/zeek/bin/zeekctl netstats + docker exec so-zeek env -i PATH=/bin:/usr/bin:/sbin:/usr/sbin:/opt/bin:/usr/local/bin:/usr/local/sbin runuser -l zeek -c '/opt/zeek/bin/zeekctl netstats' echo } diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index c1b649610..1ea63b70b 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -120,6 +120,72 @@ check_sudoers() { fi } +check_log_size_limit() { + local wait_for_enter=false + + local num_minion_pillars + num_minion_pillars=$(find /opt/so/saltstack/local/pillar/minions/ -type f | wc -l) + + if [[ $num_minion_pillars -gt 1 ]]; then + if find /opt/so/saltstack/local/pillar/minions/ -type f | grep -q "_heavynode"; then + wait_for_enter=true + echo "[INFO] The value of log_size_limit in any heavy node minion pillars may be incorrect." + echo " -> We recommend checking and adjusting the values as necessary." + echo " -> Minion pillar directory: /opt/so/saltstack/local/pillar/minions/" + fi + else + local minion_id + minion_id=$(lookup_salt_value "id" "" "grains") + + local minion_arr + IFS='_' read -ra minion_arr <<< "$minion_id" + + local node_type="${minion_arr[0]}" + + local current_limit + current_limit=$(lookup_pillar "log_size_limit" "elasticsearch") + + local percent + case $node_type in + 'standalone' | 'eval') + percent=50 + ;; + *) + percent=80 + ;; + esac + + local disk_dir="/" + if [ -d /nsm ]; then + disk_dir="/nsm" + fi + + local disk_size_1k + disk_size_1k=$(df $disk_dir | grep -v "^Filesystem" | awk '{print $2}') + + local ratio="1048576" + + local disk_size_gb + disk_size_gb=$( echo "$disk_size_1k" "$ratio" | awk '{print($1/$2)}' ) + + local new_limit + new_limit=$( echo "$disk_size_gb" "$percent" | awk '{printf("%.0f", $1 * ($2/100))}') + + if [[ $current_limit != "$new_limit" ]]; then + wait_for_enter=true + echo "[WARNING] The value of log_size_limit (${current_limit}) does not match the recommended value of ${new_limit}." + echo " -> We recommend checking and adjusting the value as necessary." + echo " -> File: /opt/so/saltstack/local/pillar/minions/${minion_id}.sls" + fi + + if [[ $wait_for_enter == true ]]; then + echo "" + read -n 1 -s -r -p "Press any key to continue..." + echo "" # Since read doesn't print a newline, print one for it + fi + fi +} + clean_dockers() { # Place Holder for cleaning up old docker images echo "Trying to clean up old dockers." @@ -159,7 +225,7 @@ generate_and_clean_tarballs() { local new_version new_version=$(cat $UPDATE_DIR/VERSION) [ -d /opt/so/repo ] || mkdir -p /opt/so/repo - tar -cxf "/opt/so/repo/$new_version.tar.gz" "$UPDATE_DIR" + tar -czf "/opt/so/repo/$new_version.tar.gz" "$UPDATE_DIR" find "/opt/so/repo" -type f -not -name "$new_version.tar.gz" -exec rm -rf {} \; } @@ -338,8 +404,22 @@ up_2.3.2X_to_2.3.30() { # Replace any curly brace scalars with the same scalar in single quotes readarray -t minion_pillars <<< "$(find /opt/so/saltstack/local/pillar/minions -type f -name '*.sls')" for pillar in "${minion_pillars[@]}"; do - sed -i -r "s/ (\{\{.*}})$/ '\1'/g" "$pillar" + sed -i -r "s/ (\{\{.*}})$/ '\1'/g" "$pillar" done + +# Change the IMAGEREPO + sed -i "/ imagerepo: 'securityonion'/c\ imagerepo: 'security-onion-solutions'" /opt/so/saltstack/local/pillar/global.sls + sed -i "/ imagerepo: securityonion/c\ imagerepo: 'security-onion-solutions'" /opt/so/saltstack/local/pillar/global.sls + +# Strelka rule repo pillar addition + if [ $is_airgap -eq 0 ]; then + # Add manager as default Strelka YARA rule repo + sed -i "/^strelka:/a \\ repos: \n - https://$HOSTNAME/repo/rules/strelka" /opt/so/saltstack/local/pillar/global.sls; + else + # Add Github repo for Strelka YARA rules + sed -i "/^strelka:/a \\ repos: \n - https://github.com/Neo23x0/signature-base" /opt/so/saltstack/local/pillar/global.sls; + fi + check_log_size_limit } space_check() { @@ -662,6 +742,8 @@ fi check_sudoers + + } main "$@" | tee /dev/fd/3 diff --git a/salt/docker_clean/init.sls b/salt/docker_clean/init.sls index 21e672c00..006d108eb 100644 --- a/salt/docker_clean/init.sls +++ b/salt/docker_clean/init.sls @@ -3,7 +3,7 @@ {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} -{% set OLDVERSIONS = ['2.0.0-rc.1','2.0.1-rc.1','2.0.2-rc.1','2.0.3-rc.1','2.1.0-rc.2','2.2.0-rc.3','2.3.0','2.3.1','2.3.2']%} +{% set OLDVERSIONS = ['2.0.0-rc.1','2.0.1-rc.1','2.0.2-rc.1','2.0.3-rc.1','2.1.0-rc.2','2.2.0-rc.3','2.3.0','2.3.1','2.3.2','2.3.10','2.3.20']%} {% for VERSION in OLDVERSIONS %} remove_images_{{ VERSION }}: @@ -45,6 +45,41 @@ remove_images_{{ VERSION }}: - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-thehive-es:{{ VERSION }}' - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-wazuh:{{ VERSION }}' - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-zeek:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-acng:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-thehive-cortex:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-curator:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-domainstats:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-elastalert:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-elasticsearch:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-filebeat:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-fleet:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-fleet-launcher:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-freqserver:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-grafana:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-idstools:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-influxdb:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-kibana:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-kratos:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-logstash:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-minio:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-mysql:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-nginx:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-pcaptools:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-playbook:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-redis:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-soc:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-soctopus:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-steno:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-strelka-frontend:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-strelka-manager:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-strelka-backend:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-strelka-filestream:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-suricata:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-telegraf:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-thehive:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-thehive-es:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-wazuh:{{ VERSION }}' + - '{{ MANAGER }}:5000/securityonion/so-zeek:{{ VERSION }}' {% endfor %} {% else %} @@ -53,4 +88,4 @@ remove_images_{{ VERSION }}: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} \ No newline at end of file +{% endif %} diff --git a/salt/registry/bin/so-buildregistry b/salt/registry/bin/so-buildregistry deleted file mode 100644 index 01756fc67..000000000 --- a/salt/registry/bin/so-buildregistry +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -VERSION=HH1.1.4 -TARBALL=/nsm/docker-registry/docker/so-dockers-$VERSION.tar - -# See if the tarball is there. If so do soemthing otherwise peace out. -if [ -f "$TARBALL" ]; then - cd /nsm/docker-registry/docker - tar xvf so-dockers-$VERSION.tar -fi - -exit 0 diff --git a/salt/registry/init.sls b/salt/registry/init.sls index d733b6f54..1cec55fd2 100644 --- a/salt/registry/init.sls +++ b/salt/registry/init.sls @@ -29,17 +29,6 @@ dockerregistryconf: - name: /opt/so/conf/docker-registry/etc/config.yml - source: salt://registry/etc/config.yml -# Copy the registry script -#dockerregistrybuild: -# file.managed: -# - name: /opt/so/conf/docker-registry/so-buildregistry -# - source: salt://registry/bin/so-buildregistry -# - mode: 755 - -#dockerexpandregistry: -# cmd.run: -# - name: /opt/so/conf/docker-registry/so-buildregistry - # Install the registry container so-dockerregistry: docker_container.running: diff --git a/salt/zeek/cron/packetloss.sh b/salt/zeek/cron/packetloss.sh index c8750dd92..0a3df7a18 100755 --- a/salt/zeek/cron/packetloss.sh +++ b/salt/zeek/cron/packetloss.sh @@ -1,2 +1,2 @@ #!/bin/bash -/usr/bin/docker exec so-zeek /opt/zeek/bin/zeekctl netstats | awk '{print $(NF-2),$(NF-1),$NF}' | awk -F '[ =]' '{RCVD += $2;DRP += $4;TTL += $6} END { print "rcvd: " RCVD, "dropped: " DRP, "total: " TTL}' >> /nsm/zeek/logs/packetloss.log 2>&1 +/usr/bin/docker exec so-zeek env -i PATH=/bin:/usr/bin:/sbin:/usr/sbin:/opt/bin:/usr/local/bin:/usr/local/sbin runuser -l zeek -c '/opt/zeek/bin/zeekctl netstats' | awk '{print $(NF-2),$(NF-1),$NF}' | awk -F '[ =]' '{RCVD += $2;DRP += $4;TTL += $6} END { print "rcvd: " RCVD, "dropped: " DRP, "total: " TTL}' >> /nsm/zeek/logs/packetloss.log 2>&1 diff --git a/salt/zeek/init.sls b/salt/zeek/init.sls index 8cb9f5d21..fe6478464 100644 --- a/salt/zeek/init.sls +++ b/salt/zeek/init.sls @@ -73,6 +73,14 @@ zeekpolicysync: - group: 939 - template: jinja +# Ensure the zeek spool tree (and state.db) ownership is correct +zeekspoolownership: + file.directory: + - name: /nsm/zeek/spool + - user: 937 + - recurse: + - user + # Sync Intel zeekintelloadsync: file.managed: diff --git a/setup/automation/distributed-net-ubuntu-manager-suricata b/setup/automation/distributed-net-ubuntu-manager-suricata new file mode 100644 index 000000000..d1fdf158d --- /dev/null +++ b/setup/automation/distributed-net-ubuntu-manager-suricata @@ -0,0 +1,77 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +TESTING=true + +address_type=DHCP +ADMINUSER=onionuser +ADMINPASS1=onionuser +ADMINPASS2=onionuser +ALLOW_CIDR=0.0.0.0/0 +ALLOW_ROLE=a +BASICZEEK=7 +BASICSURI=7 +# BLOGS= +#BNICS=eth1 +ZEEKVERSION=SURICATA +# CURCLOSEDAYS= +# EVALADVANCED=BASIC +GRAFANA=1 +# HELIXAPIKEY= +HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 +HNSENSOR=inherit +HOSTNAME=distributed-manager +install_type=MANAGER +# LSINPUTBATCHCOUNT= +# LSINPUTTHREADS= +# LSPIPELINEBATCH= +# LSPIPELINEWORKERS= +MANAGERADV=BASIC +MANAGERUPDATES=1 +# MDNS= +# MGATEWAY= +# MIP= +# MMASK= +MNIC=ens18 +# MSEARCH= +# MSRV= +# MTU= +NIDS=Suricata +# NODE_ES_HEAP_SIZE= +# NODE_LS_HEAP_SIZE= +NODESETUP=NODEBASIC +NSMSETUP=BASIC +NODEUPDATES=MANAGER +# OINKCODE= +OSQUERY=1 +# PATCHSCHEDULEDAYS= +# PATCHSCHEDULEHOURS= +PATCHSCHEDULENAME=auto +PLAYBOOK=1 +# REDIRECTHOST= +REDIRECTINFO=IP +RULESETUP=ETOPEN +# SHARDCOUNT= +# SKIP_REBOOT= +SOREMOTEPASS1=onionuser +SOREMOTEPASS2=onionuser +STRELKA=1 +THEHIVE=1 +WAZUH=1 +WEBUSER=onionuser@somewhere.invalid +WEBPASSWD1=0n10nus3r +WEBPASSWD2=0n10nus3r diff --git a/setup/automation/standalone-iso-suricata b/setup/automation/standalone-iso-suricata new file mode 100644 index 000000000..f4697f308 --- /dev/null +++ b/setup/automation/standalone-iso-suricata @@ -0,0 +1,77 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +TESTING=true + +address_type=DHCP +ADMINUSER=onionuser +ADMINPASS1=onionuser +ADMINPASS2=onionuser +ALLOW_CIDR=0.0.0.0/0 +ALLOW_ROLE=a +BASICZEEK=2 +BASICSURI=2 +# BLOGS= +BNICS=eth1 +ZEEKVERSION=SURICATA +# CURCLOSEDAYS= +# EVALADVANCED=BASIC +GRAFANA=1 +# HELIXAPIKEY= +HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 +HNSENSOR=inherit +HOSTNAME=standalone +install_type=STANDALONE +# LSINPUTBATCHCOUNT= +# LSINPUTTHREADS= +# LSPIPELINEBATCH= +# LSPIPELINEWORKERS= +MANAGERADV=BASIC +MANAGERUPDATES=1 +# MDNS= +# MGATEWAY= +# MIP= +# MMASK= +MNIC=eth0 +# MSEARCH= +# MSRV= +# MTU= +NIDS=Suricata +# NODE_ES_HEAP_SIZE= +# NODE_LS_HEAP_SIZE= +NODESETUP=NODEBASIC +NSMSETUP=BASIC +NODEUPDATES=MANAGER +# OINKCODE= +OSQUERY=1 +# PATCHSCHEDULEDAYS= +# PATCHSCHEDULEHOURS= +PATCHSCHEDULENAME=auto +PLAYBOOK=1 +# REDIRECTHOST= +REDIRECTINFO=IP +RULESETUP=ETOPEN +# SHARDCOUNT= +# SKIP_REBOOT= +SOREMOTEPASS1=onionuser +SOREMOTEPASS2=onionuser +STRELKA=1 +THEHIVE=1 +WAZUH=1 +WEBUSER=onionuser@somewhere.invalid +WEBPASSWD1=0n10nus3r +WEBPASSWD2=0n10nus3r diff --git a/setup/so-functions b/setup/so-functions index 4410e7443..0ad993d3d 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1558,9 +1558,17 @@ manager_global() { " node_checkin_interval_ms: $NODE_CHECKIN_INTERVAL_MS"\ "strelka:"\ " enabled: $STRELKA"\ - " rules: 1"\ - " repos:"\ - " - https://github.com/Neo23x0/signature-base"\ + " rules: 1" >> "$global_pillar" + if [[ $is_airgap ]]; then + printf '%s\n'\ + " repos:"\ + " - 'https://$HOSTNAME/repo/rules/strelka'" >> "$global_pillar" + else + printf '%s\n'\ + " repos:"\ + " - 'https://github.com/Neo23x0/signature-base'" >> "$global_pillar" + fi + printf '%s\n'\ "curator:"\ " hot_warm: False"\ "elastic:"\ @@ -2283,28 +2291,32 @@ sensor_pillar() { } set_default_log_size() { - local percentage + local percentage case $install_type in STANDALONE | EVAL | HEAVYNODE) percentage=50 - ;; + ;; *) - percentage=80 - ;; - esac + percentage=80 + ;; + esac local disk_dir="/" if [ -d /nsm ]; then disk_dir="/nsm" fi + if [ -d /nsm/elasticsearch ]; then + disk_dir="/nsm/elasticsearch" + fi + local disk_size_1k disk_size_1k=$(df $disk_dir | grep -v "^Filesystem" | awk '{print $2}') - local ratio="1048576" + local ratio="1048576" - local disk_size_gb - disk_size_gb=$( echo "$disk_size_1k" "$ratio" | awk '{print($1/$2)}' ) + local disk_size_gb + disk_size_gb=$( echo "$disk_size_1k" "$ratio" | awk '{print($1/$2)}' ) log_size_limit=$( echo "$disk_size_gb" "$percentage" | awk '{printf("%.0f", $1 * ($2/100))}') }