Merge pull request #3140 from Security-Onion-Solutions/issue/3130

Issue/3130
2025-12-30 21:03:10 +01:00 · 2021-02-25 11:27:46 -05:00
parent 81331264e7 fcd3f81400
commit 18365ed87d
10 changed files with 309 additions and 41 deletions
--- a/salt/common/tools/sbin/so-zeek-stats
+++ b/salt/common/tools/sbin/so-zeek-stats
@@ -24,11 +24,11 @@ show_stats() {
  echo
  echo "Average throughput:"
  echo
-  docker exec -it so-zeek /opt/zeek/bin/zeekctl capstats
+  docker exec so-zeek env -i PATH=/bin:/usr/bin:/sbin:/usr/sbin:/opt/bin:/usr/local/bin:/usr/local/sbin runuser -l zeek -c '/opt/zeek/bin/zeekctl capstats'
  echo
  echo "Average packet loss:"
  echo
-  docker exec -it so-zeek /opt/zeek/bin/zeekctl netstats
+  docker exec so-zeek env -i PATH=/bin:/usr/bin:/sbin:/usr/sbin:/opt/bin:/usr/local/bin:/usr/local/sbin runuser -l zeek -c '/opt/zeek/bin/zeekctl netstats'
  echo
 }

--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -120,6 +120,72 @@ check_sudoers() {
 	fi
 }

+check_log_size_limit() {
+  local wait_for_enter=false
+
+  local num_minion_pillars
+  num_minion_pillars=$(find /opt/so/saltstack/local/pillar/minions/ -type f | wc -l)
+  
+  if [[ $num_minion_pillars -gt 1 ]]; then
+    if find /opt/so/saltstack/local/pillar/minions/ -type f | grep -q "_heavynode"; then
+      wait_for_enter=true
+      echo "[INFO] The value of log_size_limit in any heavy node minion pillars may be incorrect."
+      echo " -> We recommend checking and adjusting the values as necessary."
+      echo " -> Minion pillar directory: /opt/so/saltstack/local/pillar/minions/"
+    fi
+  else
+    local minion_id
+    minion_id=$(lookup_salt_value "id" "" "grains")
+
+    local minion_arr
+    IFS='_' read -ra minion_arr <<< "$minion_id"
+
+    local node_type="${minion_arr[0]}"
+    
+    local current_limit
+    current_limit=$(lookup_pillar "log_size_limit" "elasticsearch")
+    
+    local percent
+    case $node_type in
+      'standalone' | 'eval')
+        percent=50
+      ;;
+      *)
+        percent=80
+      ;;
+    esac
+
+    local disk_dir="/"
+    if [ -d /nsm ]; then
+      disk_dir="/nsm"
+    fi
+
+    local disk_size_1k
+    disk_size_1k=$(df $disk_dir | grep -v "^Filesystem" | awk '{print $2}')
+
+    local ratio="1048576"
+
+    local disk_size_gb
+    disk_size_gb=$( echo "$disk_size_1k" "$ratio" | awk '{print($1/$2)}' )
+
+    local new_limit
+    new_limit=$( echo "$disk_size_gb" "$percent" | awk '{printf("%.0f", $1 * ($2/100))}')
+
+    if [[ $current_limit != "$new_limit" ]]; then
+    wait_for_enter=true
+      echo "[WARNING] The value of log_size_limit (${current_limit}) does not match the recommended value of ${new_limit}."
+      echo " -> We recommend checking and adjusting the value as necessary."
+      echo " -> File: /opt/so/saltstack/local/pillar/minions/${minion_id}.sls"
+    fi
+
+    if [[ $wait_for_enter == true ]]; then
+      echo ""
+      read -n 1 -s -r -p "Press any key to continue..."
+      echo "" # Since read doesn't print a newline, print one for it
+    fi
+  fi
+}
+
 clean_dockers() {
  # Place Holder for cleaning up old docker images
  echo "Trying to clean up old dockers."
@@ -159,7 +225,7 @@ generate_and_clean_tarballs() {
  local new_version
  new_version=$(cat $UPDATE_DIR/VERSION)
  [ -d /opt/so/repo ] || mkdir -p /opt/so/repo
-  tar -cxf "/opt/so/repo/$new_version.tar.gz" "$UPDATE_DIR"
+  tar -czf "/opt/so/repo/$new_version.tar.gz" "$UPDATE_DIR"
  find "/opt/so/repo" -type f -not -name "$new_version.tar.gz" -exec rm -rf {} \;
 }

@@ -338,8 +404,22 @@ up_2.3.2X_to_2.3.30() {
  # Replace any curly brace scalars with the same scalar in single quotes
  readarray -t minion_pillars <<< "$(find /opt/so/saltstack/local/pillar/minions -type f -name '*.sls')"
  for pillar in "${minion_pillars[@]}"; do
-    sed -i -r "s/ (\{\{.*}})$/ '\1'/g" "$pillar" 
+    sed -i -r "s/ (\{\{.*}})$/ '\1'/g" "$pillar"
  done
+
+# Change the IMAGEREPO
+  sed -i "/  imagerepo: 'securityonion'/c\  imagerepo: 'security-onion-solutions'" /opt/so/saltstack/local/pillar/global.sls
+  sed -i "/  imagerepo: securityonion/c\  imagerepo: 'security-onion-solutions'" /opt/so/saltstack/local/pillar/global.sls 
+
+# Strelka rule repo pillar addition
+  if [ $is_airgap -eq 0 ]; then
+      # Add manager as default Strelka YARA rule repo
+      sed -i "/^strelka:/a \\  repos: \n    - https://$HOSTNAME/repo/rules/strelka" /opt/so/saltstack/local/pillar/global.sls;
+  else
+      # Add Github repo for Strelka YARA rules
+      sed -i "/^strelka:/a \\  repos: \n    - https://github.com/Neo23x0/signature-base" /opt/so/saltstack/local/pillar/global.sls;
+  fi
+  check_log_size_limit
 }

 space_check() {
@@ -662,6 +742,8 @@ fi

 check_sudoers

+
+
 }

 main "$@" | tee /dev/fd/3
--- a/salt/docker_clean/init.sls
+++ b/salt/docker_clean/init.sls
@@ -3,7 +3,7 @@

 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 {% set MANAGER = salt['grains.get']('master') %}
-{% set OLDVERSIONS = ['2.0.0-rc.1','2.0.1-rc.1','2.0.2-rc.1','2.0.3-rc.1','2.1.0-rc.2','2.2.0-rc.3','2.3.0','2.3.1','2.3.2']%}
+{% set OLDVERSIONS = ['2.0.0-rc.1','2.0.1-rc.1','2.0.2-rc.1','2.0.3-rc.1','2.1.0-rc.2','2.2.0-rc.3','2.3.0','2.3.1','2.3.2','2.3.10','2.3.20']%}

 {% for VERSION in OLDVERSIONS %}
 remove_images_{{ VERSION }}:
@@ -45,6 +45,41 @@ remove_images_{{ VERSION }}:
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-thehive-es:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-wazuh:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-zeek:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-acng:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-thehive-cortex:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-curator:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-domainstats:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-elastalert:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-elasticsearch:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-filebeat:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-fleet:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-fleet-launcher:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-freqserver:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-grafana:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-idstools:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-influxdb:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-kibana:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-kratos:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-logstash:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-minio:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-mysql:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-nginx:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-pcaptools:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-playbook:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-redis:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-soc:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-soctopus:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-steno:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-strelka-frontend:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-strelka-manager:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-strelka-backend:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-strelka-filestream:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-suricata:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-telegraf:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-thehive:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-thehive-es:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-wazuh:{{ VERSION }}'
+      - '{{ MANAGER }}:5000/securityonion/so-zeek:{{ VERSION }}'
 {% endfor %}

 {% else %}
@@ -53,4 +88,4 @@ remove_images_{{ VERSION }}:
  test.fail_without_changes:
    - name: {{sls}}_state_not_allowed

-{% endif %}
+{% endif %}
--- a/salt/registry/bin/so-buildregistry
+++ b/salt/registry/bin/so-buildregistry
@@ -1,12 +0,0 @@
-#!/bin/bash
-
-VERSION=HH1.1.4
-TARBALL=/nsm/docker-registry/docker/so-dockers-$VERSION.tar
-
-# See if the tarball is there. If so do soemthing otherwise peace out.
-if [ -f "$TARBALL" ]; then
-  cd /nsm/docker-registry/docker
-  tar xvf so-dockers-$VERSION.tar
-fi
-
-exit 0
--- a/salt/registry/init.sls
+++ b/salt/registry/init.sls
@@ -29,17 +29,6 @@ dockerregistryconf:
    - name: /opt/so/conf/docker-registry/etc/config.yml
    - source: salt://registry/etc/config.yml

-# Copy the registry script
-#dockerregistrybuild:
-#  file.managed:
-#    - name: /opt/so/conf/docker-registry/so-buildregistry
-#    - source: salt://registry/bin/so-buildregistry
-#    - mode: 755
-
-#dockerexpandregistry:
-# cmd.run:
-#   - name: /opt/so/conf/docker-registry/so-buildregistry
-
 # Install the registry container
 so-dockerregistry:
  docker_container.running:
--- a/salt/zeek/cron/packetloss.sh
+++ b/salt/zeek/cron/packetloss.sh
@@ -1,2 +1,2 @@
 #!/bin/bash
-/usr/bin/docker exec so-zeek /opt/zeek/bin/zeekctl netstats | awk '{print $(NF-2),$(NF-1),$NF}' | awk -F '[ =]' '{RCVD += $2;DRP += $4;TTL += $6} END { print "rcvd: " RCVD, "dropped: " DRP, "total: " TTL}' >> /nsm/zeek/logs/packetloss.log 2>&1
+/usr/bin/docker exec so-zeek env -i PATH=/bin:/usr/bin:/sbin:/usr/sbin:/opt/bin:/usr/local/bin:/usr/local/sbin runuser -l zeek -c '/opt/zeek/bin/zeekctl netstats' | awk '{print $(NF-2),$(NF-1),$NF}' | awk -F '[ =]' '{RCVD += $2;DRP += $4;TTL += $6} END { print "rcvd: " RCVD, "dropped: " DRP, "total: " TTL}' >> /nsm/zeek/logs/packetloss.log 2>&1
--- a/salt/zeek/init.sls
+++ b/salt/zeek/init.sls
@@ -73,6 +73,14 @@ zeekpolicysync:
    - group: 939
    - template: jinja

+# Ensure the zeek spool tree (and state.db) ownership is correct
+zeekspoolownership:
+  file.directory:
+    - name: /nsm/zeek/spool
+    - user: 937
+    - recurse:
+      - user
+
 # Sync Intel
 zeekintelloadsync:
  file.managed:
--- a/setup/automation/distributed-net-ubuntu-manager-suricata
+++ b/setup/automation/distributed-net-ubuntu-manager-suricata
@@ -0,0 +1,77 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+TESTING=true
+
+address_type=DHCP
+ADMINUSER=onionuser
+ADMINPASS1=onionuser
+ADMINPASS2=onionuser
+ALLOW_CIDR=0.0.0.0/0
+ALLOW_ROLE=a
+BASICZEEK=7
+BASICSURI=7
+# BLOGS=
+#BNICS=eth1
+ZEEKVERSION=SURICATA
+# CURCLOSEDAYS=
+# EVALADVANCED=BASIC
+GRAFANA=1
+# HELIXAPIKEY=
+HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12
+HNSENSOR=inherit
+HOSTNAME=distributed-manager
+install_type=MANAGER
+# LSINPUTBATCHCOUNT=
+# LSINPUTTHREADS=
+# LSPIPELINEBATCH=
+# LSPIPELINEWORKERS=
+MANAGERADV=BASIC
+MANAGERUPDATES=1
+# MDNS=
+# MGATEWAY=
+# MIP=
+# MMASK=
+MNIC=ens18
+# MSEARCH=
+# MSRV=
+# MTU=
+NIDS=Suricata
+# NODE_ES_HEAP_SIZE=
+# NODE_LS_HEAP_SIZE=
+NODESETUP=NODEBASIC
+NSMSETUP=BASIC
+NODEUPDATES=MANAGER
+# OINKCODE=
+OSQUERY=1
+# PATCHSCHEDULEDAYS=
+# PATCHSCHEDULEHOURS=
+PATCHSCHEDULENAME=auto
+PLAYBOOK=1
+# REDIRECTHOST=
+REDIRECTINFO=IP
+RULESETUP=ETOPEN
+# SHARDCOUNT=
+# SKIP_REBOOT=
+SOREMOTEPASS1=onionuser
+SOREMOTEPASS2=onionuser
+STRELKA=1
+THEHIVE=1
+WAZUH=1
+WEBUSER=onionuser@somewhere.invalid
+WEBPASSWD1=0n10nus3r
+WEBPASSWD2=0n10nus3r
--- a/setup/automation/standalone-iso-suricata
+++ b/setup/automation/standalone-iso-suricata
@@ -0,0 +1,77 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+TESTING=true
+
+address_type=DHCP
+ADMINUSER=onionuser
+ADMINPASS1=onionuser
+ADMINPASS2=onionuser
+ALLOW_CIDR=0.0.0.0/0
+ALLOW_ROLE=a
+BASICZEEK=2
+BASICSURI=2
+# BLOGS=
+BNICS=eth1
+ZEEKVERSION=SURICATA
+# CURCLOSEDAYS=
+# EVALADVANCED=BASIC
+GRAFANA=1
+# HELIXAPIKEY=
+HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12
+HNSENSOR=inherit
+HOSTNAME=standalone
+install_type=STANDALONE
+# LSINPUTBATCHCOUNT=
+# LSINPUTTHREADS=
+# LSPIPELINEBATCH=
+# LSPIPELINEWORKERS=
+MANAGERADV=BASIC
+MANAGERUPDATES=1
+# MDNS=
+# MGATEWAY=
+# MIP=
+# MMASK=
+MNIC=eth0
+# MSEARCH=
+# MSRV=
+# MTU=
+NIDS=Suricata
+# NODE_ES_HEAP_SIZE=
+# NODE_LS_HEAP_SIZE=
+NODESETUP=NODEBASIC
+NSMSETUP=BASIC
+NODEUPDATES=MANAGER
+# OINKCODE=
+OSQUERY=1
+# PATCHSCHEDULEDAYS=
+# PATCHSCHEDULEHOURS=
+PATCHSCHEDULENAME=auto
+PLAYBOOK=1
+# REDIRECTHOST=
+REDIRECTINFO=IP
+RULESETUP=ETOPEN
+# SHARDCOUNT=
+# SKIP_REBOOT=
+SOREMOTEPASS1=onionuser
+SOREMOTEPASS2=onionuser
+STRELKA=1
+THEHIVE=1
+WAZUH=1
+WEBUSER=onionuser@somewhere.invalid
+WEBPASSWD1=0n10nus3r
+WEBPASSWD2=0n10nus3r
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -1558,9 +1558,17 @@ manager_global() {
 		"  node_checkin_interval_ms: $NODE_CHECKIN_INTERVAL_MS"\
 		"strelka:"\
 		"  enabled: $STRELKA"\
-		"  rules: 1"\
-		"  repos:"\
-		"    - https://github.com/Neo23x0/signature-base"\
+		"  rules: 1" >> "$global_pillar"
+		if [[ $is_airgap ]]; then
+		    printf '%s\n'\
+			    "  repos:"\
+			    "    - 'https://$HOSTNAME/repo/rules/strelka'" >> "$global_pillar"
+	    else 
+		    printf '%s\n'\
+		        "  repos:"\
+			    "    - 'https://github.com/Neo23x0/signature-base'" >> "$global_pillar"
+	    fi
+	printf '%s\n'\
 		"curator:"\
 		"  hot_warm: False"\
 		"elastic:"\
@@ -2283,28 +2291,32 @@ sensor_pillar() {
 }

 set_default_log_size() {
-    local percentage
+	local percentage

 	case $install_type in
 		STANDALONE | EVAL | HEAVYNODE)
 			percentage=50
-			;;
+		;;
 		*)
-            percentage=80
-			;;
-		esac
+			percentage=80
+		;;
+	esac

 	local disk_dir="/"
 	if [ -d /nsm ]; then
 		disk_dir="/nsm"
 	fi
+	if [ -d /nsm/elasticsearch ]; then
+		disk_dir="/nsm/elasticsearch"
+	fi
+	
 	local disk_size_1k
 	disk_size_1k=$(df $disk_dir | grep -v "^Filesystem" | awk '{print $2}')

-    local ratio="1048576"
+	local ratio="1048576"

-    local disk_size_gb
-    disk_size_gb=$( echo "$disk_size_1k" "$ratio" | awk '{print($1/$2)}' )
+	local disk_size_gb
+	disk_size_gb=$( echo "$disk_size_1k" "$ratio" | awk '{print($1/$2)}' )

 	log_size_limit=$( echo "$disk_size_gb" "$percentage" | awk '{printf("%.0f", $1 * ($2/100))}')
 }