Merge branch 'dev' of https://github.com/Security-Onion-Solutions/securityonion-saltstack into dev

.gitignore

@@ -1,2 +1,59 @@
+
+# Created by https://www.gitignore.io/api/macos,windows
+# Edit at https://www.gitignore.io/?templates=macos,windows
+
+### macOS ###
+# General
 .DS_Store
-.idea
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+### Windows ###
+# Windows thumbnail cache files
+Thumbs.db
+Thumbs.db:encryptable
+ehthumbs.db
+ehthumbs_vista.db
+
+# Dump file
+*.stackdump
+
+# Folder config file
+[Dd]esktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows Installer files
+*.cab
+*.msi
+*.msix
+*.msm
+*.msp
+
+# Windows shortcuts
+*.lnk
+
+# End of https://www.gitignore.io/api/macos,windows

salt/fleet/init.sls

@@ -13,6 +13,9 @@
   {% set MAINIP = salt['pillar.get']('static:masterip') %}
 {% endif %}
 
+include:
+  - mysql
+
 #{% if grains.id.split('_')|last in ['master', 'eval', 'fleet'] %}
 #so/fleet:
 #  event.send:
@@ -86,6 +89,8 @@ fleetdb:
     - connection_port: 3306
     - connection_user: root
     - connection_pass: {{ MYSQLPASS }}
+    - require: 
+      - sls: mysql
 
 fleetdbuser:
   mysql_user.present:
@@ -95,6 +100,8 @@ fleetdbuser:
     - connection_port: 3306
     - connection_user: root
     - connection_pass: {{ MYSQLPASS }}
+    - require: 
+      - fleetdb
 
 fleetdbpriv:
   mysql_grants.present:
@@ -106,6 +113,8 @@ fleetdbpriv:
     - connection_port: 3306
     - connection_user: root
     - connection_pass: {{ MYSQLPASS }}
+    - require: 
+      - fleetdb
 
 
 {% if FLEETPASS == None or FLEETJWT == None %}

salt/mysql/init.sls

@@ -85,4 +85,9 @@ so-mysql:
       - /opt/so/log/mysql:/var/log/mysql:rw
     - watch:
       - /opt/so/conf/mysql/etc
+  cmd.run:
+    - name: until nc -z {{ MAINIP }} 3306; do sleep 1; done
+    - timeout: 120
+    - onchanges:
+      - docker_container: so-mysql
 {% endif %}

salt/nginx/etc/nginx.conf.so-eval

@@ -146,6 +146,20 @@ http {
 
         }
 
+        location /cyberchef/ {
+          auth_request          /auth/sessions/whoami;
+          proxy_read_timeout    90;
+          proxy_connect_timeout 90;
+          proxy_set_header      Host $host;
+          proxy_set_header      X-Real-IP $remote_addr;
+          proxy_set_header      X-Forwarded-For $proxy_add_x_forwarded_for;
+          proxy_set_header      Proxy "";
+        }
+
+        location /cyberchef {
+          rewrite               ^ /cyberchef/ permanent;
+        }
+
         location /packages/ {
           try_files $uri =206;
           auth_request          /auth/sessions/whoami;

salt/nginx/etc/nginx.conf.so-master

@@ -146,6 +146,20 @@ http {
 
         }
 
+        location /cyberchef/ {
+          auth_request          /auth/sessions/whoami;
+          proxy_read_timeout    90;
+          proxy_connect_timeout 90;
+          proxy_set_header      Host $host;
+          proxy_set_header      X-Real-IP $remote_addr;
+          proxy_set_header      X-Forwarded-For $proxy_add_x_forwarded_for;
+          proxy_set_header      Proxy "";
+        }
+
+        location /cyberchef {
+          rewrite               ^ /cyberchef/ permanent;
+        }
+
         location /packages/ {
           try_files $uri =206;
           auth_request          /auth/sessions/whoami;

salt/nginx/etc/nginx.conf.so-mastersearch

@@ -146,6 +146,20 @@ http {
 
         }
 
+        location /cyberchef/ {
+          auth_request          /auth/sessions/whoami;
+          proxy_read_timeout    90;
+          proxy_connect_timeout 90;
+          proxy_set_header      Host $host;
+          proxy_set_header      X-Real-IP $remote_addr;
+          proxy_set_header      X-Forwarded-For $proxy_add_x_forwarded_for;
+          proxy_set_header      Proxy "";
+        }
+
+        location /cyberchef {
+          rewrite               ^ /cyberchef/ permanent;
+        }
+
         location /packages/ {
           try_files $uri =206;
           auth_request          /auth/sessions/whoami;

setup/install_scripts/99-so-checksum-offload-disable

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-if [[ "$DEVICE_IFACE" != "$MAININT" && "$DEVICE_IFACE" != *"docker"* ]]; then
+if [[ "$DEVICE_IFACE" != "$MNIC" && "$DEVICE_IFACE" != *"docker"* ]]; then
 	for i in rx tx sg tso ufo gso gro lro; do
 		ethtool -K "$DEVICE_IFACE" "$i" off;
   	done

setup/public_keys/salt.pem

@@ -0,0 +1,31 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v2.0.22 (GNU/Linux)
+
+mQENBFOpvpgBCADkP656H41i8fpplEEB8IeLhugyC2rTEwwSclb8tQNYtUiGdna9
+m38kb0OS2DDrEdtdQb2hWCnswxaAkUunb2qq18vd3dBvlnI+C4/xu5ksZZkRj+fW
+tArNR18V+2jkwcG26m8AxIrT+m4M6/bgnSfHTBtT5adNfVcTHqiT1JtCbQcXmwVw
+WbqS6v/LhcsBE//SHne4uBCK/GHxZHhQ5jz5h+3vWeV4gvxS3Xu6v1IlIpLDwUts
+kT1DumfynYnnZmWTGc6SYyIFXTPJLtnoWDb9OBdWgZxXfHEcBsKGha+bXO+m2tHA
+gNneN9i5f8oNxo5njrL8jkCckOpNpng18BKXABEBAAG0MlNhbHRTdGFjayBQYWNr
+YWdpbmcgVGVhbSA8cGFja2FnaW5nQHNhbHRzdGFjay5jb20+iQE4BBMBAgAiBQJT
+qb6YAhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRAOCKFJ3le/vhkqB/0Q
+WzELZf4d87WApzolLG+zpsJKtt/ueXL1W1KA7JILhXB1uyvVORt8uA9FjmE083o1
+yE66wCya7V8hjNn2lkLXboOUd1UTErlRg1GYbIt++VPscTxHxwpjDGxDB1/fiX2o
+nK5SEpuj4IeIPJVE/uLNAwZyfX8DArLVJ5h8lknwiHlQLGlnOu9ulEAejwAKt9CU
+4oYTszYM4xrbtjB/fR+mPnYh2fBoQO4d/NQiejIEyd9IEEMd/03AJQBuMux62tjA
+/NwvQ9eqNgLw9NisFNHRWtP4jhAOsshv1WW+zPzu3ozoO+lLHixUIz7fqRk38q8Q
+9oNR31KvrkSNrFbA3D89uQENBFOpvpgBCADJ79iH10AfAfpTBEQwa6vzUI3Eltqb
+9aZ0xbZV8V/8pnuU7rqM7Z+nJgldibFk4gFG2bHCG1C5aEH/FmcOMvTKDhJSFQUx
+uhgxttMArXm2c22OSy1hpsnVG68G32Nag/QFEJ++3hNnbyGZpHnPiYgej3FrerQJ
+zv456wIsxRDMvJ1NZQB3twoCqwapC6FJE2hukSdWB5yCYpWlZJXBKzlYz/gwD/Fr
+GL578WrLhKw3UvnJmlpqQaDKwmV2s7MsoZogC6wkHE92kGPG2GmoRD3ALjmCvN1E
+PsIsQGnwpcXsRpYVCoW7e2nW4wUf7IkFZ94yOCmUq6WreWI4NggRcFC5ABEBAAGJ
+AR8EGAECAAkFAlOpvpgCGwwACgkQDgihSd5Xv74/NggA08kEdBkiWWwJZUZEy7cK
+WWcgjnRuOHd4rPeT+vQbOWGu6x4bxuVf9aTiYkf7ZjVF2lPn97EXOEGFWPZeZbH4
+vdRFH9jMtP+rrLt6+3c9j0M8SIJYwBL1+CNpEC/BuHj/Ra/cmnG5ZNhYebm76h5f
+T9iPW9fFww36FzFka4VPlvA4oB7ebBtquFg3sdQNU/MmTVV4jPFWXxh4oRDDR+8N
+1bcPnbB11b5ary99F/mqr7RgQ+YFF0uKRE3SKa7a+6cIuHEZ7Za+zhPaQlzAOZlx
+fuBmScum8uQTrEF5+Um5zkwC7EXTdH1co/+/V/fpOtxIg4XO4kcugZefVm5ERfVS
+MA==
+=dtMN
+-----END PGP PUBLIC KEY BLOCK-----

setup/so-common-functions

@@ -0,0 +1,68 @@
+#!/bin/bash
+
+source ./so-variables
+
+# Helper functions
+
+filter_unused_nics() {
+
+	if [[ $MNIC ]]; then local grep_string="$MNIC\|bond0"; else local grep_string="bond0"; fi
+
+	# If we call this function and NICs have already been assigned to the bond interface then add them to the grep search string
+	if [[ $BNICS ]]; then
+		grep_string="$grep_string"
+		for BONDNIC in "${BNICS[@]}"; do
+			grep_string="$grep_string\|$BONDNIC"
+		done
+	fi
+
+	# Finally, set filtered_nics to any NICs we aren't using (and ignore interfaces that aren't of use)
+	filtered_nics=$(ip link| awk -F: '$0 !~ "lo|vir|veth|br|docker|wl|^[^0-9]"{print $2}' | grep -vwe "$grep_string"  | sed 's/ //g')
+	readarray -t filtered_nics <<< "$filtered_nics"
+	
+	nic_list=()
+	for nic in "${filtered_nics[@]}"; do
+		nic_list+=("$nic" "" "OFF")
+	done
+
+	export nic_list
+}
+
+calculate_useable_cores() {
+
+	# Calculate reasonable core usage
+	local cores_for_bro=$(( (num_cpu_cores/2) - 1 ))
+	local lb_procs_round
+	lb_procs_round=$(printf "%.0f\n" $cores_for_bro)
+
+	if [ "$lb_procs_round" -lt 1 ]; then lb_procs=1; else lb_procs=$lb_procs_round; fi
+    export lb_procs
+}
+
+set_defaul_log_size() {
+    local percentage
+
+	case $INSTALLTYPE in
+		EVAL | HEAVYNODE)
+			percentage=50
+			;;
+		*)
+            percentage=80
+			;;
+		esac
+
+	local disk_dir="/"
+	if [ -d /nsm ]; then
+		disk_dir="/nsm"
+	fi
+	local disk_size_1k
+	disk_size_1k=$(df $disk_dir | grep -v "^Filesystem" | awk '{print $2}')
+
+    local ratio="1048576"
+
+    local disk_size_gb
+    disk_size_gb=$( echo "$disk_size_1k" "$ratio" | awk '{print($1/$2)}' )
+
+	log_size_limit=$( echo "$disk_size_gb" "$percentage" | awk '{printf("%.0f", $1 * ($2/100))}')
+	export log_size_limit
+}

setup/so-variables

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+total_mem=$(grep MemTotal /proc/meminfo | awk '{print $2}' | sed -r 's/.{3}$//')
+export total_mem
+
+num_cpu_cores=$(nproc)
+export num_cpu_cores
+
+readarray -t cpu_core_list <<< "$(grep "processor" /proc/cpuinfo | grep -v "KVM" | awk '{print $3}')"
+export cpu_core_list
+
+random_uid=$(</dev/urandom tr -dc 'a-zA-Z0-9' | fold -w 16 | head -n 1)
+export random_uid
+
+node_es_port=9200
+export node_es_port
+
+setup_log="/root/sosetup.log"
+export setup_log
+
+filesystem_root=$(df / | awk '$3 ~ /[0-9]+/ { print $2 * 1000 }')
+export filesystem_root
+
+mkdir -p /nsm
+filesystem_nsm=$(df /nsm | awk '$3 ~ /[0-9]+/ { print $2 * 1000 }')
+export filesystem_nsm
+
+mkdir -p /root/installtmp/pillar/minions
+export temp_install_dir=/root/installtmp
+
+export percentage_str='Getting started'
+
+export DEBIAN_FRONTEND=noninteractive

setup/yum_repos/salt-2019-2.repo

@@ -0,0 +1,6 @@
+[saltstack-repo]
+name=SaltStack repo for RHEL/CentOS $releasever PY3
+baseurl=https://repo.saltstack.com/py3/redhat/$releasever/$basearch/2019.2
+enabled=1
+gpgcheck=1
+gpgkey=https://repo.saltstack.com/py3/redhat/$releasever/$basearch/2019.2/SALTSTACK-GPG-KEY.pub

setup/yum_repos/salt-latest.repo

@@ -0,0 +1,7 @@
+[salt-latest]
+name=SaltStack Latest Release Channel for RHEL/Centos $releasever
+baseurl=https://repo.saltstack.com/py3/redhat/7/$basearch/latest
+failovermethod=priority
+enabled=1
+gpgcheck=1
+gpgkey=https://repo.saltstack.com/py3/redhat/$releasever/$basearch/latest/SALTSTACK-GPG-KEY.pub

setup/yum_repos/wazuh.repo

@@ -0,0 +1,7 @@
+[wazuh_repo]
+gpgcheck=1
+gpgkey=https://packages.wazuh.com/key/GPG-KEY-WAZUH
+enabled=1
+name=Wazuh repository
+baseurl=https://packages.wazuh.com/3.x/yum/
+protect=1