Merge remote-tracking branch 'remotes/origin/dev' into issue/1049

2025-12-06 17:22:49 +01:00 · 2020-08-07 10:27:11 -04:00
parent d7801acea5 b534d2b975
commit fce22c1cc4
92 changed files with 569 additions and 297 deletions
--- a/setup/automation/aws_eval_defaults
+++ b/setup/automation/aws_eval_defaults
@@ -0,0 +1,77 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+TESTING=true
+
+address_type=DHCP
+ADMINUSER=onionuser
+ADMINPASS1=onionuser
+ADMINPASS2=onionuser
+ALLOW_CIDR=0.0.0.0/0
+ALLOW_ROLE=a
+BASICZEEK=7
+BASICSURI=7
+# BLOGS=
+BNICS=ens6
+ZEEKVERSION=ZEEK
+# CURCLOSEDAYS=
+# EVALADVANCED=BASIC
+GRAFANA=1
+# HELIXAPIKEY=
+HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12
+HNSENSOR=inherit
+HOSTNAME=eval-aws
+install_type=EVAL
+# LSINPUTBATCHCOUNT=
+# LSINPUTTHREADS=
+# LSPIPELINEBATCH=
+# LSPIPELINEWORKERS=
+MANAGERADV=BASIC
+MANAGERUPDATES=1
+# MDNS=
+# MGATEWAY=
+# MIP=
+# MMASK=
+MNIC=ens5
+# MSEARCH=
+# MSRV=
+# MTU=
+NIDS=Suricata
+# NODE_ES_HEAP_SIZE=
+# NODE_LS_HEAP_SIZE=
+NODESETUP=NODEBASIC
+NSMSETUP=BASIC
+NODEUPDATES=MANAGER
+# OINKCODE=
+OSQUERY=1
+# PATCHSCHEDULEDAYS=
+# PATCHSCHEDULEHOURS=
+PATCHSCHEDULENAME=auto
+PLAYBOOK=1
+# REDIRECTHOST=
+REDIRECTINFO=HOSTNAME
+RULESETUP=ETOPEN
+# SHARDCOUNT=
+SKIP_REBOOT=0
+SOREMOTEPASS1=onionuser
+SOREMOTEPASS2=onionuser
+STRELKA=1
+THEHIVE=1
+WAZUH=1
+WEBUSER=onionuser@somewhere.invalid
+WEBPASSWD1=0n10nus3r
+WEBPASSWD2=0n10nus3r
--- a/setup/automation/aws_manager_defaults
+++ b/setup/automation/aws_manager_defaults
@@ -0,0 +1,77 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+TESTING=true
+
+address_type=DHCP
+ADMINUSER=onionuser
+ADMINPASS1=onionuser
+ADMINPASS2=onionuser
+ALLOW_CIDR=0.0.0.0/0
+ALLOW_ROLE=a
+BASICZEEK=7
+BASICSURI=7
+# BLOGS=
+BNICS=ens6
+ZEEKVERSION=ZEEK
+# CURCLOSEDAYS=
+# EVALADVANCED=BASIC
+GRAFANA=1
+# HELIXAPIKEY=
+HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12
+HNSENSOR=inherit
+HOSTNAME=manager-aws
+install_type=MANAGER
+# LSINPUTBATCHCOUNT=
+# LSINPUTTHREADS=
+# LSPIPELINEBATCH=
+# LSPIPELINEWORKERS=
+MANAGERADV=BASIC
+MANAGERUPDATES=1
+# MDNS=
+# MGATEWAY=
+# MIP=
+# MMASK=
+MNIC=ens5
+# MSEARCH=
+# MSRV=
+# MTU=
+NIDS=Suricata
+# NODE_ES_HEAP_SIZE=
+# NODE_LS_HEAP_SIZE=
+NODESETUP=NODEBASIC
+NSMSETUP=BASIC
+NODEUPDATES=MANAGER
+# OINKCODE=
+OSQUERY=1
+# PATCHSCHEDULEDAYS=
+# PATCHSCHEDULEHOURS=
+PATCHSCHEDULENAME=auto
+PLAYBOOK=1
+# REDIRECTHOST=
+REDIRECTINFO=HOSTNAME
+RULESETUP=ETOPEN
+# SHARDCOUNT=
+SKIP_REBOOT=0
+SOREMOTEPASS1=onionuser
+SOREMOTEPASS2=onionuser
+STRELKA=1
+THEHIVE=1
+WAZUH=1
+WEBUSER=onionuser@somewhere.invalid
+WEBPASSWD1=0n10nus3r
+WEBPASSWD2=0n10nus3r
--- a/setup/automation/aws_standalone_defaults
+++ b/setup/automation/aws_standalone_defaults
@@ -34,7 +34,7 @@ GRAFANA=1
 # HELIXAPIKEY=
 HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12
 HNSENSOR=inherit
-HOSTNAME=standalone
+HOSTNAME=standalone-aws
 install_type=STANDALONE
 # LSINPUTBATCHCOUNT=
 # LSINPUTTHREADS=
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -27,7 +27,7 @@ accept_salt_key_remote() {
 	echo "Accept the key remotely on the manager" >> "$setup_log" 2>&1
 	# Delete the key just in case.
 	ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo salt-key -d "$MINION_ID" -y
-	salt-call state.apply ca
+	salt-call state.apply ca >> /dev/null 2>&1
 	ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo salt-key -a "$MINION_ID" -y

 }
@@ -743,6 +743,7 @@ docker_seed_registry() {
 				"so-grafana:$VERSION" \
 				"so-influxdb:$VERSION" \
 				"so-kibana:$VERSION" \
+				"so-minio:$VERSION" \
 				"so-mysql:$VERSION" \
 				"so-pcaptools:$VERSION" \
 				"so-playbook:$VERSION" \
@@ -960,8 +961,8 @@ manager_pillar() {
 	cat "$pillar_file" >> "$setup_log" 2>&1
  }

-manager_static() {
-	local static_pillar="$local_salt_dir/pillar/static.sls"
+manager_global() {
+	local global_pillar="$local_salt_dir/pillar/global.sls"

 	if [ -z "$SENSOR_CHECKIN_INTERVAL_MS" ]; then
 		SENSOR_CHECKIN_INTERVAL_MS=10000
@@ -970,9 +971,9 @@ manager_static() {
 		fi
 	fi

-	# Create a static file for global values
+	# Create a global file for global values
 	printf '%s\n'\
-		"static:"\
+		"global:"\
 		"  soversion: $SOVERSION"\
 		"  hnmanager: $HNMANAGER"\
 		"  ntpserver: $NTPSERVER"\
@@ -1000,8 +1001,9 @@ manager_static() {
 		"  fleet_ip: N/A"\
 		"  sensoronikey: $SENSORONIKEY"\
 		"  wazuh: $WAZUH"\
-        "  managerupdate: $MANAGERUPDATES"\
+                "  managerupdate: $MANAGERUPDATES"\
 		"  imagerepo: $IMAGEREPO"\
+		"  pipeline: minio"\
 		"pcap:"\
 		"  sensor_checkin_interval_ms: $SENSOR_CHECKIN_INTERVAL_MS"\
 		"strelka:"\
@@ -1018,9 +1020,9 @@ manager_static() {
 		"  discovery_nodes: 1"\
 		"  hot_warm_enabled: False"\
 		"  cluster_routing_allocation_disk.threshold_enabled: true"\
-        "  cluster_routing_allocation_disk_watermark_low: 95%"\
-        "  cluster_routing_allocation_disk_watermark_high: 98%"\
-        "  cluster_routing_allocation_disk_watermark_flood_stage: 98%"\
+                "  cluster_routing_allocation_disk_watermark_low: 95%"\
+                "  cluster_routing_allocation_disk_watermark_high: 98%"\
+                "  cluster_routing_allocation_disk_watermark_flood_stage: 98%"\
 		"  index_settings:"\
 		"    so-beats:"\
 		"      shards: 1"\
@@ -1071,10 +1073,19 @@ manager_static() {
 		"      shards: 5"\
 		"      warm: 7"\
 		"      close: 365"\
-		"      delete: 45" > "$static_pillar"
-
+		"      delete: 45"\
+		"minio:"\
+		"  access_key: $ACCESS_KEY"\
+		"  access_secret: $ACCESS_SECRET"\
+		"s3_settings:"\ 
+		"  size_file: 2048"\
+		"  time_file: 1"\
+		"  upload_queue_size: 4"\
+		"  encoding: gzip"\
+		"  interval: 5" > "$global_pillar"
+		
 	printf '%s\n'  '----' >> "$setup_log" 2>&1
-	cat "$static_pillar" >> "$setup_log" 2>&1
+	cat "$global_pillar" >> "$setup_log" 2>&1
 }

 minio_generate_keys() {
@@ -1474,10 +1485,6 @@ sensor_pillar() {
 	if [ "$HNSENSOR" != 'inherit' ]; then
 		echo "  hnsensor: $HNSENSOR" >> "$pillar_file"
 	fi
-	printf '%s\n'\
-		"  access_key: $ACCESS_KEY"\
-		"  access_secret: $ACCESS_SECRET"\
-		"" >> "$pillar_file"

 	printf '%s\n'  '----' >> "$setup_log" 2>&1
 	cat "$pillar_file" >> "$setup_log" 2>&1
@@ -1553,13 +1560,13 @@ set_initial_firewall_policy() {
            $default_salt_dir/salt/common/tools/sbin/so-firewall --apply includehost search_node "$MAINIP"
 			case "$install_type" in 
 				'EVAL')
-					$default_salt_dir/pillar/data/addtotab.sh evaltab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm" $INTERFACE True
+					$default_salt_dir/pillar/data/addtotab.sh evaltab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm" "$INTERFACE" True
 					;;
 				'MANAGERSEARCH')
 					$default_salt_dir/pillar/data/addtotab.sh managersearchtab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm"
 					;;
 				'STANDALONE')
-					$default_salt_dir/pillar/data/addtotab.sh standalonetab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm" $INTERFACE
+					$default_salt_dir/pillar/data/addtotab.sh standalonetab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm" "$INTERFACE"
 					;;
 			esac
 			;;
@@ -1573,7 +1580,7 @@ set_initial_firewall_policy() {
 			case "$install_type" in
 				'SENSOR')
 					ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/salt/common/tools/sbin/so-firewall --apply includehost sensor "$MAINIP"
-					ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/pillar/data/addtotab.sh sensorstab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm" $INTERFACE
+					ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/pillar/data/addtotab.sh sensorstab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm" "$INTERFACE"
 					;;
 				'SEARCHNODE')
 					ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/salt/common/tools/sbin/so-firewall --apply includehost search_node "$MAINIP"
@@ -1582,7 +1589,7 @@ set_initial_firewall_policy() {
 				'HEAVYNODE')
 					ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/salt/common/tools/sbin/so-firewall includehost sensor "$MAINIP"
 					ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/salt/common/tools/sbin/so-firewall --apply includehost search_node "$MAINIP"
-					ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/pillar/data/addtotab.sh sensorstab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm" $INTERFACE
+					ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/pillar/data/addtotab.sh sensorstab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm" "$INTERFACE"
 					ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/pillar/data/addtotab.sh nodestab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm"
 					;;
 				'FLEET')
--- a/setup/so-setup
+++ b/setup/so-setup
@@ -463,8 +463,9 @@ fi
 		set_progress_str 11 'Updating sudoers file for soremote user'
 		update_sudoers >> $setup_log 2>&1
 		
-		set_progress_str 12 'Generating manager static pillar'
-		manager_static >> $setup_log 2>&1
+		set_progress_str 12 'Generating manager global pillar'
+		minio_generate_keys
+		manager_global >> $setup_log 2>&1
 		
 		set_progress_str 13 'Generating manager pillar'
 		manager_pillar >> $setup_log 2>&1
@@ -619,7 +620,7 @@ fi

 		if [[ $is_fleet_standalone && $FLEETCUSTOMHOSTNAME != '' ]]; then
 			set_progress_str 77 "$(print_salt_state_apply 'fleet.event_update-custom-hostname')"
-			pillar_override="{\"static\":{\"fleet_custom_hostname\": \"$FLEETCUSTOMHOSTNAME\"}}"
+			pillar_override="{\"global\":{\"fleet_custom_hostname\": \"$FLEETCUSTOMHOSTNAME\"}}"
 			salt-call state.apply -l info fleet.event_update-custom-hostname pillar="$pillar_override" >> $setup_log 2>&1
 		fi