Merge remote-tracking branch 'remotes/origin/dev' into bugfix/playbook-mysql

pillar/firewall/analyst.sls

@@ -1,3 +0,0 @@
-analyst:
-  - 127.0.0.1
-  

pillar/firewall/beats_endpoint.sls

@@ -1,3 +0,0 @@
-beats_endpoint:
-  - 127.0.0.1
-  

pillar/firewall/forward_nodes.sls

@@ -1,3 +0,0 @@
-forward_nodes:
-  - 127.0.0.1
-  

pillar/firewall/masterfw.sls

@@ -1,2 +0,0 @@
-masterfw:
-  - 127.0.0.1

pillar/firewall/minions.sls

@@ -1,3 +0,0 @@
-minions:
-  - 127.0.0.1
-  

pillar/firewall/osquery_endpoint.sls

@@ -1,3 +0,0 @@
-osquery_endpoint:
-  - 127.0.0.1
-  

pillar/firewall/search_nodes.sls

@@ -1,2 +0,0 @@
-search_nodes:
-  - 127.0.0.1

pillar/firewall/wazuh_endpoint.sls

@@ -1,2 +0,0 @@
-wazuh_endpoint:
-  - 127.0.0.1

salt/kibana/bin/so-kibana-config-load

@@ -5,23 +5,7 @@
 {%- set FLEET = salt['pillar.get']('static:fleet_ip', '') %}
 {%- set KRATOS = salt['pillar.get']('kratos:redirect', '') %}
 
-
 KIBANA_VERSION="7.6.1"
-MAX_WAIT=120
-
-# Check to see if Kibana is available
-until curl "{{ MASTER }}:5601/nonexistenturl" 2>&1 |grep -q "Not Found" ; do
-                wait_step=$(( ${wait_step} + 1 ))
-                echo "Waiting on Kibana ({{ MASTER }}:5601)...Attempt #$wait_step"
-                if [ ${wait_step} -gt ${MAX_WAIT} ]; then
-                                echo "ERROR: Kibana not available for more than ${MAX_WAIT} seconds."
-                                exit 5
-                fi
-                sleep 1s;
-done
-
-# Sleep additional JIC server is not ready
-sleep 30s
 
 # Copy template file
 cp /opt/so/conf/kibana/saved_objects.ndjson.template /opt/so/conf/kibana/saved_objects.ndjson

salt/kibana/init.sls

@@ -66,13 +66,6 @@ kibanabin:
     - mode: 755
     - template: jinja
 
-kibanadashtemplate:
-  file.managed:
-    - name: /opt/so/conf/kibana/saved_objects.ndjson.template
-    - source: salt://kibana/files/saved_objects.ndjson
-    - user: 932
-    - group: 939
-
 # Start the kibana docker
 so-kibana:
   docker_container.running:
@@ -91,12 +84,27 @@ so-kibana:
     - port_bindings:
       - 0.0.0.0:5601:5601
 
+kibanadashtemplate:
+  file.managed:
+    - name: /opt/so/conf/kibana/saved_objects.ndjson.template
+    - source: salt://kibana/files/saved_objects.ndjson
+    - user: 932
+    - group: 939
+
+wait_for_kibana:
+  module.run:
+    - http.wait_for_successful_query:
+      - url: "http://{{MASTER}}:5601/api/saved_objects/_find?type=config"
+      - wait_for: 180
+    - onchanges:
+      - file: kibanadashtemplate
+
 so-kibana-config-load:
   cmd.run:
     - name: /usr/sbin/so-kibana-config-load
     - cwd: /opt/so
     - onchanges:
-      - file: kibanadashtemplate
+      - wait_for_kibana
 
 
 # Keep the setting correct

setup/so-functions

@@ -19,7 +19,7 @@ source ./so-whiptail
 source ./so-variables
 source ./so-common-functions
 
-SOVERSION=1.2.2
+SOVERSION=1.3.0
 
 accept_salt_key_remote() {
 	systemctl restart salt-minion
@@ -626,9 +626,9 @@ docker_registry() {
 docker_seed_registry() {
 	local VERSION="HH$SOVERSION"
 
-	if ! [ -f /nsm/docker-registry/docker/so-dockers-"$VERSION".tar ]; then
+	if ! [ -f /nsm/docker-registry/docker/registry.tar ]; then
 		local TRUSTED_CONTAINERS=(\
-			"so-core:$VERSION" \
+			"so-nginx:$VERSION" \
 			"so-filebeat:$VERSION" \
 			"so-logstash:$VERSION" \
 			"so-idstools:$VERSION" \
@@ -686,8 +686,8 @@ docker_seed_registry() {
 			} >> "$setup_log" 2>&1
 		done
 	else
-		tar xvf /nsm/docker-registry/docker/so-dockers-$VERSION.tar >> "$setup_log" 2>&1
+		tar xvf /nsm/docker-registry/docker/registry.tar >> "$setup_log" 2>&1
-		rm /nsm/docker-registry/docker/so-dockers-$VERSION.tar >> "$setup_log" 2>&1
+		rm /nsm/docker-registry/docker/registry.tar >> "$setup_log" 2>&1
 	fi
 
 }
@@ -705,9 +705,20 @@ fireeye_pillar() {
 
 }
 
-# Run a salt command to generate the minion key
+# Generate Firewall Templates
-salt_firstcheckin() {
+firewall_generate_templates() {
-	salt-call state.show_top >> /dev/null # send output to /dev/null because we don't actually care about the ouput
+
+	local firewall_pillar_path=/opt/so/saltstack/pillar/firewall
+	mkdir -p "$firewall_pillar_path"
+
+	for i in analyst beats_endpoint forward_nodes masterfw minions osquery_endpoint search_nodes wazuh_endpoint
+	do
+	  printf '%s\n'\
+	  "$i:"\
+	  "  - 127.0.0.1"\
+	  "" > "$firewall_pillar_path"/$i.sls
+	  echo "Added $i Template"
+	done
 }
 
 fleet_pillar() {
@@ -734,10 +745,6 @@ generate_passwords(){
   KRATOSKEY=$(tr -dc 'a-zA-Z0-9' < /dev/urandom  | fold -w 20 | head -n 1)
 }
 
-set_main_ip() {
-	MAINIP=$(ip route get 1 | awk '{print $7;exit}')
-}
-
 get_redirect() {
 	whiptail_set_redirect_info
 	whiptail_set_redirect
@@ -755,15 +762,6 @@ got_root() {
 	fi
 }
 
-install_cleanup() {
-	echo "Installer removing the following files:"
-	ls -lR "$temp_install_dir"
-
-	# Clean up after ourselves
-	rm -rf "$temp_install_dir"
-
-}
-
 get_minion_type() {
 	local minion_type
 	case "$install_type" in
@@ -780,9 +778,13 @@ get_minion_type() {
 	echo "$minion_type"
 }
 
-set_base_heapsizes() {
+install_cleanup() {
-	es_heapsize
+	echo "Installer removing the following files:"
-	ls_heapsize
+	ls -lR "$temp_install_dir"
+
+	# Clean up after ourselves
+	rm -rf "$temp_install_dir"
+
 }
 
 master_pillar() {
@@ -1173,6 +1175,20 @@ salt_checkin() {
 	} >> "$setup_log" 2>&1
 }
 
+# Run a salt command to generate the minion key
+salt_firstcheckin() {
+	salt-call state.show_top >> /dev/null # send output to /dev/null because we don't actually care about the ouput
+}
+
+set_base_heapsizes() {
+	es_heapsize
+	ls_heapsize
+}
+
+set_main_ip() {
+	MAINIP=$(ip route get 1 | awk '{print $7;exit}')
+}
+
 setup_salt_master_dirs() {
 	# Create salt paster directories
 	mkdir -p /opt/so/saltstack/salt

setup/so-setup

@@ -62,6 +62,11 @@ if [ "$install_type" = 'EVAL' ]; then
 	is_master=true
 	is_sensor=true
 	is_eval=true
+elif [ "$install_type" = 'STANDALONE' ]; then
+	is_master=true
+	is_distmaster=true
+	is_node=true
+	is_sensor=true
 elif [ "$install_type" = 'MASTERSEARCH' ]; then
 	is_master=true
 	is_distmaster=true
@@ -73,7 +78,7 @@ elif [ "$install_type" = 'SENSOR' ]; then
 	is_sensor=true
 	is_minion=true
 	STRELKA=1
-elif [[ "$install_type" =~ ^('SEARCHNODE'|'PARSINGNODE'|'HOTNODE'|'WARMNODE')$ ]]; then
+elif [[ "$install_type" =~ ^('SEARCHNODE'|'HOTNODE'|'WARMNODE')$ ]]; then
 	is_node=true
 	is_minion=true
 elif [ "$install_type" = 'HEAVYNODE' ]; then
@@ -294,6 +299,7 @@ export percentage=0
 		set_progress_str 10 'Configuring Salt master'
 		copy_master_config 2>> "$setup_log"
 		setup_salt_master_dirs 2>> "$setup_log"
+		firewall_generate_templates 2>> "$setup_log"
 		
 		set_progress_str 11 'Updating sudoers file for soremote user'
 		update_sudoers 2>> "$setup_log"

setup/so-whiptail

@@ -390,15 +390,15 @@ whiptail_install_type() {
 	"SEARCHNODE" "Add a Search Node with parsing" OFF \
 	"MASTER" "Start a new grid" OFF \
 	"EVAL" "Evaluate all the things" OFF \
+	"PROD" "Standalone full install of everything" OFF \
 	"MASTERSEARCH" "Master + Search Node" OFF \
 	"HEAVYNODE" "Sensor + Search Node" OFF \
 	"HELIXSENSOR" "Connect this sensor to FireEye Helix" OFF \
 	"FLEET" "Dedicated Fleet Osquery Node" OFF \
-	"HOTNODE" "TODO Add Hot Node (Search Node without Parsing)" OFF \
+	"HOTNODE" "TODO Add Hot Node (Uses Elastic Clustering)" OFF \
 	"WARMNODE" "TODO Add Warm Node to existing Hot or Search node" OFF \
-	"WAZUH" "TODO Stand Alone Wazuh Node" OFF \
+	"WAZUH" "TODO Stand Alone Wazuh Server" OFF \
-	"STRELKA" "TODO Stand Alone Strelka Node" OFF \
+	"STRELKA" "TODO Stand Alone Strelka Node" OFF 3>&1 1>&2 2>&3 )
-	"PARSINGNODE" "TODO Add a dedicated Parsing Node" OFF 3>&1 1>&2 2>&3 )
 
 	local exitstatus=$?
 	whiptail_check_exitstatus $exitstatus