From 96666ab30712e9c451253f9b689aa6caa1340c26 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Tue, 7 Dec 2021 10:19:32 -0500
Subject: [PATCH 01/47] add receiver node

---
 .../assigned_hostgroups.local.map.yaml        |  3 ++-
 files/firewall/hostgroups.local.yaml          |  4 ++++
 pillar/logstash/receiver.sls                  | 10 ++++++++
 pillar/top.sls                                |  8 +++++++
 .../config/so/9999_output_redis.conf.jinja    |  2 +-
 salt/top.sls                                  | 24 +++++++++++++++++++
 setup/so-functions                            |  5 +++-
 setup/so-whiptail                             |  3 ++-
 8 files changed, 55 insertions(+), 4 deletions(-)
 create mode 100644 pillar/logstash/receiver.sls

diff --git a/files/firewall/assigned_hostgroups.local.map.yaml b/files/firewall/assigned_hostgroups.local.map.yaml
index 50ef751a4..ee871ad80 100644
--- a/files/firewall/assigned_hostgroups.local.map.yaml
+++ b/files/firewall/assigned_hostgroups.local.map.yaml
@@ -16,6 +16,7 @@ role:
   import:
   manager:
   managersearch:
+  receiver:
   standalone:
   searchnode:
-  sensor:
\ No newline at end of file
+  sensor:
diff --git a/files/firewall/hostgroups.local.yaml b/files/firewall/hostgroups.local.yaml
index d02d7c785..334b090d1 100644
--- a/files/firewall/hostgroups.local.yaml
+++ b/files/firewall/hostgroups.local.yaml
@@ -44,6 +44,10 @@ firewall:
       ips:
         delete:
         insert:
+    receiver:
+      ips:
+        delete:
+        insert:
     search_node:
       ips:
         delete:
diff --git a/pillar/logstash/receiver.sls b/pillar/logstash/receiver.sls
new file mode 100644
index 000000000..fc0788824
--- /dev/null
+++ b/pillar/logstash/receiver.sls
@@ -0,0 +1,10 @@
+{%- set PIPELINE = salt['pillar.get']('global:pipeline', 'redis') %}
+logstash:
+  pipelines:
+    manager:
+      config:
+        - so/0009_input_beats.conf      
+        - so/0010_input_hhbeats.conf
+        - so/0011_input_endgame.conf
+        - so/9999_output_redis.conf.jinja
+        
\ No newline at end of file
diff --git a/pillar/top.sls b/pillar/top.sls
index 5401b83e3..a81fdc862 100644
--- a/pillar/top.sls
+++ b/pillar/top.sls
@@ -104,6 +104,14 @@ base:
     - minions.{{ grains.id }}
     - data.nodestab
 
+  '*_receiver':
+    - logstash
+    - logstash.receiver
+    - elasticsearch.auth
+    - global
+    - minions.{{ grains.id }}
+    - data.receivertab
+
   '*_import':
     - zeeklogs
     - secrets
diff --git a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja
index 6b9c62e2f..eac5fe304 100644
--- a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja
@@ -1,4 +1,4 @@
-{%- if grains.role in ['so-heavynode'] %}
+{%- if grains.role in ['so-heavynode', 'so-receiver'] %}
   {%- set HOST = salt['grains.get']('host') %}
 {%- else %}
   {%- set HOST = salt['grains.get']('master') %}
diff --git a/salt/top.sls b/salt/top.sls
index a8f2018a6..18f37e713 100644
--- a/salt/top.sls
+++ b/salt/top.sls
@@ -478,3 +478,27 @@ base:
     - docker_clean
     - pipeline.load
     - learn
+
+  '*_receiver and G@saltversion:{{saltversion}}':
+    - match: compound
+    - ca
+    - ssl
+    - telegraf
+    - firewall
+    {%- if WAZUH != 0 %}
+    - wazuh
+    {%- endif %}
+    {%- if LOGSTASH %}
+    - logstash
+    {%- endif %}
+    {%- if REDIS %}
+    - redis
+    {%- endif %}
+    {%- if FILEBEAT %}
+    - filebeat
+    {%- endif %}
+    {%- if FLEETMANAGER or FLEETNODE %}
+    - fleet.install_package
+    {%- endif %}
+    - schedule
+    - docker_clean
diff --git a/setup/so-functions b/setup/so-functions
index daf609f67..20818aa72 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -2665,7 +2665,7 @@ set_initial_firewall_policy() {
 			$default_salt_dir/salt/common/tools/sbin/so-firewall includehost minion "$MAINIP"
             $default_salt_dir/salt/common/tools/sbin/so-firewall --apply includehost sensor "$MAINIP"
 			;;
-		'SENSOR' | 'SEARCHNODE' | 'HEAVYNODE' | 'FLEET')
+		'SENSOR' | 'SEARCHNODE' | 'HEAVYNODE' | 'FLEET' | 'RECEIVER')
 			$sshcmd -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/salt/common/tools/sbin/so-firewall includehost minion "$MAINIP"
 			case "$install_type" in
 				'SENSOR')
@@ -2685,6 +2685,9 @@ set_initial_firewall_policy() {
 				'FLEET')
 				    $sshcmd -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/salt/common/tools/sbin/so-firewall --apply includehost beats_endpoint_ssl "$MAINIP"
 					;;
+				'RECEIVER')
+					$sshcmd -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/salt/common/tools/sbin/so-firewall --apply includehost receiver "$MAINIP"
+					$sshcmd -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/pillar/data/addtotab.sh receivertab "$MINION_ID" "$MAINIP"
 			esac
 			;;
 		'PARSINGNODE')
diff --git a/setup/so-whiptail b/setup/so-whiptail
index c49e7396a..bdaa50849 100755
--- a/setup/so-whiptail
+++ b/setup/so-whiptail
@@ -796,11 +796,12 @@ whiptail_install_type_dist_existing() {
 	Note: Heavy nodes (HEAVYNODE) are NOT recommended for most users.
 	EOM
 	
-	install_type=$(whiptail --title "$whiptail_title" --radiolist "$node_msg" 18 58 4 \
+	install_type=$(whiptail --title "$whiptail_title" --radiolist "$node_msg" 18 58 5 \
 		"SENSOR" "Create a forward only sensor " ON \
 		"SEARCHNODE" "Add a search node with parsing " OFF \
 		"FLEET" "Dedicated Fleet Osquery Node " OFF \
 		"HEAVYNODE" "Sensor + Search Node " OFF \
+		"RECEIVER" "Receiver Node " OFF \
 		3>&1 1>&2 2>&3
 		# "HOTNODE" "Add Hot Node (Uses Elastic Clustering)" OFF \ # TODO
 		# "WARMNODE" "Add Warm Node to existing Hot or Search node" OFF \ # TODO

From ba30c59ec7696eeabb1fbb8a1c16bd864323906c Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Tue, 7 Dec 2021 10:56:35 -0500
Subject: [PATCH 02/47] add receiver node

---
 setup/so-functions | 2 +-
 setup/so-setup     | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/setup/so-functions b/setup/so-functions
index 20818aa72..9720cc757 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -1490,7 +1490,7 @@ get_redirect() {
 get_minion_type() {
 	local minion_type
 	case "$install_type" in
-		'EVAL' | 'MANAGERSEARCH' | 'MANAGER' | 'SENSOR' | 'HEAVYNODE' | 'FLEET' | 'STANDALONE' | 'IMPORT')
+		'EVAL' | 'MANAGERSEARCH' | 'MANAGER' | 'SENSOR' | 'HEAVYNODE' | 'FLEET' | 'STANDALONE' | 'IMPORT' | 'RECEIVER')
 			minion_type=$(echo "$install_type" | tr '[:upper:]' '[:lower:]')
 			;;
 		'HELIXSENSOR')
diff --git a/setup/so-setup b/setup/so-setup
index 159367793..9d3701fce 100755
--- a/setup/so-setup
+++ b/setup/so-setup
@@ -255,6 +255,8 @@ elif [ "$install_type" = 'HELIXSENSOR' ]; then
 	is_helix=true
 elif [ "$install_type" = 'IMPORT' ]; then
 	is_import=true
+elif [ "$install_type" = 'RECEIVER' ]; then
+	is_minion=true
 elif [ "$install_type" = 'ANALYST' ]; then
 	cd .. || exit 255
 	exec bash so-analyst-install

From f3ec5df4479c3b14cf24c6d10a4361497f1404ee Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Tue, 7 Dec 2021 11:13:51 -0500
Subject: [PATCH 03/47] add receiver node

---
 pillar/top.sls     | 2 +-
 setup/so-functions | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/pillar/top.sls b/pillar/top.sls
index a81fdc862..b3cc8dc8f 100644
--- a/pillar/top.sls
+++ b/pillar/top.sls
@@ -110,7 +110,7 @@ base:
     - elasticsearch.auth
     - global
     - minions.{{ grains.id }}
-    - data.receivertab
+    - data.receiverstab
 
   '*_import':
     - zeeklogs
diff --git a/setup/so-functions b/setup/so-functions
index 8a0e59842..527d914e5 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -99,7 +99,7 @@ addtotab_generate_templates() {
 
 	local addtotab_path=$local_salt_dir/pillar/data
 
-	for i in evaltab managersearchtab managertab nodestab sensorstab standalonetab; do
+	for i in evaltab managersearchtab managertab nodestab sensorstab standalonetab receiverstab; do
 		printf '%s\n'\
 		"$i:"\
 		"" > "$addtotab_path"/$i.sls
@@ -2687,7 +2687,7 @@ set_initial_firewall_policy() {
 					;;
 				'RECEIVER')
 					$sshcmd -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/salt/common/tools/sbin/so-firewall --apply includehost receiver "$MAINIP"
-					$sshcmd -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/pillar/data/addtotab.sh receivertab "$MINION_ID" "$MAINIP"
+					$sshcmd -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/pillar/data/addtotab.sh receiverstab "$MINION_ID" "$MAINIP"
 			esac
 			;;
 		'PARSINGNODE')

From 06010bd157ce8e465521792e2e2470c38089060e Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Tue, 7 Dec 2021 13:34:06 -0500
Subject: [PATCH 04/47] add so-receiver to allowed_states

---
 salt/allowed_states.map.jinja | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja
index 0175953b0..204b0d3b4 100644
--- a/salt/allowed_states.map.jinja
+++ b/salt/allowed_states.map.jinja
@@ -205,9 +205,17 @@
           'tcpreplay',
           'docker_clean'
           ],
+      'so-receiver': [
+          'ca',
+          'ssl',
+          'telegraf',
+          'firewall',
+          'schedule',
+          'docker_clean'
+          ],
   }, grain='role') %}
 
-  {% if FILEBEAT and grains.role in ['so-helixsensor', 'so-eval', 'so-manager', 'so-standalone', 'so-node', 'so-managersearch', 'so-heavynode', 'so-import'] %}
+  {% if FILEBEAT and grains.role in ['so-helixsensor', 'so-eval', 'so-manager', 'so-standalone', 'so-node', 'so-managersearch', 'so-heavynode', 'so-import', 'so-receiver'] %}
     {% do allowed_states.append('filebeat') %}
   {% endif %}
 
@@ -215,7 +223,7 @@
     {% do allowed_states.append('mysql') %}
   {% endif %}
 
-  {% if (FLEETMANAGER or FLEETNODE) and grains.role in ['so-sensor', 'so-eval', 'so-manager', 'so-standalone', 'so-node', 'so-managersearch', 'so-heavynode'] %}
+  {% if (FLEETMANAGER or FLEETNODE) and grains.role in ['so-sensor', 'so-eval', 'so-manager', 'so-standalone', 'so-node', 'so-managersearch', 'so-heavynode', 'so-receiver'] %}
     {% do allowed_states.append('fleet.install_package') %}
   {% endif %}
 
@@ -235,7 +243,7 @@
     {% do allowed_states.append('strelka') %}
   {% endif %}
 
-  {% if WAZUH and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-node', 'so-managersearch', 'so-heavynode']%}
+  {% if WAZUH and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-node', 'so-managersearch', 'so-heavynode', 'so-receiver']%}
     {% do allowed_states.append('wazuh') %}
   {% endif %}
 
@@ -280,11 +288,11 @@
     {% do allowed_states.append('domainstats') %}
   {% endif %}
 
-  {% if LOGSTASH and grains.role in ['so-helixsensor', 'so-manager', 'so-standalone', 'so-node', 'so-managersearch', 'so-heavynode'] %}
+  {% if LOGSTASH and grains.role in ['so-helixsensor', 'so-manager', 'so-standalone', 'so-node', 'so-managersearch', 'so-heavynode', 'so-receiver'] %}
     {% do allowed_states.append('logstash') %}
   {% endif %}
 
-  {% if REDIS and grains.role in ['so-manager', 'so-standalone', 'so-managersearch', 'so-heavynode'] %}
+  {% if REDIS and grains.role in ['so-manager', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-receiver'] %}
     {% do allowed_states.append('redis') %}
   {% endif %}
 

From f8da5c7fe99ff90a0b546aac1b37e146b923c3dd Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Tue, 7 Dec 2021 15:59:11 -0500
Subject: [PATCH 05/47] start of fw rules for receiver

---
 salt/firewall/assigned_hostgroups.map.yaml | 66 ++++++++++++++++++++++
 1 file changed, 66 insertions(+)

diff --git a/salt/firewall/assigned_hostgroups.map.yaml b/salt/firewall/assigned_hostgroups.map.yaml
index d5fca081f..e56b86277 100644
--- a/salt/firewall/assigned_hostgroups.map.yaml
+++ b/salt/firewall/assigned_hostgroups.map.yaml
@@ -604,3 +604,69 @@ role:
           minion:
             portgroups:
               - {{ portgroups.salt_manager }}
+
+  receiver:
+    chain:
+      DOCKER-USER:
+        hostgroups:
+          manager:
+            portgroups:
+              - {{ portgroups.elasticsearch_rest }}
+              - {{ portgroups.elasticsearch_node }}
+          sensor:
+            portgroups:
+              - {{ portgroups.beats_5044 }}
+              - {{ portgroups.beats_5644 }}
+          search_node:
+            portgroups:
+              - {{ portgroups.redis }}
+              - {{ portgroups.minio }}
+              - {{ portgroups.elasticsearch_node }}
+              - {{ portgroups.beats_5644 }}
+          heavy_node:
+            portgroups:
+              - {{ portgroups.redis }}
+              - {{ portgroups.minio }}
+              - {{ portgroups.elasticsearch_node }}
+              - {{ portgroups.beats_5644 }}
+          self:
+            portgroups:
+              - {{ portgroups.syslog}}
+          syslog:
+            portgroups:
+              - {{ portgroups.syslog }}
+          beats_endpoint:
+            portgroups:
+              - {{ portgroups.beats_5044 }}
+          beats_endpoint_ssl:
+            portgroups:
+              - {{ portgroups.beats_5644 }}
+          elasticsearch_rest:
+            portgroups:
+              - {{ portgroups.elasticsearch_rest }}
+          endgame:
+            portgroups:
+              - {{ portgroups.endgame }}
+          osquery_endpoint:
+            portgroups:
+              - {{ portgroups.fleet_api }}
+          wazuh_agent:
+            portgroups:
+              - {{ portgroups.wazuh_agent  }}
+          wazuh_api:
+            portgroups:
+              - {{ portgroups.wazuh_api }}
+          wazuh_authd:
+            portgroups:
+              - {{ portgroups.wazuh_authd }}
+      INPUT:
+        hostgroups:
+          anywhere:
+            portgroups:
+              - {{ portgroups.ssh }}
+          dockernet:
+            portgroups:
+              - {{ portgroups.all }}
+          localhost:
+            portgroups:
+              - {{ portgroups.all }}

From 429b9cab2f03464755e17925bcbce9eb374eb91c Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Tue, 7 Dec 2021 16:22:07 -0500
Subject: [PATCH 06/47] set ip for ossec.conf

---
 salt/wazuh/files/agent/ossec.conf | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/salt/wazuh/files/agent/ossec.conf b/salt/wazuh/files/agent/ossec.conf
index 7e33f5599..6ae873875 100644
--- a/salt/wazuh/files/agent/ossec.conf
+++ b/salt/wazuh/files/agent/ossec.conf
@@ -4,6 +4,9 @@
   {%- set ip = salt['pillar.get']('elasticsearch:mainip', '') %}
 {%- elif grains['role'] == 'so-sensor' %}
   {%- set ip = salt['pillar.get']('sensor:mainip', '') %}
+{%- else %}
+  {%- set mainint = salt['pillar.get']('host:mainint') %}
+  {%- set ip = salt['grains.get']('ip_interfaces:' ~ mainint)[0] %}
 {%- endif %}
 <!--
   Wazuh - Agent Configuration

From 8c95d0f36b64607030865b6def9af43305450f6a Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Tue, 7 Dec 2021 16:50:41 -0500
Subject: [PATCH 07/47] set ip for wazuh-register-agent and dont apply nginx in
 setup for receiver

---
 salt/wazuh/files/agent/wazuh-register-agent | 3 +++
 setup/so-setup                              | 3 ++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/salt/wazuh/files/agent/wazuh-register-agent b/salt/wazuh/files/agent/wazuh-register-agent
index 8128aaa8e..dac66c12b 100755
--- a/salt/wazuh/files/agent/wazuh-register-agent
+++ b/salt/wazuh/files/agent/wazuh-register-agent
@@ -4,6 +4,9 @@
   {%- set ip = salt['pillar.get']('elasticsearch:mainip', '') %}
 {%- elif grains['role'] == 'so-sensor' %}
   {%- set ip = salt['pillar.get']('sensor:mainip', '') %}
+{%- else %}
+  {%- set mainint = salt['pillar.get']('host:mainint') %}
+  {%- set ip = salt['grains.get']('ip_interfaces:' ~ mainint)[0] %}
 {%- endif %}
 #!/bin/bash
 
diff --git a/setup/so-setup b/setup/so-setup
index 9d3701fce..741ecb34b 100755
--- a/setup/so-setup
+++ b/setup/so-setup
@@ -257,6 +257,7 @@ elif [ "$install_type" = 'IMPORT' ]; then
 	is_import=true
 elif [ "$install_type" = 'RECEIVER' ]; then
 	is_minion=true
+	is_receiver=true
 elif [ "$install_type" = 'ANALYST' ]; then
 	cd .. || exit 255
 	exec bash so-analyst-install
@@ -783,7 +784,7 @@ echo "1" > /root/accept_changes
 	set_progress_str 62 "$(print_salt_state_apply 'common')"
 	salt-call state.apply -l info common >> $setup_log 2>&1
 
-	if [[ ! $is_helix ]]; then
+	if [[ ! $is_helix || ! $is_receiver ]]; then
 		set_progress_str 62 "$(print_salt_state_apply 'nginx')"
 		salt-call state.apply -l info nginx >> $setup_log 2>&1
 	fi

From c80059efb0349c439677431fa486718734848192 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Tue, 7 Dec 2021 17:11:15 -0500
Subject: [PATCH 08/47] change from || to &&

---
 setup/so-setup | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup/so-setup b/setup/so-setup
index 741ecb34b..5c1194df7 100755
--- a/setup/so-setup
+++ b/setup/so-setup
@@ -784,7 +784,7 @@ echo "1" > /root/accept_changes
 	set_progress_str 62 "$(print_salt_state_apply 'common')"
 	salt-call state.apply -l info common >> $setup_log 2>&1
 
-	if [[ ! $is_helix || ! $is_receiver ]]; then
+	if [[ ! $is_helix && ! $is_receiver ]]; then
 		set_progress_str 62 "$(print_salt_state_apply 'nginx')"
 		salt-call state.apply -l info nginx >> $setup_log 2>&1
 	fi

From 1ef63f3a23ed69a539c012031be1f61b16997912 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Wed, 8 Dec 2021 09:08:46 -0500
Subject: [PATCH 09/47] ssl things for so-receiver

---
 salt/logstash/init.sls | 4 ++++
 salt/ssl/init.sls      | 4 ++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls
index 069b2f7bd..5160e6607 100644
--- a/salt/logstash/init.sls
+++ b/salt/logstash/init.sls
@@ -46,7 +46,9 @@
 
 include:
   - ssl
+{% if grains.role not in ['so-receiver'] %}
   - elasticsearch
+{% endif %}
 
 # Create the logstash group
 logstashgroup:
@@ -210,8 +212,10 @@ so-logstash:
   {% else %}
       - x509: pki_public_ca_crt
   {% endif %}
+  {% if grains.role not in ['so-receiver'] %}
       - file: cacertz
       - file: capemz
+  {% endif %}
 
 append_so-logstash_so-status.conf:
   file.append:
diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls
index 43d789e75..7d485a895 100644
--- a/salt/ssl/init.sls
+++ b/salt/ssl/init.sls
@@ -115,7 +115,7 @@ influxkeyperms:
     - mode: 640
     - group: 939
 
-{% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-fleet'] %}
+{% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-fleet', 'so-receiver'] %}
 # Create a cert for Redis encryption
 redis_key:
   x509.private_key_managed:
@@ -484,7 +484,7 @@ fleetkeyperms:
     - group: 939
 
 {% endif %}
-{% if grains['role'] in ['so-sensor', 'so-manager', 'so-node', 'so-eval', 'so-helix', 'so-managersearch', 'so-heavynode', 'so-fleet', 'so-standalone', 'so-import'] %}
+{% if grains['role'] in ['so-sensor', 'so-manager', 'so-node', 'so-eval', 'so-helix', 'so-managersearch', 'so-heavynode', 'so-fleet', 'so-standalone', 'so-import', 'so-receiver'] %}
    
 fbcertdir:
   file.directory:

From 59464af10c55b6270859851c5b80fce3bc08a4c5 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Wed, 8 Dec 2021 09:41:17 -0500
Subject: [PATCH 10/47] filebeat certs for logstash on so-receiver

---
 salt/logstash/init.sls | 6 ++++--
 salt/ssl/init.sls      | 2 +-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls
index 5160e6607..cb8f2a261 100644
--- a/salt/logstash/init.sls
+++ b/salt/logstash/init.sls
@@ -172,7 +172,7 @@ so-logstash:
       - /nsm/logstash:/usr/share/logstash/data:rw
       - /opt/so/log/logstash:/var/log/logstash:rw
       - /sys/fs/cgroup:/sys/fs/cgroup:ro
-  {% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode'] %}
+  {% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-receiver'] %}
       - /etc/pki/filebeat.crt:/usr/share/logstash/filebeat.crt:ro
       - /etc/pki/filebeat.p8:/usr/share/logstash/filebeat.key:ro
   {% endif %}
@@ -182,8 +182,10 @@ so-logstash:
   {% else %}
       - /etc/pki/ca.crt:/usr/share/filebeat/ca.crt:ro
   {% endif %}
+  {% if grains.role not in ['so-receiver'] %}
       - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro
       - /opt/so/conf/ca/tls-ca-bundle.pem:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem:ro
+  {% endif %}
   {%- if grains['role'] == 'so-eval' %}
       - /nsm/zeek:/nsm/zeek:ro
       - /nsm/suricata:/suricata:ro
@@ -204,7 +206,7 @@ so-logstash:
       - file: es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}
   {% endfor %}
     - require:
-  {% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode'] %}
+  {% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-receiver'] %}
       - x509: etc_filebeat_crt
   {% endif %}
   {% if grains['role'] == 'so-heavynode' %}
diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls
index 7d485a895..d2edb51c4 100644
--- a/salt/ssl/init.sls
+++ b/salt/ssl/init.sls
@@ -164,7 +164,7 @@ rediskeyperms:
     - group: 939
 {% endif %}
 
-{% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode'] %}
+{% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-receiver'] %}
 etc_filebeat_key:
   x509.private_key_managed:
     - name: /etc/pki/filebeat.key

From ecc8594d443a1bbb8090313b06ae38a4891ab8bc Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Wed, 8 Dec 2021 13:32:56 -0500
Subject: [PATCH 11/47] prevent so-receiver from getting extra keys/certs

---
 salt/ssl/init.sls | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls
index d2edb51c4..6641ff186 100644
--- a/salt/ssl/init.sls
+++ b/salt/ssl/init.sls
@@ -225,7 +225,7 @@ chownilogstashfilebeatp8:
     - user: 931
     - group: 939
 
-  {% if grains.role != 'so-heavynode' %}
+  {% if grains.role not in  ['so-heavynode', 'so-receiver']%}
 # Create Symlinks to the keys so I can distribute it to all the things
 filebeatdir:
   file.directory:

From b4bc32d3cab429886809be6141c04df5b090fc88 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Wed, 8 Dec 2021 14:33:15 -0500
Subject: [PATCH 12/47] set logstash pillar and enable avanced ls menu for
 so-receiver

---
 setup/so-functions | 28 ++++++++++++++--------------
 setup/so-setup     | 26 +++++++++++++++++++++-----
 2 files changed, 35 insertions(+), 19 deletions(-)

diff --git a/setup/so-functions b/setup/so-functions
index 527d914e5..b47f585cd 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -1327,15 +1327,6 @@ elasticsearch_pillar() {
 		"  log_size_limit: $log_size_limit"\
 		"  node_route_type: 'hot'"\
 		"" >> "$pillar_file"
-	
-	printf '%s\n'\
-		"logstash_settings:"\
-		"  ls_pipeline_batch_size: $LSPIPELINEBATCH"\
-		"  ls_input_threads: $LSINPUTTHREADS"\
-		"  lsheap: $NODE_LS_HEAP_SIZE"\
-		"  ls_pipeline_workers: $num_cpu_cores"\
-		"" >> "$pillar_file"
-
 }
 
 es_heapsize() {
@@ -1554,6 +1545,20 @@ import_registry_docker() {
 	fi
 }
 
+logstash_pillar() {
+
+	local pillar_file=$temp_install_dir/pillar/minions/$MINION_ID.sls
+
+	# Create the logstash pillar
+	printf '%s\n'\
+		"logstash_settings:"\
+		"  ls_pipeline_batch_size: $LSPIPELINEBATCH"\
+		"  ls_input_threads: $LSINPUTTHREADS"\
+		"  lsheap: $NODE_LS_HEAP_SIZE"\
+		"  ls_pipeline_workers: $num_cpu_cores"\
+		"" >> "$pillar_file"
+}
+
 # Set Logstash heap size based on total memory
 ls_heapsize() {
 
@@ -2407,11 +2412,6 @@ securityonion_repo() {
 	fi
 }
 
-set_base_heapsizes() {
-	es_heapsize
-	ls_heapsize
-}
-
 set_network_dev_status_list() {
 	readarray -t nmcli_dev_status_list <<< "$(nmcli -t -f DEVICE,STATE -c no dev status)"
 	export nmcli_dev_status_list
diff --git a/setup/so-setup b/setup/so-setup
index 5c1194df7..e890853f5 100755
--- a/setup/so-setup
+++ b/setup/so-setup
@@ -464,8 +464,14 @@ if [[ $is_helix ||  $is_manager || $is_import ]]; then
 	collect_homenet_mngr
 fi
 
+#set base elasticsearch heap size
 if [[ $is_helix || $is_manager || $is_node || $is_import ]]; then
-	set_base_heapsizes
+	es_heapsize
+fi
+
+#set base logstash heap size
+if [[ $is_helix || $is_manager || $is_node || $is_import || $is_receiver ]]; then
+	ls_heapsize
 fi
 
 if [[ $is_manager && ! $is_eval ]]; then
@@ -544,17 +550,21 @@ fi
 
 [[ $is_iso ]] && collect_ntp_servers
 
-if [[ $is_node && ! $is_eval ]]; then
+if [[ ($is_node || $is_receiver) && ! $is_eval ]]; then
 	whiptail_node_advanced
 	if [ "$NODESETUP" == 'NODEADVANCED' ]; then
-		collect_node_es_heap
+		if [[ ! $is_receiver ]]; then 
+			collect_node_es_heap
+			collect_es_space_limit
+		fi
 		collect_node_ls_heap
 		collect_node_ls_pipeline_worker_count
 		collect_node_ls_pipeline_batch_size
 		collect_node_ls_input
-		collect_es_space_limit
 	else
-		NODE_ES_HEAP_SIZE=$ES_HEAP_SIZE
+		if [[ ! $is_receiver ]]; then
+			NODE_ES_HEAP_SIZE=$ES_HEAP_SIZE
+		fi
 		NODE_LS_HEAP_SIZE=$LS_HEAP_SIZE
 		LSPIPELINEWORKERS=$num_cpu_cores
 		LSPIPELINEBATCH=125
@@ -727,6 +737,12 @@ echo "1" > /root/accept_changes
 		fi
 	fi
 
+	if [[ $is_node || $is_receiver ]]; then
+		set_progress_str 19 'Generating logstash pillar'
+		logstash_pillar >> $setup_log 2>&1
+	fi
+
+
 	if [[ $is_minion ]]; then
 		set_progress_str 20 'Accepting Salt key on manager'
 		retry 20 10	accept_salt_key_remote "going to be accepted" >> $setup_log 2>&1

From 7390b03dc197e4547f54dc018cc4f7e483c95a53 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Wed, 8 Dec 2021 14:58:34 -0500
Subject: [PATCH 13/47] dont show es options in final whiptail setup
 confirmation

---
 setup/so-whiptail | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/setup/so-whiptail b/setup/so-whiptail
index bdaa50849..62df1b5d9 100755
--- a/setup/so-whiptail
+++ b/setup/so-whiptail
@@ -563,19 +563,23 @@ whiptail_end_settings() {
 
 	if [[ $NODESETUP == 'NODEADVANCED' ]]; then
 		__append_end_msg "Advanced Node Settings:"
-		__append_end_msg "  Elasticsearch Heap Size: $NODE_ES_HEAP_SIZE"
+		if [[ ! $is_receiver ]]; then
+			__append_end_msg "  Elasticsearch Heap Size: $NODE_ES_HEAP_SIZE"
+			__append_end_msg "  Elasticsearch Storage Space: ${log_size_limit}GB"
+		fi
 		__append_end_msg "  Logstash Heap Size: $NODE_LS_HEAP_SIZE"
 		__append_end_msg "  Logstash Worker Count: $LSPIPELINEWORKERS"
 		__append_end_msg "  Logstash Batch Size: $LSPIPELINEBATCH"
 		__append_end_msg "  Logstash Input Threads: $LSINPUTTHREADS"
-		__append_end_msg "  Elasticsearch Storage Space: ${log_size_limit}GB"
 	else
-		__append_end_msg "Elasticsearch Heap Size: $NODE_ES_HEAP_SIZE"
+		if [[ ! $is_receiver ]]; then
+			__append_end_msg "Elasticsearch Heap Size: $NODE_ES_HEAP_SIZE"
+			__append_end_msg "Elasticsearch Storage Space: ${log_size_limit}GB"
+		fi
 		__append_end_msg "Logstash Heap Size: $NODE_LS_HEAP_SIZE"
 		__append_end_msg "Logstash Worker Count: $LSPIPELINEWORKERS"
 		__append_end_msg "Logstash Batch Size: $LSPIPELINEBATCH"
 		__append_end_msg "Logstash Input Threads: $LSINPUTTHREADS"
-		__append_end_msg "Elasticsearch Storage Space: ${log_size_limit}GB"
 	fi
 
 

From a7f0d8155586e449278c9873e992f26610b308ea Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Thu, 9 Dec 2021 13:07:00 -0500
Subject: [PATCH 14/47] SSL modifications

---
 salt/allowed_states.map.jinja | 4 ----
 salt/top.sls                  | 4 ----
 setup/so-functions            | 6 +++---
 3 files changed, 3 insertions(+), 11 deletions(-)

diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja
index 0175953b0..f3c46edb8 100644
--- a/salt/allowed_states.map.jinja
+++ b/salt/allowed_states.map.jinja
@@ -50,7 +50,6 @@
           'learn'
           ],
       'so-heavynode': [
-          'ca',
           'ssl',
           'nginx',
           'telegraf',
@@ -80,7 +79,6 @@
           'docker_clean'
           ],
       'so-fleet': [
-          'ca',
           'ssl',
           'nginx',
           'telegraf',
@@ -157,7 +155,6 @@
           'learn'
           ],
       'so-node': [
-          'ca',
           'ssl',
           'nginx',
           'telegraf',
@@ -191,7 +188,6 @@
           'learn'
           ],
       'so-sensor': [
-          'ca',
           'ssl',
           'telegraf',
           'firewall',
diff --git a/salt/top.sls b/salt/top.sls
index a8f2018a6..2b617f52d 100644
--- a/salt/top.sls
+++ b/salt/top.sls
@@ -63,7 +63,6 @@ base:
 
   '*_sensor and G@saltversion:{{saltversion}}':
     - match: compound
-    - ca
     - ssl
     - sensoroni
     - telegraf
@@ -298,7 +297,6 @@ base:
 
   '*_searchnode and G@saltversion:{{saltversion}}':
     - match: compound
-    - ca
     - ssl
     - sensoroni
     - nginx
@@ -391,7 +389,6 @@ base:
 
   '*_heavynode and G@saltversion:{{saltversion}}':
     - match: compound
-    - ca
     - ssl
     - sensoroni
     - nginx
@@ -433,7 +430,6 @@ base:
   
   '*_fleet and G@saltversion:{{saltversion}}':
     - match: compound
-    - ca
     - ssl
     - sensoroni
     - nginx
diff --git a/setup/so-functions b/setup/so-functions
index 3d88a7fdb..c3a286402 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -77,7 +77,7 @@ accept_salt_key_remote() {
 	echo "Accept the key remotely on the manager" >> "$setup_log" 2>&1
 	# Delete the key just in case.
 	$sshcmd -i /root/.ssh/so.key soremote@"$MSRV" sudo salt-key -d "$MINION_ID" -y
-	salt-call state.apply ca >> /dev/null 2>&1
+	salt-call state.show_top >> /dev/null 2>&1
 	$sshcmd -i /root/.ssh/so.key soremote@"$MSRV" sudo salt-key -a "$MINION_ID" -y
 }
 
@@ -2345,13 +2345,13 @@ salt_checkin() {
 			;;
 		*)
 			{
-				salt-call state.apply ca;
+				#salt-call state.apply ca;
 				salt-call state.apply ssl;
 			} >> "$setup_log" 2>&1
 			;;
 	esac
 	{
-		salt-call state.apply ca;
+		#salt-call state.apply ca;
 		salt-call state.apply ssl;
 		salt-call saltutil.sync_modules;
 	} >> "$setup_log" 2>&1

From b92cbb01b3430aaa5bf3286befcce38261ab1bae Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Thu, 9 Dec 2021 13:13:01 -0500
Subject: [PATCH 15/47] SSL modifications

---
 salt/ssl/init.sls | 64 +++++++++++++++--------------------------------
 1 file changed, 20 insertions(+), 44 deletions(-)

diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls
index 43d789e75..deb156edc 100644
--- a/salt/ssl/init.sls
+++ b/salt/ssl/init.sls
@@ -18,6 +18,8 @@
 {% if grains.id.split('_')|last in ['manager', 'eval', 'standalone', 'import', 'helixsensor'] %}
     {% set trusttheca_text = salt['cp.get_file_str']('/etc/pki/ca.crt')|replace('\n', '') %}
     {% set ca_server = grains.id %}
+include:
+  - ca
 {% else %}
     {% set x509dict = salt['mine.get']('*', 'x509.get_pem_entries') %}
     {% for host in x509dict %}
@@ -30,9 +32,6 @@
     {% set ca_server = global_ca_server[0] %}
 {% endif %}
 
-include:
-  - ca
-
 # Trust the CA
 trusttheca:
   x509.pem_managed:
@@ -70,7 +69,7 @@ removeesp12dir:
 influxdb_key:
   x509.private_key_managed:
     - name: /etc/pki/influxdb.key
-    - CN: {{ manager }}
+    - CN: {{ HOSTNAME }}
     - bits: 4096
     - days_remaining: 0
     - days_valid: 820
@@ -92,8 +91,8 @@ influxdb_crt:
     - ca_server: {{ ca_server }}
     - signing_policy: influxdb
     - public_key: /etc/pki/influxdb.key
-    - CN: {{ manager }}
-    - subjectAltName: DNS:{{ manager }}, IP:{{ managerip }} 
+    - CN: {{ HOSTNAME }}
+    - subjectAltName: DNS:{{ HOSTNAME }}, IP:{{ MAINIP }} 
     - days_remaining: 0
     - days_valid: 820
     - backup: True
@@ -115,12 +114,12 @@ influxkeyperms:
     - mode: 640
     - group: 939
 
-{% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-fleet'] %}
+{% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-fleet', 'so-receiver'] %}
 # Create a cert for Redis encryption
 redis_key:
   x509.private_key_managed:
     - name: /etc/pki/redis.key
-    - CN: {{ COMMONNAME }}
+    - CN: {{ HOSTNAME }}
     - bits: 4096
     - days_remaining: 0
     - days_valid: 820
@@ -139,9 +138,10 @@ redis_crt:
   x509.certificate_managed:
     - name: /etc/pki/redis.crt
     - ca_server: {{ ca_server }}
+    - subjectAltName: DNS:{{ HOSTNAME }}, IP:{{ MAINIP }}
     - signing_policy: registry
     - public_key: /etc/pki/redis.key
-    - CN: {{ COMMONNAME }}
+    - CN: {{ HOSTNAME }}
     - days_remaining: 0
     - days_valid: 820
     - backup: True
@@ -164,7 +164,7 @@ rediskeyperms:
     - group: 939
 {% endif %}
 
-{% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode'] %}
+{% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-receiver'] %}
 etc_filebeat_key:
   x509.private_key_managed:
     - name: /etc/pki/filebeat.key
@@ -190,7 +190,8 @@ etc_filebeat_crt:
     - ca_server: {{ ca_server }}
     - signing_policy: filebeat
     - public_key: /etc/pki/filebeat.key
-    - CN: {{ COMMONNAME }}
+    - CN: {{ HOSTNAME }}
+    - subjectAltName: DNS:{{ HOSTNAME }}, IP:{{ MAINIP }}
     - days_remaining: 0
     - days_valid: 820
     - backup: True
@@ -310,33 +311,6 @@ minio_key:
         attempts: 5
         interval: 30
 
-# Create a cert for minio
-minio_crt:
-  x509.certificate_managed:
-    - name: /etc/pki/minio.crt
-    - ca_server: {{ ca_server }}
-    - signing_policy: registry
-    - public_key: /etc/pki/minio.key
-    - CN: {{ manager }}
-    - days_remaining: 0
-    - days_valid: 820
-    - backup: True
-    - unless:
-      # https://github.com/saltstack/salt/issues/52167
-      # Will trigger 5 days (432000 sec) from cert expiration
-      - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/minio.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
-    - timeout: 30
-    - retry:
-        attempts: 5
-        interval: 30
-
-miniokeyperms:
-  file.managed:
-    - replace: False
-    - name: /etc/pki/minio.key
-    - mode: 640
-    - group: 939
-  {% endif %}
 # Create a cert for elasticsearch
 /etc/pki/elasticsearch.key:
   x509.private_key_managed:
@@ -360,7 +334,8 @@ miniokeyperms:
     - ca_server: {{ ca_server }}
     - signing_policy: registry
     - public_key: /etc/pki/elasticsearch.key
-    - CN: {{ COMMONNAME }}
+    - CN: {{ HOSTNAME }}
+    - subjectAltName: DNS:{{ HOSTNAME }}, IP:{{ MAINIP }}
     - days_remaining: 0
     - days_valid: 820
     - backup: True
@@ -418,7 +393,7 @@ managerssl_crt:
     - ca_server: {{ ca_server }}
     - signing_policy: managerssl
     - public_key: /etc/pki/managerssl.key
-    - CN: {{ manager }}
+    - CN: {{ HOSTNAME }}
     - subjectAltName: DNS:{{ HOSTNAME }}, IP:{{ MAINIP }} {% if CUSTOM_FLEET_HOSTNAME != None %},DNS:{{ CUSTOM_FLEET_HOSTNAME }} {% endif %}
     - days_remaining: 0
     - days_valid: 820
@@ -443,7 +418,7 @@ msslkeyperms:
 fleet_key:
   x509.private_key_managed:
     - name: /etc/pki/fleet.key
-    - CN: {{ manager }}
+    - CN: {{ HOSTNAME }}
     - bits: 4096
     - days_remaining: 0
     - days_valid: 820
@@ -462,8 +437,8 @@ fleet_crt:
   x509.certificate_managed:
     - name: /etc/pki/fleet.crt
     - signing_private_key: /etc/pki/fleet.key
-    - CN: {{ manager }}
-    - subjectAltName: DNS:{{ manager }},IP:{{ managerip }}{% if CUSTOM_FLEET_HOSTNAME != None %},DNS:{{ CUSTOM_FLEET_HOSTNAME }}{% endif %}
+    - CN: {{ HOSTNAME }}
+    - subjectAltName: DNS:{{ HOSTNAME }},IP:{{ MAINIP }}{% if CUSTOM_FLEET_HOSTNAME != None %},DNS:{{ CUSTOM_FLEET_HOSTNAME }}{% endif %}
     - days_remaining: 0
     - days_valid: 820
     - backup: True
@@ -516,7 +491,7 @@ conf_filebeat_crt:
     - ca_server: {{ ca_server }}
     - signing_policy: filebeat
     - public_key: /opt/so/conf/filebeat/etc/pki/filebeat.key
-    - CN: {{ COMMONNAME }}
+    - CN: {{ HOSTNAME }}
     - subjectAltName: DNS:{{ HOSTNAME }}, IP:{{ MAINIP }}
     - days_remaining: 0
     - days_valid: 820
@@ -675,6 +650,7 @@ fleetkeyperms:
     - signing_policy: registry
     - public_key: /etc/pki/elasticsearch.key
     - CN: {{ HOSTNAME }}
+    - subjectAltName: DNS:{{ HOSTNAME }}, IP:{{ MAINIP }}
     - days_remaining: 0
     - days_valid: 820
     - backup: True

From d94496bb90aa514c3fbe596f033491a958187b43 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Thu, 9 Dec 2021 13:24:20 -0500
Subject: [PATCH 16/47] remove minio_key and add missing endif

---
 salt/ssl/init.sls | 18 +-----------------
 1 file changed, 1 insertion(+), 17 deletions(-)

diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls
index 7a1bb0c99..3b4bb2bcf 100644
--- a/salt/ssl/init.sls
+++ b/salt/ssl/init.sls
@@ -293,23 +293,7 @@ regkeyperms:
     - mode: 640
     - group: 939
 
-minio_key:
-  x509.private_key_managed:
-    - name: /etc/pki/minio.key
-    - CN: {{ manager }}
-    - bits: 4096
-    - days_remaining: 0
-    - days_valid: 820
-    - backup: True
-    - new: True
-    {% if salt['file.file_exists']('/etc/pki/minio.key') -%}
-    - prereq:
-      - x509: /etc/pki/minio.crt
-    {%- endif %}
-    - timeout: 30
-    - retry:
-        attempts: 5
-        interval: 30
+  {% endif %}
 
 # Create a cert for elasticsearch
 /etc/pki/elasticsearch.key:

From 54c32acdbf730f810f792e018a15fee0e599bec7 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Thu, 9 Dec 2021 15:26:00 -0500
Subject: [PATCH 17/47] dont call logstash_pillar if manager or helix

---
 setup/so-setup | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup/so-setup b/setup/so-setup
index e890853f5..4aaab8ade 100755
--- a/setup/so-setup
+++ b/setup/so-setup
@@ -737,7 +737,7 @@ echo "1" > /root/accept_changes
 		fi
 	fi
 
-	if [[ $is_node || $is_receiver ]]; then
+	if [[ ($is_node || $is_receiver) && !($is_manager || $is_helix) ]]; then
 		set_progress_str 19 'Generating logstash pillar'
 		logstash_pillar >> $setup_log 2>&1
 	fi

From fe7247f876eda74ac82f4a5046475eac7f41dc59 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Fri, 10 Dec 2021 15:28:40 -0500
Subject: [PATCH 18/47] update fw for receiver and add mine_functions for
 ip_addr

---
 salt/firewall/assigned_hostgroups.map.yaml | 3 +--
 salt/salt/etc/minion.d/mine_functions.conf | 3 +++
 salt/salt/minion.sls                       | 9 +++++++++
 3 files changed, 13 insertions(+), 2 deletions(-)
 create mode 100644 salt/salt/etc/minion.d/mine_functions.conf

diff --git a/salt/firewall/assigned_hostgroups.map.yaml b/salt/firewall/assigned_hostgroups.map.yaml
index e56b86277..e9727f331 100644
--- a/salt/firewall/assigned_hostgroups.map.yaml
+++ b/salt/firewall/assigned_hostgroups.map.yaml
@@ -620,17 +620,16 @@ role:
           search_node:
             portgroups:
               - {{ portgroups.redis }}
-              - {{ portgroups.minio }}
               - {{ portgroups.elasticsearch_node }}
               - {{ portgroups.beats_5644 }}
           heavy_node:
             portgroups:
               - {{ portgroups.redis }}
-              - {{ portgroups.minio }}
               - {{ portgroups.elasticsearch_node }}
               - {{ portgroups.beats_5644 }}
           self:
             portgroups:
+              - {{ portgroups.redis }}
               - {{ portgroups.syslog}}
           syslog:
             portgroups:
diff --git a/salt/salt/etc/minion.d/mine_functions.conf b/salt/salt/etc/minion.d/mine_functions.conf
new file mode 100644
index 000000000..42a543841
--- /dev/null
+++ b/salt/salt/etc/minion.d/mine_functions.conf
@@ -0,0 +1,3 @@
+mine_functions:
+  network.ip_addrs:
+    - interface: {{ pillar.host.mainint }}
diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls
index 04fc1769c..22ea149bf 100644
--- a/salt/salt/minion.sls
+++ b/salt/salt/minion.sls
@@ -76,14 +76,23 @@ salt_minion_service_unit_file:
       - module: systemd_reload
     - listen_in:
       - service: salt_minion_service
+
 {% endif %}
 
+mine_functions:
+  file.managed:
+    - name: /etc/salt/minion.d/mine_functions.conf
+    - source: salt://salt/etc/mine_functions.conf
+    - template: jinja
+
 # this has to be outside the if statement above since there are <requisite>_in calls to this state
 salt_minion_service:
   service.running:
     - name: salt-minion
     - enable: True
     - onlyif: test "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}"
+    - watch:
+      - file: mine_functions
 
 patch_pkg:
   pkg.installed:

From 86f67198bfe55a62a56a4fa089ca62261359390d Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Fri, 10 Dec 2021 17:43:06 -0500
Subject: [PATCH 19/47] loadbalance filebeat if across managers and receivers

---
 salt/filebeat/etc/filebeat.yml | 14 +++++++++++++-
 salt/salt/minion.sls           |  2 +-
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml
index 0c27e3c1b..02c094f80 100644
--- a/salt/filebeat/etc/filebeat.yml
+++ b/salt/filebeat/etc/filebeat.yml
@@ -321,7 +321,19 @@ output.logstash:
   enabled: true
 
   # The Logstash hosts
-  hosts: ["{{ MANAGER }}:5644"]
+  hosts:
+{%- set LOADBALANCE = ['false'] %}
+{%- if grains.role not in ['so-heavynode', 'so-import', 'so-helix', 'so-eval'] %}
+{%-   for minionid, ip in salt['mine.get']('G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver', 'network.ip_addrs', tgt_type='compound') | dictsort() %}
+    - "{{ minionid.split('_')[0] }}:5644" #{{ ip[0] }}
+{%-     if loop.index == 2 %}
+{%-       do LOADBALANCE.insert(0, 'true') %}
+{%-     endif %}
+{%-   endfor %}
+  loadbalance: {{ LOADBALANCE[0] }}
+{%- else %}
+    - "{{ grains.host }}:5644"
+{%- endif %}
 
   # Number of workers per Logstash host.
   worker: {{ FBLSWORKERS }}
diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls
index 22ea149bf..38f8889c3 100644
--- a/salt/salt/minion.sls
+++ b/salt/salt/minion.sls
@@ -82,7 +82,7 @@ salt_minion_service_unit_file:
 mine_functions:
   file.managed:
     - name: /etc/salt/minion.d/mine_functions.conf
-    - source: salt://salt/etc/mine_functions.conf
+    - source: salt://salt/etc/minion.d/mine_functions.conf
     - template: jinja
 
 # this has to be outside the if statement above since there are <requisite>_in calls to this state

From 8d0872bce53c3597c82b61af4261ef721e1169f5 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Mon, 13 Dec 2021 15:48:30 -0500
Subject: [PATCH 20/47] create node_data pillar from mine data, use node_data
 pillar for filebeat config

---
 pillar/node_data/ips.sls                   | 23 ++++++++++++++++++++++
 pillar/node_data/test_ping.sls             | 23 ++++++++++++++++++++++
 pillar/top.sls                             |  2 ++
 salt/filebeat/etc/filebeat.yml             |  8 ++++++--
 salt/salt/etc/minion.d/mine_functions.conf |  1 +
 5 files changed, 55 insertions(+), 2 deletions(-)
 create mode 100644 pillar/node_data/ips.sls
 create mode 100644 pillar/node_data/test_ping.sls

diff --git a/pillar/node_data/ips.sls b/pillar/node_data/ips.sls
new file mode 100644
index 000000000..40551eafc
--- /dev/null
+++ b/pillar/node_data/ips.sls
@@ -0,0 +1,23 @@
+{% set node_types = {} %}
+{% for minionid, ip in salt.saltutil.runner('mine.get', tgt='*', fun='network.ip_addrs', tgt_type='glob') | dictsort() %}
+{%   set hostname = minionid.split('_')[0] %}
+{%   set node_type = minionid.split('_')[1] %}
+{%   if node_type not in node_types.keys() %}
+{%     do node_types.update({node_type: {hostname: ip[0]}}) %}
+{%   else %}
+{%     if hostname not in node_types[node_type] %}
+{%       do node_types[node_type].update({hostname: ip[0]}) %}
+{%     else %}
+{%       do node_types[node_type][hostname].update(ip[0]) %}
+{%     endif %}
+{%   endif %}
+{% endfor %}
+
+node_data:
+{% for node_type, values in node_types.items() %}
+  {{node_type}}:
+{%   for hostname, ip in values.items() %}
+    {{hostname}}:
+      ip: {{ip}}
+{%   endfor %}
+{% endfor %}
diff --git a/pillar/node_data/test_ping.sls b/pillar/node_data/test_ping.sls
new file mode 100644
index 000000000..81cf66004
--- /dev/null
+++ b/pillar/node_data/test_ping.sls
@@ -0,0 +1,23 @@
+{% set node_types = {} %}
+{% for minionid, test_ping in salt.saltutil.runner('mine.get', tgt='*', fun='test.ping', tgt_type='glob') | dictsort() %}
+{%   set node_type = minionid.split('_')[1] %}
+{%   set hostname = minionid.split('_')[0] %}
+{%   if node_type not in node_types.keys() %}
+{%     do node_types.update({node_type: {hostname: test_ping}}) %}
+{%   else %}
+{%     if hostname not in node_types[node_type] %}
+{%       do node_types[node_type].update({hostname: test_ping}) %}
+{%     else %}
+{%       do node_types[node_type][hostname].update(test_ping) %}
+{%     endif %}
+{%   endif %}
+{% endfor %}
+
+node_data:
+{% for node_type, values in node_types.items() %}
+  {{node_type}}:
+{%   for hostname, test_ping in values.items() %}
+    {{hostname}}:
+      test_ping: {{test_ping}}
+{%   endfor %}
+{% endfor %}
diff --git a/pillar/top.sls b/pillar/top.sls
index b3cc8dc8f..6517fce6d 100644
--- a/pillar/top.sls
+++ b/pillar/top.sls
@@ -2,6 +2,8 @@ base:
   '*':
     - patch.needs_restarting
     - logrotate
+    - node_data.ips
+    - node_data.test_ping
 
   '*_eval or *_helixsensor or *_heavynode or *_sensor or *_standalone or *_import':
     - match: compound
diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml
index 02c094f80..f29205743 100644
--- a/salt/filebeat/etc/filebeat.yml
+++ b/salt/filebeat/etc/filebeat.yml
@@ -324,8 +324,12 @@ output.logstash:
   hosts:
 {%- set LOADBALANCE = ['false'] %}
 {%- if grains.role not in ['so-heavynode', 'so-import', 'so-helix', 'so-eval'] %}
-{%-   for minionid, ip in salt['mine.get']('G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver', 'network.ip_addrs', tgt_type='compound') | dictsort() %}
-    - "{{ minionid.split('_')[0] }}:5644" #{{ ip[0] }}
+{%-   for node_type, node_details in salt['pillar.get']('node_data') | dictsort() %}
+{%-     if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %}
+{%-       for hostname in node_type.keys() %}
+    - "{{ hostname }}:5644" #{{ node_details[hostname][ip] }}
+{%-       endfor %}
+{%-     endif %}
 {%-     if loop.index == 2 %}
 {%-       do LOADBALANCE.insert(0, 'true') %}
 {%-     endif %}
diff --git a/salt/salt/etc/minion.d/mine_functions.conf b/salt/salt/etc/minion.d/mine_functions.conf
index 42a543841..023ba0308 100644
--- a/salt/salt/etc/minion.d/mine_functions.conf
+++ b/salt/salt/etc/minion.d/mine_functions.conf
@@ -1,3 +1,4 @@
 mine_functions:
+  test.ping: []
   network.ip_addrs:
     - interface: {{ pillar.host.mainint }}

From 6de2f5bd03b3bc0afbce3671581d5abd7adc8640 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Mon, 13 Dec 2021 15:55:09 -0500
Subject: [PATCH 21/47] fix node_data

---
 salt/filebeat/etc/filebeat.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml
index f29205743..e11dabb8b 100644
--- a/salt/filebeat/etc/filebeat.yml
+++ b/salt/filebeat/etc/filebeat.yml
@@ -323,10 +323,11 @@ output.logstash:
   # The Logstash hosts
   hosts:
 {%- set LOADBALANCE = ['false'] %}
+{%- set node_data = salt['pillar.get']('node_data') | dictsort() %}
 {%- if grains.role not in ['so-heavynode', 'so-import', 'so-helix', 'so-eval'] %}
-{%-   for node_type, node_details in salt['pillar.get']('node_data') | dictsort() %}
+{%-   for node_type, node_details in node_data %}
 {%-     if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %}
-{%-       for hostname in node_type.keys() %}
+{%-       for hostname in node_data[node_type].keys() %}
     - "{{ hostname }}:5644" #{{ node_details[hostname][ip] }}
 {%-       endfor %}
 {%-     endif %}

From 067e79894f7f6cfad3677a4a198cf66f8d1676d9 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Mon, 13 Dec 2021 16:26:38 -0500
Subject: [PATCH 22/47] fix loop for node_data

---
 salt/filebeat/etc/filebeat.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml
index e11dabb8b..f04149958 100644
--- a/salt/filebeat/etc/filebeat.yml
+++ b/salt/filebeat/etc/filebeat.yml
@@ -322,13 +322,13 @@ output.logstash:
 
   # The Logstash hosts
   hosts:
-{%- set LOADBALANCE = ['false'] %}
-{%- set node_data = salt['pillar.get']('node_data') | dictsort() %}
 {%- if grains.role not in ['so-heavynode', 'so-import', 'so-helix', 'so-eval'] %}
-{%-   for node_type, node_details in node_data %}
+{%-   set LOADBALANCE = ['false'] %}
+{%-   set node_data = salt['pillar.get']('node_data') %}
+{%-   for node_type, node_details in node_data.items() %}
 {%-     if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %}
 {%-       for hostname in node_data[node_type].keys() %}
-    - "{{ hostname }}:5644" #{{ node_details[hostname][ip] }}
+    - "{{ hostname }}:5644" #{{ node_details[hostname].ip }}
 {%-       endfor %}
 {%-     endif %}
 {%-     if loop.index == 2 %}

From 6518691c55100e8d7eb2ff6e42a706af04add2cd Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Mon, 13 Dec 2021 18:16:25 -0500
Subject: [PATCH 23/47] sort the items

---
 salt/filebeat/etc/filebeat.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml
index f04149958..6b6adfd6f 100644
--- a/salt/filebeat/etc/filebeat.yml
+++ b/salt/filebeat/etc/filebeat.yml
@@ -325,7 +325,7 @@ output.logstash:
 {%- if grains.role not in ['so-heavynode', 'so-import', 'so-helix', 'so-eval'] %}
 {%-   set LOADBALANCE = ['false'] %}
 {%-   set node_data = salt['pillar.get']('node_data') %}
-{%-   for node_type, node_details in node_data.items() %}
+{%-   for node_type, node_details in node_data.items() | sort %}
 {%-     if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %}
 {%-       for hostname in node_data[node_type].keys() %}
     - "{{ hostname }}:5644" #{{ node_details[hostname].ip }}

From c490a3be36695401421b710ba1dc902a08f4c08d Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Tue, 14 Dec 2021 13:32:42 -0500
Subject: [PATCH 24/47] move node_data pillar to logstash:nodes, set extra
 hosts for filebeat docker

---
 pillar/logstash/nodes.sls                  | 29 ++++++++++++++++++++++
 pillar/node_data/ips.sls                   | 22 +++++++++++-----
 pillar/node_data/test_ping.sls             | 23 -----------------
 pillar/top.sls                             |  3 +--
 salt/filebeat/etc/filebeat.yml             |  4 +--
 salt/filebeat/init.sls                     |  6 ++---
 salt/filebeat/map.jinja                    | 17 +++++++++++++
 salt/salt/etc/minion.d/mine_functions.conf |  1 -
 8 files changed, 67 insertions(+), 38 deletions(-)
 create mode 100644 pillar/logstash/nodes.sls
 delete mode 100644 pillar/node_data/test_ping.sls

diff --git a/pillar/logstash/nodes.sls b/pillar/logstash/nodes.sls
new file mode 100644
index 000000000..3658065cb
--- /dev/null
+++ b/pillar/logstash/nodes.sls
@@ -0,0 +1,29 @@
+{% set node_types = {} %}
+{% for minionid, ip in salt.saltutil.runner(
+    'mine.get',
+    tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-node or G@role:so-heavynode or G@role:so-receiver or G@role:so-helix ',
+    fun='network.ip_addrs',
+    tgt_type='compound') | dictsort() 
+%}
+{%   set hostname = minionid.split('_')[0] %}
+{%   set node_type = minionid.split('_')[1] %}
+{%   if node_type not in node_types.keys() %}
+{%     do node_types.update({node_type: {hostname: ip[0]}}) %}
+{%   else %}
+{%     if hostname not in node_types[node_type] %}
+{%       do node_types[node_type].update({hostname: ip[0]}) %}
+{%     else %}
+{%       do node_types[node_type][hostname].update(ip[0]) %}
+{%     endif %}
+{%   endif %}
+{% endfor %}
+
+logstash:
+  nodes:
+{% for node_type, values in node_types.items() %}
+    {{node_type}}:
+{%   for hostname, ip in values.items() %}
+      {{hostname}}:
+        ip: {{ip}}
+{%   endfor %}
+{% endfor %}
diff --git a/pillar/node_data/ips.sls b/pillar/node_data/ips.sls
index 40551eafc..233038e66 100644
--- a/pillar/node_data/ips.sls
+++ b/pillar/node_data/ips.sls
@@ -1,23 +1,33 @@
 {% set node_types = {} %}
+{% set manage_alived = salt.saltutil.runner('manage.alived', show_ip=True) %}
+{% set manager = grains.master %}
+{% set manager_type = manager.split('_')|last %}
 {% for minionid, ip in salt.saltutil.runner('mine.get', tgt='*', fun='network.ip_addrs', tgt_type='glob') | dictsort() %}
 {%   set hostname = minionid.split('_')[0] %}
 {%   set node_type = minionid.split('_')[1] %}
+{%   set is_alive = False %}
+{%   if minionid in manage_alived.keys() %}
+{%     if ip[0] == manage_alived[minionid] %}
+{%       set is_alive = True %}
+{%     endif %}
+{%   endif %}
 {%   if node_type not in node_types.keys() %}
-{%     do node_types.update({node_type: {hostname: ip[0]}}) %}
+{%     do node_types.update({node_type: {hostname: {'ip':ip[0], 'alive':is_alive }}}) %}
 {%   else %}
 {%     if hostname not in node_types[node_type] %}
-{%       do node_types[node_type].update({hostname: ip[0]}) %}
+{%       do node_types[node_type].update({hostname: {'ip':ip[0], 'alive':is_alive}}) %}
 {%     else %}
-{%       do node_types[node_type][hostname].update(ip[0]) %}
+{%       do node_types[node_type][hostname].update({'ip':ip[0], 'alive':is_alive}) %}
 {%     endif %}
 {%   endif %}
 {% endfor %}
 
 node_data:
-{% for node_type, values in node_types.items() %}
+{% for node_type, host_values in node_types.items() %}
   {{node_type}}:
-{%   for hostname, ip in values.items() %}
+{%   for hostname, details in host_values.items() %}
     {{hostname}}:
-      ip: {{ip}}
+      ip: {{details.ip}}
+      alive: {{ details.alive }}
 {%   endfor %}
 {% endfor %}
diff --git a/pillar/node_data/test_ping.sls b/pillar/node_data/test_ping.sls
deleted file mode 100644
index 81cf66004..000000000
--- a/pillar/node_data/test_ping.sls
+++ /dev/null
@@ -1,23 +0,0 @@
-{% set node_types = {} %}
-{% for minionid, test_ping in salt.saltutil.runner('mine.get', tgt='*', fun='test.ping', tgt_type='glob') | dictsort() %}
-{%   set node_type = minionid.split('_')[1] %}
-{%   set hostname = minionid.split('_')[0] %}
-{%   if node_type not in node_types.keys() %}
-{%     do node_types.update({node_type: {hostname: test_ping}}) %}
-{%   else %}
-{%     if hostname not in node_types[node_type] %}
-{%       do node_types[node_type].update({hostname: test_ping}) %}
-{%     else %}
-{%       do node_types[node_type][hostname].update(test_ping) %}
-{%     endif %}
-{%   endif %}
-{% endfor %}
-
-node_data:
-{% for node_type, values in node_types.items() %}
-  {{node_type}}:
-{%   for hostname, test_ping in values.items() %}
-    {{hostname}}:
-      test_ping: {{test_ping}}
-{%   endfor %}
-{% endfor %}
diff --git a/pillar/top.sls b/pillar/top.sls
index 6517fce6d..b6c3b744a 100644
--- a/pillar/top.sls
+++ b/pillar/top.sls
@@ -2,8 +2,7 @@ base:
   '*':
     - patch.needs_restarting
     - logrotate
-    - node_data.ips
-    - node_data.test_ping
+    - logstash.nodes
 
   '*_eval or *_helixsensor or *_heavynode or *_sensor or *_standalone or *_import':
     - match: compound
diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml
index 6b6adfd6f..05605b96d 100644
--- a/salt/filebeat/etc/filebeat.yml
+++ b/salt/filebeat/etc/filebeat.yml
@@ -322,9 +322,9 @@ output.logstash:
 
   # The Logstash hosts
   hosts:
-{%- if grains.role not in ['so-heavynode', 'so-import', 'so-helix', 'so-eval'] %}
+{%- if grains.role in ['so-sensor', 'so-fleet', 'so-node'] %}
 {%-   set LOADBALANCE = ['false'] %}
-{%-   set node_data = salt['pillar.get']('node_data') %}
+{%-   set node_data = salt['pillar.get']('logstash:nodes') %}
 {%-   for node_type, node_details in node_data.items() | sort %}
 {%-     if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %}
 {%-       for hostname in node_data[node_type].keys() %}
diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls
index e5d7228dc..54c8a4273 100644
--- a/salt/filebeat/init.sls
+++ b/salt/filebeat/init.sls
@@ -17,12 +17,10 @@
 {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 {% set LOCALHOSTNAME = salt['grains.get']('host') %}
-{% set MAININT = salt['pillar.get']('host:mainint') %}
-{% set LOCALHOSTIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] %}
 {% set MANAGER = salt['grains.get']('master') %}
-{% set MANAGERIP = salt['pillar.get']('global:managerip', '') %}
 {% from 'filebeat/map.jinja' import THIRDPARTY with context %}
 {% from 'filebeat/map.jinja' import SO with context %}
+{% from 'filebeat/map.jinja' import EXTRA_HOSTS with context %}
 {% set ES_INCLUDED_NODES = ['so-eval', 'so-standalone', 'so-managersearch', 'so-node', 'so-heavynode', 'so-import'] %}
 
 include:
@@ -111,7 +109,7 @@ so-filebeat:
     - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-filebeat:{{ VERSION }}
     - hostname: so-filebeat
     - user: root
-    - extra_hosts: {{ MANAGER }}:{{ MANAGERIP }},{{ LOCALHOSTNAME }}:{{ LOCALHOSTIP }}
+    - extra_hosts: {{ EXTRA_HOSTS }}
     - binds:
       - /nsm:/nsm:ro
       - /opt/so/log/filebeat:/usr/share/filebeat/logs:rw
diff --git a/salt/filebeat/map.jinja b/salt/filebeat/map.jinja
index 6ae6e7cff..c18fe582c 100644
--- a/salt/filebeat/map.jinja
+++ b/salt/filebeat/map.jinja
@@ -4,3 +4,20 @@
 {% import_yaml 'filebeat/securityoniondefaults.yaml' as SODEFAULTS %}
 {% set SO = SODEFAULTS.securityonion_filebeat %}
 {#% set SO = salt['pillar.get']('filebeat:third_party_filebeat', default=SODEFAULTS.third_party_filebeat, merge=True) %#}
+
+{% set role = grains.role %}
+{% set EXTRA_HOSTS = [] %}
+{% set mainint = salt['pillar.get']('host:mainint') %}
+{% set localhostip = salt['grains.get']('ip_interfaces').get(mainint)[0] %}
+{% if role in ['so-sensor', 'so-fleet', 'so-node' ] %}
+  {% set node_data = salt['pillar.get']('logstash:nodes') %}
+  {% for node_type, node_details in node_data.items() | sort %}
+    {% if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %}
+      {% for hostname in node_data[node_type].keys() %}
+        {% do EXTRA_HOSTS.append({hostname:node_details[hostname].ip}) %}
+      {% endfor %}
+    {% endif %}
+  {% endfor %}
+{% else %}
+  {% do EXTRA_HOSTS.append({grains.host:localhostip}) %}
+{% endif %}
diff --git a/salt/salt/etc/minion.d/mine_functions.conf b/salt/salt/etc/minion.d/mine_functions.conf
index 023ba0308..42a543841 100644
--- a/salt/salt/etc/minion.d/mine_functions.conf
+++ b/salt/salt/etc/minion.d/mine_functions.conf
@@ -1,4 +1,3 @@
 mine_functions:
-  test.ping: []
   network.ip_addrs:
     - interface: {{ pillar.host.mainint }}

From 6962e3f9b36e95247ae2a11d22bdc4fd262abe59 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Tue, 14 Dec 2021 14:52:15 -0500
Subject: [PATCH 25/47] fix logstash certs mapped into container

---
 salt/logstash/init.sls | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls
index cb8f2a261..cb733a7fb 100644
--- a/salt/logstash/init.sls
+++ b/salt/logstash/init.sls
@@ -177,12 +177,12 @@ so-logstash:
       - /etc/pki/filebeat.p8:/usr/share/logstash/filebeat.key:ro
   {% endif %}
       - /opt/so/conf/logstash/etc/certs:/usr/share/logstash/certs:ro
-  {% if grains['role'] == 'so-heavynode' %}
-      - /etc/ssl/certs/intca.crt:/usr/share/filebeat/ca.crt:ro
-  {% else %}
+  {% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-receiver'] %}
       - /etc/pki/ca.crt:/usr/share/filebeat/ca.crt:ro
+  {% else %}
+      - /etc/ssl/certs/intca.crt:/usr/share/filebeat/ca.crt:ro
   {% endif %}
-  {% if grains.role not in ['so-receiver'] %}
+  {% if grains.role in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-receiver'] %}
       - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro
       - /opt/so/conf/ca/tls-ca-bundle.pem:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem:ro
   {% endif %}
@@ -209,12 +209,12 @@ so-logstash:
   {% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-receiver'] %}
       - x509: etc_filebeat_crt
   {% endif %}
-  {% if grains['role'] == 'so-heavynode' %}
-      - x509: trusttheca
-  {% else %}
+  {% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import'] %}
       - x509: pki_public_ca_crt
+  {% else %}
+      - x509: trusttheca
   {% endif %}
-  {% if grains.role not in ['so-receiver'] %}
+  {% if grains.role in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import'] %}
       - file: cacertz
       - file: capemz
   {% endif %}

From a31f034f2e85690cef3c8cb65559aa670e80ebd8 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Tue, 14 Dec 2021 15:02:59 -0500
Subject: [PATCH 26/47] remove receiver add node for cacerts and tls-ca-bundle
 for logstash bind

---
 salt/logstash/init.sls | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls
index cb733a7fb..623bcddf6 100644
--- a/salt/logstash/init.sls
+++ b/salt/logstash/init.sls
@@ -182,7 +182,7 @@ so-logstash:
   {% else %}
       - /etc/ssl/certs/intca.crt:/usr/share/filebeat/ca.crt:ro
   {% endif %}
-  {% if grains.role in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-receiver'] %}
+  {% if grains.role in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-node'] %}
       - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro
       - /opt/so/conf/ca/tls-ca-bundle.pem:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem:ro
   {% endif %}

From d0b6d5bba6e8acd0725f64fddc3c6699385d06da Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Tue, 14 Dec 2021 15:33:06 -0500
Subject: [PATCH 27/47] remove so-eval from lists since it doesnt run logstash

---
 salt/logstash/init.sls | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls
index 623bcddf6..53a710f9b 100644
--- a/salt/logstash/init.sls
+++ b/salt/logstash/init.sls
@@ -172,17 +172,17 @@ so-logstash:
       - /nsm/logstash:/usr/share/logstash/data:rw
       - /opt/so/log/logstash:/var/log/logstash:rw
       - /sys/fs/cgroup:/sys/fs/cgroup:ro
-  {% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-receiver'] %}
+      - /opt/so/conf/logstash/etc/certs:/usr/share/logstash/certs:ro
+  {% if grains['role'] in ['so-manager', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-receiver'] %}
       - /etc/pki/filebeat.crt:/usr/share/logstash/filebeat.crt:ro
       - /etc/pki/filebeat.p8:/usr/share/logstash/filebeat.key:ro
   {% endif %}
-      - /opt/so/conf/logstash/etc/certs:/usr/share/logstash/certs:ro
-  {% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-receiver'] %}
+  {% if grains['role'] in ['so-manager', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import'] %}
       - /etc/pki/ca.crt:/usr/share/filebeat/ca.crt:ro
   {% else %}
       - /etc/ssl/certs/intca.crt:/usr/share/filebeat/ca.crt:ro
   {% endif %}
-  {% if grains.role in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-node'] %}
+  {% if grains.role in ['so-manager', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-node'] %}
       - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro
       - /opt/so/conf/ca/tls-ca-bundle.pem:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem:ro
   {% endif %}
@@ -206,15 +206,15 @@ so-logstash:
       - file: es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}
   {% endfor %}
     - require:
-  {% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-receiver'] %}
+  {% if grains['role'] in ['so-manager', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-receiver'] %}
       - x509: etc_filebeat_crt
   {% endif %}
-  {% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import'] %}
+  {% if grains['role'] in ['so-manager', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import'] %}
       - x509: pki_public_ca_crt
   {% else %}
       - x509: trusttheca
   {% endif %}
-  {% if grains.role in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import'] %}
+  {% if grains.role in ['so-manager', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import'] %}
       - file: cacertz
       - file: capemz
   {% endif %}

From 841b91e0529c6cc0b752b26f03133fa7596fcd43 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Tue, 14 Dec 2021 16:05:47 -0500
Subject: [PATCH 28/47] exclude elasticsearch and managerssl keys and certs
 from receiver

---
 salt/ssl/init.sls | 10 ++++++----
 salt/top.sls      |  1 -
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls
index 3b4bb2bcf..c4c280da4 100644
--- a/salt/ssl/init.sls
+++ b/salt/ssl/init.sls
@@ -210,7 +210,6 @@ etc_filebeat_crt:
     - onchanges:
       - x509: etc_filebeat_key
 
-
 fbperms:
   file.managed:
     - replace: False
@@ -226,7 +225,7 @@ chownilogstashfilebeatp8:
     - user: 931
     - group: 939
 
-  {% if grains.role not in  ['so-heavynode', 'so-receiver']%}
+  {% if grains.role not in ['so-heavynode', 'so-receiver'] %}
 # Create Symlinks to the keys so I can distribute it to all the things
 filebeatdir:
   file.directory:
@@ -294,7 +293,7 @@ regkeyperms:
     - group: 939
 
   {% endif %}
-
+  {% if grains.role not in ['so-receiver'] %}
 # Create a cert for elasticsearch
 /etc/pki/elasticsearch.key:
   x509.private_key_managed:
@@ -338,7 +337,7 @@ regkeyperms:
     - onchanges:
       - x509: /etc/pki/elasticsearch.key
 
-ealstickeyperms:
+elastickeyperms:
   file.managed:
     - replace: False
     - name: /etc/pki/elasticsearch.key
@@ -398,6 +397,8 @@ msslkeyperms:
     - mode: 640
     - group: 939
 
+  {% endif %}
+
 # Create a private key and cert for OSQuery
 fleet_key:
   x509.private_key_managed:
@@ -443,6 +444,7 @@ fleetkeyperms:
     - group: 939
 
 {% endif %}
+
 {% if grains['role'] in ['so-sensor', 'so-manager', 'so-node', 'so-eval', 'so-helix', 'so-managersearch', 'so-heavynode', 'so-fleet', 'so-standalone', 'so-import', 'so-receiver'] %}
    
 fbcertdir:
diff --git a/salt/top.sls b/salt/top.sls
index 1f0a92898..aad8dc7ef 100644
--- a/salt/top.sls
+++ b/salt/top.sls
@@ -477,7 +477,6 @@ base:
 
   '*_receiver and G@saltversion:{{saltversion}}':
     - match: compound
-    - ca
     - ssl
     - telegraf
     - firewall

From a31d61e151513f546079a7986e8a0ac8468e5976 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Tue, 14 Dec 2021 16:43:04 -0500
Subject: [PATCH 29/47] handle ca for redis

---
 salt/redis/init.sls | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/salt/redis/init.sls b/salt/redis/init.sls
index d52c49d5b..6b893fbf8 100644
--- a/salt/redis/init.sls
+++ b/salt/redis/init.sls
@@ -66,7 +66,11 @@ so-redis:
       - /opt/so/conf/redis/working:/redis:rw
       - /etc/pki/redis.crt:/certs/redis.crt:ro
       - /etc/pki/redis.key:/certs/redis.key:ro
+  {% if grains['role'] in ['so-manager', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import'] %}
       - /etc/pki/ca.crt:/certs/ca.crt:ro
+  {% else %}
+      - /etc/ssl/certs/intca.crt:/certs/ca.crt:ro
+  {% endif %}
     - entrypoint: "redis-server /usr/local/etc/redis/redis.conf"
     - watch:
       - file: /opt/so/conf/redis/etc
@@ -74,7 +78,11 @@ so-redis:
       - file: redisconf
       - x509: redis_crt
       - x509: redis_key
+  {% if grains['role'] in ['so-manager', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import'] %}
       - x509: pki_public_ca_crt
+  {% else %}
+      - x509: trusttheca
+  {% endif %}
 
 append_so-redis_so-status.conf:
   file.append:

From 4da017d61cbf9f47b864f78ac9f19f5c78c3d460 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Tue, 14 Dec 2021 17:05:30 -0500
Subject: [PATCH 30/47] change extra_hosts for docker container

---
 salt/logstash/init.sls | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls
index 53a710f9b..9bd77828d 100644
--- a/salt/logstash/init.sls
+++ b/salt/logstash/init.sls
@@ -36,9 +36,9 @@
   {% set DOCKER_OPTIONS = salt['pillar.get']('logstash:docker_options', {}) %}
   {% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %}
 
-  {% if grains.role in ['so-heavynode'] %}
+  {% if grains.role in ['so-heavynode', 'so-receiver'] %}
     {% set EXTRAHOSTHOSTNAME = salt['grains.get']('host') %}
-    {% set EXTRAHOSTIP = salt['pillar.get']('sensor:mainip') %}
+    {% set EXTRAHOSTIP = salt['grains.get']('ip_interfaces').get(salt['pillar.get']('host:mainint'))[0] %}
   {% else %}
     {% set EXTRAHOSTHOSTNAME = MANAGER %}
     {% set EXTRAHOSTIP = MANAGERIP %}

From 55b74abcc55975fe107efe1abe653f15c40edea6 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Tue, 14 Dec 2021 18:49:30 -0500
Subject: [PATCH 31/47] extra_hosts and redis_input for logstash

---
 salt/logstash/init.sls                           |  8 ++++----
 salt/logstash/map.jinja                          | 16 ++++++++++++++++
 .../config/so/0900_input_redis.conf.jinja        | 14 +++++++-------
 3 files changed, 27 insertions(+), 11 deletions(-)
 create mode 100644 salt/logstash/map.jinja

diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls
index 9bd77828d..34c2eca32 100644
--- a/salt/logstash/init.sls
+++ b/salt/logstash/init.sls
@@ -36,13 +36,14 @@
   {% set DOCKER_OPTIONS = salt['pillar.get']('logstash:docker_options', {}) %}
   {% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %}
 
-  {% if grains.role in ['so-heavynode', 'so-receiver'] %}
+  {# if grains.role in ['so-heavynode', 'so-receiver'] %}
     {% set EXTRAHOSTHOSTNAME = salt['grains.get']('host') %}
     {% set EXTRAHOSTIP = salt['grains.get']('ip_interfaces').get(salt['pillar.get']('host:mainint'))[0] %}
   {% else %}
     {% set EXTRAHOSTHOSTNAME = MANAGER %}
     {% set EXTRAHOSTIP = MANAGERIP %}
-  {% endif %}
+  {% endif #}
+  {% from 'logstash/map.jinja' import EXTRA_HOSTS with context %}
 
 include:
   - ssl
@@ -155,8 +156,7 @@ so-logstash:
     - hostname: so-logstash
     - name: so-logstash
     - user: logstash
-    - extra_hosts:
-      - {{ EXTRAHOSTHOSTNAME }}:{{ EXTRAHOSTIP }}
+    - extra_hosts: {{ EXTRA_HOSTS }}
     - environment:
       - LS_JAVA_OPTS=-Xms{{ lsheap }} -Xmx{{ lsheap }}
     - port_bindings:
diff --git a/salt/logstash/map.jinja b/salt/logstash/map.jinja
new file mode 100644
index 000000000..85c733789
--- /dev/null
+++ b/salt/logstash/map.jinja
@@ -0,0 +1,16 @@
+{% set role = grains.role %}
+{% set EXTRA_HOSTS = [] %}
+{% set mainint = salt['pillar.get']('host:mainint') %}
+{% set localhostip = salt['grains.get']('ip_interfaces').get(mainint)[0] %}
+{% if role in ['so-node'] %}
+  {% set node_data = salt['pillar.get']('logstash:nodes') %}
+  {% for node_type, node_details in node_data.items() | sort %}
+    {% if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %}
+      {% for hostname in node_data[node_type].keys() %}
+        {% do EXTRA_HOSTS.append({hostname:node_details[hostname].ip}) %}
+      {% endfor %}
+    {% endif %}
+  {% endfor %}
+{% else %}
+  {% do EXTRA_HOSTS.append({grains.host:localhostip}) %}
+{% endif %}
diff --git a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja
index 35f77c5a0..995c2ce5e 100644
--- a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja
+++ b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja
@@ -1,13 +1,12 @@
-{%- if grains.role in ['so-heavynode'] %}
-  {%- set HOST = salt['grains.get']('host') %}
-{%- else %}
-  {%- set HOST = salt['grains.get']('master') %}
-{%- endif %}
-  {%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %}
+{%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %}
 {%- set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %}
+
+{%- from 'logstash/map.jinja' import EXTRA_HOSTS with context %}
+
+{%- for host in EXTRA_HOSTS.keys %}
 input {
 	redis {
-		host => '{{ HOST }}'
+		host => '{{ host }}'
 		port => 9696
 		ssl => true
 		data_type => 'list'
@@ -17,3 +16,4 @@ input {
 		batch_count => {{ BATCH }}
 	}
 }
+{%- endfor %}

From 15b8d80b7130df45d722287ffc898f88448deeee Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Tue, 14 Dec 2021 18:51:43 -0500
Subject: [PATCH 32/47] fix host for input_redis

---
 salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja
index 995c2ce5e..fa43ff5a4 100644
--- a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja
+++ b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja
@@ -3,10 +3,10 @@
 
 {%- from 'logstash/map.jinja' import EXTRA_HOSTS with context %}
 
-{%- for host in EXTRA_HOSTS.keys %}
+{%- for host_ip in EXTRA_HOSTS %}
 input {
 	redis {
-		host => '{{ host }}'
+		host => '{{ host_ip.split(':')[0] }}'
 		port => 9696
 		ssl => true
 		data_type => 'list'

From 0c6aba16ec2e0553fb45bbbbb98399e1d1c9b059 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Tue, 14 Dec 2021 23:42:37 -0500
Subject: [PATCH 33/47] fix redis input

---
 .../config/so/0900_input_redis.conf.jinja     | 33 ++++++++++---------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja
index fa43ff5a4..2d7a2d4fe 100644
--- a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja
+++ b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja
@@ -1,19 +1,20 @@
-{%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %}
-{%- set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %}
+{% set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') -%}
+{% set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) -%}
+{% from 'logstash/map.jinja' import REDIS_NODES with context -%}
 
-{%- from 'logstash/map.jinja' import EXTRA_HOSTS with context %}
-
-{%- for host_ip in EXTRA_HOSTS %}
+{% for index in range(REDIS_NODES|length) -%}
+{%   for host in REDIS_NODES[index] -%}
 input {
-	redis {
-		host => '{{ host_ip.split(':')[0] }}'
-		port => 9696
-		ssl => true
-		data_type => 'list'
-		key => 'logstash:unparsed'
-		type => 'redis-input'
-		threads => {{ THREADS }}
-		batch_count => {{ BATCH }}
-	}
+        redis {
+                host => '{{ host }}'
+                port => 9696
+                ssl => true
+                data_type => 'list'
+                key => 'logstash:unparsed'
+                type => 'redis-input'
+                threads => {{ THREADS }}
+                batch_count => {{ BATCH }}
+        }
 }
-{%- endfor %}
+{%   endfor %}
+{% endfor -%}

From 024860d0ae308981628d26c81a3b7a74adf93d90 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Tue, 14 Dec 2021 23:43:06 -0500
Subject: [PATCH 34/47] rename EXTRA_NODES to LOGSTASH_NODES AND REDIS_NODES

---
 salt/filebeat/init.sls  | 4 ++--
 salt/filebeat/map.jinja | 6 +++---
 salt/logstash/init.sls  | 4 ++--
 salt/logstash/map.jinja | 6 +++---
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls
index 54c8a4273..82a7b7871 100644
--- a/salt/filebeat/init.sls
+++ b/salt/filebeat/init.sls
@@ -20,7 +20,7 @@
 {% set MANAGER = salt['grains.get']('master') %}
 {% from 'filebeat/map.jinja' import THIRDPARTY with context %}
 {% from 'filebeat/map.jinja' import SO with context %}
-{% from 'filebeat/map.jinja' import EXTRA_HOSTS with context %}
+{% from 'filebeat/map.jinja' import LOGSTASH_NODES with context %}
 {% set ES_INCLUDED_NODES = ['so-eval', 'so-standalone', 'so-managersearch', 'so-node', 'so-heavynode', 'so-import'] %}
 
 include:
@@ -109,7 +109,7 @@ so-filebeat:
     - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-filebeat:{{ VERSION }}
     - hostname: so-filebeat
     - user: root
-    - extra_hosts: {{ EXTRA_HOSTS }}
+    - extra_hosts: {{ LOGSTASH_NODES }}
     - binds:
       - /nsm:/nsm:ro
       - /opt/so/log/filebeat:/usr/share/filebeat/logs:rw
diff --git a/salt/filebeat/map.jinja b/salt/filebeat/map.jinja
index c18fe582c..b0286bb68 100644
--- a/salt/filebeat/map.jinja
+++ b/salt/filebeat/map.jinja
@@ -6,7 +6,7 @@
 {#% set SO = salt['pillar.get']('filebeat:third_party_filebeat', default=SODEFAULTS.third_party_filebeat, merge=True) %#}
 
 {% set role = grains.role %}
-{% set EXTRA_HOSTS = [] %}
+{% set LOGSTASH_NODES = [] %}
 {% set mainint = salt['pillar.get']('host:mainint') %}
 {% set localhostip = salt['grains.get']('ip_interfaces').get(mainint)[0] %}
 {% if role in ['so-sensor', 'so-fleet', 'so-node' ] %}
@@ -14,10 +14,10 @@
   {% for node_type, node_details in node_data.items() | sort %}
     {% if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %}
       {% for hostname in node_data[node_type].keys() %}
-        {% do EXTRA_HOSTS.append({hostname:node_details[hostname].ip}) %}
+        {% do LOGSTASH_NODES.append({hostname:node_details[hostname].ip}) %}
       {% endfor %}
     {% endif %}
   {% endfor %}
 {% else %}
-  {% do EXTRA_HOSTS.append({grains.host:localhostip}) %}
+  {% do LOGSTASH_NODES.append({grains.host:localhostip}) %}
 {% endif %}
diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls
index 34c2eca32..abfe1fe17 100644
--- a/salt/logstash/init.sls
+++ b/salt/logstash/init.sls
@@ -43,7 +43,7 @@
     {% set EXTRAHOSTHOSTNAME = MANAGER %}
     {% set EXTRAHOSTIP = MANAGERIP %}
   {% endif #}
-  {% from 'logstash/map.jinja' import EXTRA_HOSTS with context %}
+  {% from 'logstash/map.jinja' import REDIS_NODES with context %}
 
 include:
   - ssl
@@ -156,7 +156,7 @@ so-logstash:
     - hostname: so-logstash
     - name: so-logstash
     - user: logstash
-    - extra_hosts: {{ EXTRA_HOSTS }}
+    - extra_hosts: {{ REDIS_NODES }}
     - environment:
       - LS_JAVA_OPTS=-Xms{{ lsheap }} -Xmx{{ lsheap }}
     - port_bindings:
diff --git a/salt/logstash/map.jinja b/salt/logstash/map.jinja
index 85c733789..63e047281 100644
--- a/salt/logstash/map.jinja
+++ b/salt/logstash/map.jinja
@@ -1,5 +1,5 @@
 {% set role = grains.role %}
-{% set EXTRA_HOSTS = [] %}
+{% set REDIS_NODES = [] %}
 {% set mainint = salt['pillar.get']('host:mainint') %}
 {% set localhostip = salt['grains.get']('ip_interfaces').get(mainint)[0] %}
 {% if role in ['so-node'] %}
@@ -7,10 +7,10 @@
   {% for node_type, node_details in node_data.items() | sort %}
     {% if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %}
       {% for hostname in node_data[node_type].keys() %}
-        {% do EXTRA_HOSTS.append({hostname:node_details[hostname].ip}) %}
+        {% do REDIS_NODES.append({hostname:node_details[hostname].ip}) %}
       {% endfor %}
     {% endif %}
   {% endfor %}
 {% else %}
-  {% do EXTRA_HOSTS.append({grains.host:localhostip}) %}
+  {% do REDIS_NODES.append({grains.host:localhostip}) %}
 {% endif %}

From 176ef852c8a49ccf5ac7411a6bfc8e8ba2f097ec Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Wed, 15 Dec 2021 08:28:40 -0500
Subject: [PATCH 35/47] clean up assinged hostgroups for receiver

---
 salt/firewall/assigned_hostgroups.map.yaml | 18 +-----------------
 1 file changed, 1 insertion(+), 17 deletions(-)

diff --git a/salt/firewall/assigned_hostgroups.map.yaml b/salt/firewall/assigned_hostgroups.map.yaml
index e9727f331..4467b30a9 100644
--- a/salt/firewall/assigned_hostgroups.map.yaml
+++ b/salt/firewall/assigned_hostgroups.map.yaml
@@ -609,10 +609,6 @@ role:
     chain:
       DOCKER-USER:
         hostgroups:
-          manager:
-            portgroups:
-              - {{ portgroups.elasticsearch_rest }}
-              - {{ portgroups.elasticsearch_node }}
           sensor:
             portgroups:
               - {{ portgroups.beats_5044 }}
@@ -620,12 +616,6 @@ role:
           search_node:
             portgroups:
               - {{ portgroups.redis }}
-              - {{ portgroups.elasticsearch_node }}
-              - {{ portgroups.beats_5644 }}
-          heavy_node:
-            portgroups:
-              - {{ portgroups.redis }}
-              - {{ portgroups.elasticsearch_node }}
               - {{ portgroups.beats_5644 }}
           self:
             portgroups:
@@ -640,18 +630,12 @@ role:
           beats_endpoint_ssl:
             portgroups:
               - {{ portgroups.beats_5644 }}
-          elasticsearch_rest:
-            portgroups:
-              - {{ portgroups.elasticsearch_rest }}
           endgame:
             portgroups:
               - {{ portgroups.endgame }}
-          osquery_endpoint:
-            portgroups:
-              - {{ portgroups.fleet_api }}
           wazuh_agent:
             portgroups:
-              - {{ portgroups.wazuh_agent  }}
+              - {{ portgroups.wazuh_agent }}
           wazuh_api:
             portgroups:
               - {{ portgroups.wazuh_api }}

From d9a384cc29e4214443fd2d5f88ca20387a1a0b35 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Wed, 15 Dec 2021 09:30:15 -0500
Subject: [PATCH 36/47] remove global:pipeline pillar call from logstash
 pipeline pillars

---
 pillar/logstash/manager.sls  | 1 -
 pillar/logstash/receiver.sls | 3 +--
 pillar/logstash/search.sls   | 1 -
 3 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/pillar/logstash/manager.sls b/pillar/logstash/manager.sls
index fc0788824..00d82f86a 100644
--- a/pillar/logstash/manager.sls
+++ b/pillar/logstash/manager.sls
@@ -1,4 +1,3 @@
-{%- set PIPELINE = salt['pillar.get']('global:pipeline', 'redis') %}
 logstash:
   pipelines:
     manager:
diff --git a/pillar/logstash/receiver.sls b/pillar/logstash/receiver.sls
index fc0788824..09c2549e6 100644
--- a/pillar/logstash/receiver.sls
+++ b/pillar/logstash/receiver.sls
@@ -1,7 +1,6 @@
-{%- set PIPELINE = salt['pillar.get']('global:pipeline', 'redis') %}
 logstash:
   pipelines:
-    manager:
+    receiver:
       config:
         - so/0009_input_beats.conf      
         - so/0010_input_hhbeats.conf
diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls
index a0ddf946e..917657e1f 100644
--- a/pillar/logstash/search.sls
+++ b/pillar/logstash/search.sls
@@ -1,4 +1,3 @@
-{%- set PIPELINE = salt['pillar.get']('global:pipeline', 'minio') %}
 logstash:
   pipelines:
     search:

From 759bf9837e960be9a25695d224cedc294eaea13a Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Wed, 15 Dec 2021 09:31:03 -0500
Subject: [PATCH 37/47] pillar top clean up for receiver and logstash.nodes

---
 pillar/top.sls | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pillar/top.sls b/pillar/top.sls
index b6c3b744a..37bd53f5e 100644
--- a/pillar/top.sls
+++ b/pillar/top.sls
@@ -2,6 +2,8 @@ base:
   '*':
     - patch.needs_restarting
     - logrotate
+
+  '* and not *_eval and not *_import':
     - logstash.nodes
 
   '*_eval or *_helixsensor or *_heavynode or *_sensor or *_standalone or *_import':
@@ -111,7 +113,6 @@ base:
     - elasticsearch.auth
     - global
     - minions.{{ grains.id }}
-    - data.receiverstab
 
   '*_import':
     - zeeklogs

From ea89d2074b34d449c72d11018eeb5a5856c6e100 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Wed, 15 Dec 2021 09:32:12 -0500
Subject: [PATCH 38/47] remove ca from allowed_hosts on so-receiver

---
 salt/allowed_states.map.jinja | 1 -
 1 file changed, 1 deletion(-)

diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja
index d1660954f..bdff03c43 100644
--- a/salt/allowed_states.map.jinja
+++ b/salt/allowed_states.map.jinja
@@ -202,7 +202,6 @@
           'docker_clean'
           ],
       'so-receiver': [
-          'ca',
           'ssl',
           'telegraf',
           'firewall',

From ce0a39db4be42507ed0d1d0caf73b9400da4318c Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Wed, 15 Dec 2021 09:43:46 -0500
Subject: [PATCH 39/47] remove old EXTRAHOSTNAME EXTRAHOSTIP from being set for
 logstash

---
 salt/logstash/init.sls | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls
index abfe1fe17..a86ec358b 100644
--- a/salt/logstash/init.sls
+++ b/salt/logstash/init.sls
@@ -36,13 +36,6 @@
   {% set DOCKER_OPTIONS = salt['pillar.get']('logstash:docker_options', {}) %}
   {% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %}
 
-  {# if grains.role in ['so-heavynode', 'so-receiver'] %}
-    {% set EXTRAHOSTHOSTNAME = salt['grains.get']('host') %}
-    {% set EXTRAHOSTIP = salt['grains.get']('ip_interfaces').get(salt['pillar.get']('host:mainint'))[0] %}
-  {% else %}
-    {% set EXTRAHOSTHOSTNAME = MANAGER %}
-    {% set EXTRAHOSTIP = MANAGERIP %}
-  {% endif #}
   {% from 'logstash/map.jinja' import REDIS_NODES with context %}
 
 include:

From 6ab2bdef0c85e3576c456bf47059e989d377b8a8 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Wed, 15 Dec 2021 10:45:54 -0500
Subject: [PATCH 40/47] add sensoroni state to receiver node

---
 salt/top.sls | 1 +
 1 file changed, 1 insertion(+)

diff --git a/salt/top.sls b/salt/top.sls
index aad8dc7ef..4fd8c1fd3 100644
--- a/salt/top.sls
+++ b/salt/top.sls
@@ -478,6 +478,7 @@ base:
   '*_receiver and G@saltversion:{{saltversion}}':
     - match: compound
     - ssl
+    - sensoroni
     - telegraf
     - firewall
     {%- if WAZUH != 0 %}

From cf2f4bad09ecb32978ea3e78963474e76dc23364 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Wed, 15 Dec 2021 15:27:23 -0500
Subject: [PATCH 41/47] have standalone and managersearch pull from redis nodes

---
 salt/logstash/map.jinja | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/logstash/map.jinja b/salt/logstash/map.jinja
index 63e047281..5f27a17e2 100644
--- a/salt/logstash/map.jinja
+++ b/salt/logstash/map.jinja
@@ -2,7 +2,7 @@
 {% set REDIS_NODES = [] %}
 {% set mainint = salt['pillar.get']('host:mainint') %}
 {% set localhostip = salt['grains.get']('ip_interfaces').get(mainint)[0] %}
-{% if role in ['so-node'] %}
+{% if role in ['so-node', 'so-standalone', 'so-managersearch'] %}
   {% set node_data = salt['pillar.get']('logstash:nodes') %}
   {% for node_type, node_details in node_data.items() | sort %}
     {% if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %}

From 522bc1d2b84bcddc5bca7641b4e267b3544e249f Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Wed, 15 Dec 2021 16:21:08 -0500
Subject: [PATCH 42/47] fix loadbalance logic and whitespace for filebeat.yml

---
 salt/filebeat/etc/filebeat.yml | 35 ++++++++++++++++++----------------
 1 file changed, 19 insertions(+), 16 deletions(-)

diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml
index 05605b96d..a0f973053 100644
--- a/salt/filebeat/etc/filebeat.yml
+++ b/salt/filebeat/etc/filebeat.yml
@@ -322,24 +322,27 @@ output.logstash:
 
   # The Logstash hosts
   hosts:
-{%- if grains.role in ['so-sensor', 'so-fleet', 'so-node'] %}
-{%-   set LOADBALANCE = ['false'] %}
-{%-   set node_data = salt['pillar.get']('logstash:nodes') %}
-{%-   for node_type, node_details in node_data.items() | sort %}
-{%-     if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %}
-{%-       for hostname in node_data[node_type].keys() %}
+{%- if grains.role in ['so-sensor', 'so-fleet', 'so-node'] -%}
+{%   set LOGSTASH = namespace() -%}
+{%   set LOGSTASH.count = 0 -%}
+{%   set LOGSTASH.loadbalance = false -%}
+{%   set node_data = salt['pillar.get']('logstash:nodes') -%}
+{%   for node_type, node_details in node_data.items() | sort -%}
+{%     if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] -%}
+{%       for hostname in node_data[node_type].keys() -%}
+{%         set LOGSTASH.count = LOGSTASH.count + 1 %}
     - "{{ hostname }}:5644" #{{ node_details[hostname].ip }}
-{%-       endfor %}
-{%-     endif %}
-{%-     if loop.index == 2 %}
-{%-       do LOADBALANCE.insert(0, 'true') %}
-{%-     endif %}
-{%-   endfor %}
-  loadbalance: {{ LOADBALANCE[0] }}
-{%- else %}
-    - "{{ grains.host }}:5644"
-{%- endif %}
+{%-       endfor -%}
+{%     endif -%}
+{%     if LOGSTASH.count > 1 -%}
+{%       set LOGSTASH.loadbalance = true -%}
+{%     endif -%}
+{%   endfor %}
 
+  loadbalance: {{ LOGSTASH.loadbalance | lower }}
+{% else -%}
+    - "{{ grains.host }}:5644"
+{% endif %}
   # Number of workers per Logstash host.
   worker: {{ FBLSWORKERS }}
 

From f9b04ab96ae00f385b297bd4533e713175e99b29 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Wed, 15 Dec 2021 16:53:22 -0500
Subject: [PATCH 43/47] add node's own ip to FILEBEAT_EXTRA_HOSTS

---
 salt/filebeat/init.sls  | 4 ++--
 salt/filebeat/map.jinja | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls
index 82a7b7871..3fa36fa51 100644
--- a/salt/filebeat/init.sls
+++ b/salt/filebeat/init.sls
@@ -20,7 +20,7 @@
 {% set MANAGER = salt['grains.get']('master') %}
 {% from 'filebeat/map.jinja' import THIRDPARTY with context %}
 {% from 'filebeat/map.jinja' import SO with context %}
-{% from 'filebeat/map.jinja' import LOGSTASH_NODES with context %}
+{% from 'filebeat/map.jinja' import FILEBEAT_EXTRA_HOSTS with context %}
 {% set ES_INCLUDED_NODES = ['so-eval', 'so-standalone', 'so-managersearch', 'so-node', 'so-heavynode', 'so-import'] %}
 
 include:
@@ -109,7 +109,7 @@ so-filebeat:
     - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-filebeat:{{ VERSION }}
     - hostname: so-filebeat
     - user: root
-    - extra_hosts: {{ LOGSTASH_NODES }}
+    - extra_hosts: {{ FILEBEAT_EXTRA_HOSTS }}
     - binds:
       - /nsm:/nsm:ro
       - /opt/so/log/filebeat:/usr/share/filebeat/logs:rw
diff --git a/salt/filebeat/map.jinja b/salt/filebeat/map.jinja
index b0286bb68..0660bf856 100644
--- a/salt/filebeat/map.jinja
+++ b/salt/filebeat/map.jinja
@@ -6,7 +6,7 @@
 {#% set SO = salt['pillar.get']('filebeat:third_party_filebeat', default=SODEFAULTS.third_party_filebeat, merge=True) %#}
 
 {% set role = grains.role %}
-{% set LOGSTASH_NODES = [] %}
+{% set FILEBEAT_EXTRA_HOSTS = [] %}
 {% set mainint = salt['pillar.get']('host:mainint') %}
 {% set localhostip = salt['grains.get']('ip_interfaces').get(mainint)[0] %}
 {% if role in ['so-sensor', 'so-fleet', 'so-node' ] %}
@@ -14,10 +14,10 @@
   {% for node_type, node_details in node_data.items() | sort %}
     {% if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %}
       {% for hostname in node_data[node_type].keys() %}
-        {% do LOGSTASH_NODES.append({hostname:node_details[hostname].ip}) %}
+        {% do FILEBEAT_EXTRA_HOSTS.append({hostname:node_details[hostname].ip}) %}
       {% endfor %}
     {% endif %}
   {% endfor %}
-{% else %}
-  {% do LOGSTASH_NODES.append({grains.host:localhostip}) %}
 {% endif %}
+
+{% do FILEBEAT_EXTRA_HOSTS.append({grains.host:localhostip}) %}

From a7600f7f430d90610f53b0924a2974eb190b6880 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Wed, 15 Dec 2021 17:31:39 -0500
Subject: [PATCH 44/47] update scripts to use their own ip

---
 salt/common/tools/sbin/so-logstash-events         | 3 ++-
 salt/common/tools/sbin/so-logstash-pipeline-stats | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/salt/common/tools/sbin/so-logstash-events b/salt/common/tools/sbin/so-logstash-events
index 817cafb72..8ab74862b 100755
--- a/salt/common/tools/sbin/so-logstash-events
+++ b/salt/common/tools/sbin/so-logstash-events
@@ -14,7 +14,8 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>
-{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
+{% set MAININT = salt['pillar.get']('host:mainint') -%}
+{% set NODEIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] -%}
 
 . /usr/sbin/so-common
 
diff --git a/salt/common/tools/sbin/so-logstash-pipeline-stats b/salt/common/tools/sbin/so-logstash-pipeline-stats
index b82a125d2..642269267 100755
--- a/salt/common/tools/sbin/so-logstash-pipeline-stats
+++ b/salt/common/tools/sbin/so-logstash-pipeline-stats
@@ -14,7 +14,8 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>
-{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
+{% set MAININT = salt['pillar.get']('host:mainint') -%}
+{% set NODEIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] -%}
 
 . /usr/sbin/so-common
 

From bd7ef1cc5987d38d435e0b8ba19e80eab297b7e6 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Thu, 16 Dec 2021 09:19:20 -0500
Subject: [PATCH 45/47] fix whitespace control

---
 salt/filebeat/etc/filebeat.yml | 35 +++++++++++++++++-----------------
 1 file changed, 18 insertions(+), 17 deletions(-)

diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml
index a0f973053..f18a72752 100644
--- a/salt/filebeat/etc/filebeat.yml
+++ b/salt/filebeat/etc/filebeat.yml
@@ -322,27 +322,28 @@ output.logstash:
 
   # The Logstash hosts
   hosts:
-{%- if grains.role in ['so-sensor', 'so-fleet', 'so-node'] -%}
-{%   set LOGSTASH = namespace() -%}
-{%   set LOGSTASH.count = 0 -%}
-{%   set LOGSTASH.loadbalance = false -%}
-{%   set node_data = salt['pillar.get']('logstash:nodes') -%}
-{%   for node_type, node_details in node_data.items() | sort -%}
-{%     if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] -%}
-{%       for hostname in node_data[node_type].keys() -%}
-{%         set LOGSTASH.count = LOGSTASH.count + 1 %}
+{%- if grains.role in ['so-sensor', 'so-fleet', 'so-node'] %}
+{%-   set LOGSTASH = namespace() %}
+{%-   set LOGSTASH.count = 0 %}
+{%-   set LOGSTASH.loadbalance = false %}
+{%-   set node_data = salt['pillar.get']('logstash:nodes') %}
+{%-   for node_type, node_details in node_data.items() | sort -%}
+{%-     if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %}
+{%-       for hostname in node_data[node_type].keys() %}
+{%-         set LOGSTASH.count = LOGSTASH.count + 1 %}
     - "{{ hostname }}:5644" #{{ node_details[hostname].ip }}
-{%-       endfor -%}
-{%     endif -%}
-{%     if LOGSTASH.count > 1 -%}
-{%       set LOGSTASH.loadbalance = true -%}
-{%     endif -%}
-{%   endfor %}
+{%-       endfor %}
+{%-     endif %}
+{%-     if LOGSTASH.count > 1 %}
+{%-       set LOGSTASH.loadbalance = true %}
+{%-     endif %}
+{%-   endfor %}
 
   loadbalance: {{ LOGSTASH.loadbalance | lower }}
-{% else -%}
+{%- else %}
     - "{{ grains.host }}:5644"
-{% endif %}
+{%- endif %}
+
   # Number of workers per Logstash host.
   worker: {{ FBLSWORKERS }}
 

From 2e4ed8062ed738241a9685f60d6b061f82ee93ed Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Thu, 16 Dec 2021 11:11:01 -0500
Subject: [PATCH 46/47] simplify wazuh agent ip logic

---
 salt/firewall/assigned_hostgroups.map.yaml          |  2 +-
 .../pipelines/config/so/0009_input_beats.conf       |  2 +-
 salt/wazuh/files/agent/ossec.conf                   | 13 +++----------
 3 files changed, 5 insertions(+), 12 deletions(-)

diff --git a/salt/firewall/assigned_hostgroups.map.yaml b/salt/firewall/assigned_hostgroups.map.yaml
index 4467b30a9..e2fbfc737 100644
--- a/salt/firewall/assigned_hostgroups.map.yaml
+++ b/salt/firewall/assigned_hostgroups.map.yaml
@@ -611,7 +611,6 @@ role:
         hostgroups:
           sensor:
             portgroups:
-              - {{ portgroups.beats_5044 }}
               - {{ portgroups.beats_5644 }}
           search_node:
             portgroups:
@@ -621,6 +620,7 @@ role:
             portgroups:
               - {{ portgroups.redis }}
               - {{ portgroups.syslog}}
+              - {{ portgroups.beats_5644 }}
           syslog:
             portgroups:
               - {{ portgroups.syslog }}
diff --git a/salt/logstash/pipelines/config/so/0009_input_beats.conf b/salt/logstash/pipelines/config/so/0009_input_beats.conf
index 9ca55b184..8643a64b4 100644
--- a/salt/logstash/pipelines/config/so/0009_input_beats.conf
+++ b/salt/logstash/pipelines/config/so/0009_input_beats.conf
@@ -8,4 +8,4 @@ filter {
   mutate {
     rename => {"@metadata" => "metadata"}
   }
-}
\ No newline at end of file
+}
diff --git a/salt/wazuh/files/agent/ossec.conf b/salt/wazuh/files/agent/ossec.conf
index 6ae873875..136b998b1 100644
--- a/salt/wazuh/files/agent/ossec.conf
+++ b/salt/wazuh/files/agent/ossec.conf
@@ -1,13 +1,6 @@
-{%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone'] %}
-  {%- set ip = salt['pillar.get']('global:managerip', '') %}
-{%- elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %}
-  {%- set ip = salt['pillar.get']('elasticsearch:mainip', '') %}
-{%- elif grains['role'] == 'so-sensor' %}
-  {%- set ip = salt['pillar.get']('sensor:mainip', '') %}
-{%- else %}
-  {%- set mainint = salt['pillar.get']('host:mainint') %}
-  {%- set ip = salt['grains.get']('ip_interfaces:' ~ mainint)[0] %}
-{%- endif %}
+{% set mainint = salt['pillar.get']('host:mainint') -%}
+{% set ip = salt['grains.get']('ip_interfaces').get(mainint)[0] -%}
+
 <!--
   Wazuh - Agent Configuration
   More info at: https://documentation.wazuh.com

From b4b8b91ccd360e8c0baef4a7e59972ae51a77890 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Thu, 16 Dec 2021 11:24:35 -0500
Subject: [PATCH 47/47] simplify ip logic wazuh-register-agent, mine_interval
 to 35 minutes

---
 salt/salt/etc/minion.d/mine_functions.conf  |  1 +
 salt/wazuh/files/agent/wazuh-register-agent | 13 +++----------
 2 files changed, 4 insertions(+), 10 deletions(-)

diff --git a/salt/salt/etc/minion.d/mine_functions.conf b/salt/salt/etc/minion.d/mine_functions.conf
index 42a543841..8570e7a86 100644
--- a/salt/salt/etc/minion.d/mine_functions.conf
+++ b/salt/salt/etc/minion.d/mine_functions.conf
@@ -1,3 +1,4 @@
+mine_interval: 35
 mine_functions:
   network.ip_addrs:
     - interface: {{ pillar.host.mainint }}
diff --git a/salt/wazuh/files/agent/wazuh-register-agent b/salt/wazuh/files/agent/wazuh-register-agent
index dac66c12b..98ab9fb13 100755
--- a/salt/wazuh/files/agent/wazuh-register-agent
+++ b/salt/wazuh/files/agent/wazuh-register-agent
@@ -1,13 +1,6 @@
-{%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone'] %}
-  {%- set ip = salt['pillar.get']('global:managerip', '') %}
-{%- elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %}
-  {%- set ip = salt['pillar.get']('elasticsearch:mainip', '') %}
-{%- elif grains['role'] == 'so-sensor' %}
-  {%- set ip = salt['pillar.get']('sensor:mainip', '') %}
-{%- else %}
-  {%- set mainint = salt['pillar.get']('host:mainint') %}
-  {%- set ip = salt['grains.get']('ip_interfaces:' ~ mainint)[0] %}
-{%- endif %}
+{% set mainint = salt['pillar.get']('host:mainint') -%}
+{% set ip = salt['grains.get']('ip_interfaces').get(mainint)[0] -%}
+
 #!/bin/bash
 
 ###