Merge kaffytaffy

Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com>

pillar/top.sls

@@ -61,7 +61,7 @@ base:
     - backup.adv_backup
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
-    - kafka.nodes
+    - kafka.*
     - stig.soc_stig
 
   '*_sensor':
@@ -177,6 +177,7 @@ base:
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
     - stig.soc_stig
+    - kafka.*
 
   '*_heavynode':
     - elasticsearch.auth
@@ -233,6 +234,15 @@ base:
     - redis.adv_redis
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
+    - kafka.*
+
+  '*_kafkanode':
+    - logstash.nodes
+    - logstash.soc_logstash
+    - logstash.adv_logstash
+    - minions.{{ grains.id }}
+    - minions.adv_{{ grains.id }}
+    - secrets
     - kafka.nodes
     - secrets
 

salt/allowed_states.map.jinja

@@ -123,7 +123,8 @@
           'utility',
           'schedule',
           'docker_clean',
-          'stig'
+          'stig',
+          'kafka'
           ],
       'so-searchnode': [
           'ssl',
@@ -157,7 +158,8 @@
           'schedule',
           'tcpreplay',
           'docker_clean',
-          'stig'
+          'stig',
+          'kafka'
           ],
       'so-sensor': [
           'ssl',

salt/kafka/soc_kafka.yaml

@@ -2,6 +2,12 @@ kafka:
   enabled:
     description: Enable or disable Kafka.
     helpLink: kafka.html
+  cluster_id:
+    description: The ID of the Kafka cluster.
+    readonly: True
+    advanced: True
+    sensitive: True
+    helpLink: kafka.html
   config:
     server:
       advertised_x_listeners:

salt/kafka/storage.sls

@@ -6,17 +6,18 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
-{% set kafka_cluster_id = salt['pillar.get']('secrets:kafka_cluster_id', default=None) %}
+{%   set kafka_cluster_id = salt['pillar.get']('kafka:cluster_id', default=None) %}
 
+{%   if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone'] %}
 {%     if kafka_cluster_id is none %}
 generate_kafka_cluster_id:
   cmd.run:
     - name: /usr/sbin/so-kafka-clusterid
 {%     endif %}
+{%   endif %}
 
 {# Initialize kafka storage if it doesn't already exist. Just looking for meta.properties in /nsm/kafka/data #}
-{% if salt['file.file_exists']('/nsm/kafka/data/meta.properties') %}
-{% else %}
+{%   if not salt['file.file_exists']('/nsm/kafka/data/meta.properties') %}
 kafka_storage_init:
   cmd.run:
     - name: |

salt/manager/tools/sbin/so-kafka-clusterid

@@ -13,12 +13,12 @@ else
   source $(dirname $0)/../../../common/tools/sbin/so-common
 fi
 
-if ! grep -q "^  kafka_cluster_id:" $local_salt_dir/pillar/secrets.sls; then
+if ! grep -q "^  cluster_id:" $local_salt_dir/pillar/kafka/soc_kafka.sls; then
     kafka_cluster_id=$(get_random_value 22)
-    echo '  kafka_cluster_id: '$kafka_cluster_id >> $local_salt_dir/pillar/secrets.sls
-fi
+    echo 'kafka: ' > $local_salt_dir/pillar/kafka/soc_kafka.sls
+    echo '  cluster_id: '$kafka_cluster_id >> $local_salt_dir/pillar/kafka/soc_kafka.sls
 
-if ! grep -q "^  kafkapass:" $local_salt_dir/pillar/secrets.sls; then
+if ! grep -q "^  kafkapass:" $local_salt_dir/pillar/kafka/soc_kafka.sls; then
     kafkapass=$(get_random_value)
-    echo '  kafkapass: '$kafkapass >> $local_salt_dir/pillar/secrets.sls
+    echo '  kafkapass: '$kafkapass >> $local_salt_dir/pillar/kafka/soc_kafka.sls
 fi

salt/ssl/init.sls

@@ -664,7 +664,8 @@ elastickeyperms:
 
 {%- endif %}
 
-{% if grains['role'] in ['so-manager', 'so-searchnode', 'so-receiver'] %}
+{% if grains['role'] in ['so-manager', 'so-receiver', 'so-searchnode'] %}
+
 kafka_key:
   x509.private_key_managed:
     - name: /etc/pki/kafka.key
@@ -767,7 +768,7 @@ kafka_logstash_crt:
     - onchanges:
       - x509: /etc/pki/kafka-logstash.key
 
-{%   if grains['role'] in ['so-manager'] %}
+{%   if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-receiver'] %}
 kafka_client_key:
   x509.private_key_managed:
     - name: /etc/pki/kafka-client.key

salt/top.sls

@@ -235,8 +235,8 @@ base:
     - firewall
     - logstash
     - redis
-    - kafka
     - elasticfleet.install_agent_grid
+    - kafka
 
   '*_idh and G@saltversion:{{saltversion}}':
     - match: compound

setup/so-functions

@@ -1115,8 +1115,6 @@ generate_passwords(){
   REDISPASS=$(get_random_value)
   SOCSRVKEY=$(get_random_value 64)
   IMPORTPASS=$(get_random_value)
-  KAFKACLUSTERID=$(get_random_value 22)
-  KAFKAPASS=$(get_random_value)
 }
 
 generate_interface_vars() {
@@ -1392,7 +1390,7 @@ make_some_dirs() {
 	mkdir -p $local_salt_dir/salt/firewall/portgroups
 	mkdir -p $local_salt_dir/salt/firewall/ports
 
-	for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni soc docker zeek suricata nginx telegraf logstash soc manager kratos idstools idh elastalert stig global;do
+	for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni soc docker zeek suricata nginx telegraf logstash soc manager kratos idstools idh elastalert stig global kafka;do
 	mkdir -p $local_salt_dir/pillar/$THEDIR
 	touch $local_salt_dir/pillar/$THEDIR/adv_$THEDIR.sls
 	touch $local_salt_dir/pillar/$THEDIR/soc_$THEDIR.sls
@@ -1946,9 +1944,7 @@ secrets_pillar(){
 	printf '%s\n'\
 		"secrets:"\
 		"  import_pass: $IMPORTPASS"\
-		"  influx_pass: $INFLUXPASS"\
-		"  kafka_cluster_id: $KAFKACLUSTERID"\
-		"  kafka_pass: $KAFKAPASS" > $local_salt_dir/pillar/secrets.sls
+		"  influx_pass: $INFLUXPASS" > $local_salt_dir/pillar/secrets.sls
   fi
 }