Kafka init

Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com>

pillar/kafka/nodes.sls

@@ -0,0 +1,30 @@
+{% set current_kafkanodes = salt.saltutil.runner('mine.get', tgt='G@role:so-kafkanode', fun='network.ip_addrs', tgt_type='compound') %}
+{% set pillar_kafkanodes = salt['pillar.get']('kafka:nodes', default={}, merge=True) %}
+
+{% set existing_ids = [] %}
+{% for node in pillar_kafkanodes.values() %}
+  {% if node.get('id') %}
+    {% do existing_ids.append(node['nodeid']) %}
+  {% endif %}
+{% endfor %}
+{% set all_possible_ids = range(1, 256)|list %}
+
+{% set available_ids = [] %}
+{% for id in all_possible_ids %}
+  {% if id not in existing_ids %}
+    {% do available_ids.append(id) %}
+  {% endif %}
+{% endfor %}
+
+{% set final_nodes = pillar_kafkanodes.copy() %}
+
+{% for minionid, ip in current_kafkanodes.items() %}
+    {% set hostname = minionid.split('_')[0] %}
+    {% if hostname not in final_nodes %}
+        {% set new_id = available_ids.pop(0) %}
+        {% do final_nodes.update({hostname: {'nodeid': new_id, 'ip': ip[0]}}) %}
+    {% endif %}
+{% endfor %}
+
+kafka:
+  nodes: {{ final_nodes|tojson }}

pillar/logstash/nodes.sls

@@ -2,7 +2,7 @@
 {% set cached_grains = salt.saltutil.runner('cache.grains', tgt='*') %}
 {% for minionid, ip in salt.saltutil.runner(
     'mine.get',
-    tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-searchnode or G@role:so-heavynode or G@role:so-receiver or G@role:so-fleet ',
+    tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-searchnode or G@role:so-heavynode or G@role:so-receiver or G@role:so-fleet or G@role:so-kafkanode ',
     fun='network.ip_addrs',
     tgt_type='compound') | dictsort()
 %}

pillar/top.sls

@@ -65,6 +65,7 @@ base:
     - soctopus.adv_soctopus
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
+    - kafka.nodes
 
   '*_sensor':
     - healthcheck.sensor
@@ -241,6 +242,15 @@ base:
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
 
+  '*_kafkanode':
+    - logstash.nodes
+    - logstash.soc_logstash
+    - logstash.adv_logstash
+    - minions.{{ grains.id }}
+    - minions.adv_{{ grains.id }}
+    - secrets
+    - kafka.nodes
+
   '*_import':
     - secrets
     - elasticsearch.index_templates