Merge remote-tracking branch 'origin/reyesj2/kafka' into kaffytaffy

salt/common/tools/sbin/so-image-common

@@ -66,6 +66,7 @@ container_list() {
       "so-suricata"
       "so-telegraf"
       "so-zeek"
+      "so-kafka"
     )
   else
     TRUSTED_CONTAINERS=(

salt/kafka/config.sls

@@ -16,7 +16,7 @@
 {% set nodes = salt['pillar.get']('kafka:nodes', {}) %}
 {% set combined = [] %}
 {% for hostname, data in nodes.items() %}
-  {% do combined.append(data.nodeid ~ "@" ~ hostname) %}
+  {% do combined.append(data.nodeid ~ "@" ~ hostname ~ ":9093") %}
 {% endfor %}
 {% set kraft_controller_quorum_voters = ','.join(combined) %}
 
@@ -55,6 +55,15 @@ kafka_sbin_tools:
     - group: 960
     - file_mode: 755 #}
 
+kafka_sbin_jinja_tools:
+  file.recurse:
+    - name: /usr/sbin
+    - source: salt://kafka/tools/sbin_jinja
+    - user: 960
+    - group: 960
+    - file_mode: 755
+    - template: jinja
+
 kakfa_log_dir:
   file.directory:
     - name: /opt/so/log/kafka
@@ -69,11 +78,10 @@ kafka_data_dir:
     - group: 960
     - makedirs: True
 
-{# When docker container is created an added to registry. Update so-kafka-generate-keystore script #}
 kafka_keystore_script:
   cmd.script:
     - source: salt://kafka/tools/sbin_jinja/so-kafka-generate-keystore
-    - tempalte: jinja
+    - template: jinja
     - cwd: /opt/so
     - defaults:
         GLOBALS: {{ GLOBALS }}

salt/kafka/enabled.sls

@@ -7,6 +7,7 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKER %}
+{%   set KAFKANODES =  salt['pillar.get']('kafka:nodes', {}) %}
 
 include:
   - kafka.sostatus
@@ -15,7 +16,7 @@ include:
 
 so-kafka:
   docker_container.running:
-    - image: so-kafka
+    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kafka:{{ GLOBALS.so_version }}
     - hostname: so-kafka
     - name: so-kafka
     - networks:

salt/kafka/etc/server.properties.jinja

@@ -42,7 +42,7 @@ listener.security.protocol.map=CONTROLLER:SSL,BROKER:SSL
 
 #SSL configuration
 ssl.keystore.location=/etc/pki/kafka.jks
-ssl.keystore.pasword=changeit
+ssl.keystore.password=changeit
 ssl.keystore.type=JKS
 ssl.truststore.location=/etc/pki/java/sos/cacerts
 ssl.truststore.password=changeit

salt/kafka/storage.sls

@@ -14,7 +14,7 @@
 kafka_storage_init:
   cmd.run:
     - name: |
-        docker run -v /nsm/kafka/data:/nsm/kafka/data -v /opt/so/conf/kafka/server.properties:/kafka/config/kraft/newserver.properties --name so-kafkainit --user root --entrypoint /kafka/bin/kafka-storage.sh so-kafka format -t {{ kafka_cluster_id }} -c /kafka/config/kraft/server.properties
+        docker run -v /nsm/kafka/data:/nsm/kafka/data -v /opt/so/conf/kafka/server.properties:/kafka/config/kraft/newserver.properties --name so-kafkainit --user root --entrypoint /kafka/bin/kafka-storage.sh {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kafka:{{ GLOBALS.so_version }} format -t {{ kafka_cluster_id }} -c /kafka/config/kraft/newserver.properties
 kafka_rm_kafkainit:
   cmd.run:
     - name: |

salt/kafka/tools/sbin_jinja/so-kafka-generate-keystore

@@ -8,7 +8,7 @@
 . /usr/sbin/so-common
 
 if [ ! -f /etc/pki/kafka.jks ]; then
-    docker run -v /etc/pki/kafka.p12:/etc/pki/kafka.p12 --name so-kafka-keystore --user root --entrypoint keytool so-kafka -importkeystore -srckeystore /etc/pki/kafka.p12 -srcstoretype PKCS12 -srsstorepass changeit -destkeystore /etc/pki/kafka.jks -deststoretype JKS -deststorepass changeit -alias kafkastore -noprompt
+    docker run -v /etc/pki/kafka.p12:/etc/pki/kafka.p12 --name so-kafka-keystore --user root --entrypoint keytool {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kafka:{{ GLOBALS.so_version }} -importkeystore -srckeystore /etc/pki/kafka.p12 -srcstoretype PKCS12 -srcstorepass changeit -destkeystore /etc/pki/kafka.jks -deststoretype JKS -deststorepass changeit -noprompt
     docker cp so-kafka-keystore:/etc/pki/kafka.jks /etc/pki/kafka.jks
     docker rm so-kafka-keystore
 else

salt/logstash/config.sls

@@ -12,7 +12,7 @@
 
 include:
   - ssl
-  {% if GLOBALS.role not in ['so-receiver','so-fleet'] %}
+  {% if GLOBALS.role not in ['so-receiver','so-fleet', 'so-kafkanode'] %}
   - elasticsearch
   {% endif %}
 

setup/so-setup

@@ -629,15 +629,6 @@ if ! [[ -f $install_opt_file ]]; then
 		set_minion_info
 		whiptail_end_settings
 
-	elif [[ $is_kafka ]]; then
-		info "Setting up as node type Kafka broker"
-		#check_requirements "kafka"
-		networking_needful
-		collect_mngr_hostname
-		add_mngr_ip_to_hosts
-		check_manager_connection
-		set_minion_info
-		whiptail_end_settings
 	fi
 
 	if [[ $waitforstate ]]; then