Generate new Kafka truststore

Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com>

pillar/top.sls

@@ -233,6 +233,8 @@ base:
     - stig.soc_stig
     - soc.license
     - kafka.nodes
+    - kafka.soc_kafka
+    - kafka.adv_kafka
 
   '*_receiver':
     - logstash.nodes

salt/allowed_states.map.jinja

@@ -136,7 +136,9 @@
           'firewall',
           'schedule',
           'docker_clean',
-          'stig'
+          'stig',
+          'kafka.ca',
+          'kafka.ssl'
           ],
       'so-standalone': [
           'salt.master',

salt/kafka/ca.sls

@@ -0,0 +1,37 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states or sls in allowed_states %}
+{%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   set KAFKATRUST = salt['pillar.get']('kafka:truststore') %}
+
+kafkaconfdir:
+  file.directory:
+    - name: /opt/so/conf/kafka
+    - user: 960
+    - group: 960
+    - makedirs: True
+
+{% if GLOBALS.is_manager %}
+# Manager runs so-kafka-trust to create truststore for Kafka ssl communication
+kafka_truststore:
+  cmd.script:
+    - source: salt://kafka/tools/sbin_jinja/so-kafka-trust
+    - template: jinja
+    - cwd: /opt/so
+    - defaults:
+        GLOBALS: {{ GLOBALS }}
+        KAFKATRUST: {{ KAFKATRUST }}
+{% endif %}
+
+kafkacertz:
+  file.managed:
+    - name: /opt/so/conf/kafka/kafka-truststore.jks
+    - source: salt://kafka/files/kafka-truststore
+    - user: 960
+    - group: 931
+
+{% endif %}

salt/kafka/config.map.jinja

@@ -7,6 +7,7 @@
 
 {% set KAFKA_NODES_PILLAR = salt['pillar.get']('kafka:nodes') %}
 {% set KAFKA_PASSWORD = salt['pillar.get']('kafka:password') %}
+{% set KAFKA_TRUSTPASS = salt['pillar.get']('kafka:trustpass') %}
 
 {# Create list of KRaft controllers #}
 {% set controllers = [] %}
@@ -67,19 +68,12 @@
 
 {% endif %}
 
-{# If a password other than PLACEHOLDER isn't set remove it from the server.properties #}
-{% if KAFKAMERGED.config.broker.ssl_x_truststore_x_password == 'PLACEHOLDER' %}
-{%   do KAFKAMERGED.config.broker.pop('ssl_x_truststore_x_password') %}
-{% endif %}
-
-{% if KAFKAMERGED.config.controller.ssl_x_truststore_x_password == 'PLACEHOLDER' %}
-{%   do KAFKAMERGED.config.controller.pop('ssl_x_truststore_x_password') %}
-{% endif %}
+{# Truststore config #}
+{% do KAFKAMERGED.config.broker.update({'ssl_x_truststore_x_password': KAFKA_TRUSTPASS }) %}
+{% do KAFKAMERGED.config.controller.update({'ssl_x_truststore_x_password': KAFKA_TRUSTPASS }) %}
+{% do KAFKAMERGED.config.client.update({'ssl_x_truststore_x_password': KAFKA_TRUSTPASS }) %}
 
 {# Client properties stuff #}
-{% if KAFKAMERGED.config.client.ssl_x_truststore_x_password == 'PLACEHOLDER' %}
-{%   do KAFKAMERGED.config.client.pop('ssl_x_truststore_x_password') %}
-{% endif %}
 {% do KAFKAMERGED.config.client.update({'ssl_x_keystore_x_password': KAFKA_PASSWORD }) %}
 
 {% if 'broker' in node_type %}

salt/kafka/config.sls

@@ -7,18 +7,21 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 
-include:
-  - ssl
-
 kafka_group:
   group.present:
     - name: kafka
     - gid: 960
 
-kafka:
+kafka_user:
   user.present:
     - uid: 960
     - gid: 960
+    - home: /opt/so/conf/kafka
+    - createhome: False
+
+kafka_home_dir:
+  file.absent:
+    - name: /home/kafka
 
 kafka_sbin_tools:
   file.recurse:
@@ -28,6 +31,17 @@ kafka_sbin_tools:
     - group: 960
     - file_mode: 755
 
+kafka_sbin_jinja_tools:
+  file.recurse:
+    - name: /usr/sbin
+    - source: salt://kafka/tools/sbin_jinja
+    - user: 960
+    - group: 960
+    - file_mode: 755
+    - template: jinja
+    - defaults:
+        GLOBALS: {{ GLOBALS }}
+
 kafka_log_dir:
   file.directory:
     - name: /opt/so/log/kafka

salt/kafka/defaults.yaml

@@ -1,10 +1,11 @@
 kafka:
   enabled: False
   cluster_id:
-  password:
   controllers:
   reset:
   config:
+    password:
+    trustpass:
     broker:
       advertised_x_listeners:
       auto_x_create_x_topics_x_enable: true
@@ -30,16 +31,16 @@ kafka:
       ssl_x_keystore_x_location: /etc/pki/kafka.p12
       ssl_x_keystore_x_type: PKCS12
       ssl_x_keystore_x_password:
-      ssl_x_truststore_x_location: /etc/pki/java/sos/cacerts
-      ssl_x_truststore_x_password: PLACEHOLDER
-      ssl_x_truststore_x_type: PEM
+      ssl_x_truststore_x_location: /etc/pki/kafka-truststore.jks
+      ssl_x_truststore_x_type: JKS
+      ssl_x_truststore_x_password:
       transaction_x_state_x_log_x_min_x_isr: 1
       transaction_x_state_x_log_x_replication_x_factor: 1
     client:
       security_x_protocol: SSL
-      ssl_x_truststore_x_location: /etc/pki/java/sos/cacerts
-      ssl_x_truststore_x_password: PLACEHOLDER
-      ssl_x_truststore_x_type: PEM
+      ssl_x_truststore_x_location: /etc/pki/kafka-truststore.jks
+      ssl_x_truststore_x_type: JKS
+      ssl_x_truststore_x_password:
       ssl_x_keystore_x_location: /etc/pki/kafka.p12
       ssl_x_keystore_x_type: PKCS12
       ssl_x_keystore_x_password:
@@ -57,6 +58,6 @@ kafka:
       ssl_x_keystore_x_location: /etc/pki/kafka.p12
       ssl_x_keystore_x_type: PKCS12
       ssl_x_keystore_x_password:
-      ssl_x_truststore_x_location: /etc/pki/java/sos/cacerts
-      ssl_x_truststore_x_password: PLACEHOLDER
-      ssl_x_truststore_x_type: PEM
+      ssl_x_truststore_x_location: /etc/pki/kafka-truststore.jks
+      ssl_x_truststore_x_type: JKS
+      ssl_x_truststore_x_password:

salt/kafka/enabled.sls

@@ -17,10 +17,11 @@
 {%   if 'gmd' in salt['pillar.get']('features', []) %}
 
 include:
-  - elasticsearch.ca
-  - kafka.sostatus
+  - kafka.ca
   - kafka.config
+  - kafka.ssl
   - kafka.storage
+  - kafka.sostatus
 
 so-kafka:
   docker_container.running:
@@ -49,7 +50,7 @@ so-kafka:
       {% endfor %}
     - binds:
       - /etc/pki/kafka.p12:/etc/pki/kafka.p12:ro
-      - /etc/pki/tls/certs/intca.crt:/etc/pki/java/sos/cacerts:ro
+      - /opt/so/conf/kafka/kafka-truststore.jks:/etc/pki/kafka-truststore.jks:ro
       - /nsm/kafka/data/:/nsm/kafka/data/:rw
       - /opt/so/log/kafka:/opt/kafka/logs/:rw
       - /opt/so/conf/kafka/server.properties:/opt/kafka/config/kraft/server.properties:ro
@@ -58,6 +59,9 @@ so-kafka:
       {% for sc in ['server', 'client'] %}
       - file: kafka_kraft_{{sc}}_properties
       {% endfor %}
+      - file: kafkacertz
+    - require:
+      - file: kafkacertz
 
 delete_so-kafka_so-status.disabled:
   file.uncomment:

salt/kafka/soc_kafka.yaml

@@ -8,19 +8,25 @@ kafka:
     advanced: True
     sensitive: True
     helpLink: kafka.html
-  password:
-    description: The password to use for the Kafka certificates.
-    sensitive: True
-    helpLink: kafka.html
   controllers:
     description: A comma-separated list of hostnames that will act as Kafka controllers. These hosts will be responsible for managing the Kafka cluster. Note that only manager and receiver nodes are eligible to run Kafka. This configuration needs to be set before enabling Kafka. Failure to do so may result in Kafka topics becoming unavailable requiring manual intervention to restore functionality or reset Kafka, either of which can result in data loss.
-    forcedType: "string"
+    forcedType: string
     helpLink: kafka.html
   reset:
     description: Disable and reset the Kafka cluster. This will remove all Kafka data including logs that may have not yet been ingested into Elasticsearch and reverts the grid to using REDIS as the global pipeline. This is useful when testing different Kafka configurations such as rearranging Kafka brokers / controllers allowing you to reset the cluster rather than manually fixing any issues arising from attempting to reassign a Kafka broker into a controller. Enter 'YES_RESET_KAFKA' and submit to disable and reset Kafka. Make any configuration changes required and re-enable Kafka when ready. This action CANNOT be reversed.
     advanced: True
     helpLink: kafka.html
   config:
+    password:
+      description: The password used for the Kafka certificates.
+      readonly: True
+      sensitive: True
+      helpLink: kafka.html
+    trustpass:
+      description: The password used for the Kafka truststore.
+      readonly: True
+      sensitive: True
+      helpLink: kafka.html
     broker:
       advertised_x_listeners:
         description: Specify the list of listeners (hostname and port) that Kafka brokers provide to clients for communication.
@@ -128,6 +134,10 @@ kafka:
         description: The trust store file location within the Docker container.
         title: ssl.truststore.location
         helpLink: kafka.html
+      ssl_x_truststore_x_type:
+        description: The trust store file format.
+        title: ssl.truststore.type
+        helpLink: kafka.html
       ssl_x_truststore_x_password:
         description: The trust store file password. If null, the trust store file is still use, but integrity checking is disabled. Invalid for PEM format.
         title: ssl.truststore.password
@@ -167,6 +177,10 @@ kafka:
         description: The trust store file location within the Docker container.
         title: ssl.truststore.location
         helpLink: kafka.html
+      ssl_x_truststore_x_type:
+        description: The trust store file format.
+        title: ssl.truststore.type
+        helpLink: kafka.html
       ssl_x_truststore_x_password:
         description: The trust store file password. If null, the trust store file is still use, but integrity checking is disabled. Invalid for PEM format.
         title: ssl.truststore.password

salt/kafka/tools/sbin_jinja/so-kafka-trust

@@ -0,0 +1,13 @@
+#!/bin/bash
+#
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+{% set TRUSTPASS = salt['pillar.get']('kafka:trustpass') %}
+
+if [ ! -f /opt/so/saltstack/local/salt/kafka/files/kafka-truststore ]; then
+    docker run -v /etc/pki/ca.crt:/etc/pki/ca.crt --name so-kafkatrust --user root --entrypoint /opt/java/openjdk/bin/keytool {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kafka:{{ GLOBALS.so_version }} -import -file /etc/pki/ca.crt -alias SOS -keystore /etc/pki/kafka-truststore -storepass {{ TRUSTPASS }} -storetype jks -noprompt
+    docker cp so-kafkatrust:/etc/pki/kafka-truststore /opt/so/saltstack/local/salt/kafka/files/kafka-truststore
+    docker rm so-kafkatrust
+fi

salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja

@@ -1,4 +1,5 @@
 {%- set kafka_password = salt['pillar.get']('kafka:password') %}
+{%- set kafka_trustpass = salt['pillar.get']('kafka:trustpass') %}
 {%- set kafka_brokers = salt['pillar.get']('kafka:nodes', {}) %}
 {%- set brokers = [] %}
 
@@ -22,8 +23,8 @@ input {
         ssl_keystore_location => '/usr/share/logstash/kafka-logstash.p12'
         ssl_keystore_password => '{{ kafka_password }}'
         ssl_keystore_type => 'PKCS12'
-        ssl_truststore_location => '/etc/pki/ca-trust/extracted/java/cacerts'
-        ssl_truststore_password => 'changeit'
+        ssl_truststore_location => '/etc/pki/kafka-truststore.jks'
+        ssl_truststore_password => '{{ kafka_trustpass }}'
         decorate_events => true
         tags => [ "elastic-agent", "input-{{ GLOBALS.hostname}}", "kafka" ]
     }

salt/manager/tools/sbin/soup

@@ -673,7 +673,10 @@ up_to_2.4.80() {
 }
 
 up_to_2.4.90() {
-  echo "Nothing to apply"
+  kafkatrust=$(get_random_value)
+  echo '  trustpass: '$kafkatrust >> /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls
+
+
   INSTALLEDVERSION=2.4.90
 }
 

setup/so-functions

@@ -1180,13 +1180,15 @@ kibana_pillar() {
 kafka_pillar() {
 	KAFKACLUSTERID=$(get_random_value 22)
 	KAFKAPASS=$(get_random_value)
+	KAFKATRUST=$(get_random_value)
 	logCmd "mkdir -p $local_salt_dir/pillar/kafka"
 	logCmd "touch $adv_kafka_pillar_file"
 	logCmd "touch $kafka_pillar_file"
 	printf '%s\n'\
 		"kafka:"\
 		"  cluster_id: $KAFKACLUSTERID"\
-		"  password: $KAFKAPASS" > $kafka_pillar_file
+		"  password: $KAFKAPASS"\
+		"  trustpass: $KAFKATRUST" > $kafka_pillar_file
 }
 
 logrotate_pillar() {