From a54a72c2696f32b9d565efce7d9748b3bd148070 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 12 Apr 2024 11:19:20 -0400 Subject: [PATCH] move kafka_cluster_id to kafka:cluster_id --- pillar/top.sls | 4 +++- salt/allowed_states.map.jinja | 20 +++++++------------- salt/kafka/soc_kafka.yaml | 6 ++++++ salt/kafka/storage.sls | 15 ++++++++------- salt/manager/tools/sbin/so-kafka-clusterid | 11 ++++++----- salt/ssl/init.sls | 4 ++-- 6 files changed, 32 insertions(+), 28 deletions(-) diff --git a/pillar/top.sls b/pillar/top.sls index 61f4f338f..170b3f759 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -61,7 +61,7 @@ base: - backup.adv_backup - minions.{{ grains.id }} - minions.adv_{{ grains.id }} - - kafka.nodes + - kafka.* - stig.soc_stig '*_sensor': @@ -177,6 +177,7 @@ base: - minions.{{ grains.id }} - minions.adv_{{ grains.id }} - stig.soc_stig + - kafka.* '*_heavynode': - elasticsearch.auth @@ -233,6 +234,7 @@ base: - redis.adv_redis - minions.{{ grains.id }} - minions.adv_{{ grains.id }} + - kafka.* '*_kafkanode': - logstash.nodes diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 6fa60c2ea..091cb3786 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -123,7 +123,8 @@ 'utility', 'schedule', 'docker_clean', - 'stig' + 'stig', + 'kafka' ], 'so-searchnode': [ 'ssl', @@ -157,7 +158,8 @@ 'schedule', 'tcpreplay', 'docker_clean', - 'stig' + 'stig', + 'kafka' ], 'so-sensor': [ 'ssl', @@ -188,16 +190,8 @@ 'telegraf', 'firewall', 'schedule', - 'docker_clean' - ], - 'so-kafkanode': [ - 'kafka', - 'logstash', - 'ssl', - 'telegraf', - 'firewall', - 'schedule', - 'docker_clean' + 'docker_clean', + 'kafka' ], 'so-desktop': [ 'ssl', @@ -214,7 +208,7 @@ {% do allowed_states.append('strelka') %} {% endif %} - {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-searchnode', 'so-managersearch', 'so-heavynode', 'so-import', 'so-kafkanode'] %} + {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-searchnode', 'so-managersearch', 'so-heavynode', 'so-import'] %} {% do allowed_states.append('elasticsearch') %} {% endif %} diff --git a/salt/kafka/soc_kafka.yaml b/salt/kafka/soc_kafka.yaml index 2fec8c302..8a6c516a9 100644 --- a/salt/kafka/soc_kafka.yaml +++ b/salt/kafka/soc_kafka.yaml @@ -2,6 +2,12 @@ kafka: enabled: description: Enable or disable Kafka. helpLink: kafka.html + cluster_id: + description: The ID of the Kafka cluster. + readonly: True + advanced: True + sensitive: True + helpLink: kafka.html config: server: advertised_x_listeners: diff --git a/salt/kafka/storage.sls b/salt/kafka/storage.sls index e99455e3d..fbb7c7328 100644 --- a/salt/kafka/storage.sls +++ b/salt/kafka/storage.sls @@ -6,17 +6,18 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls.split('.')[0] in allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} -{% set kafka_cluster_id = salt['pillar.get']('secrets:kafka_cluster_id', default=None) %} +{% set kafka_cluster_id = salt['pillar.get']('kafka:cluster_id', default=None) %} -{% if kafka_cluster_id is none %} +{% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone'] %} +{% if kafka_cluster_id is none %} generate_kafka_cluster_id: cmd.run: - name: /usr/sbin/so-kafka-clusterid -{% endif %} +{% endif %} +{% endif %} {# Initialize kafka storage if it doesn't already exist. Just looking for meta.properties in /nsm/kafka/data #} -{% if salt['file.file_exists']('/nsm/kafka/data/meta.properties') %} -{% else %} +{% if not salt['file.file_exists']('/nsm/kafka/data/meta.properties') %} kafka_storage_init: cmd.run: - name: | @@ -25,7 +26,7 @@ kafka_rm_kafkainit: cmd.run: - name: | docker rm so-kafkainit -{% endif %} +{% endif %} {% else %} @@ -34,4 +35,4 @@ kafka_rm_kafkainit: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} \ No newline at end of file +{% endif %} diff --git a/salt/manager/tools/sbin/so-kafka-clusterid b/salt/manager/tools/sbin/so-kafka-clusterid index 719973247..def454372 100644 --- a/salt/manager/tools/sbin/so-kafka-clusterid +++ b/salt/manager/tools/sbin/so-kafka-clusterid @@ -13,10 +13,11 @@ else source $(dirname $0)/../../../common/tools/sbin/so-common fi -if ! grep -q "^ kafka_cluster_id:" $local_salt_dir/pillar/secrets.sls; then +if ! grep -q "^ cluster_id:" $local_salt_dir/pillar/kafka/soc_kafka.sls; then kafka_cluster_id=$(get_random_value 22) - echo ' kafka_cluster_id: '$kafka_cluster_id >> $local_salt_dir/pillar/secrets.sls + echo 'kafka: ' > $local_salt_dir/pillar/kafka/soc_kafka.sls + echo ' cluster_id: '$kafka_cluster_id >> $local_salt_dir/pillar/kafka/soc_kafka.sls else - echo 'kafka_cluster_id exists' - salt-call pillar.get secrets -fi \ No newline at end of file + echo 'kafka:cluster_id pillar exists' + salt-call pillar.get kafka:cluster_id +fi diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 90f9cc64f..0aa06bc8e 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -666,7 +666,7 @@ elastickeyperms: # Roles will need to be modified. Below is just for testing encrypted kafka pipelines # Remove so-manager. Just inplace for testing -{% if grains['role'] in ['so-manager', 'so-kafkanode', 'so-searchnode'] %} +{% if grains['role'] in ['so-manager', 'so-receiver', 'so-searchnode'] %} # Create a cert for Redis encryption kafka_key: x509.private_key_managed: @@ -770,7 +770,7 @@ kafka_logstash_crt: - onchanges: - x509: /etc/pki/kafka-logstash.key -{% if grains['role'] in ['so-manager'] %} +{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-receiver'] %} kafka_client_key: x509.private_key_managed: - name: /etc/pki/kafka-client.key