diff --git a/pillar/top.sls b/pillar/top.sls index 817767bf7..837e15d28 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -61,7 +61,7 @@ base: - backup.adv_backup - minions.{{ grains.id }} - minions.adv_{{ grains.id }} - - kafka.nodes + - kafka.* - stig.soc_stig '*_sensor': @@ -177,6 +177,7 @@ base: - minions.{{ grains.id }} - minions.adv_{{ grains.id }} - stig.soc_stig + - kafka.* '*_heavynode': - elasticsearch.auth @@ -233,6 +234,15 @@ base: - redis.adv_redis - minions.{{ grains.id }} - minions.adv_{{ grains.id }} + - kafka.* + + '*_kafkanode': + - logstash.nodes + - logstash.soc_logstash + - logstash.adv_logstash + - minions.{{ grains.id }} + - minions.adv_{{ grains.id }} + - secrets - kafka.nodes - secrets diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 0fa968658..091cb3786 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -123,7 +123,8 @@ 'utility', 'schedule', 'docker_clean', - 'stig' + 'stig', + 'kafka' ], 'so-searchnode': [ 'ssl', @@ -157,7 +158,8 @@ 'schedule', 'tcpreplay', 'docker_clean', - 'stig' + 'stig', + 'kafka' ], 'so-sensor': [ 'ssl', diff --git a/salt/kafka/soc_kafka.yaml b/salt/kafka/soc_kafka.yaml index 2fec8c302..8a6c516a9 100644 --- a/salt/kafka/soc_kafka.yaml +++ b/salt/kafka/soc_kafka.yaml @@ -2,6 +2,12 @@ kafka: enabled: description: Enable or disable Kafka. helpLink: kafka.html + cluster_id: + description: The ID of the Kafka cluster. + readonly: True + advanced: True + sensitive: True + helpLink: kafka.html config: server: advertised_x_listeners: diff --git a/salt/kafka/storage.sls b/salt/kafka/storage.sls index e99455e3d..fbb7c7328 100644 --- a/salt/kafka/storage.sls +++ b/salt/kafka/storage.sls @@ -6,17 +6,18 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls.split('.')[0] in allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} -{% set kafka_cluster_id = salt['pillar.get']('secrets:kafka_cluster_id', default=None) %} +{% set kafka_cluster_id = salt['pillar.get']('kafka:cluster_id', default=None) %} -{% if kafka_cluster_id is none %} +{% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone'] %} +{% if kafka_cluster_id is none %} generate_kafka_cluster_id: cmd.run: - name: /usr/sbin/so-kafka-clusterid -{% endif %} +{% endif %} +{% endif %} {# Initialize kafka storage if it doesn't already exist. Just looking for meta.properties in /nsm/kafka/data #} -{% if salt['file.file_exists']('/nsm/kafka/data/meta.properties') %} -{% else %} +{% if not salt['file.file_exists']('/nsm/kafka/data/meta.properties') %} kafka_storage_init: cmd.run: - name: | @@ -25,7 +26,7 @@ kafka_rm_kafkainit: cmd.run: - name: | docker rm so-kafkainit -{% endif %} +{% endif %} {% else %} @@ -34,4 +35,4 @@ kafka_rm_kafkainit: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} \ No newline at end of file +{% endif %} diff --git a/salt/manager/tools/sbin/so-kafka-clusterid b/salt/manager/tools/sbin/so-kafka-clusterid index fcbe3ba42..adddfe3ce 100644 --- a/salt/manager/tools/sbin/so-kafka-clusterid +++ b/salt/manager/tools/sbin/so-kafka-clusterid @@ -13,12 +13,12 @@ else source $(dirname $0)/../../../common/tools/sbin/so-common fi -if ! grep -q "^ kafka_cluster_id:" $local_salt_dir/pillar/secrets.sls; then +if ! grep -q "^ cluster_id:" $local_salt_dir/pillar/kafka/soc_kafka.sls; then kafka_cluster_id=$(get_random_value 22) - echo ' kafka_cluster_id: '$kafka_cluster_id >> $local_salt_dir/pillar/secrets.sls -fi + echo 'kafka: ' > $local_salt_dir/pillar/kafka/soc_kafka.sls + echo ' cluster_id: '$kafka_cluster_id >> $local_salt_dir/pillar/kafka/soc_kafka.sls -if ! grep -q "^ kafkapass:" $local_salt_dir/pillar/secrets.sls; then +if ! grep -q "^ kafkapass:" $local_salt_dir/pillar/kafka/soc_kafka.sls; then kafkapass=$(get_random_value) - echo ' kafkapass: '$kafkapass >> $local_salt_dir/pillar/secrets.sls -fi \ No newline at end of file + echo ' kafkapass: '$kafkapass >> $local_salt_dir/pillar/kafka/soc_kafka.sls +fi diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index f337d62cb..853afb2b3 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -664,7 +664,8 @@ elastickeyperms: {%- endif %} -{% if grains['role'] in ['so-manager', 'so-searchnode', 'so-receiver'] %} +{% if grains['role'] in ['so-manager', 'so-receiver', 'so-searchnode'] %} + kafka_key: x509.private_key_managed: - name: /etc/pki/kafka.key @@ -767,7 +768,7 @@ kafka_logstash_crt: - onchanges: - x509: /etc/pki/kafka-logstash.key -{% if grains['role'] in ['so-manager'] %} +{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-receiver'] %} kafka_client_key: x509.private_key_managed: - name: /etc/pki/kafka-client.key diff --git a/salt/top.sls b/salt/top.sls index ec5e4d738..e4cd067c3 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -235,8 +235,8 @@ base: - firewall - logstash - redis - - kafka - elasticfleet.install_agent_grid + - kafka '*_idh and G@saltversion:{{saltversion}}': - match: compound diff --git a/setup/so-functions b/setup/so-functions index 038a4deb4..2332ab94c 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1115,8 +1115,6 @@ generate_passwords(){ REDISPASS=$(get_random_value) SOCSRVKEY=$(get_random_value 64) IMPORTPASS=$(get_random_value) - KAFKACLUSTERID=$(get_random_value 22) - KAFKAPASS=$(get_random_value) } generate_interface_vars() { @@ -1392,7 +1390,7 @@ make_some_dirs() { mkdir -p $local_salt_dir/salt/firewall/portgroups mkdir -p $local_salt_dir/salt/firewall/ports - for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni soc docker zeek suricata nginx telegraf logstash soc manager kratos idstools idh elastalert stig global;do + for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni soc docker zeek suricata nginx telegraf logstash soc manager kratos idstools idh elastalert stig global kafka;do mkdir -p $local_salt_dir/pillar/$THEDIR touch $local_salt_dir/pillar/$THEDIR/adv_$THEDIR.sls touch $local_salt_dir/pillar/$THEDIR/soc_$THEDIR.sls @@ -1946,9 +1944,7 @@ secrets_pillar(){ printf '%s\n'\ "secrets:"\ " import_pass: $IMPORTPASS"\ - " influx_pass: $INFLUXPASS"\ - " kafka_cluster_id: $KAFKACLUSTERID"\ - " kafka_pass: $KAFKAPASS" > $local_salt_dir/pillar/secrets.sls + " influx_pass: $INFLUXPASS" > $local_salt_dir/pillar/secrets.sls fi }