From 284c1be85fc058945e772b5ee02898d13594e070 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 10 Jun 2024 11:08:54 -0400 Subject: [PATCH] Update Kafka controller(s) via SOC UI Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/kafka/config.map.jinja | 2 -- salt/kafka/config.sls | 6 ++++++ salt/kafka/defaults.yaml | 2 +- salt/kafka/enabled.sls | 2 -- salt/kafka/nodes.map.jinja | 5 +++-- salt/kafka/soc_kafka.yaml | 5 ++--- salt/salt/files/engines.conf | 18 ++++++++++++++++++ 7 files changed, 30 insertions(+), 10 deletions(-) diff --git a/salt/kafka/config.map.jinja b/salt/kafka/config.map.jinja index e5b77db11..88d27c1a8 100644 --- a/salt/kafka/config.map.jinja +++ b/salt/kafka/config.map.jinja @@ -7,8 +7,6 @@ {% set KAFKA_NODES_PILLAR = salt['pillar.get']('kafka:nodes') %} -{% set KAFKA_CONTROLLERS_PILLAR = salt['pillar.get']('kafka:kafka_controllers', default=None) %} - {# Create list of KRaft controllers #} {% set controllers = [] %} diff --git a/salt/kafka/config.sls b/salt/kafka/config.sls index 5cf6f8201..165daf7eb 100644 --- a/salt/kafka/config.sls +++ b/salt/kafka/config.sls @@ -66,6 +66,12 @@ kafka_kraft_{{sc}}_properties: - show_changes: False {% endfor %} +reset_quorum_on_changes: + cmd.run: + - name: rm -f /nsm/kafka/data/__cluster_metadata-0/quorum-state + - watch: + - file: /opt/so/conf/kafka/server.properties + {% else %} {{sls}}_state_not_allowed: diff --git a/salt/kafka/defaults.yaml b/salt/kafka/defaults.yaml index 56ad9252f..f45560e60 100644 --- a/salt/kafka/defaults.yaml +++ b/salt/kafka/defaults.yaml @@ -2,7 +2,7 @@ kafka: enabled: False cluster_id: kafka_pass: - kafka_controllers: [] + kafka_controllers: config: broker: advertised_x_listeners: diff --git a/salt/kafka/enabled.sls b/salt/kafka/enabled.sls index e90a314d2..75cf71148 100644 --- a/salt/kafka/enabled.sls +++ b/salt/kafka/enabled.sls @@ -53,9 +53,7 @@ so-kafka: - /nsm/kafka/data/:/nsm/kafka/data/:rw - /opt/so/log/kafka:/opt/kafka/logs/:rw - /opt/so/conf/kafka/server.properties:/opt/kafka/config/kraft/server.properties:ro - {% if GLOBALS.is_manager %} - /opt/so/conf/kafka/client.properties:/opt/kafka/config/kraft/client.properties - {% endif %} - watch: {% for sc in ['server', 'client'] %} - file: kafka_kraft_{{sc}}_properties diff --git a/salt/kafka/nodes.map.jinja b/salt/kafka/nodes.map.jinja index fa33adda5..c0b98de14 100644 --- a/salt/kafka/nodes.map.jinja +++ b/salt/kafka/nodes.map.jinja @@ -68,14 +68,15 @@ {# Update the process_x_roles value for any host in the kafka_controllers_pillar configured from SOC UI #} {% set ns = namespace(has_controller=false) %} {% if KAFKA_CONTROLLERS_PILLAR != none %} -{% for hostname in KAFKA_CONTROLLERS_PILLAR %} +{% set KAFKA_CONTROLLERS_PILLAR_LIST = KAFKA_CONTROLLERS_PILLAR.split(',') %} +{% for hostname in KAFKA_CONTROLLERS_PILLAR_LIST %} {% if hostname in COMBINED_KAFKANODES %} {% do COMBINED_KAFKANODES[hostname].update({'role': 'controller'}) %} {% set ns.has_controller = true %} {% endif %} {% endfor %} {% for hostname in COMBINED_KAFKANODES %} -{% if hostname not in KAFKA_CONTROLLERS_PILLAR %} +{% if hostname not in KAFKA_CONTROLLERS_PILLAR_LIST %} {% do COMBINED_KAFKANODES[hostname].update({'role': 'broker'}) %} {% endif %} {% endfor %} diff --git a/salt/kafka/soc_kafka.yaml b/salt/kafka/soc_kafka.yaml index b1de1f243..05f047c4a 100644 --- a/salt/kafka/soc_kafka.yaml +++ b/salt/kafka/soc_kafka.yaml @@ -13,9 +13,8 @@ kafka: sensitive: True helpLink: kafka.html kafka_controllers: - description: A list of Security Onion grid members that should act as controllers for this Kafka cluster. By default, the grid manager will use a 'combined' role where it will act as both a broker and controller. Keep total Kafka controllers to an odd number and ensure you do not assign ALL your Kafka nodes as controllers or this Kafka cluster will not start. - forcedType: "[]string" - multiline: True + description: A comma-seperated list of Security Onion grid members that should act as controllers for this Kafka cluster. By default, the grid manager will use a 'combined' role where it will act as both a broker and controller. Keep total Kafka controllers to an odd number and ensure you do not assign ALL your Kafka nodes as controllers or this Kafka cluster will not start. + forcedType: "string" helpLink: kafka.html config: broker: diff --git a/salt/salt/files/engines.conf b/salt/salt/files/engines.conf index 69d596ed0..de5685fff 100644 --- a/salt/salt/files/engines.conf +++ b/salt/salt/files/engines.conf @@ -57,4 +57,22 @@ engines: cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver' saltutil.kill_all_jobs - cmd.run: cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver' state.highstate + - files: + - /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls + - /opt/so/saltstack/local/pillar/kafka/adv_kafka.sls + pillar: kafka.kafka_controllers + default: '' + actions: + from: + '*': + to: + '*': + - cmd.run: + cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver' saltutil.kill_all_jobs + - cmd.run: + cmd: salt-call state.apply kafka.nodes + - cmd.run: + cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver' state.apply kafka + - cmd.run: + cmd: salt-call state.apply elasticfleet interval: 10