Kafka init

Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com>

salt/logstash/defaults.yaml

@@ -19,6 +19,8 @@ logstash:
         - search
       fleet:
         - fleet
+      kafkanode:
+        - kafkanode
   defined_pipelines:
     fleet:
       - so/0012_input_elastic_agent.conf.jinja
@@ -37,6 +39,8 @@ logstash:
       - so/0900_input_redis.conf.jinja
       - so/9805_output_elastic_agent.conf.jinja
       - so/9900_output_endgame.conf.jinja
+    kafkanode:
+      - so/0899_output_kafka.conf.jinja
     custom0: []
     custom1: []
     custom2: []

salt/logstash/enabled.sls

@@ -75,10 +75,13 @@ so-logstash:
       {% else %}
       - /etc/pki/tls/certs/intca.crt:/usr/share/filebeat/ca.crt:ro
       {% endif %}
-      {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-searchnode'] %}
+      {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-searchnode', 'so-kafkanode' ] %}
       - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro
       - /opt/so/conf/ca/tls-ca-bundle.pem:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem:ro
       {% endif %}
+      {% if GLOBALS.role in ['so-kafkanode'] %}
+      - /etc/pki/kafka-logstash.p12:/usr/share/logstash/kafka-logstash.p12:ro
+      {% endif %}
       {% if GLOBALS.role == 'so-eval' %}
       - /nsm/zeek:/nsm/zeek:ro
       - /nsm/suricata:/suricata:ro

salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja

@@ -0,0 +1,26 @@
+{% set kafka_brokers = salt['pillar.get']('logstash:nodes:kafkanode', {}) %}
+{% set broker_ips = [] %}
+{% for node, node_data in kafka_brokers.items() %}
+  {% do broker_ips.append(node_data['ip'] + ":9092") %}
+{% endfor %}
+
+{% set bootstrap_servers = "','".join(broker_ips) %}
+
+
+#Run on searchnodes ingest kafka topic(s) group_id allows load balancing of event ingest to all searchnodes
+input {
+    kafka {
+        codec => json
+        #Can ingest multiple topics. Set to a value from SOC UI?
+        topics => ['logstash-topic',]
+        group_id => 'searchnodes'
+        security_protocol => 'SSL'
+        bootstrap_servers => {{ bootstrap_servers }}
+        ssl_keystore_location => '/usr/share/logstash/kafka-logstash.p12'
+        ssl_keystore_password => ''
+        ssl_keystore_type => 'PKCS12'
+        ssl_truststore_location => '/etc/pki/ca-trust/extracted/java/cacerts'
+        # Set password as a pillar to avoid bad optics? This is default truststore for grid
+        ssl_truststore_password => 'changeit'
+    }
+}

salt/logstash/pipelines/config/so/0899_output_kafka.conf.jinja

@@ -0,0 +1,22 @@
+{% set kafka_brokers = salt['pillar.get']('logstash:nodes:kafkanode', {}) %}
+{% set broker_ips = [] %}
+{% for node, node_data in kafka_brokers.items() %}
+  {% do broker_ips.append(node_data['ip'] + ":9092") %}
+{% endfor %}
+
+{% set bootstrap_servers = "','".join(broker_ips) %}
+
+#Run on kafka broker logstash writes to topic 'logstash-topic'
+output {
+    kafka {
+        codec => json
+        topic_id => 'logstash-topic'
+        bootstrap_servers => '{{ bootstrap_servers }}'
+        security_protocol => 'SSL'
+        ssl_keystore_location => '/usr/share/logstash/kafka-logstash.p12'
+        ssl_keystore_password => ''
+        ssl_keystore_type => 'PKCS12'
+        ssl_truststore_location => '/etc/pki/ca-trust/extracted/java/cacerts'
+        ssl_truststore_password => 'changeit'
+    }
+}

salt/logstash/soc_logstash.yaml

@@ -16,6 +16,7 @@ logstash:
       manager: *assigned_pipelines
       managersearch: *assigned_pipelines
       fleet: *assigned_pipelines
+      kafkanode: *assigned_pipelines
   defined_pipelines:
     receiver: &defined_pipelines
       description: List of pipeline configurations assign to this group.
@@ -26,6 +27,7 @@ logstash:
     fleet: *defined_pipelines
     manager: *defined_pipelines
     search: *defined_pipelines
+    kafkanode: *defined_pipelines
     custom0: *defined_pipelines
     custom1: *defined_pipelines
     custom2: *defined_pipelines