CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2025-12-18 23:13:20 +01:00
Code Issues Packages Projects Releases Wiki Activity

Append '-securityonion' to all Security Onion related Kafka topics. Adjust logstash to ingest all topics ending in '-securityonion' to avoid having to manually list topic names

Browse Source 
Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com>
This commit is contained in:
reyesj2
2024-06-12 12:05:16 -04:00
parent 83412b813f
commit e8106befe9
2 changed files with 10 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
salt/kafka/tools/sbin_jinja/so-kafka-fleet-output-policy
View File

@@ -17,7 +17,7 @@ if ! echo "$output" | grep -q "so-manager_kafka"; then
--arg KAFKACA "$KAFKACA" \ --arg KAFKACA "$KAFKACA" \
--arg MANAGER_IP "{{ GLOBALS.manager_ip }}:9092" \ --arg MANAGER_IP "{{ GLOBALS.manager_ip }}:9092" \
--arg KAFKA_OUTPUT_VERSION "$KAFKA_OUTPUT_VERSION" \ --arg KAFKA_OUTPUT_VERSION "$KAFKA_OUTPUT_VERSION" \
'{ "name": "grid-kafka", "id": "so-manager_kafka", "type": "kafka", "hosts": [ $MANAGER_IP ], "is_default": false, "is_default_monitoring": false, "config_yaml": "", "ssl": { "certificate_authorities": [ $KAFKACA ], "certificate": $KAFKACRT, "key": $KAFKAKEY, "verification_mode": "full" }, "proxy_id": null, "client_id": "Elastic", "version": $KAFKA_OUTPUT_VERSION, "compression": "none", "auth_type": "ssl", "partition": "round_robin", "round_robin": { "group_events": 1 }, "topics":[{"topic":"%{[event.module]}-topic","when":{"type":"regexp","condition":"event.module:.+"}},{"topic":"default-topic"}], "headers": [ { "key": "", "value": "" } ], "timeout": 30, "broker_timeout": 30, "required_acks": 1 }' '{ "name": "grid-kafka", "id": "so-manager_kafka", "type": "kafka", "hosts": [ $MANAGER_IP ], "is_default": false, "is_default_monitoring": false, "config_yaml": "", "ssl": { "certificate_authorities": [ $KAFKACA ], "certificate": $KAFKACRT, "key": $KAFKAKEY, "verification_mode": "full" }, "proxy_id": null, "client_id": "Elastic", "version": $KAFKA_OUTPUT_VERSION, "compression": "none", "auth_type": "ssl", "partition": "round_robin", "round_robin": { "group_events": 1 }, "topics":[{"topic":"%{[event.module]}-securityonion","when":{"type":"regexp","condition":"event.module:.+"}},{"topic":"default-securityonion"}], "headers": [ { "key": "", "value": "" } ], "timeout": 30, "broker_timeout": 30, "required_acks": 1 }'
) )
curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" 2&1> /dev/null curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" 2&1> /dev/null
fi fi

18
salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja
View File

@@ -1,17 +1,17 @@
{% set kafka_brokers = salt['pillar.get']('kafka:nodes', {}) %} {%- set kafka_brokers = salt['pillar.get']('kafka:nodes', {}) %}
{% set brokers = [] %} {%- set brokers = [] %}
{% for key, values in kafka_brokers.items() %} {%- for key, values in kafka_brokers.items() %}
{% if 'broker' in values['role'] %} {%- if 'broker' in values['role'] %}
{% do brokers.append(key ~ ':9092') %} {%- do brokers.append(key ~ ':9092') %}
{% endif %} {%- endif %}
{% endfor %} {%- endfor %}
{% set bootstrap_servers = ','.join(brokers) %} {%- set bootstrap_servers = ','.join(brokers) %}
input { input {
kafka { kafka {
codec => json codec => json
topics => ['default-topic', 'kratos-topic', 'soc-topic', 'strelka-topic', 'suricata-topic', 'zeek-topic', 'rita-topic', 'opencanary-topic', 'syslog-topic'] topics_pattern => '.*-securityonion$'
group_id => 'searchnodes' group_id => 'searchnodes'
consumer_threads => 3 consumer_threads => 3
client_id => '{{ GLOBALS.hostname }}' client_id => '{{ GLOBALS.hostname }}'