CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2025-12-09 18:52:52 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch '2.4/dev' into jertel/wip

Browse Source
This commit is contained in:
Jason Ertel
2025-04-16 15:55:34 -04:00
parent 366e39950a 7f5cde9a1c
commit 2f8d8d2d96
23 changed files with 421 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
salt/docker/defaults.yaml
View File

@@ -200,6 +200,7 @@ docker:
final_octet: 88
port_bindings:
- 0.0.0.0:9092:9092
- 0.0.0.0:29092:29092
- 0.0.0.0:9093:9093
- 0.0.0.0:8778:8778
custom_bind_mounts: []

3
salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json
View File

@@ -31,7 +31,8 @@
],
"tags": [
"so-grid-node"
]
],
"processors": "- if:\n contains:\n message: \"salt-minion\"\n then: \n - dissect:\n tokenizer: \"%{} %{} %{} %{} %{} %{}: [%{log.level}] %{*}\"\n field: \"message\"\n trim_values: \"all\"\n target_prefix: \"\"\n - drop_event:\n when:\n equals:\n log.level: \"INFO\""
}
}
}

2
salt/elasticfleet/tools/sbin_jinja/so-kafka-fleet-output-policy
View File

@@ -32,7 +32,7 @@ if ! echo "$output" | grep -q "so-manager_kafka"; then
--arg KAFKACA "$KAFKACA" \
--arg MANAGER_IP "{{ GLOBALS.manager_ip }}:9092" \
--arg KAFKA_OUTPUT_VERSION "$KAFKA_OUTPUT_VERSION" \
'{ "name": "grid-kafka", "id": "so-manager_kafka", "type": "kafka", "hosts": [ $MANAGER_IP ], "is_default": false, "is_default_monitoring": false, "config_yaml": "", "ssl": { "certificate_authorities": [ $KAFKACA ], "certificate": $KAFKACRT, "key": $KAFKAKEY, "verification_mode": "full" }, "proxy_id": null, "client_id": "Elastic", "version": $KAFKA_OUTPUT_VERSION, "compression": "none", "auth_type": "ssl", "partition": "round_robin", "round_robin": { "group_events": 1 }, "topics":[{"topic":"%{[event.module]}-securityonion","when":{"type":"regexp","condition":"event.module:.+"}},{"topic":"default-securityonion"}], "headers": [ { "key": "", "value": "" } ], "timeout": 30, "broker_timeout": 30, "required_acks": 1 }'
'{ "name": "grid-kafka", "id": "so-manager_kafka", "type": "kafka", "hosts": [ $MANAGER_IP ], "is_default": false, "is_default_monitoring": false, "config_yaml": "", "ssl": { "certificate_authorities": [ $KAFKACA ], "certificate": $KAFKACRT, "key": $KAFKAKEY, "verification_mode": "full" }, "proxy_id": null, "client_id": "Elastic", "version": $KAFKA_OUTPUT_VERSION, "compression": "none", "auth_type": "ssl", "partition": "round_robin", "round_robin": { "group_events": 10 }, "topics":[{"topic":"default-securityonion"}], "headers": [ { "key": "", "value": "" } ], "timeout": 30, "broker_timeout": 30, "required_acks": 1 }'
)
curl -sK /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" -o /dev/null
refresh_output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs" | jq -r .items[].id)

24
salt/elasticsearch/defaults.yaml
View File

@@ -162,6 +162,7 @@ elasticsearch:
- http-mappings
- dtc-http-mappings
- log-mappings
- metadata-mappings
- network-mappings
- dtc-network-mappings
- observer-mappings
@@ -274,7 +275,7 @@ elasticsearch:
number_of_replicas: 0
auto_expand_replicas: 0-2
number_of_shards: 1
refresh_interval: 30s
refresh_interval: 1s
sort:
field: '@timestamp'
order: desc
@@ -316,6 +317,7 @@ elasticsearch:
- http-mappings
- dtc-http-mappings
- log-mappings
- metadata-mappings
- network-mappings
- dtc-network-mappings
- observer-mappings
@@ -427,6 +429,7 @@ elasticsearch:
- http-mappings
- dtc-http-mappings
- log-mappings
- metadata-mappings
- network-mappings
- dtc-network-mappings
- observer-mappings
@@ -534,6 +537,7 @@ elasticsearch:
- http-mappings
- dtc-http-mappings
- log-mappings
- metadata-mappings
- network-mappings
- dtc-network-mappings
- observer-mappings
@@ -697,6 +701,7 @@ elasticsearch:
- client-mappings
- device-mappings
- network-mappings
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
- so-fleet_agent_id_verification-1
data_stream:
@@ -768,6 +773,7 @@ elasticsearch:
- http-mappings
- dtc-http-mappings
- log-mappings
- metadata-mappings
- network-mappings
- dtc-network-mappings
- observer-mappings
@@ -878,6 +884,7 @@ elasticsearch:
- http-mappings
- dtc-http-mappings
- log-mappings
- metadata-mappings
- network-mappings
- dtc-network-mappings
- observer-mappings
@@ -998,6 +1005,7 @@ elasticsearch:
index_template:
composed_of:
- so-data-streams-mappings
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
- so-fleet_agent_id_verification-1
- so-logs-mappings
@@ -2832,6 +2840,7 @@ elasticsearch:
- http-mappings
- dtc-http-mappings
- log-mappings
- metadata-mappings
- network-mappings
- dtc-network-mappings
- observer-mappings
@@ -3062,6 +3071,7 @@ elasticsearch:
- event-mappings
- logs-system.syslog@package
- logs-system.syslog@custom
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
- so-fleet_agent_id_verification-1
- so-system-mappings
@@ -3421,6 +3431,7 @@ elasticsearch:
- dtc-http-mappings
- log-mappings
- logstash-mappings
- metadata-mappings
- network-mappings
- dtc-network-mappings
- observer-mappings
@@ -3505,6 +3516,7 @@ elasticsearch:
composed_of:
- metrics-endpoint.metadata@package
- metrics-endpoint.metadata@custom
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
- so-fleet_agent_id_verification-1
data_stream:
@@ -3551,6 +3563,7 @@ elasticsearch:
composed_of:
- metrics-endpoint.metrics@package
- metrics-endpoint.metrics@custom
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
- so-fleet_agent_id_verification-1
data_stream:
@@ -3597,6 +3610,7 @@ elasticsearch:
composed_of:
- metrics-endpoint.policy@package
- metrics-endpoint.policy@custom
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
- so-fleet_agent_id_verification-1
data_stream:
@@ -3645,6 +3659,7 @@ elasticsearch:
- metrics-fleet_server.agent_status@package
- metrics-fleet_server.agent_status@custom
- ecs@mappings
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
- so-fleet_agent_id_verification-1
data_stream:
@@ -3668,6 +3683,7 @@ elasticsearch:
- metrics-fleet_server.agent_versions@package
- metrics-fleet_server.agent_versions@custom
- ecs@mappings
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
- so-fleet_agent_id_verification-1
data_stream:
@@ -3715,6 +3731,7 @@ elasticsearch:
- http-mappings
- dtc-http-mappings
- log-mappings
- metadata-mappings
- network-mappings
- dtc-network-mappings
- observer-mappings
@@ -3827,6 +3844,7 @@ elasticsearch:
- http-mappings
- dtc-http-mappings
- log-mappings
- metadata-mappings
- network-mappings
- dtc-network-mappings
- observer-mappings
@@ -3939,6 +3957,7 @@ elasticsearch:
- http-mappings
- dtc-http-mappings
- log-mappings
- metadata-mappings
- network-mappings
- dtc-network-mappings
- observer-mappings
@@ -4051,6 +4070,7 @@ elasticsearch:
- http-mappings
- dtc-http-mappings
- log-mappings
- metadata-mappings
- network-mappings
- dtc-network-mappings
- observer-mappings
@@ -4163,6 +4183,7 @@ elasticsearch:
- http-mappings
- dtc-http-mappings
- log-mappings
- metadata-mappings
- network-mappings
- dtc-network-mappings
- observer-mappings
@@ -4276,6 +4297,7 @@ elasticsearch:
- http-mappings
- dtc-http-mappings
- log-mappings
- metadata-mappings
- network-mappings
- dtc-network-mappings
- observer-mappings

2
salt/elasticsearch/templates/component/ecs/log.json
View File

@@ -29,7 +29,7 @@
"file": {
"properties": {
"line": {
"type": "integer"
"type": "long"
},
"name": {
"ignore_above": 1024,

26
salt/elasticsearch/templates/component/ecs/metadata.json Normal file
View File

@@ -0,0 +1,26 @@
{
"template": {
"mappings": {
"dynamic_templates": [],
"properties": {
"metadata": {
"properties": {
"kafka": {
"properties": {
"timestamp": {
"type": "date"
}
}
}
}
}
}
}
},
"_meta": {
"_meta": {
"documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-log.html",
"ecs_version": "1.12.2"
}
}
}

15
salt/elasticsearch/templates/component/elastic-agent/so-fleet_globals-1.json
View File

@@ -5,6 +5,7 @@
"managed_by": "security_onion",
"managed": true
},
"date_detection": false,
"dynamic_templates": [
{
"strings_as_keyword": {
@@ -16,7 +17,19 @@
}
}
],
"date_detection": false
"properties": {
"metadata": {
"properties": {
"kafka": {
"properties": {
"timestamp": {
"type": "date"
}
}
}
}
}
}
}
},
"_meta": {

76
salt/elasticsearch/templates/component/elastic-agent/so-fleet_integrations.ip_mappings-1.json
View File

@@ -1,37 +1,59 @@
{
"template": {
"mappings": {
"properties": {
"host": {
"properties":{
"ip": {
"type": "ip"
}
}
},
"related": {
"properties":{
"ip": {
"type": "ip"
}
"template": {
"mappings": {
"properties": {
"host": {
"properties": {
"ip": {
"type": "ip"
}
},
"destination": {
"properties":{
"ip": {
"type": "ip"
}
}
},
"related": {
"properties": {
"ip": {
"type": "ip"
}
},
"source": {
"properties":{
"ip": {
"type": "ip"
}
},
"destination": {
"properties": {
"ip": {
"type": "ip"
}
}
},
"source": {
"properties": {
"ip": {
"type": "ip"
}
}
},
"metadata": {
"properties": {
"input": {
"properties": {
"beats": {
"properties": {
"host": {
"properties": {
"ip": {
"type": "ip"
}
}
}
}
}
}
}
}
}
}
}
},
"_meta": {
"managed_by": "security_onion",
"managed": true
}
}

13
salt/firewall/defaults.yaml
View File

@@ -11,6 +11,7 @@ firewall:
endgame: []
eval: []
external_suricata: []
external_kafka: []
fleet: []
heavynode: []
idh: []
@@ -103,6 +104,10 @@ firewall:
tcp:
- 9092
udp: []
kafka_external_access:
tcp:
- 29092
udp: []
kibana:
tcp:
- 5601
@@ -473,6 +478,8 @@ firewall:
external_suricata:
portgroups:
- external_suricata
external_kafka:
portgroups: []
desktop:
portgroups:
- docker_registry
@@ -668,6 +675,8 @@ firewall:
external_suricata:
portgroups:
- external_suricata
external_kafka:
portgroups: []
desktop:
portgroups:
- docker_registry
@@ -867,6 +876,8 @@ firewall:
external_suricata:
portgroups:
- external_suricata
external_kafka:
portgroups: []
strelka_frontend:
portgroups:
- strelka_frontend
@@ -1337,6 +1348,8 @@ firewall:
endgame:
portgroups:
- endgame
external_kafka:
portgroups: []
receiver:
portgroups: []
customhostgroup0:

47
salt/firewall/map.jinja
View File

@@ -21,25 +21,38 @@
{# Only add Kafka firewall items when Kafka enabled #}
{% set role = GLOBALS.role.split('-')[1] %}
{% if GLOBALS.pipeline == 'KAFKA' and role in ['manager', 'managersearch', 'standalone'] %}
{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[role].portgroups.append('kafka_controller') %}
{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.receiver.portgroups.append('kafka_controller') %}
{% endif %}
{% if GLOBALS.pipeline == 'KAFKA' %}
{% set KAFKA_EXTERNAL_ACCESS = salt['pillar.get']('kafka:config:external_access:enabled', default=False) %}
{% set kafka_node_type = salt['pillar.get']('kafka:nodes:'+ GLOBALS.hostname + ':role') %}
{% if GLOBALS.pipeline == 'KAFKA' and role == 'receiver' %}
{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.self.portgroups.append('kafka_controller') %}
{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.standalone.portgroups.append('kafka_controller') %}
{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.manager.portgroups.append('kafka_controller') %}
{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.managersearch.portgroups.append('kafka_controller') %}
{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.receiver.portgroups.append('kafka_controller') %}
{% endif %}
{% if role in ['manager', 'managersearch', 'standalone'] %}
{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[role].portgroups.append('kafka_controller') %}
{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.receiver.portgroups.append('kafka_controller') %}
{% endif %}
{% if GLOBALS.pipeline == 'KAFKA' and role in ['manager', 'managersearch', 'standalone', 'receiver'] %}
{% for r in ['manager', 'managersearch', 'standalone', 'receiver', 'fleet', 'idh', 'sensor', 'searchnode','heavynode', 'elastic_agent_endpoint', 'desktop'] %}
{% if FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[r] is defined %}
{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[r].portgroups.append('kafka_data') %}
{% if role == 'receiver' %}
{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.self.portgroups.append('kafka_controller') %}
{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.standalone.portgroups.append('kafka_controller') %}
{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.manager.portgroups.append('kafka_controller') %}
{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.managersearch.portgroups.append('kafka_controller') %}
{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.receiver.portgroups.append('kafka_controller') %}
{% endif %}
{% if role in ['manager', 'managersearch', 'standalone', 'receiver'] %}
{% for r in ['manager', 'managersearch', 'standalone', 'receiver', 'fleet', 'idh', 'sensor', 'searchnode','heavynode', 'elastic_agent_endpoint', 'desktop'] %}
{% if FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[r] is defined %}
{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[r].portgroups.append('kafka_data') %}
{% endif %}
{% endfor %}
{% endif %}
{% if KAFKA_EXTERNAL_ACCESS %}
{# Kafka external access only applies for Kafka nodes with the broker role. #}
{% if role in ['manager', 'managersearch', 'standalone', 'receiver'] and 'broker' in kafka_node_type %}
{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.external_kafka.portgroups.append('kafka_external_access') %}
{% endif %}
{% endfor %}
{% endif %}
{% endif %}
{% set FIREWALL_MERGED = salt['pillar.get']('firewall', FIREWALL_DEFAULT.firewall, merge=True) %}
{% set FIREWALL_MERGED = salt['pillar.get']('firewall', FIREWALL_DEFAULT.firewall, merge=True) %}

4
salt/firewall/soc_firewall.yaml
View File

@@ -33,6 +33,7 @@ firewall:
endgame: *hostgroupsettingsadv
eval: *hostgroupsettings
external_suricata: *hostgroupsettings
external_kafka: *hostgroupsettings
fleet: *hostgroupsettings
heavynode: *hostgroupsettings
idh: *hostgroupsettings
@@ -130,6 +131,9 @@ firewall:
kafka_data:
tcp: *tcpsettings
udp: *udpsettings
kafka_external_access:
tcp: *tcpsettings
udp: *udpsettings
kibana:
tcp: *tcpsettings
udp: *udpsettings

20
salt/idstools/config.sls
View File

@@ -24,13 +24,23 @@ idstools_sbin:
- group: 939
- file_mode: 755
idstools_sbin_jinja:
file.recurse:
- name: /usr/sbin
- source: salt://idstools/tools/sbin_jinja
# If this is used, exclude so-rule-update
#idstools_sbin_jinja:
# file.recurse:
# - name: /usr/sbin
# - source: salt://idstools/tools/sbin_jinja
# - user: 934
# - group: 939
# - file_mode: 755
# - template: jinja
idstools_so-rule-update:
file.managed:
- name: /usr/sbin/so-rule-update
- source: salt://idstools/tools/sbin_jinja/so-rule-update
- user: 934
- group: 939
- file_mode: 755
- mode: 755
- template: jinja
suricatacustomdirsfile:

2
salt/idstools/enabled.sls
View File

@@ -55,6 +55,7 @@ so-idstools:
{% endif %}
- watch:
- file: idstoolsetcsync
- file: idstools_so-rule-update
delete_so-idstools_so-status.disabled:
file.uncomment:
@@ -76,6 +77,7 @@ run_so-rule-update:
- require:
- docker_container: so-idstools
- onchanges:
- file: idstools_so-rule-update
- file: idstoolsetcsync
- file: synclocalnidsrules
- order: last

24
salt/kafka/config.map.jinja
View File

@@ -8,6 +8,7 @@
{% set KAFKA_NODES_PILLAR = salt['pillar.get']('kafka:nodes') %}
{% set KAFKA_PASSWORD = salt['pillar.get']('kafka:config:password') %}
{% set KAFKA_TRUSTPASS = salt['pillar.get']('kafka:config:trustpass') %}
{% set KAFKA_EXTERNAL_ACCESS = salt['pillar.get']('kafka:config:external_access:enabled', default=False) %}
{# Create list of KRaft controllers #}
{% set controllers = [] %}
@@ -15,7 +16,7 @@
{# Check for Kafka nodes with controller in process_x_roles #}
{% for node in KAFKA_NODES_PILLAR %}
{% if 'controller' in KAFKA_NODES_PILLAR[node].role %}
{% do controllers.append(KAFKA_NODES_PILLAR[node].nodeid ~ "@" ~ node ~ ":9093") %}
{% do controllers.append(KAFKA_NODES_PILLAR[node].nodeid ~ "@" ~ KAFKA_NODES_PILLAR[node].ip ~ ":9093") %}
{% endif %}
{% endfor %}
@@ -28,7 +29,15 @@
{# Generate server.properties for 'broker' , 'controller', 'broker,controller' node types
anything above this line is a configuration needed for ALL Kafka nodes #}
{% if node_type == 'broker' %}
{% do KAFKAMERGED.config.broker.update({'advertised_x_listeners': 'BROKER://'+ GLOBALS.node_ip +':9092' }) %}
{% if KAFKA_EXTERNAL_ACCESS %}
{% do KAFKAMERGED.config.broker.update({'advertised_x_listeners': 'BROKER://'+ GLOBALS.node_ip +':9092' + ',' + 'EXTERNAL_ACCESS://' + GLOBALS.node_ip + ':29092' }) %}
{% do KAFKAMERGED.config.broker.update({'listeners': KAFKAMERGED.config.broker.listeners + ',' + KAFKAMERGED.config.external_access.listeners }) %}
{% do KAFKAMERGED.config.broker.update({'listener_x_security_x_protocol_x_map': KAFKAMERGED.config.broker.listener_x_security_x_protocol_x_map + ',' + KAFKAMERGED.config.external_access.listener_x_security_x_protocol_x_map }) %}
{% do KAFKAMERGED.config.broker.update({'sasl_x_enabled_x_mechanisms': KAFKAMERGED.config.external_access.sasl_x_enabled_x_mechanisms }) %}
{% do KAFKAMERGED.config.broker.update({'sasl_x_mechanism_x_inter_x_broker_x_protocol': KAFKAMERGED.config.external_access.sasl_x_mechanism_x_inter_x_broker_x_protocol }) %}
{% else %}
{% do KAFKAMERGED.config.broker.update({'advertised_x_listeners': 'BROKER://'+ GLOBALS.node_ip +':9092' }) %}
{% endif %}
{% do KAFKAMERGED.config.broker.update({'controller_x_quorum_x_voters': kafka_controller_quorum_voters }) %}
{% do KAFKAMERGED.config.broker.update({'node_x_id': salt['pillar.get']('kafka:nodes:'+ GLOBALS.hostname +':nodeid') }) %}
{% do KAFKAMERGED.config.broker.update({'ssl_x_keystore_x_password': KAFKA_PASSWORD }) %}
@@ -42,6 +51,7 @@
{% endif %}
{% if node_type == 'controller' %}
{% do KAFKAMERGED.config.controller.update({'advertised_x_listeners': 'CONTROLLER://' + GLOBALS.node_ip + ':9093'}) %}
{% do KAFKAMERGED.config.controller.update({'controller_x_quorum_x_voters': kafka_controller_quorum_voters }) %}
{% do KAFKAMERGED.config.controller.update({'node_x_id': salt['pillar.get']('kafka:nodes:'+ GLOBALS.hostname +':nodeid') }) %}
{% do KAFKAMERGED.config.controller.update({'ssl_x_keystore_x_password': KAFKA_PASSWORD }) %}
@@ -50,7 +60,15 @@
{# Kafka nodes of this type are not recommended for use outside of development / testing. #}
{% if node_type == 'broker,controller' %}
{% do KAFKAMERGED.config.broker.update({'advertised_x_listeners': 'BROKER://'+ GLOBALS.node_ip +':9092' }) %}
{% if KAFKA_EXTERNAL_ACCESS %}
{% do KAFKAMERGED.config.broker.update({'advertised_x_listeners': 'BROKER://'+ GLOBALS.node_ip +':9092' + ',' + 'CONTROLLER://'+ GLOBALS.node_ip +':9093' + ',' + 'EXTERNAL_ACCESS://' + GLOBALS.node_ip + ':29092' }) %}
{% do KAFKAMERGED.config.broker.update({'listeners': KAFKAMERGED.config.broker.listeners + ',' + KAFKAMERGED.config.external_access.listeners }) %}
{% do KAFKAMERGED.config.broker.update({'listener_x_security_x_protocol_x_map': KAFKAMERGED.config.broker.listener_x_security_x_protocol_x_map + ',' + KAFKAMERGED.config.external_access.listener_x_security_x_protocol_x_map }) %}
{% do KAFKAMERGED.config.broker.update({'sasl_x_enabled_x_mechanisms': KAFKAMERGED.config.external_access.sasl_x_enabled_x_mechanisms }) %}
{% do KAFKAMERGED.config.broker.update({'sasl_x_mechanism_x_inter_x_broker_x_protocol': KAFKAMERGED.config.external_access.sasl_x_mechanism_x_inter_x_broker_x_protocol }) %}
{% else %}
{% do KAFKAMERGED.config.broker.update({'advertised_x_listeners': 'BROKER://'+ GLOBALS.node_ip +':9092' + ',' + 'CONTROLLER://'+ GLOBALS.node_ip +':9093' }) %}
{% endif %}
{% do KAFKAMERGED.config.broker.update({'controller_x_listener_x_names': KAFKAMERGED.config.controller.controller_x_listener_x_names }) %}
{% do KAFKAMERGED.config.broker.update({'controller_x_quorum_x_voters': kafka_controller_quorum_voters }) %}
{% do KAFKAMERGED.config.broker.update({'node_x_id': salt['pillar.get']('kafka:nodes:'+ GLOBALS.hostname +':nodeid') }) %}

27
salt/kafka/config.sls
View File

@@ -6,6 +6,8 @@
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% set KAFKA_EXTERNAL_ACCESS = salt['pillar.get']('kafka:config:external_access:enabled', default=False) %}
{% set KAFKA_EXTERNAL_USERS = salt['pillar.get']('kafka:config:external_access:remote_users', default=None) %}
kafka_group:
group.present:
@@ -69,6 +71,29 @@ kafka_kraft_{{sc}}_properties:
- show_changes: False
{% endfor %}
{% if KAFKA_EXTERNAL_ACCESS and KAFKA_EXTERNAL_USERS != None %}
kafka_server_jaas_properties:
file.managed:
- source: salt://kafka/etc/jaas.conf.jinja
- name: /opt/so/conf/kafka/kafka_server_jaas.conf
- template: jinja
- user: 960
- group: 960
- show_changes: False
{% else %}
remove_kafka_server_jaas_properties:
file.absent:
- name: /opt/so/conf/kafka/kafka_server_jaas.conf
{% endif %}
kafka_log4j_properties:
file.managed:
- source: salt://kafka/etc/log4j.properties
- name: /opt/so/conf/kafka/log4j.properties
- user: 960
- group: 960
- show_changes: False
reset_quorum_on_changes:
cmd.run:
- name: rm -f /nsm/kafka/data/__cluster_metadata-0/quorum-state
@@ -81,4 +106,4 @@ reset_quorum_on_changes:
test.fail_without_changes:
- name: {{sls}}_state_not_allowed
{% endif %}
{% endif %}

11
salt/kafka/defaults.yaml
View File

@@ -21,7 +21,7 @@ kafka:
log_x_segment_x_bytes: 1073741824
node_x_id:
num_x_io_x_threads: 8
num_x_network_x_threads: 3
num_x_network_x_threads: 5
num_x_partitions: 3
num_x_recovery_x_threads_x_per_x_data_x_dir: 1
offsets_x_topic_x_replication_x_factor: 1
@@ -46,6 +46,7 @@ kafka:
ssl_x_keystore_x_type: PKCS12
ssl_x_keystore_x_password:
controller:
advertsied_x_listeners:
controller_x_listener_x_names: CONTROLLER
controller_x_quorum_x_voters:
listeners: CONTROLLER://0.0.0.0:9093
@@ -61,4 +62,10 @@ kafka:
ssl_x_keystore_x_password:
ssl_x_truststore_x_location: /etc/pki/kafka-truststore.jks
ssl_x_truststore_x_type: JKS
ssl_x_truststore_x_password:
ssl_x_truststore_x_password:
external_access:
enabled: False
listeners: EXTERNAL_ACCESS://0.0.0.0:29092
listener_x_security_x_protocol_x_map: EXTERNAL_ACCESS:SASL_SSL
sasl_x_enabled_x_mechanisms: PLAIN
sasl_x_mechanism_x_inter_x_broker_x_protocol: SSL

14
salt/kafka/enabled.sls
View File

@@ -14,6 +14,7 @@
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'docker/docker.map.jinja' import DOCKER %}
{% set KAFKANODES = salt['pillar.get']('kafka:nodes') %}
{% set KAFKA_EXTERNAL_ACCESS = salt['pillar.get']('kafka:config:external_access:enabled', default=False) %}
{% if 'gmd' in salt['pillar.get']('features', []) %}
include:
@@ -34,7 +35,7 @@ so-kafka:
- user: kafka
- environment:
KAFKA_HEAP_OPTS: -Xmx2G -Xms1G
KAFKA_OPTS: -javaagent:/opt/jolokia/agents/jolokia-agent-jvm-javaagent.jar=port=8778,host={{ DOCKER.containers['so-kafka'].ip }},policyLocation=file:/opt/jolokia/jolokia.xml
KAFKA_OPTS: "-javaagent:/opt/jolokia/agents/jolokia-agent-jvm-javaagent.jar=port=8778,host={{ DOCKER.containers['so-kafka'].ip }},policyLocation=file:/opt/jolokia/jolokia.xml {%- if KAFKA_EXTERNAL_ACCESS %} -Djava.security.auth.login.config=/opt/kafka/config/kafka_server_jaas.conf {% endif -%}"
- extra_hosts:
{% for node in KAFKANODES %}
- {{ node }}:{{ KAFKANODES[node].ip }}
@@ -54,11 +55,18 @@ so-kafka:
- /nsm/kafka/data/:/nsm/kafka/data/:rw
- /opt/so/log/kafka:/opt/kafka/logs/:rw
- /opt/so/conf/kafka/server.properties:/opt/kafka/config/kraft/server.properties:ro
- /opt/so/conf/kafka/client.properties:/opt/kafka/config/kraft/client.properties
- /opt/so/conf/kafka/client.properties:/opt/kafka/config/kraft/client.properties:ro
- /opt/so/conf/kafka/log4j.properties:/opt/kafka/config/log4j.properties:ro
{% if KAFKA_EXTERNAL_ACCESS %}
- /opt/so/conf/kafka/kafka_server_jaas.conf:/opt/kafka/config/kafka_server_jaas.conf:ro
{% endif %}
- watch:
{% for sc in ['server', 'client'] %}
- file: kafka_kraft_{{sc}}_properties
{% endfor %}
{% if KAFKA_EXTERNAL_ACCESS %}
- file: kafka_server_jaas_properties
{% endif %}
- file: kafkacertz
- require:
- file: kafkacertz
@@ -87,4 +95,4 @@ include:
test.fail_without_changes:
- name: {{sls}}_state_not_allowed
{% endif %}
{% endif %}

16
salt/kafka/etc/jaas.conf.jinja Normal file
View File

@@ -0,0 +1,16 @@
{% set KAFKA_EXTERNAL_USERS = salt['pillar.get']('kafka:config:external_access:remote_users') -%}
{%- set valid_users = [] -%}
{%- for item, user in KAFKA_EXTERNAL_USERS.items() -%}
{% if 'password' in user and user.password is not none and user.password.strip() != "" -%}
{% do valid_users.append('user_' ~ user.username ~ '="' ~ user.password ~ '"') -%}
{% endif -%}
{%- endfor -%}
KafkaServer {
org.apache.kafka.common.security.plain.PlainLoginModule required
{% for user_entry in valid_users -%}
{{ user_entry }}{{ ";" if loop.last }}
{% endfor %}
};

101
salt/kafka/etc/log4j.properties Normal file
View File

@@ -0,0 +1,101 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Unspecified loggers and loggers with additivity=true output to server.log and stdout
# Note that INFO only applies to unspecified loggers, the log level of the child logger is used otherwise
log4j.rootLogger=INFO, stdout, kafkaAppender
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n
log4j.appender.kafkaAppender=org.apache.log4j.RollingFileAppender
log4j.appender.kafkaAppender.File=${kafka.logs.dir}/server.log
log4j.appender.kafkaAppender.MaxFileSize=10MB
log4j.appender.kafkaAppender.MaxBackupIndex=10
log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout
log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
log4j.appender.stateChangeAppender=org.apache.log4j.RollingFileAppender
log4j.appender.stateChangeAppender.File=${kafka.logs.dir}/state-change.log
log4j.appender.stateChangeAppender.MaxFileSize=10MB
log4j.appender.stateChangeAppender.MaxBackupIndex=10
log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout
log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
log4j.appender.requestAppender=org.apache.log4j.RollingFileAppender
log4j.appender.requestAppender.File=${kafka.logs.dir}/kafka-request.log
log4j.appender.requestAppender.MaxFileSize=10MB
log4j.appender.requestAppender.MaxBackupIndex=10
log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout
log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
log4j.appender.cleanerAppender=org.apache.log4j.RollingFileAppender
log4j.appender.cleanerAppender.File=${kafka.logs.dir}/log-cleaner.log
log4j.appender.cleanerAppender.MaxFileSize=10MB
log4j.appender.cleanerAppender.MaxBackupIndex=10
log4j.appender.cleanerAppender.layout=org.apache.log4j.PatternLayout
log4j.appender.cleanerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
log4j.appender.controllerAppender=org.apache.log4j.RollingFileAppender
log4j.appender.controllerAppender.File=${kafka.logs.dir}/controller.log
log4j.appender.controllerAppender.MaxFileSize=10MB
log4j.appender.controllerAppender.MaxBackupIndex=10
log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout
log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
log4j.appender.authorizerAppender=org.apache.log4j.RollingFileAppender
log4j.appender.authorizerAppender.File=${kafka.logs.dir}/kafka-authorizer.log
log4j.appender.authorizerAppender.MaxFileSize=10MB
log4j.appender.authorizerAppender.MaxBackupIndex=10
log4j.appender.authorizerAppender.layout=org.apache.log4j.PatternLayout
log4j.appender.authorizerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
# Change the line below to adjust ZK client logging
log4j.logger.org.apache.zookeeper=INFO
# Change the two lines below to adjust the general broker logging level (output to server.log and stdout)
log4j.logger.kafka=INFO
log4j.logger.org.apache.kafka=INFO
# Change to DEBUG or TRACE to enable request logging
log4j.logger.kafka.request.logger=WARN, requestAppender
log4j.additivity.kafka.request.logger=false
# Uncomment the lines below and change log4j.logger.kafka.network.RequestChannel$ to TRACE for additional output
# related to the handling of requests
#log4j.logger.kafka.network.Processor=TRACE, requestAppender
#log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender
#log4j.additivity.kafka.server.KafkaApis=false
log4j.logger.kafka.network.RequestChannel$=WARN, requestAppender
log4j.additivity.kafka.network.RequestChannel$=false
# Change the line below to adjust KRaft mode controller logging
log4j.logger.org.apache.kafka.controller=INFO, controllerAppender
log4j.additivity.org.apache.kafka.controller=false
# Change the line below to adjust ZK mode controller logging
log4j.logger.kafka.controller=TRACE, controllerAppender
log4j.additivity.kafka.controller=false
log4j.logger.kafka.log.LogCleaner=INFO, cleanerAppender
log4j.additivity.kafka.log.LogCleaner=false
log4j.logger.state.change.logger=INFO, stateChangeAppender
log4j.additivity.state.change.logger=false
# Access denials are logged at INFO level, change to DEBUG to also log allowed accesses
log4j.logger.kafka.authorizer.logger=INFO, authorizerAppender
log4j.additivity.kafka.authorizer.logger=false

54
salt/kafka/soc_kafka.yaml
View File

@@ -34,10 +34,6 @@ kafka:
sensitive: True
helpLink: kafka.html
broker:
advertised_x_listeners:
description: Specify the list of listeners (hostname and port) that Kafka brokers provide to clients for communication.
title: advertised.listeners
helpLink: kafka.html
auto_x_create_x_topics_x_enable:
description: Enable the auto creation of topics.
title: auto.create.topics.enable
@@ -226,4 +222,52 @@ kafka:
description: The role performed by controller node.
title: process.roles
readonly: True
helpLink: kafka.html
helpLink: kafka.html
external_access:
enabled:
description: Enables or disables access to Kafka topics using user/password authentication. Used for producing / consuming messages via an external client.
forcedType: bool
helpLink: kafka.html
listeners:
description: Set of URIs that is listened on and the listener names in a comma-seperated list.
title: listeners
readonly: True
advanced: True
helpLink: kafka.html
listener_x_security_x_protocol_x_map:
description: External listener name and mapped security protocol.
title: listener.security.protocol.map
readonly: True
advanced: True
helpLink: kafka.html
sasl_x_enabled_x_mechanisms:
description: SASL/PLAIN is a simple username/password authentication mechanism, used with TLS to implement secure authentication.
title: sasl.enabled.mechanisms
readonly: True
advanced: True
helpLink: kafka.html
sasl_x_mechanism_x_inter_x_broker_x_protocol:
description: SASL mechanism used for inter-broker communication
title: sasl.mechanism.inter.broker.protocol
readonly: True
advanced: True
helpLink: kafka.html
remote_users:
user01: &remote_user
username:
description: Username to be used for custom account
forcedType: string
global: True
password:
description: Password to be used for custom account
forcedType: string
global: True
sensitive: True
user02: *remote_user
user03: *remote_user
user04: *remote_user
user05: *remote_user
user06: *remote_user
user07: *remote_user
user08: *remote_user
user09: *remote_user

2
salt/logstash/etc/log4j2.properties
View File

@@ -23,6 +23,8 @@ appender.rolling.policies.type = Policies
appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
appender.rolling.policies.time.interval = 1
appender.rolling.policies.time.modulate = true
appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
appender.rolling.policies.size.size = 1GB
appender.rolling.strategy.type = DefaultRolloverStrategy
appender.rolling.strategy.action.type = Delete
appender.rolling.strategy.action.basepath = /var/log/logstash

5
salt/manager/tools/sbin/soup
View File

@@ -746,8 +746,6 @@ up_to_2.4.90() {
so-yaml.py remove /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.password
so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.password "$kafkatrimpass"
so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.trustpass "$kafkatrust"
echo "If the Detection index exists, update the refresh_interval"
so-elasticsearch-query so-detection*/_settings -X PUT -d '{"index":{"refresh_interval":"1s"}}'
INSTALLEDVERSION=2.4.90
}
@@ -807,7 +805,8 @@ up_to_2.4.141() {
}
up_to_2.4.150() {
echo "Nothing to do for 2.4.150"
echo "If the Detection indices exists, update the refresh_interval"
so-elasticsearch-query so-detection*/_settings -X PUT -d '{"index":{"refresh_interval":"1s"}}'
INSTALLEDVERSION=2.4.150
}

3
salt/salt/master/mine_update_highstate.sls
View File

@@ -19,8 +19,11 @@ salt.master.mine_update_highstate.update_mine_all_minions:
# Run highstate on the original minion
# we can use concurrent on this highstate because no other highstate would be running when this is called
# this state will run onlyif there is not an instance of it already running
salt.master.mine_update_highstate.run_highstate_on_{{ MINION_ID }}:
salt.state:
- tgt: {{ MINION_ID }}
- highstate: True
- concurrent: True
- onlyif:
- 'ps -ef | grep -v grep | grep "/usr/bin/salt-minion.*ProcessPayload.*jid=.*Minion._thread_return" | wc -l | grep -q "^0$"'