diff --git a/salt/docker/defaults.yaml b/salt/docker/defaults.yaml index 7c776937d..2d7ad4e1c 100644 --- a/salt/docker/defaults.yaml +++ b/salt/docker/defaults.yaml @@ -200,6 +200,7 @@ docker: final_octet: 88 port_bindings: - 0.0.0.0:9092:9092 + - 0.0.0.0:29092:29092 - 0.0.0.0:9093:9093 - 0.0.0.0:8778:8778 custom_bind_mounts: [] diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json index 98204e894..4ee4d2b65 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json @@ -31,7 +31,8 @@ ], "tags": [ "so-grid-node" - ] + ], + "processors": "- if:\n contains:\n message: \"salt-minion\"\n then: \n - dissect:\n tokenizer: \"%{} %{} %{} %{} %{} %{}: [%{log.level}] %{*}\"\n field: \"message\"\n trim_values: \"all\"\n target_prefix: \"\"\n - drop_event:\n when:\n equals:\n log.level: \"INFO\"" } } } diff --git a/salt/elasticfleet/tools/sbin_jinja/so-kafka-fleet-output-policy b/salt/elasticfleet/tools/sbin_jinja/so-kafka-fleet-output-policy index 7727f7db1..a5ea79922 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-kafka-fleet-output-policy +++ b/salt/elasticfleet/tools/sbin_jinja/so-kafka-fleet-output-policy @@ -32,7 +32,7 @@ if ! echo "$output" | grep -q "so-manager_kafka"; then --arg KAFKACA "$KAFKACA" \ --arg MANAGER_IP "{{ GLOBALS.manager_ip }}:9092" \ --arg KAFKA_OUTPUT_VERSION "$KAFKA_OUTPUT_VERSION" \ - '{ "name": "grid-kafka", "id": "so-manager_kafka", "type": "kafka", "hosts": [ $MANAGER_IP ], "is_default": false, "is_default_monitoring": false, "config_yaml": "", "ssl": { "certificate_authorities": [ $KAFKACA ], "certificate": $KAFKACRT, "key": $KAFKAKEY, "verification_mode": "full" }, "proxy_id": null, "client_id": "Elastic", "version": $KAFKA_OUTPUT_VERSION, "compression": "none", "auth_type": "ssl", "partition": "round_robin", "round_robin": { "group_events": 1 }, "topics":[{"topic":"%{[event.module]}-securityonion","when":{"type":"regexp","condition":"event.module:.+"}},{"topic":"default-securityonion"}], "headers": [ { "key": "", "value": "" } ], "timeout": 30, "broker_timeout": 30, "required_acks": 1 }' + '{ "name": "grid-kafka", "id": "so-manager_kafka", "type": "kafka", "hosts": [ $MANAGER_IP ], "is_default": false, "is_default_monitoring": false, "config_yaml": "", "ssl": { "certificate_authorities": [ $KAFKACA ], "certificate": $KAFKACRT, "key": $KAFKAKEY, "verification_mode": "full" }, "proxy_id": null, "client_id": "Elastic", "version": $KAFKA_OUTPUT_VERSION, "compression": "none", "auth_type": "ssl", "partition": "round_robin", "round_robin": { "group_events": 10 }, "topics":[{"topic":"default-securityonion"}], "headers": [ { "key": "", "value": "" } ], "timeout": 30, "broker_timeout": 30, "required_acks": 1 }' ) curl -sK /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" -o /dev/null refresh_output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs" | jq -r .items[].id) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 7b38ed0bb..023d270f4 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -162,6 +162,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -274,7 +275,7 @@ elasticsearch: number_of_replicas: 0 auto_expand_replicas: 0-2 number_of_shards: 1 - refresh_interval: 30s + refresh_interval: 1s sort: field: '@timestamp' order: desc @@ -316,6 +317,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -427,6 +429,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -534,6 +537,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -697,6 +701,7 @@ elasticsearch: - client-mappings - device-mappings - network-mappings + - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 - so-fleet_agent_id_verification-1 data_stream: @@ -768,6 +773,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -878,6 +884,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -998,6 +1005,7 @@ elasticsearch: index_template: composed_of: - so-data-streams-mappings + - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 - so-fleet_agent_id_verification-1 - so-logs-mappings @@ -2832,6 +2840,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -3062,6 +3071,7 @@ elasticsearch: - event-mappings - logs-system.syslog@package - logs-system.syslog@custom + - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 - so-fleet_agent_id_verification-1 - so-system-mappings @@ -3421,6 +3431,7 @@ elasticsearch: - dtc-http-mappings - log-mappings - logstash-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -3505,6 +3516,7 @@ elasticsearch: composed_of: - metrics-endpoint.metadata@package - metrics-endpoint.metadata@custom + - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 - so-fleet_agent_id_verification-1 data_stream: @@ -3551,6 +3563,7 @@ elasticsearch: composed_of: - metrics-endpoint.metrics@package - metrics-endpoint.metrics@custom + - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 - so-fleet_agent_id_verification-1 data_stream: @@ -3597,6 +3610,7 @@ elasticsearch: composed_of: - metrics-endpoint.policy@package - metrics-endpoint.policy@custom + - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 - so-fleet_agent_id_verification-1 data_stream: @@ -3645,6 +3659,7 @@ elasticsearch: - metrics-fleet_server.agent_status@package - metrics-fleet_server.agent_status@custom - ecs@mappings + - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 - so-fleet_agent_id_verification-1 data_stream: @@ -3668,6 +3683,7 @@ elasticsearch: - metrics-fleet_server.agent_versions@package - metrics-fleet_server.agent_versions@custom - ecs@mappings + - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 - so-fleet_agent_id_verification-1 data_stream: @@ -3715,6 +3731,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -3827,6 +3844,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -3939,6 +3957,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -4051,6 +4070,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -4163,6 +4183,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -4276,6 +4297,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings diff --git a/salt/elasticsearch/templates/component/ecs/log.json b/salt/elasticsearch/templates/component/ecs/log.json index e79661b5e..4f51be287 100644 --- a/salt/elasticsearch/templates/component/ecs/log.json +++ b/salt/elasticsearch/templates/component/ecs/log.json @@ -29,7 +29,7 @@ "file": { "properties": { "line": { - "type": "integer" + "type": "long" }, "name": { "ignore_above": 1024, diff --git a/salt/elasticsearch/templates/component/ecs/metadata.json b/salt/elasticsearch/templates/component/ecs/metadata.json new file mode 100644 index 000000000..55da6f07c --- /dev/null +++ b/salt/elasticsearch/templates/component/ecs/metadata.json @@ -0,0 +1,26 @@ +{ + "template": { + "mappings": { + "dynamic_templates": [], + "properties": { + "metadata": { + "properties": { + "kafka": { + "properties": { + "timestamp": { + "type": "date" + } + } + } + } + } + } + } + }, + "_meta": { + "_meta": { + "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-log.html", + "ecs_version": "1.12.2" + } + } +} \ No newline at end of file diff --git a/salt/elasticsearch/templates/component/elastic-agent/so-fleet_globals-1.json b/salt/elasticsearch/templates/component/elastic-agent/so-fleet_globals-1.json index 5df7e7fe9..183031d4e 100644 --- a/salt/elasticsearch/templates/component/elastic-agent/so-fleet_globals-1.json +++ b/salt/elasticsearch/templates/component/elastic-agent/so-fleet_globals-1.json @@ -5,6 +5,7 @@ "managed_by": "security_onion", "managed": true }, + "date_detection": false, "dynamic_templates": [ { "strings_as_keyword": { @@ -16,7 +17,19 @@ } } ], - "date_detection": false + "properties": { + "metadata": { + "properties": { + "kafka": { + "properties": { + "timestamp": { + "type": "date" + } + } + } + } + } + } } }, "_meta": { diff --git a/salt/elasticsearch/templates/component/elastic-agent/so-fleet_integrations.ip_mappings-1.json b/salt/elasticsearch/templates/component/elastic-agent/so-fleet_integrations.ip_mappings-1.json index 3777e670c..d6f516272 100644 --- a/salt/elasticsearch/templates/component/elastic-agent/so-fleet_integrations.ip_mappings-1.json +++ b/salt/elasticsearch/templates/component/elastic-agent/so-fleet_integrations.ip_mappings-1.json @@ -1,37 +1,59 @@ { - "template": { - "mappings": { - "properties": { - "host": { - "properties":{ - "ip": { - "type": "ip" - } - } - }, - "related": { - "properties":{ - "ip": { - "type": "ip" - } + "template": { + "mappings": { + "properties": { + "host": { + "properties": { + "ip": { + "type": "ip" } - }, - "destination": { - "properties":{ - "ip": { - "type": "ip" - } + } + }, + "related": { + "properties": { + "ip": { + "type": "ip" } - }, - "source": { - "properties":{ - "ip": { - "type": "ip" + } + }, + "destination": { + "properties": { + "ip": { + "type": "ip" + } + } + }, + "source": { + "properties": { + "ip": { + "type": "ip" + } + } + }, + "metadata": { + "properties": { + "input": { + "properties": { + "beats": { + "properties": { + "host": { + "properties": { + "ip": { + "type": "ip" + } + } + } + } + } } } } } } } + }, + "_meta": { + "managed_by": "security_onion", + "managed": true } - \ No newline at end of file +} \ No newline at end of file diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index b9bfdbf63..e92a75bcb 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -11,6 +11,7 @@ firewall: endgame: [] eval: [] external_suricata: [] + external_kafka: [] fleet: [] heavynode: [] idh: [] @@ -103,6 +104,10 @@ firewall: tcp: - 9092 udp: [] + kafka_external_access: + tcp: + - 29092 + udp: [] kibana: tcp: - 5601 @@ -473,6 +478,8 @@ firewall: external_suricata: portgroups: - external_suricata + external_kafka: + portgroups: [] desktop: portgroups: - docker_registry @@ -668,6 +675,8 @@ firewall: external_suricata: portgroups: - external_suricata + external_kafka: + portgroups: [] desktop: portgroups: - docker_registry @@ -867,6 +876,8 @@ firewall: external_suricata: portgroups: - external_suricata + external_kafka: + portgroups: [] strelka_frontend: portgroups: - strelka_frontend @@ -1337,6 +1348,8 @@ firewall: endgame: portgroups: - endgame + external_kafka: + portgroups: [] receiver: portgroups: [] customhostgroup0: diff --git a/salt/firewall/map.jinja b/salt/firewall/map.jinja index fe04d7ad3..4347d2b31 100644 --- a/salt/firewall/map.jinja +++ b/salt/firewall/map.jinja @@ -21,25 +21,38 @@ {# Only add Kafka firewall items when Kafka enabled #} {% set role = GLOBALS.role.split('-')[1] %} -{% if GLOBALS.pipeline == 'KAFKA' and role in ['manager', 'managersearch', 'standalone'] %} -{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[role].portgroups.append('kafka_controller') %} -{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.receiver.portgroups.append('kafka_controller') %} -{% endif %} +{% if GLOBALS.pipeline == 'KAFKA' %} +{% set KAFKA_EXTERNAL_ACCESS = salt['pillar.get']('kafka:config:external_access:enabled', default=False) %} +{% set kafka_node_type = salt['pillar.get']('kafka:nodes:'+ GLOBALS.hostname + ':role') %} -{% if GLOBALS.pipeline == 'KAFKA' and role == 'receiver' %} -{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.self.portgroups.append('kafka_controller') %} -{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.standalone.portgroups.append('kafka_controller') %} -{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.manager.portgroups.append('kafka_controller') %} -{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.managersearch.portgroups.append('kafka_controller') %} -{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.receiver.portgroups.append('kafka_controller') %} -{% endif %} +{% if role in ['manager', 'managersearch', 'standalone'] %} +{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[role].portgroups.append('kafka_controller') %} +{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.receiver.portgroups.append('kafka_controller') %} +{% endif %} -{% if GLOBALS.pipeline == 'KAFKA' and role in ['manager', 'managersearch', 'standalone', 'receiver'] %} -{% for r in ['manager', 'managersearch', 'standalone', 'receiver', 'fleet', 'idh', 'sensor', 'searchnode','heavynode', 'elastic_agent_endpoint', 'desktop'] %} -{% if FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[r] is defined %} -{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[r].portgroups.append('kafka_data') %} +{% if role == 'receiver' %} +{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.self.portgroups.append('kafka_controller') %} +{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.standalone.portgroups.append('kafka_controller') %} +{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.manager.portgroups.append('kafka_controller') %} +{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.managersearch.portgroups.append('kafka_controller') %} +{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.receiver.portgroups.append('kafka_controller') %} +{% endif %} + +{% if role in ['manager', 'managersearch', 'standalone', 'receiver'] %} +{% for r in ['manager', 'managersearch', 'standalone', 'receiver', 'fleet', 'idh', 'sensor', 'searchnode','heavynode', 'elastic_agent_endpoint', 'desktop'] %} +{% if FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[r] is defined %} +{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[r].portgroups.append('kafka_data') %} +{% endif %} +{% endfor %} +{% endif %} + +{% if KAFKA_EXTERNAL_ACCESS %} +{# Kafka external access only applies for Kafka nodes with the broker role. #} +{% if role in ['manager', 'managersearch', 'standalone', 'receiver'] and 'broker' in kafka_node_type %} +{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.external_kafka.portgroups.append('kafka_external_access') %} {% endif %} -{% endfor %} +{% endif %} + {% endif %} -{% set FIREWALL_MERGED = salt['pillar.get']('firewall', FIREWALL_DEFAULT.firewall, merge=True) %} +{% set FIREWALL_MERGED = salt['pillar.get']('firewall', FIREWALL_DEFAULT.firewall, merge=True) %} \ No newline at end of file diff --git a/salt/firewall/soc_firewall.yaml b/salt/firewall/soc_firewall.yaml index 222bcc8a2..6f0208b39 100644 --- a/salt/firewall/soc_firewall.yaml +++ b/salt/firewall/soc_firewall.yaml @@ -33,6 +33,7 @@ firewall: endgame: *hostgroupsettingsadv eval: *hostgroupsettings external_suricata: *hostgroupsettings + external_kafka: *hostgroupsettings fleet: *hostgroupsettings heavynode: *hostgroupsettings idh: *hostgroupsettings @@ -130,6 +131,9 @@ firewall: kafka_data: tcp: *tcpsettings udp: *udpsettings + kafka_external_access: + tcp: *tcpsettings + udp: *udpsettings kibana: tcp: *tcpsettings udp: *udpsettings diff --git a/salt/idstools/config.sls b/salt/idstools/config.sls index 6d4b1036e..a44b02807 100644 --- a/salt/idstools/config.sls +++ b/salt/idstools/config.sls @@ -24,13 +24,23 @@ idstools_sbin: - group: 939 - file_mode: 755 -idstools_sbin_jinja: - file.recurse: - - name: /usr/sbin - - source: salt://idstools/tools/sbin_jinja +# If this is used, exclude so-rule-update +#idstools_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://idstools/tools/sbin_jinja +# - user: 934 +# - group: 939 +# - file_mode: 755 +# - template: jinja + +idstools_so-rule-update: + file.managed: + - name: /usr/sbin/so-rule-update + - source: salt://idstools/tools/sbin_jinja/so-rule-update - user: 934 - group: 939 - - file_mode: 755 + - mode: 755 - template: jinja suricatacustomdirsfile: diff --git a/salt/idstools/enabled.sls b/salt/idstools/enabled.sls index 5e4c4c066..365b38772 100644 --- a/salt/idstools/enabled.sls +++ b/salt/idstools/enabled.sls @@ -55,6 +55,7 @@ so-idstools: {% endif %} - watch: - file: idstoolsetcsync + - file: idstools_so-rule-update delete_so-idstools_so-status.disabled: file.uncomment: @@ -76,6 +77,7 @@ run_so-rule-update: - require: - docker_container: so-idstools - onchanges: + - file: idstools_so-rule-update - file: idstoolsetcsync - file: synclocalnidsrules - order: last diff --git a/salt/kafka/config.map.jinja b/salt/kafka/config.map.jinja index 1e43a3ec7..b8e299838 100644 --- a/salt/kafka/config.map.jinja +++ b/salt/kafka/config.map.jinja @@ -8,6 +8,7 @@ {% set KAFKA_NODES_PILLAR = salt['pillar.get']('kafka:nodes') %} {% set KAFKA_PASSWORD = salt['pillar.get']('kafka:config:password') %} {% set KAFKA_TRUSTPASS = salt['pillar.get']('kafka:config:trustpass') %} +{% set KAFKA_EXTERNAL_ACCESS = salt['pillar.get']('kafka:config:external_access:enabled', default=False) %} {# Create list of KRaft controllers #} {% set controllers = [] %} @@ -15,7 +16,7 @@ {# Check for Kafka nodes with controller in process_x_roles #} {% for node in KAFKA_NODES_PILLAR %} {% if 'controller' in KAFKA_NODES_PILLAR[node].role %} -{% do controllers.append(KAFKA_NODES_PILLAR[node].nodeid ~ "@" ~ node ~ ":9093") %} +{% do controllers.append(KAFKA_NODES_PILLAR[node].nodeid ~ "@" ~ KAFKA_NODES_PILLAR[node].ip ~ ":9093") %} {% endif %} {% endfor %} @@ -28,7 +29,15 @@ {# Generate server.properties for 'broker' , 'controller', 'broker,controller' node types anything above this line is a configuration needed for ALL Kafka nodes #} {% if node_type == 'broker' %} -{% do KAFKAMERGED.config.broker.update({'advertised_x_listeners': 'BROKER://'+ GLOBALS.node_ip +':9092' }) %} +{% if KAFKA_EXTERNAL_ACCESS %} +{% do KAFKAMERGED.config.broker.update({'advertised_x_listeners': 'BROKER://'+ GLOBALS.node_ip +':9092' + ',' + 'EXTERNAL_ACCESS://' + GLOBALS.node_ip + ':29092' }) %} +{% do KAFKAMERGED.config.broker.update({'listeners': KAFKAMERGED.config.broker.listeners + ',' + KAFKAMERGED.config.external_access.listeners }) %} +{% do KAFKAMERGED.config.broker.update({'listener_x_security_x_protocol_x_map': KAFKAMERGED.config.broker.listener_x_security_x_protocol_x_map + ',' + KAFKAMERGED.config.external_access.listener_x_security_x_protocol_x_map }) %} +{% do KAFKAMERGED.config.broker.update({'sasl_x_enabled_x_mechanisms': KAFKAMERGED.config.external_access.sasl_x_enabled_x_mechanisms }) %} +{% do KAFKAMERGED.config.broker.update({'sasl_x_mechanism_x_inter_x_broker_x_protocol': KAFKAMERGED.config.external_access.sasl_x_mechanism_x_inter_x_broker_x_protocol }) %} +{% else %} +{% do KAFKAMERGED.config.broker.update({'advertised_x_listeners': 'BROKER://'+ GLOBALS.node_ip +':9092' }) %} +{% endif %} {% do KAFKAMERGED.config.broker.update({'controller_x_quorum_x_voters': kafka_controller_quorum_voters }) %} {% do KAFKAMERGED.config.broker.update({'node_x_id': salt['pillar.get']('kafka:nodes:'+ GLOBALS.hostname +':nodeid') }) %} {% do KAFKAMERGED.config.broker.update({'ssl_x_keystore_x_password': KAFKA_PASSWORD }) %} @@ -42,6 +51,7 @@ {% endif %} {% if node_type == 'controller' %} +{% do KAFKAMERGED.config.controller.update({'advertised_x_listeners': 'CONTROLLER://' + GLOBALS.node_ip + ':9093'}) %} {% do KAFKAMERGED.config.controller.update({'controller_x_quorum_x_voters': kafka_controller_quorum_voters }) %} {% do KAFKAMERGED.config.controller.update({'node_x_id': salt['pillar.get']('kafka:nodes:'+ GLOBALS.hostname +':nodeid') }) %} {% do KAFKAMERGED.config.controller.update({'ssl_x_keystore_x_password': KAFKA_PASSWORD }) %} @@ -50,7 +60,15 @@ {# Kafka nodes of this type are not recommended for use outside of development / testing. #} {% if node_type == 'broker,controller' %} -{% do KAFKAMERGED.config.broker.update({'advertised_x_listeners': 'BROKER://'+ GLOBALS.node_ip +':9092' }) %} +{% if KAFKA_EXTERNAL_ACCESS %} +{% do KAFKAMERGED.config.broker.update({'advertised_x_listeners': 'BROKER://'+ GLOBALS.node_ip +':9092' + ',' + 'CONTROLLER://'+ GLOBALS.node_ip +':9093' + ',' + 'EXTERNAL_ACCESS://' + GLOBALS.node_ip + ':29092' }) %} +{% do KAFKAMERGED.config.broker.update({'listeners': KAFKAMERGED.config.broker.listeners + ',' + KAFKAMERGED.config.external_access.listeners }) %} +{% do KAFKAMERGED.config.broker.update({'listener_x_security_x_protocol_x_map': KAFKAMERGED.config.broker.listener_x_security_x_protocol_x_map + ',' + KAFKAMERGED.config.external_access.listener_x_security_x_protocol_x_map }) %} +{% do KAFKAMERGED.config.broker.update({'sasl_x_enabled_x_mechanisms': KAFKAMERGED.config.external_access.sasl_x_enabled_x_mechanisms }) %} +{% do KAFKAMERGED.config.broker.update({'sasl_x_mechanism_x_inter_x_broker_x_protocol': KAFKAMERGED.config.external_access.sasl_x_mechanism_x_inter_x_broker_x_protocol }) %} +{% else %} +{% do KAFKAMERGED.config.broker.update({'advertised_x_listeners': 'BROKER://'+ GLOBALS.node_ip +':9092' + ',' + 'CONTROLLER://'+ GLOBALS.node_ip +':9093' }) %} +{% endif %} {% do KAFKAMERGED.config.broker.update({'controller_x_listener_x_names': KAFKAMERGED.config.controller.controller_x_listener_x_names }) %} {% do KAFKAMERGED.config.broker.update({'controller_x_quorum_x_voters': kafka_controller_quorum_voters }) %} {% do KAFKAMERGED.config.broker.update({'node_x_id': salt['pillar.get']('kafka:nodes:'+ GLOBALS.hostname +':nodeid') }) %} diff --git a/salt/kafka/config.sls b/salt/kafka/config.sls index e9222388b..1bedf96a0 100644 --- a/salt/kafka/config.sls +++ b/salt/kafka/config.sls @@ -6,6 +6,8 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls.split('.')[0] in allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} +{% set KAFKA_EXTERNAL_ACCESS = salt['pillar.get']('kafka:config:external_access:enabled', default=False) %} +{% set KAFKA_EXTERNAL_USERS = salt['pillar.get']('kafka:config:external_access:remote_users', default=None) %} kafka_group: group.present: @@ -69,6 +71,29 @@ kafka_kraft_{{sc}}_properties: - show_changes: False {% endfor %} +{% if KAFKA_EXTERNAL_ACCESS and KAFKA_EXTERNAL_USERS != None %} +kafka_server_jaas_properties: + file.managed: + - source: salt://kafka/etc/jaas.conf.jinja + - name: /opt/so/conf/kafka/kafka_server_jaas.conf + - template: jinja + - user: 960 + - group: 960 + - show_changes: False +{% else %} +remove_kafka_server_jaas_properties: + file.absent: + - name: /opt/so/conf/kafka/kafka_server_jaas.conf +{% endif %} + +kafka_log4j_properties: + file.managed: + - source: salt://kafka/etc/log4j.properties + - name: /opt/so/conf/kafka/log4j.properties + - user: 960 + - group: 960 + - show_changes: False + reset_quorum_on_changes: cmd.run: - name: rm -f /nsm/kafka/data/__cluster_metadata-0/quorum-state @@ -81,4 +106,4 @@ reset_quorum_on_changes: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} +{% endif %} \ No newline at end of file diff --git a/salt/kafka/defaults.yaml b/salt/kafka/defaults.yaml index 21d6956ba..6b97ea84d 100644 --- a/salt/kafka/defaults.yaml +++ b/salt/kafka/defaults.yaml @@ -21,7 +21,7 @@ kafka: log_x_segment_x_bytes: 1073741824 node_x_id: num_x_io_x_threads: 8 - num_x_network_x_threads: 3 + num_x_network_x_threads: 5 num_x_partitions: 3 num_x_recovery_x_threads_x_per_x_data_x_dir: 1 offsets_x_topic_x_replication_x_factor: 1 @@ -46,6 +46,7 @@ kafka: ssl_x_keystore_x_type: PKCS12 ssl_x_keystore_x_password: controller: + advertsied_x_listeners: controller_x_listener_x_names: CONTROLLER controller_x_quorum_x_voters: listeners: CONTROLLER://0.0.0.0:9093 @@ -61,4 +62,10 @@ kafka: ssl_x_keystore_x_password: ssl_x_truststore_x_location: /etc/pki/kafka-truststore.jks ssl_x_truststore_x_type: JKS - ssl_x_truststore_x_password: \ No newline at end of file + ssl_x_truststore_x_password: + external_access: + enabled: False + listeners: EXTERNAL_ACCESS://0.0.0.0:29092 + listener_x_security_x_protocol_x_map: EXTERNAL_ACCESS:SASL_SSL + sasl_x_enabled_x_mechanisms: PLAIN + sasl_x_mechanism_x_inter_x_broker_x_protocol: SSL \ No newline at end of file diff --git a/salt/kafka/enabled.sls b/salt/kafka/enabled.sls index 362f7fde3..8448bd5aa 100644 --- a/salt/kafka/enabled.sls +++ b/salt/kafka/enabled.sls @@ -14,6 +14,7 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'docker/docker.map.jinja' import DOCKER %} {% set KAFKANODES = salt['pillar.get']('kafka:nodes') %} +{% set KAFKA_EXTERNAL_ACCESS = salt['pillar.get']('kafka:config:external_access:enabled', default=False) %} {% if 'gmd' in salt['pillar.get']('features', []) %} include: @@ -34,7 +35,7 @@ so-kafka: - user: kafka - environment: KAFKA_HEAP_OPTS: -Xmx2G -Xms1G - KAFKA_OPTS: -javaagent:/opt/jolokia/agents/jolokia-agent-jvm-javaagent.jar=port=8778,host={{ DOCKER.containers['so-kafka'].ip }},policyLocation=file:/opt/jolokia/jolokia.xml + KAFKA_OPTS: "-javaagent:/opt/jolokia/agents/jolokia-agent-jvm-javaagent.jar=port=8778,host={{ DOCKER.containers['so-kafka'].ip }},policyLocation=file:/opt/jolokia/jolokia.xml {%- if KAFKA_EXTERNAL_ACCESS %} -Djava.security.auth.login.config=/opt/kafka/config/kafka_server_jaas.conf {% endif -%}" - extra_hosts: {% for node in KAFKANODES %} - {{ node }}:{{ KAFKANODES[node].ip }} @@ -54,11 +55,18 @@ so-kafka: - /nsm/kafka/data/:/nsm/kafka/data/:rw - /opt/so/log/kafka:/opt/kafka/logs/:rw - /opt/so/conf/kafka/server.properties:/opt/kafka/config/kraft/server.properties:ro - - /opt/so/conf/kafka/client.properties:/opt/kafka/config/kraft/client.properties + - /opt/so/conf/kafka/client.properties:/opt/kafka/config/kraft/client.properties:ro + - /opt/so/conf/kafka/log4j.properties:/opt/kafka/config/log4j.properties:ro + {% if KAFKA_EXTERNAL_ACCESS %} + - /opt/so/conf/kafka/kafka_server_jaas.conf:/opt/kafka/config/kafka_server_jaas.conf:ro + {% endif %} - watch: {% for sc in ['server', 'client'] %} - file: kafka_kraft_{{sc}}_properties {% endfor %} + {% if KAFKA_EXTERNAL_ACCESS %} + - file: kafka_server_jaas_properties + {% endif %} - file: kafkacertz - require: - file: kafkacertz @@ -87,4 +95,4 @@ include: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} +{% endif %} \ No newline at end of file diff --git a/salt/kafka/etc/jaas.conf.jinja b/salt/kafka/etc/jaas.conf.jinja new file mode 100644 index 000000000..e12367cd9 --- /dev/null +++ b/salt/kafka/etc/jaas.conf.jinja @@ -0,0 +1,16 @@ +{% set KAFKA_EXTERNAL_USERS = salt['pillar.get']('kafka:config:external_access:remote_users') -%} + +{%- set valid_users = [] -%} + +{%- for item, user in KAFKA_EXTERNAL_USERS.items() -%} +{% if 'password' in user and user.password is not none and user.password.strip() != "" -%} +{% do valid_users.append('user_' ~ user.username ~ '="' ~ user.password ~ '"') -%} +{% endif -%} +{%- endfor -%} + +KafkaServer { + org.apache.kafka.common.security.plain.PlainLoginModule required + {% for user_entry in valid_users -%} + {{ user_entry }}{{ ";" if loop.last }} + {% endfor %} +}; \ No newline at end of file diff --git a/salt/kafka/etc/log4j.properties b/salt/kafka/etc/log4j.properties new file mode 100644 index 000000000..76194c1ee --- /dev/null +++ b/salt/kafka/etc/log4j.properties @@ -0,0 +1,101 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Unspecified loggers and loggers with additivity=true output to server.log and stdout +# Note that INFO only applies to unspecified loggers, the log level of the child logger is used otherwise +log4j.rootLogger=INFO, stdout, kafkaAppender + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.kafkaAppender=org.apache.log4j.RollingFileAppender +log4j.appender.kafkaAppender.File=${kafka.logs.dir}/server.log +log4j.appender.kafkaAppender.MaxFileSize=10MB +log4j.appender.kafkaAppender.MaxBackupIndex=10 +log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.stateChangeAppender=org.apache.log4j.RollingFileAppender +log4j.appender.stateChangeAppender.File=${kafka.logs.dir}/state-change.log +log4j.appender.stateChangeAppender.MaxFileSize=10MB +log4j.appender.stateChangeAppender.MaxBackupIndex=10 +log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.requestAppender=org.apache.log4j.RollingFileAppender +log4j.appender.requestAppender.File=${kafka.logs.dir}/kafka-request.log +log4j.appender.requestAppender.MaxFileSize=10MB +log4j.appender.requestAppender.MaxBackupIndex=10 +log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.cleanerAppender=org.apache.log4j.RollingFileAppender +log4j.appender.cleanerAppender.File=${kafka.logs.dir}/log-cleaner.log +log4j.appender.cleanerAppender.MaxFileSize=10MB +log4j.appender.cleanerAppender.MaxBackupIndex=10 +log4j.appender.cleanerAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.cleanerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.controllerAppender=org.apache.log4j.RollingFileAppender +log4j.appender.controllerAppender.File=${kafka.logs.dir}/controller.log +log4j.appender.controllerAppender.MaxFileSize=10MB +log4j.appender.controllerAppender.MaxBackupIndex=10 +log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.authorizerAppender=org.apache.log4j.RollingFileAppender +log4j.appender.authorizerAppender.File=${kafka.logs.dir}/kafka-authorizer.log +log4j.appender.authorizerAppender.MaxFileSize=10MB +log4j.appender.authorizerAppender.MaxBackupIndex=10 +log4j.appender.authorizerAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.authorizerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +# Change the line below to adjust ZK client logging +log4j.logger.org.apache.zookeeper=INFO + +# Change the two lines below to adjust the general broker logging level (output to server.log and stdout) +log4j.logger.kafka=INFO +log4j.logger.org.apache.kafka=INFO + +# Change to DEBUG or TRACE to enable request logging +log4j.logger.kafka.request.logger=WARN, requestAppender +log4j.additivity.kafka.request.logger=false + +# Uncomment the lines below and change log4j.logger.kafka.network.RequestChannel$ to TRACE for additional output +# related to the handling of requests +#log4j.logger.kafka.network.Processor=TRACE, requestAppender +#log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender +#log4j.additivity.kafka.server.KafkaApis=false +log4j.logger.kafka.network.RequestChannel$=WARN, requestAppender +log4j.additivity.kafka.network.RequestChannel$=false + +# Change the line below to adjust KRaft mode controller logging +log4j.logger.org.apache.kafka.controller=INFO, controllerAppender +log4j.additivity.org.apache.kafka.controller=false + +# Change the line below to adjust ZK mode controller logging +log4j.logger.kafka.controller=TRACE, controllerAppender +log4j.additivity.kafka.controller=false + +log4j.logger.kafka.log.LogCleaner=INFO, cleanerAppender +log4j.additivity.kafka.log.LogCleaner=false + +log4j.logger.state.change.logger=INFO, stateChangeAppender +log4j.additivity.state.change.logger=false + +# Access denials are logged at INFO level, change to DEBUG to also log allowed accesses +log4j.logger.kafka.authorizer.logger=INFO, authorizerAppender +log4j.additivity.kafka.authorizer.logger=false \ No newline at end of file diff --git a/salt/kafka/soc_kafka.yaml b/salt/kafka/soc_kafka.yaml index 8087f9bdf..cb093600f 100644 --- a/salt/kafka/soc_kafka.yaml +++ b/salt/kafka/soc_kafka.yaml @@ -34,10 +34,6 @@ kafka: sensitive: True helpLink: kafka.html broker: - advertised_x_listeners: - description: Specify the list of listeners (hostname and port) that Kafka brokers provide to clients for communication. - title: advertised.listeners - helpLink: kafka.html auto_x_create_x_topics_x_enable: description: Enable the auto creation of topics. title: auto.create.topics.enable @@ -226,4 +222,52 @@ kafka: description: The role performed by controller node. title: process.roles readonly: True - helpLink: kafka.html \ No newline at end of file + helpLink: kafka.html + external_access: + enabled: + description: Enables or disables access to Kafka topics using user/password authentication. Used for producing / consuming messages via an external client. + forcedType: bool + helpLink: kafka.html + listeners: + description: Set of URIs that is listened on and the listener names in a comma-seperated list. + title: listeners + readonly: True + advanced: True + helpLink: kafka.html + listener_x_security_x_protocol_x_map: + description: External listener name and mapped security protocol. + title: listener.security.protocol.map + readonly: True + advanced: True + helpLink: kafka.html + sasl_x_enabled_x_mechanisms: + description: SASL/PLAIN is a simple username/password authentication mechanism, used with TLS to implement secure authentication. + title: sasl.enabled.mechanisms + readonly: True + advanced: True + helpLink: kafka.html + sasl_x_mechanism_x_inter_x_broker_x_protocol: + description: SASL mechanism used for inter-broker communication + title: sasl.mechanism.inter.broker.protocol + readonly: True + advanced: True + helpLink: kafka.html + remote_users: + user01: &remote_user + username: + description: Username to be used for custom account + forcedType: string + global: True + password: + description: Password to be used for custom account + forcedType: string + global: True + sensitive: True + user02: *remote_user + user03: *remote_user + user04: *remote_user + user05: *remote_user + user06: *remote_user + user07: *remote_user + user08: *remote_user + user09: *remote_user \ No newline at end of file diff --git a/salt/logstash/etc/log4j2.properties b/salt/logstash/etc/log4j2.properties index 739756061..750a6e316 100644 --- a/salt/logstash/etc/log4j2.properties +++ b/salt/logstash/etc/log4j2.properties @@ -23,6 +23,8 @@ appender.rolling.policies.type = Policies appender.rolling.policies.time.type = TimeBasedTriggeringPolicy appender.rolling.policies.time.interval = 1 appender.rolling.policies.time.modulate = true +appender.rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.rolling.policies.size.size = 1GB appender.rolling.strategy.type = DefaultRolloverStrategy appender.rolling.strategy.action.type = Delete appender.rolling.strategy.action.basepath = /var/log/logstash diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 2325bc161..a199f4cfd 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -746,8 +746,6 @@ up_to_2.4.90() { so-yaml.py remove /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.password so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.password "$kafkatrimpass" so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.trustpass "$kafkatrust" - echo "If the Detection index exists, update the refresh_interval" - so-elasticsearch-query so-detection*/_settings -X PUT -d '{"index":{"refresh_interval":"1s"}}' INSTALLEDVERSION=2.4.90 } @@ -807,7 +805,8 @@ up_to_2.4.141() { } up_to_2.4.150() { - echo "Nothing to do for 2.4.150" + echo "If the Detection indices exists, update the refresh_interval" + so-elasticsearch-query so-detection*/_settings -X PUT -d '{"index":{"refresh_interval":"1s"}}' INSTALLEDVERSION=2.4.150 } diff --git a/salt/salt/master/mine_update_highstate.sls b/salt/salt/master/mine_update_highstate.sls index 874e6c65b..c91be20e9 100644 --- a/salt/salt/master/mine_update_highstate.sls +++ b/salt/salt/master/mine_update_highstate.sls @@ -19,8 +19,11 @@ salt.master.mine_update_highstate.update_mine_all_minions: # Run highstate on the original minion # we can use concurrent on this highstate because no other highstate would be running when this is called +# this state will run onlyif there is not an instance of it already running salt.master.mine_update_highstate.run_highstate_on_{{ MINION_ID }}: salt.state: - tgt: {{ MINION_ID }} - highstate: True - concurrent: True + - onlyif: + - 'ps -ef | grep -v grep | grep "/usr/bin/salt-minion.*ProcessPayload.*jid=.*Minion._thread_return" | wc -l | grep -q "^0$"'