diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 71253e18c..1f81e95d2 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -162,6 +162,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -316,6 +317,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -427,6 +429,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -534,6 +537,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -697,6 +701,7 @@ elasticsearch: - client-mappings - device-mappings - network-mappings + - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 - so-fleet_agent_id_verification-1 data_stream: @@ -768,6 +773,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -878,6 +884,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -998,6 +1005,7 @@ elasticsearch: index_template: composed_of: - so-data-streams-mappings + - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 - so-fleet_agent_id_verification-1 - so-logs-mappings @@ -2832,6 +2840,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -3062,6 +3071,7 @@ elasticsearch: - event-mappings - logs-system.syslog@package - logs-system.syslog@custom + - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 - so-fleet_agent_id_verification-1 - so-system-mappings @@ -3421,6 +3431,7 @@ elasticsearch: - dtc-http-mappings - log-mappings - logstash-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -3505,6 +3516,7 @@ elasticsearch: composed_of: - metrics-endpoint.metadata@package - metrics-endpoint.metadata@custom + - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 - so-fleet_agent_id_verification-1 data_stream: @@ -3551,6 +3563,7 @@ elasticsearch: composed_of: - metrics-endpoint.metrics@package - metrics-endpoint.metrics@custom + - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 - so-fleet_agent_id_verification-1 data_stream: @@ -3597,6 +3610,7 @@ elasticsearch: composed_of: - metrics-endpoint.policy@package - metrics-endpoint.policy@custom + - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 - so-fleet_agent_id_verification-1 data_stream: @@ -3645,6 +3659,7 @@ elasticsearch: - metrics-fleet_server.agent_status@package - metrics-fleet_server.agent_status@custom - ecs@mappings + - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 - so-fleet_agent_id_verification-1 data_stream: @@ -3668,6 +3683,7 @@ elasticsearch: - metrics-fleet_server.agent_versions@package - metrics-fleet_server.agent_versions@custom - ecs@mappings + - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 - so-fleet_agent_id_verification-1 data_stream: @@ -3715,6 +3731,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -3827,6 +3844,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -3939,6 +3957,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -4051,6 +4070,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -4163,6 +4183,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings @@ -4276,6 +4297,7 @@ elasticsearch: - http-mappings - dtc-http-mappings - log-mappings + - metadata-mappings - network-mappings - dtc-network-mappings - observer-mappings diff --git a/salt/elasticsearch/templates/component/ecs/log.json b/salt/elasticsearch/templates/component/ecs/log.json index e79661b5e..4f51be287 100644 --- a/salt/elasticsearch/templates/component/ecs/log.json +++ b/salt/elasticsearch/templates/component/ecs/log.json @@ -29,7 +29,7 @@ "file": { "properties": { "line": { - "type": "integer" + "type": "long" }, "name": { "ignore_above": 1024, diff --git a/salt/elasticsearch/templates/component/ecs/metadata.json b/salt/elasticsearch/templates/component/ecs/metadata.json new file mode 100644 index 000000000..55da6f07c --- /dev/null +++ b/salt/elasticsearch/templates/component/ecs/metadata.json @@ -0,0 +1,26 @@ +{ + "template": { + "mappings": { + "dynamic_templates": [], + "properties": { + "metadata": { + "properties": { + "kafka": { + "properties": { + "timestamp": { + "type": "date" + } + } + } + } + } + } + } + }, + "_meta": { + "_meta": { + "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-log.html", + "ecs_version": "1.12.2" + } + } +} \ No newline at end of file diff --git a/salt/elasticsearch/templates/component/elastic-agent/so-fleet_globals-1.json b/salt/elasticsearch/templates/component/elastic-agent/so-fleet_globals-1.json index 5df7e7fe9..183031d4e 100644 --- a/salt/elasticsearch/templates/component/elastic-agent/so-fleet_globals-1.json +++ b/salt/elasticsearch/templates/component/elastic-agent/so-fleet_globals-1.json @@ -5,6 +5,7 @@ "managed_by": "security_onion", "managed": true }, + "date_detection": false, "dynamic_templates": [ { "strings_as_keyword": { @@ -16,7 +17,19 @@ } } ], - "date_detection": false + "properties": { + "metadata": { + "properties": { + "kafka": { + "properties": { + "timestamp": { + "type": "date" + } + } + } + } + } + } } }, "_meta": { diff --git a/salt/elasticsearch/templates/component/elastic-agent/so-fleet_integrations.ip_mappings-1.json b/salt/elasticsearch/templates/component/elastic-agent/so-fleet_integrations.ip_mappings-1.json index 3777e670c..d6f516272 100644 --- a/salt/elasticsearch/templates/component/elastic-agent/so-fleet_integrations.ip_mappings-1.json +++ b/salt/elasticsearch/templates/component/elastic-agent/so-fleet_integrations.ip_mappings-1.json @@ -1,37 +1,59 @@ { - "template": { - "mappings": { - "properties": { - "host": { - "properties":{ - "ip": { - "type": "ip" - } - } - }, - "related": { - "properties":{ - "ip": { - "type": "ip" - } + "template": { + "mappings": { + "properties": { + "host": { + "properties": { + "ip": { + "type": "ip" } - }, - "destination": { - "properties":{ - "ip": { - "type": "ip" - } + } + }, + "related": { + "properties": { + "ip": { + "type": "ip" } - }, - "source": { - "properties":{ - "ip": { - "type": "ip" + } + }, + "destination": { + "properties": { + "ip": { + "type": "ip" + } + } + }, + "source": { + "properties": { + "ip": { + "type": "ip" + } + } + }, + "metadata": { + "properties": { + "input": { + "properties": { + "beats": { + "properties": { + "host": { + "properties": { + "ip": { + "type": "ip" + } + } + } + } + } } } } } } } + }, + "_meta": { + "managed_by": "security_onion", + "managed": true } - \ No newline at end of file +} \ No newline at end of file diff --git a/salt/idstools/config.sls b/salt/idstools/config.sls index 6d4b1036e..a44b02807 100644 --- a/salt/idstools/config.sls +++ b/salt/idstools/config.sls @@ -24,13 +24,23 @@ idstools_sbin: - group: 939 - file_mode: 755 -idstools_sbin_jinja: - file.recurse: - - name: /usr/sbin - - source: salt://idstools/tools/sbin_jinja +# If this is used, exclude so-rule-update +#idstools_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://idstools/tools/sbin_jinja +# - user: 934 +# - group: 939 +# - file_mode: 755 +# - template: jinja + +idstools_so-rule-update: + file.managed: + - name: /usr/sbin/so-rule-update + - source: salt://idstools/tools/sbin_jinja/so-rule-update - user: 934 - group: 939 - - file_mode: 755 + - mode: 755 - template: jinja suricatacustomdirsfile: diff --git a/salt/idstools/enabled.sls b/salt/idstools/enabled.sls index 5e4c4c066..365b38772 100644 --- a/salt/idstools/enabled.sls +++ b/salt/idstools/enabled.sls @@ -55,6 +55,7 @@ so-idstools: {% endif %} - watch: - file: idstoolsetcsync + - file: idstools_so-rule-update delete_so-idstools_so-status.disabled: file.uncomment: @@ -76,6 +77,7 @@ run_so-rule-update: - require: - docker_container: so-idstools - onchanges: + - file: idstools_so-rule-update - file: idstoolsetcsync - file: synclocalnidsrules - order: last diff --git a/salt/kafka/config.map.jinja b/salt/kafka/config.map.jinja index e1e9a30f9..b8e299838 100644 --- a/salt/kafka/config.map.jinja +++ b/salt/kafka/config.map.jinja @@ -34,7 +34,7 @@ {% do KAFKAMERGED.config.broker.update({'listeners': KAFKAMERGED.config.broker.listeners + ',' + KAFKAMERGED.config.external_access.listeners }) %} {% do KAFKAMERGED.config.broker.update({'listener_x_security_x_protocol_x_map': KAFKAMERGED.config.broker.listener_x_security_x_protocol_x_map + ',' + KAFKAMERGED.config.external_access.listener_x_security_x_protocol_x_map }) %} {% do KAFKAMERGED.config.broker.update({'sasl_x_enabled_x_mechanisms': KAFKAMERGED.config.external_access.sasl_x_enabled_x_mechanisms }) %} -{% do KAFKAMERGED.config.broker.update({'sasl_x_mechanism_x_broker_x_protocol': KAFKAMERGED.config.external_access.sasl_x_mechanism_x_broker_x_protocol }) %} +{% do KAFKAMERGED.config.broker.update({'sasl_x_mechanism_x_inter_x_broker_x_protocol': KAFKAMERGED.config.external_access.sasl_x_mechanism_x_inter_x_broker_x_protocol }) %} {% else %} {% do KAFKAMERGED.config.broker.update({'advertised_x_listeners': 'BROKER://'+ GLOBALS.node_ip +':9092' }) %} {% endif %} @@ -65,7 +65,7 @@ {% do KAFKAMERGED.config.broker.update({'listeners': KAFKAMERGED.config.broker.listeners + ',' + KAFKAMERGED.config.external_access.listeners }) %} {% do KAFKAMERGED.config.broker.update({'listener_x_security_x_protocol_x_map': KAFKAMERGED.config.broker.listener_x_security_x_protocol_x_map + ',' + KAFKAMERGED.config.external_access.listener_x_security_x_protocol_x_map }) %} {% do KAFKAMERGED.config.broker.update({'sasl_x_enabled_x_mechanisms': KAFKAMERGED.config.external_access.sasl_x_enabled_x_mechanisms }) %} -{% do KAFKAMERGED.config.broker.update({'sasl_x_mechanism_x_broker_x_protocol': KAFKAMERGED.config.external_access.sasl_x_mechanism_x_broker_x_protocol }) %} +{% do KAFKAMERGED.config.broker.update({'sasl_x_mechanism_x_inter_x_broker_x_protocol': KAFKAMERGED.config.external_access.sasl_x_mechanism_x_inter_x_broker_x_protocol }) %} {% else %} {% do KAFKAMERGED.config.broker.update({'advertised_x_listeners': 'BROKER://'+ GLOBALS.node_ip +':9092' + ',' + 'CONTROLLER://'+ GLOBALS.node_ip +':9093' }) %} {% endif %} diff --git a/salt/kafka/config.sls b/salt/kafka/config.sls index 4900cd882..1bedf96a0 100644 --- a/salt/kafka/config.sls +++ b/salt/kafka/config.sls @@ -86,6 +86,14 @@ remove_kafka_server_jaas_properties: - name: /opt/so/conf/kafka/kafka_server_jaas.conf {% endif %} +kafka_log4j_properties: + file.managed: + - source: salt://kafka/etc/log4j.properties + - name: /opt/so/conf/kafka/log4j.properties + - user: 960 + - group: 960 + - show_changes: False + reset_quorum_on_changes: cmd.run: - name: rm -f /nsm/kafka/data/__cluster_metadata-0/quorum-state diff --git a/salt/kafka/defaults.yaml b/salt/kafka/defaults.yaml index 08bcc8d18..6b97ea84d 100644 --- a/salt/kafka/defaults.yaml +++ b/salt/kafka/defaults.yaml @@ -68,4 +68,4 @@ kafka: listeners: EXTERNAL_ACCESS://0.0.0.0:29092 listener_x_security_x_protocol_x_map: EXTERNAL_ACCESS:SASL_SSL sasl_x_enabled_x_mechanisms: PLAIN - sasl_x_mechanism_x_broker_x_protocol: SSL \ No newline at end of file + sasl_x_mechanism_x_inter_x_broker_x_protocol: SSL \ No newline at end of file diff --git a/salt/kafka/enabled.sls b/salt/kafka/enabled.sls index 3bdd67208..8448bd5aa 100644 --- a/salt/kafka/enabled.sls +++ b/salt/kafka/enabled.sls @@ -56,6 +56,7 @@ so-kafka: - /opt/so/log/kafka:/opt/kafka/logs/:rw - /opt/so/conf/kafka/server.properties:/opt/kafka/config/kraft/server.properties:ro - /opt/so/conf/kafka/client.properties:/opt/kafka/config/kraft/client.properties:ro + - /opt/so/conf/kafka/log4j.properties:/opt/kafka/config/log4j.properties:ro {% if KAFKA_EXTERNAL_ACCESS %} - /opt/so/conf/kafka/kafka_server_jaas.conf:/opt/kafka/config/kafka_server_jaas.conf:ro {% endif %} diff --git a/salt/kafka/etc/log4j.properties b/salt/kafka/etc/log4j.properties new file mode 100644 index 000000000..76194c1ee --- /dev/null +++ b/salt/kafka/etc/log4j.properties @@ -0,0 +1,101 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Unspecified loggers and loggers with additivity=true output to server.log and stdout +# Note that INFO only applies to unspecified loggers, the log level of the child logger is used otherwise +log4j.rootLogger=INFO, stdout, kafkaAppender + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.kafkaAppender=org.apache.log4j.RollingFileAppender +log4j.appender.kafkaAppender.File=${kafka.logs.dir}/server.log +log4j.appender.kafkaAppender.MaxFileSize=10MB +log4j.appender.kafkaAppender.MaxBackupIndex=10 +log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.stateChangeAppender=org.apache.log4j.RollingFileAppender +log4j.appender.stateChangeAppender.File=${kafka.logs.dir}/state-change.log +log4j.appender.stateChangeAppender.MaxFileSize=10MB +log4j.appender.stateChangeAppender.MaxBackupIndex=10 +log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.requestAppender=org.apache.log4j.RollingFileAppender +log4j.appender.requestAppender.File=${kafka.logs.dir}/kafka-request.log +log4j.appender.requestAppender.MaxFileSize=10MB +log4j.appender.requestAppender.MaxBackupIndex=10 +log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.cleanerAppender=org.apache.log4j.RollingFileAppender +log4j.appender.cleanerAppender.File=${kafka.logs.dir}/log-cleaner.log +log4j.appender.cleanerAppender.MaxFileSize=10MB +log4j.appender.cleanerAppender.MaxBackupIndex=10 +log4j.appender.cleanerAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.cleanerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.controllerAppender=org.apache.log4j.RollingFileAppender +log4j.appender.controllerAppender.File=${kafka.logs.dir}/controller.log +log4j.appender.controllerAppender.MaxFileSize=10MB +log4j.appender.controllerAppender.MaxBackupIndex=10 +log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.authorizerAppender=org.apache.log4j.RollingFileAppender +log4j.appender.authorizerAppender.File=${kafka.logs.dir}/kafka-authorizer.log +log4j.appender.authorizerAppender.MaxFileSize=10MB +log4j.appender.authorizerAppender.MaxBackupIndex=10 +log4j.appender.authorizerAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.authorizerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +# Change the line below to adjust ZK client logging +log4j.logger.org.apache.zookeeper=INFO + +# Change the two lines below to adjust the general broker logging level (output to server.log and stdout) +log4j.logger.kafka=INFO +log4j.logger.org.apache.kafka=INFO + +# Change to DEBUG or TRACE to enable request logging +log4j.logger.kafka.request.logger=WARN, requestAppender +log4j.additivity.kafka.request.logger=false + +# Uncomment the lines below and change log4j.logger.kafka.network.RequestChannel$ to TRACE for additional output +# related to the handling of requests +#log4j.logger.kafka.network.Processor=TRACE, requestAppender +#log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender +#log4j.additivity.kafka.server.KafkaApis=false +log4j.logger.kafka.network.RequestChannel$=WARN, requestAppender +log4j.additivity.kafka.network.RequestChannel$=false + +# Change the line below to adjust KRaft mode controller logging +log4j.logger.org.apache.kafka.controller=INFO, controllerAppender +log4j.additivity.org.apache.kafka.controller=false + +# Change the line below to adjust ZK mode controller logging +log4j.logger.kafka.controller=TRACE, controllerAppender +log4j.additivity.kafka.controller=false + +log4j.logger.kafka.log.LogCleaner=INFO, cleanerAppender +log4j.additivity.kafka.log.LogCleaner=false + +log4j.logger.state.change.logger=INFO, stateChangeAppender +log4j.additivity.state.change.logger=false + +# Access denials are logged at INFO level, change to DEBUG to also log allowed accesses +log4j.logger.kafka.authorizer.logger=INFO, authorizerAppender +log4j.additivity.kafka.authorizer.logger=false \ No newline at end of file diff --git a/salt/kafka/soc_kafka.yaml b/salt/kafka/soc_kafka.yaml index 19ab3d2ef..cb093600f 100644 --- a/salt/kafka/soc_kafka.yaml +++ b/salt/kafka/soc_kafka.yaml @@ -228,14 +228,40 @@ kafka: description: Enables or disables access to Kafka topics using user/password authentication. Used for producing / consuming messages via an external client. forcedType: bool helpLink: kafka.html + listeners: + description: Set of URIs that is listened on and the listener names in a comma-seperated list. + title: listeners + readonly: True + advanced: True + helpLink: kafka.html + listener_x_security_x_protocol_x_map: + description: External listener name and mapped security protocol. + title: listener.security.protocol.map + readonly: True + advanced: True + helpLink: kafka.html + sasl_x_enabled_x_mechanisms: + description: SASL/PLAIN is a simple username/password authentication mechanism, used with TLS to implement secure authentication. + title: sasl.enabled.mechanisms + readonly: True + advanced: True + helpLink: kafka.html + sasl_x_mechanism_x_inter_x_broker_x_protocol: + description: SASL mechanism used for inter-broker communication + title: sasl.mechanism.inter.broker.protocol + readonly: True + advanced: True + helpLink: kafka.html remote_users: user01: &remote_user username: description: Username to be used for custom account forcedType: string + global: True password: description: Password to be used for custom account forcedType: string + global: True sensitive: True user02: *remote_user user03: *remote_user diff --git a/salt/logstash/etc/log4j2.properties b/salt/logstash/etc/log4j2.properties index 739756061..750a6e316 100644 --- a/salt/logstash/etc/log4j2.properties +++ b/salt/logstash/etc/log4j2.properties @@ -23,6 +23,8 @@ appender.rolling.policies.type = Policies appender.rolling.policies.time.type = TimeBasedTriggeringPolicy appender.rolling.policies.time.interval = 1 appender.rolling.policies.time.modulate = true +appender.rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.rolling.policies.size.size = 1GB appender.rolling.strategy.type = DefaultRolloverStrategy appender.rolling.strategy.action.type = Delete appender.rolling.strategy.action.basepath = /var/log/logstash