diff --git a/salt/kafka/disabled.sls b/salt/kafka/disabled.sls new file mode 100644 index 000000000..6658f0c5e --- /dev/null +++ b/salt/kafka/disabled.sls @@ -0,0 +1,16 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +include: + - kafka.sostatus + +so-kafka: + docker_container.absent: + - force: True + +so-kafka_so-status.disabled: + file.comment: + - name: /opt/so/conf/so-status/so-status.conf + - regex: ^so-kafka$ diff --git a/salt/kafka/etc/client.properties.jinja b/salt/kafka/etc/client.properties.jinja index 0245c3c42..91ff5f7c2 100644 --- a/salt/kafka/etc/client.properties.jinja +++ b/salt/kafka/etc/client.properties.jinja @@ -1,2 +1,7 @@ +{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one + or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at + https://securityonion.net/license; you may not use this file except in compliance with the + Elastic License 2.0. #} + {% from 'kafka/map.jinja' import KAFKAMERGED -%} {{ KAFKAMERGED.config.client | yaml(False) | replace("_x_", ".") }} diff --git a/salt/kafka/etc/server.properties.jinja b/salt/kafka/etc/server.properties.jinja index 90a80063f..df5632ba9 100644 --- a/salt/kafka/etc/server.properties.jinja +++ b/salt/kafka/etc/server.properties.jinja @@ -1,2 +1,7 @@ +{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one + or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at + https://securityonion.net/license; you may not use this file except in compliance with the + Elastic License 2.0. #} + {% from 'kafka/map.jinja' import KAFKAMERGED -%} {{ KAFKAMERGED.config.server | yaml(False) | replace("_x_", ".") }} diff --git a/salt/kafka/init.sls b/salt/kafka/init.sls index 903c66867..b4a6a28b0 100644 --- a/salt/kafka/init.sls +++ b/salt/kafka/init.sls @@ -3,7 +3,11 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{# Create map.jinja to enable / disable kafka from UI #} -{# Temporarily just enable kafka #} +{% from 'kafka/map.jinja' import KAFKAMERGED %} + include: +{% if KAFKAMERGED.enabled %} - kafka.enabled +{% else %} + - kafka.disabled +{% endif %} diff --git a/salt/kafka/map.jinja b/salt/kafka/map.jinja index f1da7ec95..771e6102b 100644 --- a/salt/kafka/map.jinja +++ b/salt/kafka/map.jinja @@ -1,3 +1,8 @@ +{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one + or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at + https://securityonion.net/license; you may not use this file except in compliance with the + Elastic License 2.0. #} + {% import_yaml 'kafka/defaults.yaml' as KAFKADEFAULTS %} {% set KAFKAMERGED = salt['pillar.get']('kafka', KAFKADEFAULTS.kafka, merge=True) %} {% from 'vars/globals.map.jinja' import GLOBALS %} diff --git a/salt/kafka/soc_kafka.yaml b/salt/kafka/soc_kafka.yaml new file mode 100644 index 000000000..c16ff212e --- /dev/null +++ b/salt/kafka/soc_kafka.yaml @@ -0,0 +1,164 @@ +kafka: + enabled: + description: Enable or disable Kafka. + helpLink: kafka.html + config: + server: + advertised_x_listeners: + description: Specify the list of listeners (hostname and port) that Kafka brokers provide to clients for communication. + title: advertised.listeners + helpLink: kafka.html + auto_x_create_x_topics_x_enable: + description: Enable the auto creation of topics. + title: auto.create.topics.enable + forcedType: bool + helpLink: kafka.html + controller_x_listener_x_names: + description: Set listeners used by the controller in a comma-seperated list. + title: controller.listener.names + helpLink: kafka.html + controller_x_quorum_x_voters: + description: A comma-seperated list of ID and endpoint information mapped for a set of voters. + title: controller.quorum.voters + helpLink: kafka.html + inter_x_broker_x_listener_x_name: + description: The name of the listener used for inter-broker communication. + title: inter.broker.listener.name + helpLink: kafka.html + listeners: + description: Set of URIs that is listened on and the listener names in a comma-seperated list. + helpLink: kafka.html + listener_x_security_x_protocol_x_map: + description: Comma-seperated mapping of listener name and security protocols. + title: listener.security.protocol.map + helpLink: kafka.html + log_x_dirs: + description: Where Kafka logs are stored within the Docker container. + title: log.dirs + helpLink: kafka.html + log_x_retention_x_check_x_interval_x_ms: + description: Frequency at which log files are checked if they are qualified for deletion. + title: log.retention.check.interval.ms + helpLink: kafka.html + log_x_retention_x_hours: + description: How long, in hours, a log file is kept. + title: log.retention.hours + forcedType: int + helpLink: kafka.html + log_x_segment_x_bytes: + description: The maximum allowable size for a log file. + title: log.segment.bytes + forcedType: int + helpLink: kafka.html + node_x_id: + description: The node ID corresponds to the roles performed by this process whenever process.roles is populated. + title: node.id + forcedType: int + readonly: True + helpLink: kafka.html + num_x_io_x_threads: + description: The number of threads used by Kafka. + title: num.io.threads + forcedType: int + helpLink: kafka.html + num_x_network_x_threads: + description: The number of threads used for network communication. + title: num.network.threads + forcedType: int + helpLink: kafka.html + num_x_partitions: + description: The number of log partitions assigned per topic. + title: num.partitions + forcedType: int + helpLink: kafka.html + num_x_recovery_x_threads_x_per_x_data_x_dir: + description: The number of threads used for log recuperation at startup and purging at shutdown. This ammount of threads is used per data directory. + title: num.recovery.threads.per.data.dir + forcedType: int + helpLink: kafka.html + offsets_x_topic_x_replication_x_factor: + description: The offsets topic replication factor. + title: offsets.topic.replication.factor + forcedType: int + helpLink: kafka.html + process_x_roles: + description: The roles the process performs. Use a comma-seperated list is multiple. + title: process.roles + helpLink: kafka.html + socket_x_receive_x_buffer_x_bytes: + description: Size, in bytes of the SO_RCVBUF buffer. A value of -1 will use the OS default. + title: socket.receive.buffer.bytes + #forcedType: int - soc needs to allow -1 as an int before we can use this + helpLink: kafka.html + socket_x_request_x_max_x_bytes: + description: The maximum bytes allowed for a request to the socket. + title: socket.request.max.bytes + forcedType: int + helpLink: kafka.html + socket_x_send_x_buffer_x_bytes: + description: Size, in bytes of the SO_SNDBUF buffer. A value of -1 will use the OS default. + title: socket.send.buffer.byte + #forcedType: int - soc needs to allow -1 as an int before we can use this + helpLink: kafka.html + ssl_x_keystore_x_location: + description: The key store file location within the Docker container. + title: ssl.keystore.location + helpLink: kafka.html + ssl_x_keystore_x_password: + description: The key store file password. Invalid for PEM format. + title: ssl.keystore.password + sensitive: True + helpLink: kafka.html + ssl_x_keystore_x_type: + description: The key store file format. + title: ssl.keystore.type + regex: ^(JKS|PKCS12|PEM)$ + helpLink: kafka.html + ssl_x_truststore_x_location: + description: The trust store file location within the Docker container. + title: ssl.truststore.location + helpLink: kafka.html + ssl_x_truststore_x_password: + description: The trust store file password. If null, the trust store file is still use, but integrity checking is disabled. Invalid for PEM format. + title: ssl.truststore.password + sensitive: True + helpLink: kafka.html + transaction_x_state_x_log_x_min_x_isr: + description: Overrides min.insync.replicas for the transaction topic. When a producer configures acks to "all" (or "-1"), this setting determines the minimum number of replicas required to acknowledge a write as successful. Failure to meet this minimum triggers an exception (either NotEnoughReplicas or NotEnoughReplicasAfterAppend). When used in conjunction, min.insync.replicas and acks enable stronger durability guarantees. For instance, creating a topic with a replication factor of 3, setting min.insync.replicas to 2, and using acks of "all" ensures that the producer raises an exception if a majority of replicas fail to receive a write. + title: transaction.state.log.min.isr + forcedType: int + helpLink: kafka.html + transaction_x_state_x_log_x_replication_x_factor: + description: Set the replication factor higher for the transaction topic to ensure availability. Internal topic creation will not proceed until the cluster size satisfies this replication factor prerequisite. + title: transaction.state.log.replication.factor + forcedType: int + helpLink: kafka.html + client: + security_x_protocol: + description: Broker communication protocol. Options are: SASL_SSL, PLAINTEXT, SSL, SASL_PLAINTEXT + title: security.protocol + regex: ^(SASL_SSL|PLAINTEXT|SSL|SASL_PLAINTEXT) + helpLink: kafka.html + ssl_x_keystore_x_location: + description: The key store file location within the Docker container. + title: ssl.keystore.location + helpLink: kafka.html + ssl_x_keystore_x_password: + description: The key store file password. Invalid for PEM format. + title: ssl.keystore.password + sensitive: True + helpLink: kafka.html + ssl_x_keystore_x_type: + description: The key store file format. + title: ssl.keystore.type + regex: ^(JKS|PKCS12|PEM)$ + helpLink: kafka.html + ssl_x_truststore_x_location: + description: The trust store file location within the Docker container. + title: ssl.truststore.location + helpLink: kafka.html + ssl_x_truststore_x_password: + description: The trust store file password. If null, the trust store file is still use, but integrity checking is disabled. Invalid for PEM format. + title: ssl.truststore.password + sensitive: True + helpLink: kafka.html