Merge pull request #13391 from Security-Onion-Solutions/2.4/dev

2.4.90

DOWNLOAD_AND_VERIFY_ISO.md

@@ -1,17 +1,17 @@
-### 2.4.80-20240624 ISO image released on 2024/06/25
+### 2.4.90-20240729 ISO image released on 2024/07/29
 
 
 ### Download and Verify
 
-2.4.80-20240624 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.4.80-20240624.iso
+2.4.90-20240729 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-2.4.90-20240729.iso
  
-MD5: 139F9762E926F9CB3C4A9528A3752C31  
-SHA1: BC6CA2C5F4ABC1A04E83A5CF8FFA6A53B1583CC9  
-SHA256: 70E90845C84FFA30AD6CF21504634F57C273E7996CA72F7250428DDBAAC5B1BD  
+MD5: 9A7714F5922EE555F08675D25E6237D5  
+SHA1: D3B331452627DB716906BA9F3922574DFA3852DC  
+SHA256: 5B0CE32543944DBC50C4E906857384211E1BE83EF409619778F18FC62017E0E0  
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.80-20240624.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.90-20240729.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS  
@@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.80-20240624.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.90-20240729.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.4.80-20240624.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.4.90-20240729.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.4.80-20240624.iso.sig securityonion-2.4.80-20240624.iso
+gpg --verify securityonion-2.4.90-20240729.iso.sig securityonion-2.4.90-20240729.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Mon 24 Jun 2024 02:42:03 PM EDT using RSA key ID FE507013
+gpg: Signature made Thu 25 Jul 2024 06:51:11 PM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

VERSION

@@ -1 +1 @@
-2.4.80
+2.4.90

pillar/elasticsearch/nodes.sls

@@ -0,0 +1,34 @@
+{% set node_types = {} %}
+{% for minionid, ip in salt.saltutil.runner(
+    'mine.get',
+    tgt='elasticsearch:enabled:true',
+    fun='network.ip_addrs',
+    tgt_type='pillar') | dictsort()
+%}
+
+# only add a node to the pillar if it returned an ip from the mine
+{%   if ip | length > 0%}
+{%     set hostname = minionid.split('_') | first %}
+{%     set node_type = minionid.split('_') | last %}
+{%     if node_type not in node_types.keys() %}
+{%       do node_types.update({node_type: {hostname: ip[0]}}) %}
+{%     else %}
+{%       if hostname not in node_types[node_type] %}
+{%         do node_types[node_type].update({hostname: ip[0]}) %}
+{%       else %}
+{%         do node_types[node_type][hostname].update(ip[0]) %}
+{%       endif %}
+{%     endif %}
+{%   endif %}
+{% endfor %}
+
+
+elasticsearch:
+  nodes:
+{% for node_type, values in node_types.items() %}
+    {{node_type}}:
+{%   for hostname, ip in values.items() %}
+      {{hostname}}:
+        ip: {{ip}}
+{%   endfor %}
+{% endfor %}

pillar/logstash/nodes.sls

@@ -1,16 +1,15 @@
 {% set node_types = {} %}
-{% set cached_grains = salt.saltutil.runner('cache.grains', tgt='*') %}
 {% for minionid, ip in salt.saltutil.runner(
     'mine.get',
-    tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-searchnode or G@role:so-heavynode or G@role:so-receiver or G@role:so-fleet ',
+    tgt='logstash:enabled:true',
     fun='network.ip_addrs',
-    tgt_type='compound') | dictsort()
+    tgt_type='pillar') | dictsort()
 %}
 
 # only add a node to the pillar if it returned an ip from the mine
 {%   if ip | length > 0%}
-{%     set hostname = cached_grains[minionid]['host'] %}
-{%     set node_type = minionid.split('_')[1] %}
+{%     set hostname = minionid.split('_') | first %}
+{%     set node_type = minionid.split('_') | last %}
 {%     if node_type not in node_types.keys() %}
 {%       do node_types.update({node_type: {hostname: ip[0]}}) %}
 {%     else %}

pillar/redis/nodes.sls

@@ -0,0 +1,34 @@
+{% set node_types = {} %}
+{% for minionid, ip in salt.saltutil.runner(
+    'mine.get',
+    tgt='redis:enabled:true',
+    fun='network.ip_addrs',
+    tgt_type='pillar') | dictsort()
+%}
+
+# only add a node to the pillar if it returned an ip from the mine
+{%   if ip | length > 0%}
+{%     set hostname = minionid.split('_') | first %}
+{%     set node_type = minionid.split('_') | last %}
+{%     if node_type not in node_types.keys() %}
+{%       do node_types.update({node_type: {hostname: ip[0]}}) %}
+{%     else %}
+{%       if hostname not in node_types[node_type] %}
+{%         do node_types[node_type].update({hostname: ip[0]}) %}
+{%       else %}
+{%         do node_types[node_type][hostname].update(ip[0]) %}
+{%       endif %}
+{%     endif %}
+{%   endif %}
+{% endfor %}
+
+
+redis:
+  nodes:
+{% for node_type, values in node_types.items() %}
+    {{node_type}}:
+{%   for hostname, ip in values.items() %}
+      {{hostname}}:
+        ip: {{ip}}
+{%   endfor %}
+{% endfor %}

pillar/top.sls

@@ -47,10 +47,12 @@ base:
     - kibana.adv_kibana
     - kratos.soc_kratos
     - kratos.adv_kratos
+    - redis.nodes
     - redis.soc_redis
     - redis.adv_redis
     - influxdb.soc_influxdb
     - influxdb.adv_influxdb
+    - elasticsearch.nodes
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
     - elasticfleet.soc_elasticfleet
@@ -147,10 +149,12 @@ base:
     - idstools.adv_idstools
     - kratos.soc_kratos
     - kratos.adv_kratos
+    - redis.nodes
     - redis.soc_redis
     - redis.adv_redis
     - influxdb.soc_influxdb
     - influxdb.adv_influxdb
+    - elasticsearch.nodes
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
     - elasticfleet.soc_elasticfleet
@@ -215,11 +219,13 @@ base:
     - logstash.nodes
     - logstash.soc_logstash
     - logstash.adv_logstash
+    - elasticsearch.nodes
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
     {% endif %}
+    - redis.nodes
     - redis.soc_redis
     - redis.adv_redis
     - minions.{{ grains.id }}
@@ -227,6 +233,8 @@ base:
     - stig.soc_stig
     - soc.license
     - kafka.nodes
+    - kafka.soc_kafka
+    - kafka.adv_kafka
 
   '*_receiver':
     - logstash.nodes

salt/allowed_states.map.jinja

@@ -136,7 +136,9 @@
           'firewall',
           'schedule',
           'docker_clean',
-          'stig'
+          'stig',
+          'kafka.ca',
+          'kafka.ssl'
           ],
       'so-standalone': [
           'salt.master',
@@ -195,7 +197,6 @@
           'schedule',
           'docker_clean',
           'kafka',
-          'elasticsearch.ca',
           'stig'
           ],
       'so-desktop': [

salt/common/tools/sbin/so-log-check

@@ -241,6 +241,7 @@ exclude_log "mysqld.log"   # MySQL is removed as of 2.4.70, logs may still be on
 exclude_log "soctopus.log" # Soctopus is removed as of 2.4.70, logs may still be on disk
 exclude_log "agentstatus.log" # ignore this log since it tracks agents in error state
 exclude_log "detections_runtime-status_yara.log" # temporarily ignore this log until Detections is more stable
+exclude_log "/nsm/kafka/data/" # ignore Kafka data directory from log check.
 
 for log_file in $(cat /tmp/log_check_files); do
     status "Checking log file $log_file"

salt/elasticsearch/config.map.jinja

@@ -1,23 +1,37 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS with context %}
 
 {% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %}
 
-{# ES_LOGSTASH_NODES is the same as LOGSTASH_NODES from logstash/map.jinja but heavynodes and fleet nodes are removed #}
-{% set ES_LOGSTASH_NODES = [] %}
-{% set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
+{# this is a list of dicts containing hostname:ip for elasticsearch nodes that need to know about each other for cluster #}
+{% set ELASTICSEARCH_SEED_HOSTS = [] %}
+{% set node_data = salt['pillar.get']('elasticsearch:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
 {% for node_type, node_details in node_data.items() | sort %}
-{%   if node_type not in ['heavynode', 'fleet'] %}
+{%   if node_type != 'heavynode' %}
 {%     for hostname in node_data[node_type].keys() %}
-{%       do ES_LOGSTASH_NODES.append({hostname:node_details[hostname].ip}) %}
+{%       do ELASTICSEARCH_SEED_HOSTS.append({hostname:node_details[hostname].ip}) %}
 {%     endfor %}
 {%   endif %}
 {% endfor %}
 
+{# this is a list of dicts containing hostname:ip of all nodes running elasticsearch #}
+{% set ELASTICSEARCH_NODES = [] %}
+{% set node_data = salt['pillar.get']('elasticsearch:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
+{% for node_type, node_details in node_data.items() %}
+{%   for hostname in node_data[node_type].keys() %}
+{%     do ELASTICSEARCH_NODES.append({hostname:node_details[hostname].ip}) %}
+{%   endfor %}
+{% endfor %}
+
 {% if grains.id.split('_') | last in ['manager','managersearch','standalone'] %}
-    {% if ES_LOGSTASH_NODES | length > 1 %}
+    {% if ELASTICSEARCH_SEED_HOSTS | length > 1 %}
       {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.update({'discovery': {'seed_hosts': []}}) %}
-      {% for NODE in ES_LOGSTASH_NODES %}
+      {% for NODE in ELASTICSEARCH_SEED_HOSTS %}
         {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.discovery.seed_hosts.append(NODE.keys()|first) %}
       {% endfor %}
     {% endif %}

salt/elasticsearch/defaults.yaml

@@ -296,7 +296,7 @@ elasticsearch:
                   limit: 1500
               number_of_replicas: 0
               number_of_shards: 1
-              refresh_interval: 30s
+              refresh_interval: 1s
               sort:
                 field: '@timestamp'
                 order: desc

salt/elasticsearch/enabled.sls

@@ -7,8 +7,8 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKER %}
-{%   from 'logstash/map.jinja' import LOGSTASH_NODES %}
-{%   from 'elasticsearch/config.map.jinja' import ES_LOGSTASH_NODES %}
+{%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
+{%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
 {%   set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %}
 {%   from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %}
@@ -27,7 +27,7 @@ so-elasticsearch:
       - sobridge:
         - ipv4_address: {{ DOCKER.containers['so-elasticsearch'].ip }}
     - extra_hosts:
-    {% for node in LOGSTASH_NODES %}
+    {% for node in ELASTICSEARCH_NODES %}
     {%   for hostname, ip in node.items() %}
       - {{hostname}}:{{ip}}
     {%   endfor %}
@@ -38,7 +38,7 @@ so-elasticsearch:
       {% endfor %}
     {% endif %}
     - environment:
-      {% if ES_LOGSTASH_NODES | length == 1 or GLOBALS.role == 'so-heavynode' %}
+      {% if ELASTICSEARCH_SEED_HOSTS | length == 1 or GLOBALS.role == 'so-heavynode' %}
       - discovery.type=single-node
       {% endif %}
       - ES_JAVA_OPTS=-Xms{{ GLOBALS.elasticsearch.es_heap }} -Xmx{{ GLOBALS.elasticsearch.es_heap }} -Des.transport.cname_in_publish_address=true -Dlog4j2.formatMsgNoLookups=true

salt/elasticsearch/template.map.jinja

@@ -1,3 +1,8 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+
 {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
 {% set DEFAULT_GLOBAL_OVERRIDES = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings.pop('global_overrides') %}
 
@@ -17,10 +22,26 @@
 {% set ES_INDEX_SETTINGS = {} %}
 {% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update(salt['defaults.merge'](ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %}
 {% for index, settings in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.items() %}
-{#   if policy isn't defined in the original index settings, then dont merge policy from the global_overrides #}
-{#   this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #}
-{%   if not ES_INDEX_SETTINGS_ORIG[index].policy is defined and ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %}
-{%     do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %}
+
+{#   prevent this action from being performed on custom defined indices. #}
+{#   the custom defined index is not present in either of the dictionaries and fails to reder. #}
+{%   if index in ES_INDEX_SETTINGS_ORIG and index in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES %}
+
+{#     dont merge policy from the global_overrides if policy isn't defined in the original index settingss #}
+{#     this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #}
+{%     if not ES_INDEX_SETTINGS_ORIG[index].policy is defined and ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %}
+{%       do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %}
+{%     endif %}
+
+{#     this prevents and index from inderiting a policy phase from global overrides if it wasnt defined in the defaults. #}
+{%     if ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %}
+{%       for phase in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.copy() %}
+{%         if ES_INDEX_SETTINGS_ORIG[index].policy.phases[phase] is not defined %}
+{%           do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.pop(phase) %}
+{%         endif %}
+{%       endfor %}
+{%     endif %}
+
 {%   endif %}
 
 {%   if settings.index_template is defined %}

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-settings

@@ -4,7 +4,7 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 {%- from 'vars/globals.map.jinja' import GLOBALS %}
-{%- set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
+{%- set node_data = salt['pillar.get']('elasticsearch:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
 
 . /usr/sbin/so-common
 

salt/idstools/tools/sbin_jinja/so-rule-update

@@ -11,8 +11,8 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
 {%- set proxy = salt['pillar.get']('manager:proxy') %}
 {%- set noproxy = salt['pillar.get']('manager:no_proxy', '') %}
 
-# Download the rules from the internet
 {%- if proxy %}
+# Download the rules from the internet
     export http_proxy={{ proxy }}
     export https_proxy={{ proxy }}
     export no_proxy="{{ noproxy }}"
@@ -20,8 +20,8 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
 
     mkdir -p /nsm/rules/suricata
     chown -R socore:socore /nsm/rules/suricata
+{%- if not GLOBALS.airgap %}
 # Download the rules from the internet
-{%- if GLOBALS.airgap != 'True' %}
 {%-   if IDSTOOLSMERGED.config.ruleset == 'ETOPEN' %}
     docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force
 {%-   elif IDSTOOLSMERGED.config.ruleset == 'ETPRO' %}

salt/kafka/ca.sls

@@ -0,0 +1,37 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states or sls in allowed_states %}
+{%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   set KAFKATRUST = salt['pillar.get']('kafka:truststore') %}
+
+kafkaconfdir:
+  file.directory:
+    - name: /opt/so/conf/kafka
+    - user: 960
+    - group: 960
+    - makedirs: True
+
+{% if GLOBALS.is_manager %}
+# Manager runs so-kafka-trust to create truststore for Kafka ssl communication
+kafka_truststore:
+  cmd.script:
+    - source: salt://kafka/tools/sbin_jinja/so-kafka-trust
+    - template: jinja
+    - cwd: /opt/so
+    - defaults:
+        GLOBALS: {{ GLOBALS }}
+        KAFKATRUST: {{ KAFKATRUST }}
+{% endif %}
+
+kafkacertz:
+  file.managed:
+    - name: /opt/so/conf/kafka/kafka-truststore.jks
+    - source: salt://kafka/files/kafka-truststore
+    - user: 960
+    - group: 931
+
+{% endif %}

salt/kafka/config.map.jinja

@@ -6,7 +6,8 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 
 {% set KAFKA_NODES_PILLAR = salt['pillar.get']('kafka:nodes') %}
-{% set KAFKA_PASSWORD = salt['pillar.get']('kafka:password') %}
+{% set KAFKA_PASSWORD = salt['pillar.get']('kafka:config:password') %}
+{% set KAFKA_TRUSTPASS = salt['pillar.get']('kafka:config:trustpass') %}
 
 {# Create list of KRaft controllers #}
 {% set controllers = [] %}
@@ -67,19 +68,12 @@
 
 {% endif %}
 
-{# If a password other than PLACEHOLDER isn't set remove it from the server.properties #}
-{% if KAFKAMERGED.config.broker.ssl_x_truststore_x_password == 'PLACEHOLDER' %}
-{%   do KAFKAMERGED.config.broker.pop('ssl_x_truststore_x_password') %}
-{% endif %}
-
-{% if KAFKAMERGED.config.controller.ssl_x_truststore_x_password == 'PLACEHOLDER' %}
-{%   do KAFKAMERGED.config.controller.pop('ssl_x_truststore_x_password') %}
-{% endif %}
+{# Truststore config #}
+{% do KAFKAMERGED.config.broker.update({'ssl_x_truststore_x_password': KAFKA_TRUSTPASS }) %}
+{% do KAFKAMERGED.config.controller.update({'ssl_x_truststore_x_password': KAFKA_TRUSTPASS }) %}
+{% do KAFKAMERGED.config.client.update({'ssl_x_truststore_x_password': KAFKA_TRUSTPASS }) %}
 
 {# Client properties stuff #}
-{% if KAFKAMERGED.config.client.ssl_x_truststore_x_password == 'PLACEHOLDER' %}
-{%   do KAFKAMERGED.config.client.pop('ssl_x_truststore_x_password') %}
-{% endif %}
 {% do KAFKAMERGED.config.client.update({'ssl_x_keystore_x_password': KAFKA_PASSWORD }) %}
 
 {% if 'broker' in node_type %}

salt/kafka/config.sls

@@ -7,18 +7,22 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 
-include:
-  - ssl
-
 kafka_group:
   group.present:
     - name: kafka
     - gid: 960
 
-kafka:
+kafka_user:
   user.present:
+    - name: kafka
     - uid: 960
     - gid: 960
+    - home: /opt/so/conf/kafka
+    - createhome: False
+
+kafka_home_dir:
+  file.absent:
+    - name: /home/kafka
 
 kafka_sbin_tools:
   file.recurse:
@@ -28,6 +32,17 @@ kafka_sbin_tools:
     - group: 960
     - file_mode: 755
 
+kafka_sbin_jinja_tools:
+  file.recurse:
+    - name: /usr/sbin
+    - source: salt://kafka/tools/sbin_jinja
+    - user: 960
+    - group: 960
+    - file_mode: 755
+    - template: jinja
+    - defaults:
+        GLOBALS: {{ GLOBALS }}
+
 kafka_log_dir:
   file.directory:
     - name: /opt/so/log/kafka

salt/kafka/defaults.yaml

@@ -1,10 +1,12 @@
 kafka:
   enabled: False
   cluster_id:
-  password:
   controllers:
   reset:
+  logstash: []
   config:
+    password:
+    trustpass:
     broker:
       advertised_x_listeners:
       auto_x_create_x_topics_x_enable: true
@@ -30,16 +32,16 @@ kafka:
       ssl_x_keystore_x_location: /etc/pki/kafka.p12
       ssl_x_keystore_x_type: PKCS12
       ssl_x_keystore_x_password:
-      ssl_x_truststore_x_location: /etc/pki/java/sos/cacerts
-      ssl_x_truststore_x_password: PLACEHOLDER
-      ssl_x_truststore_x_type: PEM
+      ssl_x_truststore_x_location: /etc/pki/kafka-truststore.jks
+      ssl_x_truststore_x_type: JKS
+      ssl_x_truststore_x_password:
       transaction_x_state_x_log_x_min_x_isr: 1
       transaction_x_state_x_log_x_replication_x_factor: 1
     client:
       security_x_protocol: SSL
-      ssl_x_truststore_x_location: /etc/pki/java/sos/cacerts
-      ssl_x_truststore_x_password: PLACEHOLDER
-      ssl_x_truststore_x_type: PEM
+      ssl_x_truststore_x_location: /etc/pki/kafka-truststore.jks
+      ssl_x_truststore_x_type: JKS
+      ssl_x_truststore_x_password:
       ssl_x_keystore_x_location: /etc/pki/kafka.p12
       ssl_x_keystore_x_type: PKCS12
       ssl_x_keystore_x_password:
@@ -57,6 +59,6 @@ kafka:
       ssl_x_keystore_x_location: /etc/pki/kafka.p12
       ssl_x_keystore_x_type: PKCS12
       ssl_x_keystore_x_password:
-      ssl_x_truststore_x_location: /etc/pki/java/sos/cacerts
-      ssl_x_truststore_x_password: PLACEHOLDER
-      ssl_x_truststore_x_type: PEM
+      ssl_x_truststore_x_location: /etc/pki/kafka-truststore.jks
+      ssl_x_truststore_x_type: JKS
+      ssl_x_truststore_x_password:

salt/kafka/disabled.sls

@@ -22,4 +22,13 @@ ensure_default_pipeline:
     - name: |
         /usr/sbin/so-yaml.py replace /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.enabled False;
         /usr/sbin/so-yaml.py replace /opt/so/saltstack/local/pillar/global/soc_global.sls global.pipeline REDIS
-{% endif %}
+{% endif %}
+
+{# If Kafka has never been manually enabled, the 'Kafka' user does not exist. In this case certs for Kafka should not exist since they'll be owned by uid 960 #}
+{% for cert in ['kafka-client.crt','kafka-client.key','kafka.crt','kafka.key','kafka-logstash.crt','kafka-logstash.key','kafka-logstash.p12','kafka.p12','elasticfleet-kafka.p8'] %}
+check_kafka_cert_{{cert}}:
+  file.absent:
+    - name: /etc/pki/{{cert}}
+    - onlyif: stat -c %U /etc/pki/{{cert}} | grep -q UNKNOWN
+    - show_changes: False
+{% endfor %}

salt/kafka/enabled.sls

@@ -17,10 +17,11 @@
 {%   if 'gmd' in salt['pillar.get']('features', []) %}
 
 include:
-  - elasticsearch.ca
-  - kafka.sostatus
+  - kafka.ca
   - kafka.config
+  - kafka.ssl
   - kafka.storage
+  - kafka.sostatus
 
 so-kafka:
   docker_container.running:
@@ -49,7 +50,7 @@ so-kafka:
       {% endfor %}
     - binds:
       - /etc/pki/kafka.p12:/etc/pki/kafka.p12:ro
-      - /etc/pki/tls/certs/intca.crt:/etc/pki/java/sos/cacerts:ro
+      - /opt/so/conf/kafka/kafka-truststore.jks:/etc/pki/kafka-truststore.jks:ro
       - /nsm/kafka/data/:/nsm/kafka/data/:rw
       - /opt/so/log/kafka:/opt/kafka/logs/:rw
       - /opt/so/conf/kafka/server.properties:/opt/kafka/config/kraft/server.properties:ro
@@ -58,6 +59,9 @@ so-kafka:
       {% for sc in ['server', 'client'] %}
       - file: kafka_kraft_{{sc}}_properties
       {% endfor %}
+      - file: kafkacertz
+    - require:
+      - file: kafkacertz
 
 delete_so-kafka_so-status.disabled:
   file.uncomment:

salt/kafka/init.sls

@@ -18,7 +18,12 @@ include:
   - kafka.nodes
 {% endif %}
 {% if GLOBALS.pipeline == "KAFKA" and KAFKAMERGED.enabled %}
+{%   if grains.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-receiver'] %}
   - kafka.enabled
+{# Searchnodes only run kafka.ssl state when Kafka is enabled #}
+{%   elif grains.role == "so-searchnode" %}
+  - kafka.ssl
+{%   endif %}
 {% else %}
   - kafka.disabled
 {% endif %}

salt/kafka/soc_kafka.yaml

@@ -8,19 +8,31 @@ kafka:
     advanced: True
     sensitive: True
     helpLink: kafka.html
-  password:
-    description: The password to use for the Kafka certificates.
-    sensitive: True
-    helpLink: kafka.html
   controllers:
     description: A comma-separated list of hostnames that will act as Kafka controllers. These hosts will be responsible for managing the Kafka cluster. Note that only manager and receiver nodes are eligible to run Kafka. This configuration needs to be set before enabling Kafka. Failure to do so may result in Kafka topics becoming unavailable requiring manual intervention to restore functionality or reset Kafka, either of which can result in data loss.
-    forcedType: "string"
+    forcedType: string
     helpLink: kafka.html
   reset:
     description: Disable and reset the Kafka cluster. This will remove all Kafka data including logs that may have not yet been ingested into Elasticsearch and reverts the grid to using REDIS as the global pipeline. This is useful when testing different Kafka configurations such as rearranging Kafka brokers / controllers allowing you to reset the cluster rather than manually fixing any issues arising from attempting to reassign a Kafka broker into a controller. Enter 'YES_RESET_KAFKA' and submit to disable and reset Kafka. Make any configuration changes required and re-enable Kafka when ready. This action CANNOT be reversed.
     advanced: True
     helpLink: kafka.html
+  logstash:
+    description: By default logstash is disabled when Kafka is enabled. This option allows you to specify any hosts you would like to re-enable logstash on alongside Kafka.
+    forcedType: "[]string"
+    multiline: True
+    advanced: True
+    helpLink: kafka.html
   config:
+    password:
+      description: The password used for the Kafka certificates.
+      readonly: True
+      sensitive: True
+      helpLink: kafka.html
+    trustpass:
+      description: The password used for the Kafka truststore.
+      readonly: True
+      sensitive: True
+      helpLink: kafka.html
     broker:
       advertised_x_listeners:
         description: Specify the list of listeners (hostname and port) that Kafka brokers provide to clients for communication.
@@ -128,6 +140,10 @@ kafka:
         description: The trust store file location within the Docker container.
         title: ssl.truststore.location
         helpLink: kafka.html
+      ssl_x_truststore_x_type:
+        description: The trust store file format.
+        title: ssl.truststore.type
+        helpLink: kafka.html
       ssl_x_truststore_x_password:
         description: The trust store file password. If null, the trust store file is still use, but integrity checking is disabled. Invalid for PEM format.
         title: ssl.truststore.password
@@ -167,6 +183,10 @@ kafka:
         description: The trust store file location within the Docker container.
         title: ssl.truststore.location
         helpLink: kafka.html
+      ssl_x_truststore_x_type:
+        description: The trust store file format.
+        title: ssl.truststore.type
+        helpLink: kafka.html
       ssl_x_truststore_x_password:
         description: The trust store file password. If null, the trust store file is still use, but integrity checking is disabled. Invalid for PEM format.
         title: ssl.truststore.password

salt/kafka/ssl.sls

@@ -0,0 +1,201 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states or sls in allowed_states %}
+{%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   set kafka_password = salt['pillar.get']('kafka:config:password') %}
+
+include:
+  - ca.dirs
+    {% set global_ca_server = [] %}
+    {% set x509dict = salt['mine.get'](GLOBALS.manager | lower~'*', 'x509.get_pem_entries') %}
+    {% for host in x509dict %}
+      {% if 'manager' in host.split('_')|last or host.split('_')|last == 'standalone' %}
+        {% do global_ca_server.append(host) %}
+      {% endif %}
+    {% endfor %}
+    {% set ca_server = global_ca_server[0] %}
+
+{% if GLOBALS.pipeline == "KAFKA" %}
+
+{%   if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone'] %}
+kafka_client_key:
+  x509.private_key_managed:
+    - name: /etc/pki/kafka-client.key
+    - keysize: 4096
+    - backup: True
+    - new: True
+    {% if salt['file.file_exists']('/etc/pki/kafka-client.key') -%}
+    - prereq:
+      - x509: /etc/pki/kafka-client.crt
+    {%- endif %}
+    - retry:
+        attempts: 5
+        interval: 30
+
+kafka_client_crt:
+  x509.certificate_managed:
+    - name: /etc/pki/kafka-client.crt
+    - ca_server: {{ ca_server }}
+    - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
+    - signing_policy: kafka
+    - private_key: /etc/pki/kafka-client.key
+    - CN: {{ GLOBALS.hostname }}
+    - days_remaining: 0
+    - days_valid: 820
+    - backup: True
+    - timeout: 30
+    - retry:
+        attempts: 5
+        interval: 30
+
+kafka_client_key_perms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/kafka-client.key
+    - mode: 640
+    - user: 960
+    - group: 939
+
+kafka_client_crt_perms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/kafka-client.crt
+    - mode: 640
+    - user: 960
+    - group: 939
+{%   endif %}
+
+{%   if GLOBALS.role in ['so-manager', 'so-managersearch','so-receiver', 'so-standalone'] %}
+kafka_key:
+  x509.private_key_managed:
+    - name: /etc/pki/kafka.key
+    - keysize: 4096
+    - backup: True
+    - new: True
+    {% if salt['file.file_exists']('/etc/pki/kafka.key') -%}
+    - prereq:
+      - x509: /etc/pki/kafka.crt
+    {%- endif %}
+    - retry:
+        attempts: 5
+        interval: 30
+
+kafka_crt:
+  x509.certificate_managed:
+    - name: /etc/pki/kafka.crt
+    - ca_server: {{ ca_server }}
+    - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
+    - signing_policy: kafka
+    - private_key: /etc/pki/kafka.key
+    - CN: {{ GLOBALS.hostname }}
+    - days_remaining: 0
+    - days_valid: 820
+    - backup: True
+    - timeout: 30
+    - retry:
+        attempts: 5
+        interval: 30
+  cmd.run:
+    - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/kafka.key -in /etc/pki/kafka.crt -export -out /etc/pki/kafka.p12 -nodes -passout pass:{{ kafka_password }}"
+    - onchanges:
+      - x509: /etc/pki/kafka.key
+kafka_key_perms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/kafka.key
+    - mode: 640
+    - user: 960
+    - group: 939
+
+kafka_crt_perms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/kafka.crt
+    - mode: 640
+    - user: 960
+    - group: 939
+
+kafka_pkcs12_perms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/kafka.p12
+    - mode: 640
+    - user: 960
+    - group: 939
+{%   endif %}
+
+# Standalone needs kafka-logstash for automated testing. Searchnode/manager search need it for logstash to consume from Kafka.
+# Manager will have cert, but be unused until a pipeline is created and logstash enabled.
+{%   if GLOBALS.role in ['so-standalone', 'so-managersearch', 'so-searchnode', 'so-manager'] %}
+kafka_logstash_key:
+  x509.private_key_managed:
+    - name: /etc/pki/kafka-logstash.key
+    - keysize: 4096
+    - backup: True
+    - new: True
+    {% if salt['file.file_exists']('/etc/pki/kafka-logstash.key') -%}
+    - prereq:
+      - x509: /etc/pki/kafka-logstash.crt
+    {%- endif %}
+    - retry:
+        attempts: 5
+        interval: 30
+
+kafka_logstash_crt:
+  x509.certificate_managed:
+    - name: /etc/pki/kafka-logstash.crt
+    - ca_server: {{ ca_server }}
+    - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
+    - signing_policy: kafka
+    - private_key: /etc/pki/kafka-logstash.key
+    - CN: {{ GLOBALS.hostname }}
+    - days_remaining: 0
+    - days_valid: 820
+    - backup: True
+    - timeout: 30
+    - retry:
+        attempts: 5
+        interval: 30
+  cmd.run:
+    - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/kafka-logstash.key -in /etc/pki/kafka-logstash.crt -export -out /etc/pki/kafka-logstash.p12 -nodes -passout pass:{{ kafka_password }}"
+    - onchanges:
+      - x509: /etc/pki/kafka-logstash.key
+
+kafka_logstash_key_perms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/kafka-logstash.key
+    - mode: 640
+    - user: 931
+    - group: 939
+
+kafka_logstash_crt_perms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/kafka-logstash.crt
+    - mode: 640
+    - user: 931
+    - group: 939
+
+kafka_logstash_pkcs12_perms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/kafka-logstash.p12
+    - mode: 640
+    - user: 931
+    - group: 939
+
+{%   endif %}
+{% endif %}
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/kafka/tools/sbin_jinja/so-kafka-trust

@@ -0,0 +1,13 @@
+#!/bin/bash
+#
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+{% set TRUSTPASS = salt['pillar.get']('kafka:config:trustpass') %}
+
+if [ ! -f /opt/so/saltstack/local/salt/kafka/files/kafka-truststore ]; then
+    docker run -v /etc/pki/ca.crt:/etc/pki/ca.crt --name so-kafkatrust --user root --entrypoint /opt/java/openjdk/bin/keytool {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kafka:{{ GLOBALS.so_version }} -import -file /etc/pki/ca.crt -alias SOS -keystore /etc/pki/kafka-truststore -storepass {{ TRUSTPASS }} -storetype jks -noprompt
+    docker cp so-kafkatrust:/etc/pki/kafka-truststore /opt/so/saltstack/local/salt/kafka/files/kafka-truststore
+    docker rm so-kafkatrust
+fi

salt/logstash/defaults.yaml

@@ -7,9 +7,7 @@ logstash:
         - search
       receiver:
         - receiver
-      heavynode:
-        - manager
-        - search
+      heavynode: []
       searchnode:
         - search
       manager:
@@ -27,7 +25,7 @@ logstash:
       - so/0011_input_endgame.conf
       - so/0012_input_elastic_agent.conf.jinja
       - so/0013_input_lumberjack_fleet.conf
-      - so/9999_output_redis.conf.jinja        
+      - so/9999_output_redis.conf.jinja
     receiver:
       - so/0011_input_endgame.conf
       - so/0012_input_elastic_agent.conf.jinja
@@ -37,7 +35,6 @@ logstash:
       - so/0900_input_redis.conf.jinja
       - so/9805_output_elastic_agent.conf.jinja
       - so/9900_output_endgame.conf.jinja
-      - so/0800_input_kafka.conf.jinja
     custom0: []
     custom1: []
     custom2: []

salt/logstash/enabled.sls

@@ -14,6 +14,11 @@
 include:
 {%   if GLOBALS.role not in ['so-receiver','so-fleet'] %}
   - elasticsearch.ca
+{%   endif %}
+{# Kafka ca runs on nodes that can run logstash for Kafka input / output. Only when Kafka is global pipeline #}
+{%   if GLOBALS.role in ['so-searchnode', 'so-manager', 'so-managersearch', 'so-receiver', 'so-standalone'] and GLOBALS.pipeline == 'KAFKA' %}
+  - kafka.ca
+  - kafka.ssl
 {%   endif %}
   - logstash.config
   - logstash.sostatus
@@ -79,8 +84,9 @@ so-logstash:
       - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro
       - /opt/so/conf/ca/tls-ca-bundle.pem:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem:ro
       {% endif %}
-      {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-searchnode'] %}
+      {% if GLOBALS.pipeline == "KAFKA" and GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-searchnode'] %}
       - /etc/pki/kafka-logstash.p12:/usr/share/logstash/kafka-logstash.p12:ro
+      - /opt/so/conf/kafka/kafka-truststore.jks:/etc/pki/kafka-truststore.jks:ro
       {% endif %}
       {% if GLOBALS.role == 'so-eval' %}
       - /nsm/zeek:/nsm/zeek:ro
@@ -105,6 +111,9 @@ so-logstash:
       - file: ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}
         {% endfor %}
       {% endfor %}
+      {% if GLOBALS.pipeline == 'KAFKA' and GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-searchnode'] %}
+      - file: kafkacertz
+      {% endif %}
     - require:
       {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-receiver'] %}
       - x509: etc_filebeat_crt
@@ -118,6 +127,9 @@ so-logstash:
       - file: cacertz
       - file: capemz
       {% endif %}
+      {% if GLOBALS.pipeline == 'KAFKA' and GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-searchnode'] %}
+      - file: kafkacertz
+      {% endif %}
 
 delete_so-logstash_so-status.disabled:
   file.uncomment:

salt/logstash/init.sls

@@ -4,13 +4,9 @@
 # Elastic License 2.0.
 
 {% from 'logstash/map.jinja' import LOGSTASH_MERGED %}
-{% from 'kafka/map.jinja' import KAFKAMERGED %}
 
 include:
-{# Disable logstash when Kafka is enabled except when the role is standalone #}
-{% if LOGSTASH_MERGED.enabled and grains.role == 'so-standalone' %}
-  - logstash.enabled
-{% elif LOGSTASH_MERGED.enabled and not KAFKAMERGED.enabled %}
+{% if LOGSTASH_MERGED.enabled %}
   - logstash.enabled
 {% else %}
   - logstash.disabled

salt/logstash/map.jinja

@@ -6,24 +6,40 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% import_yaml 'logstash/defaults.yaml' as LOGSTASH_DEFAULTS %}
 {% set LOGSTASH_MERGED = salt['pillar.get']('logstash', LOGSTASH_DEFAULTS.logstash, merge=True) %}
+{% set KAFKA_LOGSTASH = salt['pillar.get']('kafka:logstash', []) %}
 
-{% set REDIS_NODES = [] %}
-{# LOGSTASH_NODES is the same as ES_LOGSTASH_NODES from elasticsearch/config.map.jinja but heavynodes are present #}
+{# used to store the redis nodes that logstash needs to know about to pull from the queue #}
+{% set LOGSTASH_REDIS_NODES = [] %}
+{# stores all logstash nodes #}
 {% set LOGSTASH_NODES = [] %}
-{% set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
+{% set logstash_node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
+{% set redis_node_data = salt['pillar.get']('redis:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
 
-{% for node_type, node_details in node_data.items() | sort %}
+{% for node_type, node_details in redis_node_data.items() | sort %}
 {%   if GLOBALS.role in ['so-searchnode', 'so-standalone', 'so-managersearch', 'so-fleet'] %}
 {%     if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %}
-{%       for hostname in node_data[node_type].keys() %}
-{%         do REDIS_NODES.append({hostname:node_details[hostname].ip}) %}
+{%       for hostname in redis_node_data[node_type].keys() %}
+{%         do LOGSTASH_REDIS_NODES.append({hostname:node_details[hostname].ip}) %}
 {%       endfor %}
 {%     endif %}
-{%   else %}
-{%     do REDIS_NODES.append({GLOBALS.hostname:GLOBALS.node_ip}) %}
 {%   endif %}
+{% endfor %}
 
-{%   for hostname in node_data[node_type].keys() %}
+{% for node_type, node_details in logstash_node_data.items() | sort %}
+{%   for hostname in logstash_node_data[node_type].keys() %}
 {%     do LOGSTASH_NODES.append({hostname:node_details[hostname].ip}) %}
 {%   endfor %}
 {% endfor %}
+
+{# Append Kafka input pipeline when Kafka is enabled #}
+{% if GLOBALS.pipeline == 'KAFKA' %}
+{%   do LOGSTASH_MERGED.defined_pipelines.search.remove('so/0900_input_redis.conf.jinja') %}
+{%   do LOGSTASH_MERGED.defined_pipelines.search.append('so/0800_input_kafka.conf.jinja') %}
+{%   do LOGSTASH_MERGED.defined_pipelines.manager.append('so/0800_input_kafka.conf.jinja') %}
+{# Disable logstash on manager & receiver nodes unless it has an override configured #}
+{%   if not KAFKA_LOGSTASH %}
+{%     if GLOBALS.role in ['so-manager', 'so-receiver'] and GLOBALS.hostname not in KAFKA_LOGSTASH %}
+{%       do LOGSTASH_MERGED.update({'enabled': False}) %}
+{%     endif %}
+{%   endif %}
+{% endif %}

salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja

@@ -1,4 +1,5 @@
-{%- set kafka_password = salt['pillar.get']('kafka:password') %}
+{%- set kafka_password = salt['pillar.get']('kafka:config:password') %}
+{%- set kafka_trustpass = salt['pillar.get']('kafka:config:trustpass') %}
 {%- set kafka_brokers = salt['pillar.get']('kafka:nodes', {}) %}
 {%- set brokers = [] %}
 
@@ -22,8 +23,8 @@ input {
         ssl_keystore_location => '/usr/share/logstash/kafka-logstash.p12'
         ssl_keystore_password => '{{ kafka_password }}'
         ssl_keystore_type => 'PKCS12'
-        ssl_truststore_location => '/etc/pki/ca-trust/extracted/java/cacerts'
-        ssl_truststore_password => 'changeit'
+        ssl_truststore_location => '/etc/pki/kafka-truststore.jks'
+        ssl_truststore_password => '{{ kafka_trustpass }}'
         decorate_events => true
         tags => [ "elastic-agent", "input-{{ GLOBALS.hostname}}", "kafka" ]
     }

salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja

@@ -1,8 +1,8 @@
-{%- from 'logstash/map.jinja' import REDIS_NODES with context %}
+{%- from 'logstash/map.jinja' import LOGSTASH_REDIS_NODES with context %}
 {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %}
 
-{%- for index in range(REDIS_NODES|length) %}
-{%-   for host in REDIS_NODES[index] %}
+{%- for index in range(LOGSTASH_REDIS_NODES|length) %}
+{%-   for host in LOGSTASH_REDIS_NODES[index] %}
 input {
         redis {
                 host => '{{ host }}'

salt/manager/tools/sbin/so-minion

@@ -528,7 +528,6 @@ function createHEAVYNODE() {
 	pcapspace
 	add_elasticsearch_to_minion
 	add_elastic_agent_to_minion
-	add_logstash_to_minion
 	add_sensor_to_minion
 	add_strelka_to_minion
 	add_redis_to_minion

salt/manager/tools/sbin/so-user-add

@@ -1,2 +0,0 @@
-#!/bin/bash
-so-user add --email $1

salt/manager/tools/sbin/so-user-disable

@@ -1,2 +0,0 @@
-#!/bin/bash
-so-user disable --email $1

salt/manager/tools/sbin/so-user-enable

@@ -1,2 +0,0 @@
-#!/bin/bash
-so-user enable --email $1

salt/manager/tools/sbin/so-user-list

@@ -1,2 +0,0 @@
-#!/bin/bash
-so-user list

salt/manager/tools/sbin/soup

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
@@ -30,7 +30,7 @@ check_err() {
   [[ $ERR_HANDLED == true ]] && exit $exit_code
 
   if [[ $exit_code -ne 0 ]]; then
-  
+
     set +e
     systemctl_func "start" "$cron_service_name"
     systemctl_func "start" "salt-master"
@@ -108,7 +108,7 @@ add_common() {
 }
 
 airgap_mounted() {
-  # Let's see if the ISO is already mounted. 
+  # Let's see if the ISO is already mounted.
   if [[ -f /tmp/soagupdate/SecurityOnion/VERSION ]]; then
     echo "The ISO is already mounted"
   else
@@ -116,8 +116,8 @@ airgap_mounted() {
       echo "This is airgap. Ask for a location."
       echo ""
       cat << EOF
-In order for soup to proceed, the path to the downloaded Security Onion ISO file, or the path to the CD-ROM or equivalent device containing the ISO media must be provided. 
-For example, if you have copied the new Security Onion ISO file to your home directory, then the path might look like /home/myuser/securityonion-2.x.y.iso. 
+In order for soup to proceed, the path to the downloaded Security Onion ISO file, or the path to the CD-ROM or equivalent device containing the ISO media must be provided.
+For example, if you have copied the new Security Onion ISO file to your home directory, then the path might look like /home/myuser/securityonion-2.x.y.iso.
 Or, if you have burned the new ISO onto an optical disk then the path might look like /dev/cdrom.
 
 EOF
@@ -134,7 +134,7 @@ EOF
         exit 0
       else
         echo "ISO has been mounted!"
-      fi  
+      fi
     elif [[ -f $ISOLOC/SecurityOnion/VERSION ]]; then
       ln -s $ISOLOC /tmp/soagupdate
       echo "Found the update content"
@@ -149,7 +149,7 @@ EOF
         echo "Device has been mounted!"
       fi
     else
-      echo "Could not find Security Onion ISO content at ${ISOLOC}" 
+      echo "Could not find Security Onion ISO content at ${ISOLOC}"
       echo "Ensure the path you entered is correct, and that you verify the ISO that you downloaded."
       exit 0
     fi
@@ -195,7 +195,7 @@ check_airgap() {
       UPDATE_DIR=/tmp/soagupdate/SecurityOnion
       AGDOCKER=/tmp/soagupdate/docker
       AGREPO=/tmp/soagupdate/minimal/Packages
-  else 
+  else
       is_airgap=1
   fi
 }
@@ -308,6 +308,21 @@ clone_to_tmp() {
   fi
 }
 
+disable_logstash_heavynodes() {
+  c=0
+  printf "\nChecking for heavynodes and disabling Logstash if they exist\n"
+  for file in /opt/so/saltstack/local/pillar/minions/*.sls; do
+    if [[ "$file" =~ "_heavynode.sls" && ! "$file" =~ "/opt/so/saltstack/local/pillar/minions/adv_" ]]; then
+      if [ "$c" -eq 0 ]; then
+        c=$((c + 1))
+        FINAL_MESSAGE_QUEUE+=("Logstash has been disabled on all heavynodes. It can be re-enabled via Grid Configuration in SOC.")
+      fi
+      echo "Disabling Logstash for: $file"
+      so-yaml.py replace "$file" logstash.enabled False
+    fi
+  done
+}
+
 enable_highstate() {
     echo "Enabling highstate."
     salt-call state.enable highstate -l info --local
@@ -385,17 +400,18 @@ preupgrade_changes() {
     [[ "$INSTALLEDVERSION" == 2.4.50 ]] && up_to_2.4.60
     [[ "$INSTALLEDVERSION" == 2.4.60 ]] && up_to_2.4.70
     [[ "$INSTALLEDVERSION" == 2.4.70 ]] && up_to_2.4.80
+    [[ "$INSTALLEDVERSION" == 2.4.80 ]] && up_to_2.4.90
     true
 }
 
 postupgrade_changes() {
     # This function is to add any new pillar items if needed.
     echo "Running post upgrade processes."
-    
+
     [[ "$POSTVERSION" == 2.4.2 ]] && post_to_2.4.3
     [[ "$POSTVERSION" == 2.4.3 ]] && post_to_2.4.4
-    [[ "$POSTVERSION" == 2.4.4 ]] && post_to_2.4.5 
-    [[ "$POSTVERSION" == 2.4.5 ]] && post_to_2.4.10 
+    [[ "$POSTVERSION" == 2.4.4 ]] && post_to_2.4.5
+    [[ "$POSTVERSION" == 2.4.5 ]] && post_to_2.4.10
     [[ "$POSTVERSION" == 2.4.10 ]] && post_to_2.4.20
     [[ "$POSTVERSION" == 2.4.20 ]] && post_to_2.4.30
     [[ "$POSTVERSION" == 2.4.30 ]] && post_to_2.4.40
@@ -403,6 +419,7 @@ postupgrade_changes() {
     [[ "$POSTVERSION"  == 2.4.50 ]] && post_to_2.4.60
     [[ "$POSTVERSION"  == 2.4.60 ]] && post_to_2.4.70
     [[ "$POSTVERSION"  == 2.4.70 ]] && post_to_2.4.80
+    [[ "$POSTVERSION"  == 2.4.80 ]] && post_to_2.4.90
     true
 }
 
@@ -482,6 +499,11 @@ post_to_2.4.80() {
   POSTVERSION=2.4.80
 }
 
+post_to_2.4.90() {
+  disable_logstash_heavynodes
+  POSTVERSION=2.4.90
+}
+
 repo_sync() {
   echo "Sync the local repo."
   su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync."
@@ -547,13 +569,13 @@ up_to_2.4.5() {
 
 up_to_2.4.10() {
   echo "Nothing to do for 2.4.10"
-  
+
   INSTALLEDVERSION=2.4.10
 }
 
 up_to_2.4.20() {
   echo "Nothing to do for 2.4.20"
-  
+
   INSTALLEDVERSION=2.4.20
 }
 
@@ -606,7 +628,7 @@ up_to_2.4.50() {
   mkdir /opt/so/rules/nids/suri
   chown socore:socore /opt/so/rules/nids/suri
   mv -v /opt/so/rules/nids/*.rules /opt/so/rules/nids/suri/.
-  
+
   echo "Adding /nsm/elastic-fleet/artifacts to file_roots in /etc/salt/master using so-yaml"
   so-yaml.py append /etc/salt/master file_roots.base  /nsm/elastic-fleet/artifacts
 
@@ -650,6 +672,20 @@ up_to_2.4.80() {
   INSTALLEDVERSION=2.4.80
 }
 
+up_to_2.4.90() {
+  kafkatrust=$(get_random_value)
+  # rearranging the kafka pillar to reduce clutter in SOC UI
+  kafkasavedpass=$(so-yaml.py get /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.password)
+  kafkatrimpass=$(echo "$kafkasavedpass" | sed -n '1 p' )
+  echo "Making changes to the Kafka pillar layout"
+  so-yaml.py remove /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.password
+  so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.password "$kafkatrimpass"
+  so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.trustpass "$kafkatrust"
+  so-elasticsearch-query so-detection*/_settings -X PUT -d '{"index":{"refresh_interval":"1s"}}'
+
+  INSTALLEDVERSION=2.4.90
+}
+
 add_detection_test_pillars() {
   if [[ -n "$SOUP_INTERNAL_TESTING" ]]; then
     echo "Adding detection pillar values for automated testing"
@@ -679,7 +715,7 @@ Documentation: https://docs.securityonion.net/en/2.4/telemetry.html
 ASSIST_EOF
 
     echo -n "Continue the upgrade with SOC Telemetry enabled [Y/n]? "
-  
+
     read -r input
     input=$(echo "${input,,}" | xargs echo -n)
     echo ""
@@ -720,7 +756,7 @@ suricata_idstools_migration() {
   rsync -av /opt/so/rules/nids/suri/local.rules /nsm/backup/detections-migration/suricata/local-rules
   if [[ -f /opt/so/saltstack/local/salt/idstools/rules/local.rules ]]; then
       rsync -av /opt/so/saltstack/local/salt/idstools/rules/local.rules /nsm/backup/detections-migration/suricata/local-rules/local.rules.bak
-  fi 
+  fi
 
   #Tell SOC to migrate
   mkdir -p /opt/so/conf/soc/migrations
@@ -737,7 +773,7 @@ playbook_migration() {
   crontab -l | grep -v 'so-playbook-ruleupdate_cron' | crontab -
 
   if grep -A 1 'playbook:'  /opt/so/saltstack/local/pillar/minions/* | grep -q 'enabled: True'; then
-  
+
     # Check for active Elastalert rules
     active_rules_count=$(find /opt/so/rules/elastalert/playbook/ -type f \( -name "*.yaml" -o -name "*.yml" \) | wc -l)
 
@@ -829,7 +865,7 @@ upgrade_space() {
     fi
   else
       echo "You have enough space for upgrade. Proceeding with soup."
-  fi  
+  fi
 }
 
 unmount_update() {
@@ -887,7 +923,7 @@ upgrade_check() {
     fi
   else
     is_hotfix=false
-  fi 
+  fi
 
 }
 
@@ -899,7 +935,7 @@ upgrade_check_salt() {
     echo "Salt needs to be upgraded to $NEWSALTVERSION."
     UPGRADESALT=1
   fi
-}   
+}
 
 upgrade_salt() {
   SALTUPGRADED=True
@@ -1017,7 +1053,7 @@ apply_hotfix() {
       mv /etc/pki/managerssl.key /etc/pki/managerssl.key.old
       systemctl_func "start" "salt-minion"
       (wait_for_salt_minion "$MINIONID" "5" '/dev/stdout' || fail "Salt minion was not running or ready.") 2>&1 | tee -a "$SOUP_LOG"
-    fi  
+    fi
   else
    echo "No actions required. ($INSTALLEDVERSION/$HOTFIXVERSION)"
   fi
@@ -1046,7 +1082,7 @@ apply_hotfix() {
 
 main() {
   trap 'check_err $?' EXIT
-  
+
   if [ -n "$BRANCH" ]; then
     echo "SOUP will use the $BRANCH branch."
     echo ""
@@ -1230,7 +1266,7 @@ main() {
     echo "Waiting on the Salt Master service to be ready."
     check_salt_master_status || fail  "Can't access salt master or it is not ready. Check $SOUP_LOG for details."
     set -e
-    
+
     echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes."
     (wait_for_salt_minion "$MINIONID" "5" '/dev/stdout' || fail "Salt minion was not running or ready.") 2>&1 | tee -a "$SOUP_LOG"
     highstate
@@ -1281,9 +1317,9 @@ main() {
     if [[ $NUM_MINIONS -gt 1 ]]; then
 
       cat << EOF
-    
-  
-  
+
+
+
 This appears to be a distributed deployment. Other nodes should update themselves at the next Salt highstate (typically within 15 minutes). Do not manually restart anything until you know that all the search/heavy nodes in your deployment are updated. This is especially important if you are using true clustering for Elasticsearch.
 
 Each minion is on a random 15 minute check-in period and things like network bandwidth can be a factor in how long the actual upgrade takes. If you have a heavy node on a slow link, it is going to take a while to get the containers to it. Depending on what changes happened between the versions, Elasticsearch might not be able to talk to said heavy node until the update is complete.
@@ -1336,13 +1372,13 @@ while getopts ":b:f:y" opt; do
         echo "Cannot run soup in unattended mode. You must run soup manually to accept the Elastic License."
         exit 1
       else
-        UNATTENDED=true 
+        UNATTENDED=true
       fi
     ;;
     f )
       ISOLOC="$OPTARG"
     ;;
-    \? ) 
+    \? )
       echo "Usage: soup [-b] [-y] [-f <iso location>]"
       exit 1
     ;;

salt/motd/files/so_motd.jinja

@@ -15,12 +15,11 @@ Access the Security Onion web interface at https://{{ GLOBALS.url_base }}
   {%- endfor -%}
 
   {%- if minions_need_restarted | length > 0 %}
-****************************************************************************************************
-* The following nodes in your Security Onion grid may need to be restarted due to package updates. * 
-* If the node has already been patched, restarted and been up for less than 15 minutes, then it    *
-* may not have updated it's restart_needed status yet. This will cause it to be listed below, even *
-* if it has already been restarted. This feature will be improved in the future.                   *
-****************************************************************************************************
+####################################################################################################
+# The following nodes in your Security Onion grid may need to be restarted due to package updates. #
+# If a node has already been patched and restarted but has been up for less than 15 minutes,       #
+# then it may not have updated its status yet.                                                     #
+####################################################################################################
 
     {% for minion in minions_need_restarted -%}
   {{ minion }}

salt/salt/files/engines.conf

@@ -43,20 +43,20 @@ engines:
                   - cmd.run:
                       cmd: /usr/sbin/so-yaml.py replace /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.enabled True
                   - cmd.run:
-                      cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver' saltutil.kill_all_jobs
+                      cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver or G@role:so-searchnode' saltutil.kill_all_jobs
                   - cmd.run:
                       cmd: salt-call state.apply kafka.nodes
                   - cmd.run:
-                      cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver' state.highstate
+                      cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver or G@role:so-searchnode' state.highstate
               'KAFKA':
                 to:
                   'REDIS':
                   - cmd.run:
                       cmd: /usr/sbin/so-yaml.py replace /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.enabled False
                   - cmd.run:
-                      cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver' saltutil.kill_all_jobs
+                      cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver or G@role:so-searchnode' saltutil.kill_all_jobs
                   - cmd.run:
-                      cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver' state.highstate
+                      cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver or G@role:so-searchnode' state.highstate
         - files:
             - /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls
             - /opt/so/saltstack/local/pillar/kafka/adv_kafka.sls

salt/soc/defaults.map.jinja

@@ -14,7 +14,7 @@
 {% endfor %}
 
 {# add all grid heavy nodes to soc.server.modules.elastic.remoteHostUrls #}
-{% for node_type, minions in salt['pillar.get']('logstash:nodes', {}).items() %}
+{% for node_type, minions in salt['pillar.get']('elasticsearch:nodes', {}).items() %}
 {%   if node_type in ['heavynode'] %}
 {%     for m in minions.keys() %}
 {%       do SOCDEFAULTS.soc.config.server.modules.elastic.remoteHostUrls.append('https://' ~ m ~ ':9200') %}

salt/soc/defaults.yaml

@@ -96,6 +96,12 @@ soc:
         links:
           - '/#/alerts?q=rule.uuid: {:so_detection.publicId|escape} | groupby rule.name event.module* event.severity_label'
         target: ''
+      - name: actionAdd
+        description: actionAddHelp
+        icon: fa-plus
+        links:
+          - '/#/config?s=soc.config.actions'
+        target: ''
     eventFields:
       default:
         - soc_timestamp
@@ -1305,7 +1311,6 @@ soc:
         kratos:
           hostUrl:
         elastalertengine:
-          allowRegex: ''
           autoUpdateEnabled: true
           autoEnabledSigmaRules:
             default:
@@ -1321,7 +1326,6 @@ soc:
           communityRulesImportFrequencySeconds: 86400
           communityRulesImportErrorSeconds: 300
           failAfterConsecutiveErrorCount: 10
-          denyRegex: ''
           elastAlertRulesFolder: /opt/sensoroni/elastalert
           reposFolder: /opt/sensoroni/sigma/repos
           rulesFingerprintFile: /opt/sensoroni/fingerprints/sigma.fingerprint
@@ -1360,6 +1364,8 @@ soc:
           maxLogLength: 1024
           asyncThreshold: 10
           lookupTunnelParent: true
+          maxScrollSize: 10000
+          bulkIndexerWorkerCount: -1
         influxdb:
           hostUrl:
           token:
@@ -1384,7 +1390,6 @@ soc:
           userFiles:
             - rbac/users_roles
         strelkaengine:
-          allowRegex: ''
           autoEnabledYaraRules:
             - securityonion-yara
           autoUpdateEnabled: true
@@ -1392,7 +1397,6 @@ soc:
           communityRulesImportErrorSeconds: 300
           failAfterConsecutiveErrorCount: 10
           compileYaraPythonScriptPath: /opt/sensoroni/yara/compile_yara.py
-          denyRegex: ''
           reposFolder: /opt/sensoroni/yara/repos
           rulesRepos:
             default:
@@ -1407,14 +1411,14 @@ soc:
           stateFilePath: /opt/sensoroni/fingerprints/strelkaengine.state
           integrityCheckFrequencySeconds: 1200
         suricataengine:
-          allowRegex: ''
           autoUpdateEnabled: true
           communityRulesImportFrequencySeconds: 86400
           communityRulesImportErrorSeconds: 300
           customRulesets:
+          disableRegex: []
+          enableRegex: []
           failAfterConsecutiveErrorCount: 10
           communityRulesFile: /nsm/rules/suricata/emerging-all.rules
-          denyRegex: ''
           rulesFingerprintFile: /opt/sensoroni/fingerprints/emerging-all.fingerprint
           stateFilePath: /opt/sensoroni/fingerprints/suricataengine.state
           integrityCheckFrequencySeconds: 1200
@@ -2286,15 +2290,15 @@ soc:
 
               alert http $EXTERNAL_NET any -> $HOME_NET any (msg:"Example Rule Title - 'example' String Detected"; content:"example"; sid:[publicId]; rev:1;)
             strelka: |
-              /* 
+              /*
               This is a YARA rule template. Replace all template values with your own values.
               The YARA rule name is the unique identifier for the rule.
               Docs: https://yara.readthedocs.io/en/stable/writingrules.html#writing-yara-rules
-              */ 
+              */
 
               rule Example // This identifier _must_ be unique
               {
-                  meta: 
+                  meta:
                       description = "Generic YARA Rule"
                       author = "@SecurityOnion"
                       date = "YYYY-MM-DD"
@@ -2317,7 +2321,7 @@ soc:
               id: [publicId]
               status: 'experimental'
               description: |
-                  This should be a detailed description of what this Detection focuses on: what we are trying to find and why we are trying to find it. 
+                  This should be a detailed description of what this Detection focuses on: what we are trying to find and why we are trying to find it.
                   For example, from rule 97a80ec7-0e2f-4d05-9ef4-65760e634f6b: "Detects a whoami.exe executed with the /priv command line flag instructing the tool to show all current user privileges. This is often used after a privilege escalation attempt."
               references:
                 - 'https://local.invalid'
@@ -2326,7 +2330,7 @@ soc:
               tags:
                 - detection.threat_hunting
                 - attack.technique_id
-              logsource: 
+              logsource:
                 category: process_creation
                 product: windows
               detection:

salt/soc/files/soc/motd.md

@@ -1,6 +1,6 @@
 ## Getting Started
 
-New to Security Onion? Click the menu in the upper-right corner and you'll find links for [Help](/docs/) and a [Cheat Sheet](/docs/cheatsheet.pdf) that will help you best utilize Security Onion to hunt for evil! In addition, check out our free Security Onion Essentials online course, available on our [Training](https://securityonionsolutions.com/training) website.
+New to Security Onion? Click the menu in the upper-right corner and you'll find links for [Help](/docs/) and a [Cheat Sheet](/docs/cheatsheet.pdf) that will help you best utilize Security Onion to hunt for evil! In addition, check out our free Security Onion Essentials online course, available on our [Training](https://securityonion.com/training) website.
 
 If you're ready to dive in, take a look at the [Alerts](/#/alerts) interface to see what Security Onion has detected so far. If you find any false positives, then you can tune those in [Detections](/#/detections).
 
@@ -20,13 +20,17 @@ For more coverage of your enterprise, you can deploy the Elastic Agent to endpoi
 
 To see all the latest features and fixes in this version of Security Onion, click the upper-right menu and then click the [What's New](/docs/release-notes.html) link.
 
+## Security Onion Pro
+
+Need enterprise features and premium support? Check out [Security Onion Pro](https://securityonion.com/pro/)!
+
 ## Enterprise Appliances
 
-Want the best hardware for your enterprise deployment? Check out our [enterprise appliances](https://securityonionsolutions.com/hardware/)!
+Want the best hardware for your enterprise deployment? Check out our [enterprise appliances](https://securityonion.com/hardware/)!
 
 ## Premium Support
 
-Experiencing difficulties and need priority support or remote assistance? We offer a [premium support plan](https://securityonionsolutions.com/support/) to assist corporate, educational, and government organizations.
+Experiencing difficulties and need priority support or remote assistance? We offer a [premium support plan](https://securityonion.com/support/) to assist corporate, educational, and government organizations.
 
 ## Customize This Space
 

salt/soc/merged.map.jinja

@@ -5,9 +5,9 @@
 
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% from 'soc/defaults.map.jinja' import SOCDEFAULTS with context %}
-{% from 'logstash/map.jinja' import LOGSTASH_NODES %}
+{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
 {% from 'manager/map.jinja' import MANAGERMERGED %}
-{% set DOCKER_EXTRA_HOSTS = LOGSTASH_NODES %}
+{% set DOCKER_EXTRA_HOSTS = ELASTICSEARCH_NODES %}
 {% do DOCKER_EXTRA_HOSTS.append({GLOBALS.influxdb_host:pillar.node_data[GLOBALS.influxdb_host].ip}) %}
 
 {% set SOCMERGED = salt['pillar.get']('soc', SOCDEFAULTS, merge=true) %}

salt/soc/soc_soc.yaml

@@ -90,11 +90,6 @@ soc:
             helpLink: sigma.html
             forcedType: "[]string"
             multiline: True
-          allowRegex:
-            description: 'Regex used to filter imported Sigma rules. Deny regex takes precedence over the Allow regex setting.'
-            global: True
-            advanced: True
-            helpLink: sigma.html
           autoEnabledSigmaRules:
             default: &autoEnabledSigmaRules
               description: 'Sigma rules to automatically enable on initial import. Format is $Ruleset+$Level - for example, for the core community ruleset and critical level rules: core+critical. These will be applied based on role if defined and default if not.'
@@ -103,11 +98,6 @@ soc:
               helpLink: sigma.html
             so-eval: *autoEnabledSigmaRules
             so-import: *autoEnabledSigmaRules
-          denyRegex:
-            description: 'Regex used to filter imported Sigma rules. Deny regex takes precedence over the Allow regex setting.'
-            global: True
-            advanced: True
-            helpLink: sigma.html
           communityRulesImportFrequencySeconds:
             description: 'How often to check for new Sigma rules (in seconds). This applies to both Community Rule Packages and any configured Git repos.'
             global: True
@@ -174,6 +164,10 @@ soc:
           lookupTunnelParent:
             description: When true, if a pivoted event appears to be encapsulated, such as in a VXLAN packet, then SOC will pivot to the VXLAN packet stream. When false, SOC will attempt to pivot to the encapsulated packet stream itself, but at the risk that it may be unable to locate it in the stored PCAP data.
             global: True
+          maxScrollSize:
+            description: The maximum number of documents to request in a single Elasticsearch scroll request.
+          bulkIndexWorkerCount:
+            description: The number of worker threads to use when bulk indexing data into Elasticsearch. A value below 1 will default to the number of CPUs available.
         sostatus:
           refreshIntervalMs:
             description: Duration (in milliseconds) between refreshes of the grid status. Shortening this duration may not have expected results, as the backend systems feeding this sostatus data will continue their updates as scheduled.
@@ -195,21 +189,11 @@ soc:
             advanced: True
             forcedType: int
         strelkaengine:
-          allowRegex:
-            description: 'Regex used to filter imported YARA rules. Deny regex takes precedence over the Allow regex setting.'
-            global: True
-            advanced: True
-            helpLink: yara.html
           autoEnabledYaraRules:
             description: 'YARA rules to automatically enable on initial import. Format is $Ruleset - for example, for the default shipped ruleset: securityonion-yara'
             global: True
             advanced: True
             helpLink: sigma.html
-          denyRegex:
-            description: 'Regex used to filter imported YARA rules. Deny regex takes precedence over the Allow regex setting.'
-            global: True
-            advanced: True
-            helpLink: yara.html
           communityRulesImportFrequencySeconds:
             description: 'How often to check for new YARA rules (in seconds). This applies to both Community Rules and any configured Git repos.'
             global: True
@@ -228,21 +212,19 @@ soc:
               helpLink: yara.html
             airgap: *serulesRepos
         suricataengine:
-          allowRegex:
-            description: 'Regex used to filter imported Suricata rules. Deny regex takes precedence over the Allow regex setting.'
-            global: True
-            advanced: True
-            helpLink: suricata.html
-          denyRegex:
-            description: 'Regex used to filter imported Suricata rules. Deny regex takes precedence over the Allow regex setting.'
-            global: True
-            advanced: True
-            helpLink: suricata.html
           communityRulesImportFrequencySeconds:
             description: 'How often to check for new Suricata rules (in seconds).'
             global: True
             advanced: True
             helpLink: suricata.html
+          disableRegex:
+            description: A list of regular expressions used to automatically disable rules that match any of them. Each regular expression is tested against the rule's content.
+            global: True
+            forcedType: "[]string"
+          enableRegex:
+            description: A list of regular expressions used to automatically enable rules that match any of them. Each regular expression is tested against the rule's content. Takes priority over disableRegex matches.
+            global: True
+            forcedType: "[]string"
           integrityCheckFrequencySeconds:
             description: 'How often the Suricata integrity checker runs (in seconds). This verifies the integrity of deployed rules.'
             global: True

salt/ssl/init.sls

@@ -17,8 +17,6 @@
   {% set COMMONNAME = GLOBALS.manager %}
 {% endif %}
 
-{% set kafka_password = salt['pillar.get']('kafka:password') %}
-
 {% if grains.id.split('_')|last in ['manager', 'managersearch', 'eval', 'standalone', 'import'] %}
 include:
   - ca
@@ -666,7 +664,6 @@ elastickeyperms:
 {%- endif %}
 
 {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone'] %}
-
 elasticfleet_kafka_key:
   x509.private_key_managed:
     - name: /etc/pki/elasticfleet-kafka.key
@@ -696,17 +693,13 @@ elasticfleet_kafka_crt:
     - retry:
         attempts: 5
         interval: 30
-  cmd.run:
-  - name: "/usr/bin/openssl pkcs8 -in /etc/pki/elasticfleet-kafka.key -topk8 -out /etc/pki/elasticfleet-kafka.p8 -nocrypt"
-  - onchanges:
-    - x509: elasticfleet_kafka_key
 
 elasticfleet_kafka_cert_perms:
   file.managed:
     - replace: False
     - name: /etc/pki/elasticfleet-kafka.crt
     - mode: 640
-    - user: 960
+    - user: 947
     - group: 939
 
 elasticfleet_kafka_key_perms:
@@ -714,187 +707,8 @@ elasticfleet_kafka_key_perms:
     - replace: False
     - name: /etc/pki/elasticfleet-kafka.key
     - mode: 640
-    - user: 960
+    - user: 947
     - group: 939
-
-elasticfleet_kafka_pkcs8_perms:
-  file.managed:
-    - replace: False
-    - name: /etc/pki/elasticfleet-kafka.p8
-    - mode: 640
-    - user: 960
-    - group: 939
-
-kafka_client_key:
-  x509.private_key_managed:
-    - name: /etc/pki/kafka-client.key
-    - keysize: 4096
-    - backup: True
-    - new: True
-    {% if salt['file.file_exists']('/etc/pki/kafka-client.key') -%}
-    - prereq:
-      - x509: /etc/pki/kafka-client.crt
-    {%- endif %}
-    - retry:
-        attempts: 5
-        interval: 30
-
-kafka_client_crt:
-  x509.certificate_managed:
-    - name: /etc/pki/kafka-client.crt
-    - ca_server: {{ ca_server }}
-    - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
-    - signing_policy: kafka
-    - private_key: /etc/pki/kafka-client.key
-    - CN: {{ GLOBALS.hostname }}
-    - days_remaining: 0
-    - days_valid: 820
-    - backup: True
-    - timeout: 30
-    - retry:
-        attempts: 5
-        interval: 30
-
-kafka_client_key_perms:
-  file.managed:
-    - replace: False
-    - name: /etc/pki/kafka-client.key
-    - mode: 640
-    - user: 960
-    - group: 939
-
-kafka_client_crt_perms:
-  file.managed:
-    - replace: False
-    - name: /etc/pki/kafka-client.crt
-    - mode: 640
-    - user: 960
-    - group: 939
-
-{% endif %}
-
-{% if grains['role'] in ['so-manager', 'so-managersearch','so-receiver', 'so-standalone'] %}
-
-kafka_key:
-  x509.private_key_managed:
-    - name: /etc/pki/kafka.key
-    - keysize: 4096
-    - backup: True
-    - new: True
-    {% if salt['file.file_exists']('/etc/pki/kafka.key') -%}
-    - prereq:
-      - x509: /etc/pki/kafka.crt
-    {%- endif %}
-    - retry:
-        attempts: 5
-        interval: 30
-
-kafka_crt:
-  x509.certificate_managed:
-    - name: /etc/pki/kafka.crt
-    - ca_server: {{ ca_server }}
-    - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
-    - signing_policy: kafka
-    - private_key: /etc/pki/kafka.key
-    - CN: {{ GLOBALS.hostname }}
-    - days_remaining: 0
-    - days_valid: 820
-    - backup: True
-    - timeout: 30
-    - retry:
-        attempts: 5
-        interval: 30
-  cmd.run:
-    - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/kafka.key -in /etc/pki/kafka.crt -export -out /etc/pki/kafka.p12 -nodes -passout pass:{{ kafka_password }}"
-    - onchanges:
-      - x509: /etc/pki/kafka.key
-kafka_key_perms:
-  file.managed:
-    - replace: False
-    - name: /etc/pki/kafka.key
-    - mode: 640
-    - user: 960
-    - group: 939
-
-kafka_crt_perms:
-  file.managed:
-    - replace: False
-    - name: /etc/pki/kafka.crt
-    - mode: 640
-    - user: 960
-    - group: 939
-
-kafka_pkcs12_perms:
-  file.managed:
-    - replace: False
-    - name: /etc/pki/kafka.p12
-    - mode: 640
-    - user: 960
-    - group: 939
-
-{% endif %}
-
-# Standalone needs kafka-logstash for automated testing. Searchnode/manager search need it for logstash to consume from Kafka.
-# Manager will have cert, but be unused until a pipeline is created and logstash enabled.
-{% if grains['role'] in ['so-standalone', 'so-managersearch', 'so-searchnode', 'so-manager'] %}
-kafka_logstash_key:
-  x509.private_key_managed:
-    - name: /etc/pki/kafka-logstash.key
-    - keysize: 4096
-    - backup: True
-    - new: True
-    {% if salt['file.file_exists']('/etc/pki/kafka-logstash.key') -%}
-    - prereq:
-      - x509: /etc/pki/kafka-logstash.crt
-    {%- endif %}
-    - retry:
-        attempts: 5
-        interval: 30
-
-kafka_logstash_crt:
-  x509.certificate_managed:
-    - name: /etc/pki/kafka-logstash.crt
-    - ca_server: {{ ca_server }}
-    - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
-    - signing_policy: kafka
-    - private_key: /etc/pki/kafka-logstash.key
-    - CN: {{ GLOBALS.hostname }}
-    - days_remaining: 0
-    - days_valid: 820
-    - backup: True
-    - timeout: 30
-    - retry:
-        attempts: 5
-        interval: 30
-  cmd.run:
-    - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/kafka-logstash.key -in /etc/pki/kafka-logstash.crt -export -out /etc/pki/kafka-logstash.p12 -nodes -passout pass:{{ kafka_password }}"
-    - onchanges:
-      - x509: /etc/pki/kafka-logstash.key
-
-kafka_logstash_key_perms:
-  file.managed:
-    - replace: False
-    - name: /etc/pki/kafka-logstash.key
-    - mode: 640
-    - user: 960
-    - group: 939
-
-kafka_logstash_crt_perms:
-  file.managed:
-    - replace: False
-    - name: /etc/pki/kafka-logstash.crt
-    - mode: 640
-    - user: 960
-    - group: 939
-
-kafka_logstash_pkcs12_perms:
-  file.managed:
-    - replace: False
-    - name: /etc/pki/kafka-logstash.p12
-    - mode: 640
-    - user: 960
-    - group: 931
-
 {% endif %}
 
 {% else %}

salt/suricata/soc_suricata.yaml

@@ -154,12 +154,14 @@ suricata:
           description: Assign a list of hosts, or networks, using CIDR notation, to this Suricata variable. The variable can then be re-used within Suricata rules. This allows for a single adjustment to the variable that will then affect all rules referencing the variable.
           regex: ^(((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\/([0-9]|[1-2][0-9]|3[0-2]))?$|^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?))|:))|(([0-9A-Fa-f]{1,4}:){5}((:[0-9A-Fa-f]{1,4}){1,2}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){4}((:[0-9A-Fa-f]{1,4}){1,3}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){3}((:[0-9A-Fa-f]{1,4}){1,4}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){2}((:[0-9A-Fa-f]{1,4}){1,5}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){1}((:[0-9A-Fa-f]{1,4}){1,6}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(:((:[0-9A-Fa-f]{1,4}){1,7}|:)))(\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8]))?$
           regexFailureMessage: You must enter a valid IP address or CIDR.
-          helpLink: suricata.html
+          forcedType: "[]string"
           duplicates: True
+          helpLink: suricata.html
         EXTERNAL_NET: &suriaddressgroup
           description: Assign a list of hosts, or networks, or other customization, to this Suricata variable. The variable can then be re-used within Suricata rules. This allows for a single adjustment to the variable that will then affect all rules referencing the variable.
-          helpLink: suricata.html
+          forcedType: "[]string"
           duplicates: True
+          helpLink: suricata.html
         HTTP_SERVERS: *suriaddressgroup
         SMTP_SERVERS: *suriaddressgroup
         SQL_SERVERS: *suriaddressgroup
@@ -176,8 +178,9 @@ suricata:
       port-groups:
         HTTP_PORTS: &suriportgroup
           description: Assign a list of network port numbers to this Suricata variable. The variable can then be re-used within Suricata rules. This allows for a single adjustment to the variable that will then affect all rules referencing the variable.
-          helpLink: suricata.html
+          forcedType: "[]string"
           duplicates: True
+          helpLink: suricata.html
         SHELLCODE_PORTS: *suriportgroup
         ORACLE_PORTS: *suriportgroup
         SSH_PORTS: *suriportgroup

salt/top.sls

@@ -155,6 +155,7 @@ base:
     - nginx
     - elasticfleet.install_agent_grid
     - stig
+    - kafka
 
   '*_managersearch and G@saltversion:{{saltversion}}':
     - match: compound
@@ -184,6 +185,7 @@ base:
     - utility
     - elasticfleet
     - stig
+    - kafka
 
   '*_heavynode and G@saltversion:{{saltversion}}':
     - match: compound

setup/so-functions

@@ -1180,13 +1180,16 @@ kibana_pillar() {
 kafka_pillar() {
 	KAFKACLUSTERID=$(get_random_value 22)
 	KAFKAPASS=$(get_random_value)
+	KAFKATRUST=$(get_random_value)
 	logCmd "mkdir -p $local_salt_dir/pillar/kafka"
 	logCmd "touch $adv_kafka_pillar_file"
 	logCmd "touch $kafka_pillar_file"
 	printf '%s\n'\
 		"kafka:"\
 		"  cluster_id: $KAFKACLUSTERID"\
-		"  password: $KAFKAPASS" > $kafka_pillar_file
+		"  config:"\
+		"    password: $KAFKAPASS"\
+		"    trustpass: $KAFKATRUST" > $kafka_pillar_file
 }
 
 logrotate_pillar() {
@@ -1254,30 +1257,7 @@ soc_pillar() {
 		"soc:"\
 		"  config:"\
 		"    server:"\
-		"      srvKey: '$SOCSRVKEY'"\
-		"      modules:"\
-		"        elastalertengine:"\
-		"          allowRegex: '$ELASTALERT_ALLOW_REGEX'" > "$soc_pillar_file"
-	if [[ -n "$ELASTALERT_FAIL_ERROR_COUNT" ]]; then
-		printf '%s\n'\
-	    "          failAfterConsecutiveErrorCount: $ELASTALERT_FAIL_ERROR_COUNT" >> "$soc_pillar_file"
-	fi
-	
-	printf '%s\n'\
-		"        strelkaengine:"\
-		"          allowRegex: '$STRELKA_ALLOW_REGEX'" >> "$soc_pillar_file"
-	if [[ -n "$STRELKA_FAIL_ERROR_COUNT" ]]; then
-	 	printf '%s\n'\
-		"          failAfterConsecutiveErrorCount: $STRELKA_FAIL_ERROR_COUNT" >> "$soc_pillar_file"
-	fi
-	
-	printf '%s\n'\
-		"        suricataengine:"\
-		"          allowRegex: '$SURICATA_ALLOW_REGEX'" >> "$soc_pillar_file"
-	if [[ -n "$SURICATA_FAIL_ERROR_COUNT" ]]; then
-		printf '%s\n'\
-		"          failAfterConsecutiveErrorCount: $SURICATA_FAIL_ERROR_COUNT" >> "$soc_pillar_file"
-	fi
+		"      srvKey: '$SOCSRVKEY'" > "$soc_pillar_file"
 
 	if [[ $telemetry -ne 0 ]]; then
 		echo "  telemetryEnabled: false" >> $soc_pillar_file

setup/so-setup

@@ -245,12 +245,6 @@ if [ -n "$test_profile" ]; then
 	WEBUSER=onionuser@somewhere.invalid
 	WEBPASSWD1=0n10nus3r
 	WEBPASSWD2=0n10nus3r
-	STRELKA_ALLOW_REGEX="EquationGroup_Toolset_Apr17__ELV_.*"
-	STRELKA_FAIL_ERROR_COUNT=1
-	ELASTALERT_ALLOW_REGEX="Security Onion"
-	ELASTALERT_FAIL_ERROR_COUNT=1
-	SURICATA_ALLOW_REGEX="(200033\\d|2100538|2102466)"
-	SURICATA_FAIL_ERROR_COUNT=1
 
 	update_sudoers_for_testing
 fi