Merge pull request #13255 from Security-Onion-Solutions/2.4/dev

2.4.80

.github/.gitleaks.toml

@@ -536,7 +536,7 @@ secretGroup = 4
 
 [allowlist]
 description = "global allow lists"
-regexes = ['''219-09-9999''', '''078-05-1120''', '''(9[0-9]{2}|666)-\d{2}-\d{4}''', '''RPM-GPG-KEY.*''', '''.*:.*StrelkaHexDump.*''', '''.*:.*PLACEHOLDER.*''']
+regexes = ['''219-09-9999''', '''078-05-1120''', '''(9[0-9]{2}|666)-\d{2}-\d{4}''', '''RPM-GPG-KEY.*''', '''.*:.*StrelkaHexDump.*''', '''.*:.*PLACEHOLDER.*''', '''ssl_.*password''']
 paths = [
     '''gitleaks.toml''',
     '''(.*?)(jpg|gif|doc|pdf|bin|svg|socket)$''',

.github/workflows/close-threads.yml

@@ -15,6 +15,7 @@ concurrency:
 
 jobs:
   close-threads:
+    if: github.repository_owner == 'security-onion-solutions'
     runs-on: ubuntu-latest
     permissions:
       issues: write

.github/workflows/lock-threads.yml

@@ -15,6 +15,7 @@ concurrency:
 
 jobs:
   lock-threads:
+    if: github.repository_owner == 'security-onion-solutions'
     runs-on: ubuntu-latest
     steps:
       - uses: jertel/lock-threads@main

DOWNLOAD_AND_VERIFY_ISO.md

@@ -1,17 +1,17 @@
-### 2.4.60-20240320 ISO image released on 2024/03/20
+### 2.4.80-20240624 ISO image released on 2024/06/25
 
 
 ### Download and Verify
 
-2.4.60-20240320 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.4.60-20240320.iso
+2.4.80-20240624 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-2.4.80-20240624.iso
  
-MD5: 178DD42D06B2F32F3870E0C27219821E  
-SHA1: 73EDCD50817A7F6003FE405CF1808A30D034F89D  
-SHA256: DD334B8D7088A7B78160C253B680D645E25984BA5CCAB5CC5C327CA72137FC06  
+MD5: 139F9762E926F9CB3C4A9528A3752C31  
+SHA1: BC6CA2C5F4ABC1A04E83A5CF8FFA6A53B1583CC9  
+SHA256: 70E90845C84FFA30AD6CF21504634F57C273E7996CA72F7250428DDBAAC5B1BD  
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.60-20240320.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.80-20240624.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS  
@@ -25,27 +25,29 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.60-20240320.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.80-20240624.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.4.60-20240320.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.4.80-20240624.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.4.60-20240320.iso.sig securityonion-2.4.60-20240320.iso
+gpg --verify securityonion-2.4.80-20240624.iso.sig securityonion-2.4.80-20240624.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Tue 19 Mar 2024 03:17:58 PM EDT using RSA key ID FE507013
+gpg: Signature made Mon 24 Jun 2024 02:42:03 PM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.
 Primary key fingerprint: C804 A93D 36BE 0C73 3EA1  9644 7C10 60B7 FE50 7013
 ```
 
+If it fails to verify, try downloading again. If it still fails to verify, try downloading from another computer or another network.
+
 Once you've verified the ISO image, you're ready to proceed to our Installation guide:  
 https://docs.securityonion.net/en/2.4/installation.html

README.md

@@ -8,19 +8,22 @@ Alerts
 ![Alerts](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/50_alerts.png)
 
 Dashboards
-![Dashboards](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/51_dashboards.png)
+![Dashboards](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/53_dashboards.png)
 
 Hunt
-![Hunt](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/52_hunt.png)
+![Hunt](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/56_hunt.png)
+
+Detections
+![Detections](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/57_detections.png)
 
 PCAP
-![PCAP](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/53_pcap.png)
+![PCAP](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/62_pcap.png)
 
 Grid
-![Grid](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/57_grid.png)
+![Grid](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/75_grid.png)
 
 Config
-![Config](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/61_config.png)
+![Config](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/87_config.png)
 
 ### Release Notes
 

VERSION

@@ -1 +1 @@
-2.4.70
+2.4.80

pillar/kafka/nodes.sls

@@ -1,30 +1,2 @@
-{% set current_kafkanodes = salt.saltutil.runner('mine.get', tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-receiver', fun='network.ip_addrs', tgt_type='compound') %}
-{% set pillar_kafkanodes = salt['pillar.get']('kafka:nodes', default={}, merge=True) %}
-
-{% set existing_ids = [] %}
-{% for node in pillar_kafkanodes.values() %}
-  {% if node.get('id') %}
-    {% do existing_ids.append(node['nodeid']) %}
-  {% endif %}
-{% endfor %}
-{% set all_possible_ids = range(1, 256)|list %}
-
-{% set available_ids = [] %}
-{% for id in all_possible_ids %}
-  {% if id not in existing_ids %}
-    {% do available_ids.append(id) %}
-  {% endif %}
-{% endfor %}
-
-{% set final_nodes = pillar_kafkanodes.copy() %}
-
-{% for minionid, ip in current_kafkanodes.items() %}
-    {% set hostname = minionid.split('_')[0] %}
-    {% if hostname not in final_nodes %}
-        {% set new_id = available_ids.pop(0) %}
-        {% do final_nodes.update({hostname: {'nodeid': new_id, 'ip': ip[0]}}) %}
-    {% endif %}
-{% endfor %}
-
 kafka:
-  nodes: {{ final_nodes|tojson }}
+  nodes:

pillar/top.sls

@@ -226,6 +226,7 @@ base:
     - minions.adv_{{ grains.id }}
     - stig.soc_stig
     - soc.license
+    - kafka.nodes
 
   '*_receiver':
     - logstash.nodes
@@ -241,6 +242,7 @@ base:
     - kafka.nodes
     - kafka.soc_kafka
     - kafka.adv_kafka
+    - soc.license
 
   '*_import':
     - secrets

pyci.sh

@@ -15,12 +15,16 @@ TARGET_DIR=${1:-.}
 
 PATH=$PATH:/usr/local/bin
 
-if ! which pytest &> /dev/null || ! which flake8 &> /dev/null ; then
-    echo "Missing dependencies. Consider running the following command:"
-    echo "  python -m pip install flake8 pytest pytest-cov"
+if [ ! -d .venv ]; then
+    python -m venv .venv
+fi
+
+source .venv/bin/activate
+
+if ! pip install flake8 pytest pytest-cov pyyaml; then
+    echo "Unable to install dependencies."
     exit 1
 fi
 
-pip install pytest pytest-cov
 flake8 "$TARGET_DIR" "--config=${HOME_DIR}/pytest.ini"
-python3 -m pytest "--cov-config=${HOME_DIR}/pytest.ini" "--cov=$TARGET_DIR" --doctest-modules --cov-report=term --cov-fail-under=100  "$TARGET_DIR" 
+python3 -m pytest "--cov-config=${HOME_DIR}/pytest.ini" "--cov=$TARGET_DIR" --doctest-modules --cov-report=term --cov-fail-under=100  "$TARGET_DIR" 

salt/allowed_states.map.jinja

@@ -65,6 +65,7 @@
           'registry',
           'manager',
           'nginx',
+          'strelka.manager',
           'soc',
           'kratos',
           'influxdb',
@@ -91,6 +92,7 @@
           'nginx',
           'telegraf',
           'influxdb',
+          'strelka.manager',
           'soc',
           'kratos',
           'elasticfleet',
@@ -112,6 +114,7 @@
           'nginx',
           'telegraf',
           'influxdb',
+          'strelka.manager',
           'soc',
           'kratos',
           'elastic-fleet-package-registry',
@@ -192,7 +195,8 @@
           'schedule',
           'docker_clean',
           'kafka',
-          'elasticsearch.ca'
+          'elasticsearch.ca',
+          'stig'
           ],
       'so-desktop': [
           'ssl',

salt/ca/files/signing_policies.conf

@@ -1,6 +1,3 @@
-mine_functions:
-  x509.get_pem_entries: [/etc/pki/ca.crt]
-
 x509_signing_policies:
   filebeat:
     - minions: '*'

salt/common/soup_scripts.sls

@@ -1,3 +1,8 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
 {% if '2.4' in salt['cp.get_file_str']('/etc/soversion') %}
 
 {%   import_yaml '/opt/so/saltstack/local/pillar/global/soc_global.sls' as SOC_GLOBAL %}
@@ -15,6 +20,8 @@ remove_common_so-firewall:
   file.absent:
     - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-firewall
 
+# This section is used to put the scripts in place in the Salt file system
+# in case a state run tries to overwrite what we do in the next section.
 copy_so-common_common_tools_sbin:
   file.copy:
     - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-common
@@ -43,6 +50,21 @@ copy_so-firewall_manager_tools_sbin:
     - force: True
     - preserve: True
 
+copy_so-yaml_manager_tools_sbin:
+  file.copy:
+    - name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-yaml.py
+    - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-yaml.py
+    - force: True
+    - preserve: True
+
+copy_so-repo-sync_manager_tools_sbin:
+  file.copy:
+    - name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-repo-sync
+    - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-repo-sync
+    - preserve: True
+
+# This section is used to put the new script in place so that it can be called during soup.
+# It is faster than calling the states that normally manage them to put them in place.
 copy_so-common_sbin:
   file.copy:
     - name: /usr/sbin/so-common
@@ -78,6 +100,13 @@ copy_so-yaml_sbin:
     - force: True
     - preserve: True
 
+copy_so-repo-sync_sbin:
+  file.copy:
+    - name: /usr/sbin/so-repo-sync
+    - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-repo-sync
+    - force: True
+    - preserve: True
+
 {% else %}
 fix_23_soup_sbin:
   cmd.run:

salt/common/tools/sbin/so-checkin

@@ -5,8 +5,13 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-
-
 . /usr/sbin/so-common
 
-salt-call state.highstate -l info 
+cat << EOF
+
+so-checkin will run a full salt highstate to apply all salt states. If a highstate is already running, this request will be queued and so it may pause for a few minutes before you see any more output. For more information about so-checkin and salt, please see:
+https://docs.securityonion.net/en/2.4/salt.html
+
+EOF
+
+salt-call state.highstate -l info queue=True

salt/common/tools/sbin/so-common

@@ -31,6 +31,11 @@ if ! echo "$PATH" | grep -q "/usr/sbin"; then
   export PATH="$PATH:/usr/sbin"
 fi
 
+# See if a proxy is set. If so use it.
+if [ -f /etc/profile.d/so-proxy.sh ]; then
+  . /etc/profile.d/so-proxy.sh
+fi
+
 # Define a banner to separate sections
 banner="========================================================================="
 
@@ -179,6 +184,21 @@ copy_new_files() {
   cd /tmp
 }
 
+create_local_directories() {
+	echo "Creating local pillar and salt directories if needed"
+	PILLARSALTDIR=$1
+	local_salt_dir="/opt/so/saltstack/local"
+	for i in "pillar" "salt"; do
+		for d in $(find $PILLARSALTDIR/$i -type d); do
+			suffixdir=${d//$PILLARSALTDIR/}
+			if [ ! -d "$local_salt_dir/$suffixdir" ]; then
+				mkdir -pv $local_salt_dir$suffixdir
+			fi
+		done
+		chown -R socore:socore $local_salt_dir/$i
+	done
+}
+
 disable_fastestmirror() {
 	sed -i 's/enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf
 }

salt/common/tools/sbin/so-log-check

@@ -201,6 +201,10 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unknown column"               # Elastalert errors from running EQL queries
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|parsing_exception"            # Elastalert EQL parsing issue. Temp.
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|context deadline exceeded"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error running query:"         # Specific issues with detection rules
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|detect-parse"                 # Suricata encountering a malformed rule
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|integrity check failed"       # Detections: Exclude false positive due to automated testing
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|syncErrors"                   # Detections: Not an actual error
 fi
 
 RESULT=0
@@ -236,6 +240,7 @@ exclude_log "playbook.log" # Playbook is removed as of 2.4.70, logs may still be
 exclude_log "mysqld.log"   # MySQL is removed as of 2.4.70, logs may still be on disk
 exclude_log "soctopus.log" # Soctopus is removed as of 2.4.70, logs may still be on disk
 exclude_log "agentstatus.log" # ignore this log since it tracks agents in error state
+exclude_log "detections_runtime-status_yara.log" # temporarily ignore this log until Detections is more stable
 
 for log_file in $(cat /tmp/log_check_files); do
     status "Checking log file $log_file"

salt/common/tools/sbin/so-luks-tpm-regen

@@ -0,0 +1,98 @@
+#!/bin/bash
+#
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0."
+
+set -e
+# This script is intended to be used in the case the ISO install did not properly setup TPM decrypt for LUKS partitions at boot.
+if [ -z $NOROOT ]; then
+	# Check for prerequisites
+	if [ "$(id -u)" -ne 0 ]; then
+		echo "This script must be run using sudo!"
+		exit 1
+	fi
+fi
+ENROLL_TPM=N
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --enroll-tpm)
+      ENROLL_TPM=Y
+      ;;
+    *)
+      echo "Usage: $0 [options]"
+      echo ""
+      echo "where options are:"
+      echo "  --enroll-tpm            for when TPM enrollment was not selected during ISO install."
+      echo ""
+      exit 1
+      ;;
+  esac
+  shift
+done
+
+check_for_tpm() {
+  echo -n "Checking for TPM: "
+  if [ -d /sys/class/tpm/tpm0 ]; then
+    echo -e "tpm0 found."
+    TPM="yes"
+    # Check if TPM is using sha1 or sha256
+    if [ -d /sys/class/tpm/tpm0/pcr-sha1 ]; then
+      echo -e "TPM is using sha1.\n"
+      TPM_PCR="sha1"
+    elif [ -d /sys/class/tpm/tpm0/pcr-sha256 ]; then
+      echo -e "TPM is using sha256.\n"
+      TPM_PCR="sha256"
+    fi
+  else
+    echo -e "No TPM found.\n"
+    exit 1
+  fi
+}
+
+check_for_luks_partitions() {
+  echo "Checking for LUKS partitions"
+  for part in $(lsblk -o NAME,FSTYPE -ln | grep crypto_LUKS | awk '{print $1}'); do
+    echo "Found LUKS partition: $part"
+    LUKS_PARTITIONS+=("$part")
+  done
+  if [ ${#LUKS_PARTITIONS[@]} -eq 0 ]; then
+    echo -e "No LUKS partitions found.\n"
+    exit 1
+  fi
+  echo ""
+}
+
+enroll_tpm_in_luks() {
+  read -s -p "Enter the LUKS passphrase used during ISO install: " LUKS_PASSPHRASE
+  echo ""
+  for part in "${LUKS_PARTITIONS[@]}"; do
+    echo "Enrolling TPM for LUKS device: /dev/$part"
+    if [ "$TPM_PCR" == "sha1" ]; then
+      clevis luks bind -d /dev/$part tpm2 '{"pcr_bank":"sha1","pcr_ids":"7"}' <<< $LUKS_PASSPHRASE
+    elif [ "$TPM_PCR" == "sha256" ]; then
+      clevis luks bind -d /dev/$part tpm2 '{"pcr_bank":"sha256","pcr_ids":"7"}' <<< $LUKS_PASSPHRASE
+    fi
+  done
+ }
+
+regenerate_tpm_enrollment_token() {
+  for part in "${LUKS_PARTITIONS[@]}"; do
+    clevis luks regen -d /dev/$part -s 1 -q
+  done
+}
+
+check_for_tpm
+check_for_luks_partitions
+
+if [[ $ENROLL_TPM == "Y" ]]; then
+  enroll_tpm_in_luks
+else
+  regenerate_tpm_enrollment_token
+fi
+
+echo "Running dracut"
+dracut -fv
+echo -e "\nTPM configuration complete. Reboot the system to verify the TPM is correctly decrypting the LUKS partition(s) at boot.\n"

salt/common/tools/sbin_jinja/so-import-pcap

@@ -89,6 +89,7 @@ function suricata() {
     -v ${LOG_PATH}:/var/log/suricata/:rw \
     -v ${NSM_PATH}/:/nsm/:rw \
     -v "$PCAP:/input.pcap:ro" \
+    -v /dev/null:/nsm/suripcap:rw \
     -v /opt/so/conf/suricata/bpf:/etc/suricata/bpf:ro \
     {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-suricata:{{ VERSION }} \
     --runmode single -k none -r /input.pcap > $LOG_PATH/console.log 2>&1
@@ -247,7 +248,7 @@ fi
 START_OLDEST_SLASH=$(echo $START_OLDEST | sed -e 's/-/%2F/g')
 END_NEWEST_SLASH=$(echo $END_NEWEST | sed -e 's/-/%2F/g')
 if [[ $VALID_PCAPS_COUNT -gt 0 ]] || [[ $SKIPPED_PCAPS_COUNT -gt 0 ]]; then
-  URL="https://{{ URLBASE }}/#/dashboards?q=$HASH_FILTERS%20%7C%20groupby%20-sankey%20event.dataset%20event.category%2a%20%7C%20groupby%20-pie%20event.category%20%7C%20groupby%20-bar%20event.module%20%7C%20groupby%20event.dataset%20%7C%20groupby%20event.module%20%7C%20groupby%20event.category%20%7C%20groupby%20observer.name%20%7C%20groupby%20source.ip%20%7C%20groupby%20destination.ip%20%7C%20groupby%20destination.port&t=${START_OLDEST_SLASH}%2000%3A00%3A00%20AM%20-%20${END_NEWEST_SLASH}%2000%3A00%3A00%20AM&z=UTC"
+  URL="https://{{ URLBASE }}/#/dashboards?q=$HASH_FILTERS%20%7C%20groupby%20event.module*%20%7C%20groupby%20-sankey%20event.module*%20event.dataset%20%7C%20groupby%20event.dataset%20%7C%20groupby%20source.ip%20%7C%20groupby%20destination.ip%20%7C%20groupby%20destination.port%20%7C%20groupby%20network.protocol%20%7C%20groupby%20rule.name%20rule.category%20event.severity_label%20%7C%20groupby%20dns.query.name%20%7C%20groupby%20file.mime_type%20%7C%20groupby%20http.virtual_host%20http.uri%20%7C%20groupby%20notice.note%20notice.message%20notice.sub_message%20%7C%20groupby%20ssl.server_name%20%7C%20groupby%20source_geo.organization_name%20source.geo.country_name%20%7C%20groupby%20destination_geo.organization_name%20destination.geo.country_name&t=${START_OLDEST_SLASH}%2000%3A00%3A00%20AM%20-%20${END_NEWEST_SLASH}%2000%3A00%3A00%20AM&z=UTC"
 
   status "Import complete!"
   status

salt/common/tools/sbin_jinja/so-tcpreplay

@@ -10,7 +10,7 @@
 . /usr/sbin/so-common
 . /usr/sbin/so-image-common
 
-REPLAYIFACE=${REPLAYIFACE:-$(lookup_pillar interface sensor)}
+REPLAYIFACE=${REPLAYIFACE:-"{{salt['pillar.get']('sensor:interface', '')}}"}
 REPLAYSPEED=${REPLAYSPEED:-10}
 
 mkdir -p /opt/so/samples

salt/docker/defaults.yaml

@@ -180,6 +180,8 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+      ulimits:
+        - memlock=524288000
     'so-zeek':
       final_octet: 99
       custom_bind_mounts: []
@@ -190,6 +192,7 @@ docker:
       port_bindings:
         - 0.0.0.0:9092:9092
         - 0.0.0.0:9093:9093
+        - 0.0.0.0:8778:8778
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []

salt/docker/init.sls

@@ -20,30 +20,30 @@ dockergroup:
 dockerheldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 1.6.21-1
-      - docker-ce: 5:24.0.3-1~debian.12~bookworm
-      - docker-ce-cli: 5:24.0.3-1~debian.12~bookworm
-      - docker-ce-rootless-extras: 5:24.0.3-1~debian.12~bookworm
+      - containerd.io: 1.6.33-1
+      - docker-ce: 5:26.1.4-1~debian.12~bookworm
+      - docker-ce-cli: 5:26.1.4-1~debian.12~bookworm
+      - docker-ce-rootless-extras: 5:26.1.4-1~debian.12~bookworm
     - hold: True
     - update_holds: True
 {%    elif grains.oscodename == 'jammy' %}
 dockerheldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 1.6.21-1
-      - docker-ce: 5:24.0.2-1~ubuntu.22.04~jammy
-      - docker-ce-cli: 5:24.0.2-1~ubuntu.22.04~jammy
-      - docker-ce-rootless-extras: 5:24.0.2-1~ubuntu.22.04~jammy
+      - containerd.io: 1.6.33-1
+      - docker-ce: 5:26.1.4-1~ubuntu.22.04~jammy
+      - docker-ce-cli: 5:26.1.4-1~ubuntu.22.04~jammy
+      - docker-ce-rootless-extras: 5:26.1.4-1~ubuntu.22.04~jammy
     - hold: True
     - update_holds: True
 {%    else %}
 dockerheldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 1.4.9-1
-      - docker-ce: 5:20.10.8~3-0~ubuntu-focal
-      - docker-ce-cli: 5:20.10.5~3-0~ubuntu-focal
-      - docker-ce-rootless-extras: 5:20.10.5~3-0~ubuntu-focal
+      - containerd.io: 1.6.33-1
+      - docker-ce: 5:26.1.4-1~ubuntu.20.04~focal
+      - docker-ce-cli: 5:26.1.4-1~ubuntu.20.04~focal
+      - docker-ce-rootless-extras: 5:26.1.4-1~ubuntu.20.04~focal
     - hold: True
     - update_holds: True
 {% endif %}
@@ -51,10 +51,10 @@ dockerheldpackages:
 dockerheldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 1.6.21-3.1.el9
-      - docker-ce: 24.0.4-1.el9
-      - docker-ce-cli: 24.0.4-1.el9
-      - docker-ce-rootless-extras: 24.0.4-1.el9
+      - containerd.io: 1.6.33-3.1.el9
+      - docker-ce: 3:26.1.4-1.el9
+      - docker-ce-cli: 1:26.1.4-1.el9
+      - docker-ce-rootless-extras: 26.1.4-1.el9
     - hold: True
     - update_holds: True
 {% endif %}

salt/docker/soc_docker.yaml

@@ -63,6 +63,42 @@ docker:
     so-elastic-agent: *dockerOptions
     so-telegraf: *dockerOptions
     so-steno: *dockerOptions
-    so-suricata: *dockerOptions
+    so-suricata:
+      final_octet:
+        description: Last octet of the container IP address.
+        helpLink: docker.html
+        readonly: True
+        advanced: True
+        global: True
+      port_bindings:
+        description: List of port bindings for the container.
+        helpLink: docker.html
+        advanced: True
+        multiline: True
+        forcedType: "[]string"
+      custom_bind_mounts:
+        description: List of custom local volume bindings.
+        advanced: True
+        helpLink: docker.html
+        multiline: True
+        forcedType: "[]string"
+      extra_hosts:
+        description: List of additional host entries for the container.
+        advanced: True
+        helpLink: docker.html
+        multiline: True
+        forcedType: "[]string"
+      extra_env:
+        description: List of additional ENV entries for the container.
+        advanced: True
+        helpLink: docker.html
+        multiline: True
+        forcedType: "[]string"
+      ulimits:
+        description: Ulimits for the container, in bytes.
+        advanced: True
+        helpLink: docker.html
+        multiline: True
+        forcedType: "[]string"
     so-zeek: *dockerOptions
     so-kafka: *dockerOptions

salt/elastalert/config.sls

@@ -82,6 +82,36 @@ elastasomodulesync:
     - group: 933
     - makedirs: True
 
+elastacustomdir:
+  file.directory:
+    - name: /opt/so/conf/elastalert/custom
+    - user: 933
+    - group: 933
+    - makedirs: True
+
+elastacustomsync:
+  file.recurse:
+    - name: /opt/so/conf/elastalert/custom
+    - source: salt://elastalert/files/custom
+    - user: 933
+    - group: 933
+    - makedirs: True
+    - file_mode: 660
+    - show_changes: False
+
+elastapredefinedsync:
+  file.recurse:
+    - name: /opt/so/conf/elastalert/predefined
+    - source: salt://elastalert/files/predefined
+    - user: 933
+    - group: 933
+    - makedirs: True
+    - template: jinja
+    - file_mode: 660
+    - context:
+        elastalert: {{ ELASTALERTMERGED }}
+    - show_changes: False
+
 elastaconf:
   file.managed:
     - name: /opt/so/conf/elastalert/elastalert_config.yaml

salt/elastalert/defaults.yaml

@@ -1,5 +1,6 @@
 elastalert:
   enabled: False
+  alerter_parameters: ""
   config:
     rules_folder: /opt/elastalert/rules/
     scan_subdirectories: true

salt/elastalert/enabled.sls

@@ -30,6 +30,8 @@ so-elastalert:
       - /opt/so/rules/elastalert:/opt/elastalert/rules/:ro
       - /opt/so/log/elastalert:/var/log/elastalert:rw
       - /opt/so/conf/elastalert/modules/:/opt/elastalert/modules/:ro
+      - /opt/so/conf/elastalert/predefined/:/opt/elastalert/predefined/:ro
+      - /opt/so/conf/elastalert/custom/:/opt/elastalert/custom/:ro
       - /opt/so/conf/elastalert/elastalert_config.yaml:/opt/elastalert/config.yaml:ro
       {% if DOCKER.containers['so-elastalert'].custom_bind_mounts %}
         {% for BIND in DOCKER.containers['so-elastalert'].custom_bind_mounts %}

salt/elastalert/files/custom/placeholder

@@ -0,0 +1 @@
+THIS IS A PLACEHOLDER FILE

salt/elastalert/files/modules/so/playbook-es.py

@@ -1,38 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-from time import gmtime, strftime
-import requests,json
-from elastalert.alerts import Alerter
-
-import urllib3
-urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-
-class PlaybookESAlerter(Alerter):
-    """
-    Use matched data to create alerts in elasticsearch
-    """
-
-    required_options = set(['play_title','play_url','sigma_level'])
-
-    def alert(self, matches):
-       for match in matches:
-            today = strftime("%Y.%m.%d", gmtime())
-            timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S"'.000Z', gmtime())
-            headers = {"Content-Type": "application/json"}
-
-            creds = None
-            if 'es_username' in self.rule and 'es_password' in self.rule:
-                creds = (self.rule['es_username'], self.rule['es_password'])
-
-            payload = {"tags":"alert","rule": { "name": self.rule['play_title'],"case_template": self.rule['play_id'],"uuid": self.rule['play_id'],"category": self.rule['rule.category']},"event":{ "severity": self.rule['event.severity'],"module": self.rule['event.module'],"dataset": self.rule['event.dataset'],"severity_label": self.rule['sigma_level']},"kibana_pivot": self.rule['kibana_pivot'],"soc_pivot": self.rule['soc_pivot'],"play_url": self.rule['play_url'],"sigma_level": self.rule['sigma_level'],"event_data": match, "@timestamp": timestamp}
-            url = f"https://{self.rule['es_host']}:{self.rule['es_port']}/logs-playbook.alerts-so/_doc/"
-            requests.post(url, data=json.dumps(payload), headers=headers, verify=False, auth=creds)
-                            
-    def get_info(self):
-        return {'type': 'PlaybookESAlerter'} 

salt/elastalert/files/modules/so/securityonion-es.py

@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
+from time import gmtime, strftime
+import requests,json
+from elastalert.alerts import Alerter
+
+import urllib3
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+class SecurityOnionESAlerter(Alerter):
+    """
+    Use matched data to create alerts in Elasticsearch.
+    """
+
+    required_options = set(['detection_title', 'sigma_level'])
+    optional_fields = ['sigma_category', 'sigma_product', 'sigma_service']
+
+    def alert(self, matches):
+        for match in matches:
+            timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S"'.000Z', gmtime())
+            headers = {"Content-Type": "application/json"}
+
+            creds = None
+            if 'es_username' in self.rule and 'es_password' in self.rule:
+                creds = (self.rule['es_username'], self.rule['es_password'])
+
+            # Start building the rule dict
+            rule_info = {
+                "name": self.rule['detection_title'],
+                "uuid": self.rule['detection_public_id']
+            }
+
+            # Add optional fields if they are present in the rule
+            for field in self.optional_fields:
+                rule_key = field.split('_')[-1]  # Assumes field format "sigma_<key>"
+                if field in self.rule:
+                    rule_info[rule_key] = self.rule[field]
+
+            # Construct the payload with the conditional rule_info
+            payload = {
+                "tags": "alert",
+                "rule": rule_info,
+                "event": {
+                    "severity": self.rule['event.severity'],
+                    "module": self.rule['event.module'],
+                    "dataset": self.rule['event.dataset'],
+                    "severity_label": self.rule['sigma_level']
+                },
+                "sigma_level": self.rule['sigma_level'],
+                "event_data": match,
+                "@timestamp": timestamp
+            }
+            url = f"https://{self.rule['es_host']}:{self.rule['es_port']}/logs-detections.alerts-so/_doc/"
+            requests.post(url, data=json.dumps(payload), headers=headers, verify=False, auth=creds)
+
+    def get_info(self):
+        return {'type': 'SecurityOnionESAlerter'}

salt/elastalert/files/predefined/jira_auth.yaml

@@ -0,0 +1,6 @@
+{% if elastalert.get('jira_user', '') | length > 0 and elastalert.get('jira_pass', '') | length > 0 %}
+user: {{ elastalert.jira_user }}
+password: {{ elastalert.jira_pass }}
+{% else %}
+apikey: {{ elastalert.get('jira_api_key', '') }}
+{% endif %}

salt/elastalert/files/predefined/smtp_auth.yaml

@@ -0,0 +1,2 @@
+user: {{ elastalert.get('smtp_user', '') }}
+password: {{ elastalert.get('smtp_pass', '') }}

salt/elastalert/map.jinja

@@ -13,3 +13,19 @@
 {% do ELASTALERTDEFAULTS.elastalert.config.update({'es_password': pillar.elasticsearch.auth.users.so_elastic_user.pass}) %}
 
 {% set ELASTALERTMERGED = salt['pillar.get']('elastalert', ELASTALERTDEFAULTS.elastalert, merge=True) %}
+
+{% if 'ntf' in salt['pillar.get']('features', []) %}
+   {% set params = ELASTALERTMERGED.get('alerter_parameters', '') | load_yaml %}
+   {% if params != None and params | length > 0 %}
+      {% do ELASTALERTMERGED.config.update(params) %}
+   {% endif %}
+
+   {% if ELASTALERTMERGED.get('smtp_user', '') | length > 0 %}
+      {% do ELASTALERTMERGED.config.update({'smtp_auth_file': '/opt/elastalert/predefined/smtp_auth.yaml'}) %}
+   {% endif %}
+
+   {% if ELASTALERTMERGED.get('jira_user', '') | length > 0 or ELASTALERTMERGED.get('jira_key', '') | length > 0 %}
+      {% do ELASTALERTMERGED.config.update({'jira_account_file': '/opt/elastalert/predefined/jira_auth.yaml'}) %}
+   {% endif %}
+
+{% endif %}

salt/elastalert/soc_elastalert.yaml

@@ -2,6 +2,99 @@ elastalert:
   enabled:
     description: You can enable or disable Elastalert.
     helpLink: elastalert.html
+  alerter_parameters:
+    title: Alerter Parameters
+    description: Optional configuration parameters for additional alerters that can be enabled for all Sigma rules. Filter for 'Alerter' in this Configuration screen to find the setting that allows these alerters to be enabled within the SOC ElastAlert module. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key.
+    global: True
+    multiline: True
+    syntax: yaml
+    helpLink: elastalert.html
+    forcedType: string
+  jira_api_key:
+    title: Jira API Key
+    description: Optional configuration parameter for Jira API Key, used instead of the Jira username and password. Requires a valid Security Onion license key.
+    global: True
+    sensitive: True
+    helpLink: elastalert.html
+    forcedType: string
+  jira_pass:
+    title: Jira Password
+    description: Optional configuration parameter for Jira password. Requires a valid Security Onion license key.
+    global: True
+    sensitive: True
+    helpLink: elastalert.html
+    forcedType: string
+  jira_user:
+    title: Jira Username
+    description: Optional configuration parameter for Jira username. Requires a valid Security Onion license key.
+    global: True
+    helpLink: elastalert.html
+    forcedType: string
+  smtp_pass:
+    title: SMTP Password
+    description: Optional configuration parameter for SMTP password, required for authenticating email servers. Requires a valid Security Onion license key.
+    global: True
+    sensitive: True
+    helpLink: elastalert.html
+    forcedType: string
+  smtp_user:
+    title: SMTP Username
+    description: Optional configuration parameter for SMTP username, required for authenticating email servers. Requires a valid Security Onion license key.
+    global: True
+    helpLink: elastalert.html
+    forcedType: string
+  files:
+    custom:
+      alertmanager_ca__crt:
+        description: Optional custom Certificate Authority for connecting to an AlertManager server. To utilize this custom file, the alertmanager_ca_certs key must be set to /opt/elastalert/custom/alertmanager_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
+        global: True
+        file: True
+        helpLink: elastalert.html
+      gelf_ca__crt:
+        description: Optional custom Certificate Authority for connecting to a Graylog server. To utilize this custom file, the graylog_ca_certs key must be set to /opt/elastalert/custom/graylog_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
+        global: True
+        file: True
+        helpLink: elastalert.html
+      http_post_ca__crt:
+        description: Optional custom Certificate Authority for connecting to a generic HTTP server, via the legacy HTTP POST alerter. To utilize this custom file, the http_post_ca_certs key must be set to /opt/elastalert/custom/http_post2_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
+        global: True
+        file: True
+        helpLink: elastalert.html
+      http_post2_ca__crt:
+        description: Optional custom Certificate Authority for connecting to a generic HTTP server, via the newer HTTP POST 2 alerter. To utilize this custom file, the http_post2_ca_certs key must be set to /opt/elastalert/custom/http_post2_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
+        global: True
+        file: True
+        helpLink: elastalert.html
+      ms_teams_ca__crt:
+        description: Optional custom Certificate Authority for connecting to Microsoft Teams server. To utilize this custom file, the ms_teams_ca_certs key must be set to /opt/elastalert/custom/ms_teams_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
+        global: True
+        file: True
+        helpLink: elastalert.html
+      pagerduty_ca__crt:
+        description: Optional custom Certificate Authority for connecting to PagerDuty server. To utilize this custom file, the pagerduty_ca_certs key must be set to /opt/elastalert/custom/pagerduty_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
+        global: True
+        file: True
+        helpLink: elastalert.html
+      rocket_chat_ca__crt:
+        description: Optional custom Certificate Authority for connecting to PagerDuty server. To utilize this custom file, the rocket_chart_ca_certs key must be set to /opt/elastalert/custom/rocket_chat_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
+        global: True
+        file: True
+        helpLink: elastalert.html
+      smtp__crt:
+        description: Optional custom certificate for connecting to an SMTP server. To utilize this custom file, the smtp_cert_file key must be set to /opt/elastalert/custom/smtp.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
+        global: True
+        file: True
+        helpLink: elastalert.html
+      smtp__key:
+        description: Optional custom certificate key for connecting to an SMTP server. To utilize this custom file, the smtp_key_file key must be set to /opt/elastalert/custom/smtp.key in the Alerter Parameters setting. Requires a valid Security Onion license key.
+        global: True
+        file: True
+        helpLink: elastalert.html
+      slack_ca__crt:
+        description: Optional custom Certificate Authority for connecting to Slack. To utilize this custom file, the slack_ca_certs key must be set to /opt/elastalert/custom/slack_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
+        global: True
+        file: True
+        helpLink: elastalert.html
   config:
     disable_rules_on_error:
       description: Disable rules on failure.

salt/elasticfleet/defaults.yaml

@@ -37,6 +37,7 @@ elasticfleet:
     - azure
     - barracuda
     - carbonblack_edr
+    - cef
     - checkpoint
     - cisco_asa
     - cisco_duo
@@ -118,3 +119,8 @@ elasticfleet:
       base_url: https://api.platform.sublimesecurity.com
       poll_interval: 5m
       limit: 100
+    kismet:
+      base_url: http://localhost:2501
+      poll_interval: 1m
+      api_key:
+      enabled_nodes: []

salt/elasticfleet/enabled.sls

@@ -27,7 +27,9 @@ wait_for_elasticsearch_elasticfleet:
 so-elastic-fleet-auto-configure-logstash-outputs:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-outputs-update
-    - retry: True
+    - retry:
+        attempts: 4
+        interval: 30
 {% endif %}
 
 # If enabled, automatically update Fleet Server URLs & ES Connection
@@ -35,7 +37,9 @@ so-elastic-fleet-auto-configure-logstash-outputs:
 so-elastic-fleet-auto-configure-server-urls:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-urls-update
-    - retry: True
+    - retry:
+        attempts: 4
+        interval: 30
 {% endif %}
 
 # Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs
@@ -43,12 +47,16 @@ so-elastic-fleet-auto-configure-server-urls:
 so-elastic-fleet-auto-configure-elasticsearch-urls:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-es-url-update
-    - retry: True
+    - retry:
+        attempts: 4
+        interval: 30
 
 so-elastic-fleet-auto-configure-artifact-urls:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-artifacts-url-update
-    - retry: True 
+    - retry:
+        attempts: 4
+        interval: 30
 
 {% endif %}
 

salt/elasticfleet/files/integrations-optional/kismet.json

@@ -0,0 +1,36 @@
+{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
+{% raw %}
+{
+  "package": {
+    "name": "httpjson",
+    "version": ""
+  },
+  "name": "kismet-logs",
+  "namespace": "so",
+  "description": "Kismet Logs",
+  "policy_id": "FleetServer_{% endraw %}{{ NAME }}{% raw %}",
+  "inputs": {
+    "generic-httpjson": {
+      "enabled": true,
+      "streams": {
+        "httpjson.generic": {
+          "enabled": true,
+          "vars": {
+            "data_stream.dataset": "kismet",
+            "request_url": "{% endraw %}{{ ELASTICFLEETMERGED.optional_integrations.kismet.base_url }}{% raw %}/devices/last-time/-600/devices.tjson",
+            "request_interval": "{% endraw %}{{ ELASTICFLEETMERGED.optional_integrations.kismet.poll_interval }}{% raw %}",
+            "request_method": "GET",
+            "request_transforms": "- set:\r\n    target: header.Cookie\r\n    value: 'KISMET={% endraw %}{{ ELASTICFLEETMERGED.optional_integrations.kismet.api_key }}{% raw %}'",
+            "request_redirect_headers_ban_list": [],
+            "oauth_scopes": [],
+            "processors": "",
+            "tags": [],
+            "pipeline": "kismet.common"
+          }
+        }
+      }
+    }
+  },
+  "force": true
+}
+{% endraw %}

salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json

@@ -0,0 +1,35 @@
+{
+  "policy_id": "so-grid-nodes_general",
+  "package": {
+    "name": "log",
+    "version": ""
+  },
+  "name": "soc-detections-logs",
+  "description": "Security Onion Console - Detections Logs",
+  "namespace": "so",
+  "inputs": {
+    "logs-logfile": {
+      "enabled": true,
+      "streams": {
+        "log.logs": {
+          "enabled": true,
+          "vars": {
+            "paths": [
+              "/opt/so/log/soc/detections_runtime-status_sigma.log",
+              "/opt/so/log/soc/detections_runtime-status_yara.log"
+            ],
+            "exclude_files": [],
+            "ignore_older": "72h",
+            "data_stream.dataset": "soc",
+            "tags": [
+              "so-soc"
+            ],
+            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"soc\"\n    process_array: true\n    max_depth: 2\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: detections\n- rename:\n    fields:\n      - from: \"soc.fields.sourceIp\"\n        to: \"source.ip\"\n      - from: \"soc.fields.status\"\n        to: \"http.response.status_code\"\n      - from: \"soc.fields.method\"\n        to: \"http.request.method\"\n      - from: \"soc.fields.path\"\n        to: \"url.path\"\n      - from: \"soc.message\"\n        to: \"event.action\"\n      - from: \"soc.level\"\n        to: \"log.level\"\n    ignore_missing: true",
+            "custom": "pipeline: common"
+          }
+        }
+      }
+    }
+  },
+  "force": true
+}

salt/elasticfleet/soc_elasticfleet.yaml

@@ -79,3 +79,29 @@ elasticfleet:
         helpLink: elastic-fleet.html
         advanced: True
         forcedType: int
+    kismet:
+      base_url:
+        description: Base URL for Kismet.
+        global: True
+        helpLink: elastic-fleet.html
+        advanced: True
+        forcedType: string
+      poll_interval:
+        description: Poll interval for wireless device data from Kismet. Integration is currently configured to return devices seen as active by any Kismet sensor within the last 10 minutes.
+        global: True
+        helpLink: elastic-fleet.html
+        advanced: True
+        forcedType: string
+      api_key:
+        description: API key for Kismet.
+        global: True
+        helpLink: elastic-fleet.html
+        advanced: True
+        forcedType: string
+        sensitive: True
+      enabled_nodes:
+        description: Fleet nodes with the Kismet integration enabled. Enter one per line.
+        global: True
+        helpLink: elastic-fleet.html
+        advanced: True
+        forcedType: "[]string"

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers

@@ -19,7 +19,7 @@ NUM_RUNNING=$(pgrep -cf "/bin/bash /sbin/so-elastic-agent-gen-installers")
 
 for i in {1..30}
 do
-   ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key')
+   ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys?perPage=100" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key')
    FLEETHOST=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/fleet_server_hosts/grid-default' | jq -r  '.item.host_urls[]' | paste -sd ',')
 if [[ $FLEETHOST ]] && [[  $ENROLLMENTOKEN ]]; then break; else sleep 10; fi
 done
@@ -72,5 +72,5 @@ do
     printf "\n### $GOOS/$GOARCH Installer Generated...\n"
 done
 
-printf "\n### Cleaning up temp files in /nsm/elastic-agent-workspace"
+printf "\n### Cleaning up temp files in /nsm/elastic-agent-workspace\n"
 rm -rf /nsm/elastic-agent-workspace

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update

@@ -21,64 +21,104 @@ function update_logstash_outputs() {
     # Update Logstash Outputs
     curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_logstash" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" | jq
 }
+function update_kafka_outputs() {
+  # Make sure SSL configuration is included in policy updates for Kafka output. SSL is configured in so-elastic-fleet-setup
+  SSL_CONFIG=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs/so-manager_kafka" | jq -r '.item.ssl')
 
-# Get current list of Logstash Outputs
-RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_logstash')
+  JSON_STRING=$(jq -n \
+    --arg UPDATEDLIST "$NEW_LIST_JSON" \
+    --argjson SSL_CONFIG "$SSL_CONFIG" \
+    '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": $SSL_CONFIG}')
+  # Update Kafka outputs
+  curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_kafka" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" | jq
+}
 
-# Check to make sure that the server responded with good data - else, bail from script
-CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON")
-if [ "$CHECKSUM" != "so-manager_logstash" ]; then
- printf "Failed to query for current Logstash Outputs..."
- exit 1
-fi
+{% if GLOBALS.pipeline == "KAFKA" %}
+  # Get current list of Kafka Outputs
+  RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_kafka')
 
-# Get the current list of Logstash outputs & hash them
-CURRENT_LIST=$(jq -c -r '.item.hosts' <<<  "$RAW_JSON")
-CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}')
+  # Check to make sure that the server responded with good data - else, bail from script
+  CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON")
+  if [ "$CHECKSUM" != "so-manager_kafka" ]; then
+   printf "Failed to query for current Kafka Outputs..."
+   exit 1
+  fi
 
-declare -a NEW_LIST=()
+  # Get the current list of kafka outputs & hash them
+  CURRENT_LIST=$(jq -c -r '.item.hosts' <<<  "$RAW_JSON")
+  CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}')
+
+  declare -a NEW_LIST=()
+
+  # Query for the current Grid Nodes that are running kafka
+  KAFKANODES=$(salt-call --out=json pillar.get kafka:nodes | jq '.local')
+
+  # Query for Kafka nodes with Broker role and add hostname to list
+  while IFS= read -r line; do
+    NEW_LIST+=("$line")
+  done < <(jq -r 'to_entries | .[] | select(.value.role | contains("broker")) | .key + ":9092"' <<< $KAFKANODES)
+
+  {# If global pipeline isn't set to KAFKA then assume default of REDIS / logstash #}
+{% else %}
+  # Get current list of Logstash Outputs
+  RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_logstash')
+
+  # Check to make sure that the server responded with good data - else, bail from script
+  CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON")
+  if [ "$CHECKSUM" != "so-manager_logstash" ]; then
+   printf "Failed to query for current Logstash Outputs..."
+   exit 1
+  fi
+
+  # Get the current list of Logstash outputs & hash them
+  CURRENT_LIST=$(jq -c -r '.item.hosts' <<<  "$RAW_JSON")
+  CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}')
+
+  declare -a NEW_LIST=()
+
+  {# If we select to not send to manager via SOC, then omit the code that adds manager to NEW_LIST #}
+  {% if ELASTICFLEETMERGED.enable_manager_output %}
+  # Create array & add initial elements
+  if [ "{{ GLOBALS.hostname }}" = "{{ GLOBALS.url_base }}" ]; then
+      NEW_LIST+=("{{ GLOBALS.url_base }}:5055")
+  else
+      NEW_LIST+=("{{ GLOBALS.url_base }}:5055" "{{ GLOBALS.hostname }}:5055")
+  fi
+  {% endif %}
+
+  # Query for FQDN entries & add them to the list
+  {% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %}
+  CUSTOMFQDNLIST=('{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(' ') }}')
+  readarray -t -d ' ' CUSTOMFQDN < <(printf '%s' "$CUSTOMFQDNLIST")
+  for CUSTOMNAME in "${CUSTOMFQDN[@]}"
+  do
+   NEW_LIST+=("$CUSTOMNAME:5055")
+  done
+  {% endif %}
+
+  # Query for the current Grid Nodes that are running Logstash
+  LOGSTASHNODES=$(salt-call --out=json pillar.get logstash:nodes | jq '.local')
+
+  # Query for Receiver Nodes & add them to the list
+  if grep -q "receiver" <<< $LOGSTASHNODES; then
+     readarray -t RECEIVERNODES < <(jq -r ' .receiver | keys_unsorted[]'  <<< $LOGSTASHNODES)
+     for NODE in "${RECEIVERNODES[@]}"
+     do
+      NEW_LIST+=("$NODE:5055")
+     done
+  fi
+
+  # Query for Fleet Nodes & add them to the list
+  if grep -q "fleet" <<< $LOGSTASHNODES; then
+     readarray -t FLEETNODES < <(jq -r ' .fleet | keys_unsorted[]'  <<< $LOGSTASHNODES)
+     for NODE in "${FLEETNODES[@]}"
+     do
+      NEW_LIST+=("$NODE:5055")
+     done
+  fi
 
-{# If we select to not send to manager via SOC, then omit the code that adds manager to NEW_LIST #}
-{% if ELASTICFLEETMERGED.enable_manager_output %}
-# Create array & add initial elements
-if [ "{{ GLOBALS.hostname }}" = "{{ GLOBALS.url_base }}" ]; then
-    NEW_LIST+=("{{ GLOBALS.url_base }}:5055")
-else
-    NEW_LIST+=("{{ GLOBALS.url_base }}:5055" "{{ GLOBALS.hostname }}:5055")
-fi
 {% endif %}
 
-# Query for FQDN entries & add them to the list
-{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %}
-CUSTOMFQDNLIST=('{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(' ') }}')
-readarray -t -d ' ' CUSTOMFQDN < <(printf '%s' "$CUSTOMFQDNLIST")
-for CUSTOMNAME in "${CUSTOMFQDN[@]}"
-do
- NEW_LIST+=("$CUSTOMNAME:5055")
-done
-{% endif %}
-
-# Query for the current Grid Nodes that are running Logstash
-LOGSTASHNODES=$(salt-call --out=json pillar.get logstash:nodes | jq '.local')
-
-# Query for Receiver Nodes & add them to the list
-if grep -q "receiver" <<< $LOGSTASHNODES; then
-   readarray -t RECEIVERNODES < <(jq -r ' .receiver | keys_unsorted[]'  <<< $LOGSTASHNODES)
-   for NODE in "${RECEIVERNODES[@]}"
-   do
-    NEW_LIST+=("$NODE:5055")
-   done
-fi
-
-# Query for Fleet Nodes & add them to the list
-if grep -q "fleet" <<< $LOGSTASHNODES; then
-   readarray -t FLEETNODES < <(jq -r ' .fleet | keys_unsorted[]'  <<< $LOGSTASHNODES)
-   for NODE in "${FLEETNODES[@]}"
-   do
-    NEW_LIST+=("$NODE:5055")
-   done
-fi
-
 # Sort & hash the new list of Logstash Outputs
 NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${NEW_LIST[@]}")
 NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}')
@@ -87,9 +127,28 @@ NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}')
 if [ "$NEW_HASH" = "$CURRENT_HASH" ]; then
     printf "\nHashes match - no update needed.\n"
     printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n"
+
+    # Since output can be KAFKA or LOGSTASH, we need to check if the policy set as default matches the value set in GLOBALS.pipeline and update if needed
+    printf "Checking if the correct output policy is set as default\n"
+    OUTPUT_DEFAULT=$(jq -r '.item.is_default' <<< $RAW_JSON)
+    OUTPUT_DEFAULT_MONITORING=$(jq -r '.item.is_default_monitoring' <<< $RAW_JSON)
+    if [[ "$OUTPUT_DEFAULT" = "false" || "$OUTPUT_DEFAULT_MONITORING" = "false" ]]; then
+      printf "Default output policy needs to be updated.\n"
+    {%- if GLOBALS.pipeline == "KAFKA" and 'gmd' in salt['pillar.get']('features', []) %}
+      update_kafka_outputs
+    {%- else %}
+      update_logstash_outputs
+    {%- endif %}
+    else
+      printf "Default output policy is set - no update needed.\n"
+    fi
     exit 0
 else
     printf "\nHashes don't match - update needed.\n"
     printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n"
+    {%- if GLOBALS.pipeline == "KAFKA" and 'gmd' in salt['pillar.get']('features', []) %}
+    update_kafka_outputs
+    {%- else %}
     update_logstash_outputs
+    {%- endif %}
 fi

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup

@@ -77,6 +77,11 @@ curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fl
 printf "\n\n"
 {%- endif %}
 
+printf "\nCreate Kafka Output Config if node is not an Import or Eval install\n"
+{% if grains.role not in ['so-import', 'so-eval'] %}
+/usr/sbin/so-kafka-fleet-output-policy
+{% endif %}
+
 # Add Manager Hostname & URL Base to Fleet Host URLs
 printf "\nAdd SO-Manager Fleet URL\n"
 if [ "{{ GLOBALS.hostname }}" = "{{ GLOBALS.url_base }}" ]; then

salt/elasticfleet/tools/sbin_jinja/so-kafka-fleet-output-policy

@@ -0,0 +1,52 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+{% if GLOBALS.role in ['so-manager', 'so-standalone', 'so-managersearch'] %}
+
+. /usr/sbin/so-common
+
+# Check to make sure that Kibana API is up & ready
+RETURN_CODE=0
+wait_for_web_response "http://localhost:5601/api/fleet/settings" "fleet" 300 "curl -K /opt/so/conf/elasticsearch/curl.config"
+RETURN_CODE=$?
+
+if [[ "$RETURN_CODE" != "0" ]]; then
+    printf "Kibana API not accessible, can't setup Elastic Fleet output policy for Kafka..."
+    exit 1
+fi
+
+output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs" | jq -r .items[].id)
+
+if ! echo "$output" | grep -q "so-manager_kafka"; then
+  KAFKACRT=$(openssl x509 -in /etc/pki/elasticfleet-kafka.crt)
+  KAFKAKEY=$(openssl rsa -in  /etc/pki/elasticfleet-kafka.key)
+  KAFKACA=$(openssl x509 -in  /etc/pki/tls/certs/intca.crt)
+  KAFKA_OUTPUT_VERSION="2.6.0"
+  JSON_STRING=$( jq -n \
+                --arg KAFKACRT "$KAFKACRT" \
+                --arg KAFKAKEY "$KAFKAKEY" \
+                --arg KAFKACA "$KAFKACA" \
+                --arg MANAGER_IP "{{ GLOBALS.manager_ip }}:9092" \
+                --arg KAFKA_OUTPUT_VERSION "$KAFKA_OUTPUT_VERSION" \
+                    '{ "name": "grid-kafka", "id": "so-manager_kafka", "type": "kafka", "hosts": [ $MANAGER_IP ], "is_default": false, "is_default_monitoring": false, "config_yaml": "", "ssl": { "certificate_authorities": [ $KAFKACA ], "certificate": $KAFKACRT, "key": $KAFKAKEY, "verification_mode": "full" }, "proxy_id": null, "client_id": "Elastic", "version": $KAFKA_OUTPUT_VERSION, "compression": "none", "auth_type": "ssl", "partition": "round_robin", "round_robin": { "group_events": 1 }, "topics":[{"topic":"%{[event.module]}-securityonion","when":{"type":"regexp","condition":"event.module:.+"}},{"topic":"default-securityonion"}], "headers": [ { "key": "", "value": "" } ], "timeout": 30, "broker_timeout": 30, "required_acks": 1 }'
+                )
+  curl -sK /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" -o /dev/null
+  refresh_output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs" | jq -r .items[].id)
+
+  if ! echo "$refresh_output" | grep -q "so-manager_kafka"; then
+    echo -e "\nFailed to setup Elastic Fleet output policy for Kafka...\n"
+    exit 1
+  elif echo "$refresh_output" | grep -q "so-manager_kafka"; then
+    echo -e "\nSuccessfully setup Elastic Fleet output policy for Kafka...\n"
+  fi
+
+elif echo "$output" | grep -q "so-manager_kafka"; then
+  echo -e "\nElastic Fleet output policy for Kafka already exists...\n"
+fi
+{% else %}
+echo -e "\nNo update required...\n"
+{% endif %}

salt/elasticsearch/download.sls

@@ -0,0 +1,20 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+{%   from 'vars/globals.map.jinja' import GLOBALS %}
+
+so-elasticsearch_image:
+  docker_image.present:
+    - name: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elasticsearch:{{ GLOBALS.so_version }}
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/elasticsearch/enabled.sls

@@ -200,9 +200,15 @@ so-elasticsearch-roles-load:
     - require:
       - docker_container: so-elasticsearch
       - file: elasticsearch_sbin_jinja
-{% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
+
+{%     if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
+{%       if ELASTICSEARCHMERGED.index_clean %}
+{%         set ap = "present" %}
+{%       else %}
+{%         set ap = "absent" %}
+{%       endif %}
 so-elasticsearch-indices-delete:
-  cron.present:
+  cron.{{ap}}:
     - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1
     - identifier: so-elasticsearch-indices-delete
     - user: root
@@ -211,7 +217,8 @@ so-elasticsearch-indices-delete:
     - daymonth: '*'
     - month: '*'
     - dayweek: '*'
-{% endif %}
+{%     endif %}
+
 {%   endif %}
 
 {% else %}

salt/elasticsearch/files/ingest/.fleet_final_pipeline-1

@@ -80,10 +80,11 @@
     { "set":        { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } },
     { "set":        { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.dataset", "value": "import" } },
     { "set":        { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.namespace", "value": "so" } },
-    { "date":       { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp", "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } },
+    { "date":       { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp","ignore_failure": true, "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSX","yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } },
     { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true }  },
     { "set":        { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } },
     { "rename":        { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } },
+    { "set":        { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } },
     { "remove":     { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp" ], "ignore_missing": true, "ignore_failure": true } } 
   ],
   "on_failure": [

salt/elasticsearch/files/ingest/kismet.ad_hoc

@@ -0,0 +1,10 @@
+{
+  "processors": [
+    {
+      "rename": {
+        "field": "message2.kismet_device_base_macaddr",
+        "target_field": "network.wireless.bssid"
+      }
+    }
+  ]
+}

salt/elasticsearch/files/ingest/kismet.ap

@@ -0,0 +1,50 @@
+{
+  "processors": [
+    {
+      "rename": {
+        "field": "message2.dot11_device.dot11_device_last_beaconed_ssid_record.dot11_advertisedssid_cloaked",
+        "target_field": "network.wireless.ssid_cloaked",
+        "if": "ctx?.message2?.dot11_device?.dot11_device_last_beaconed_ssid_record?.dot11_advertisedssid_cloaked != null"
+      }
+    },
+    {
+      "rename": {
+        "field": "message2.dot11_device.dot11_device_last_beaconed_ssid_record.dot11_advertisedssid_ssid",
+        "target_field": "network.wireless.ssid",
+        "if": "ctx?.message2?.dot11_device?.dot11_device_last_beaconed_ssid_record?.dot11_advertisedssid_ssid != null"
+      }
+    },
+    {
+      "set": {
+        "field": "network.wireless.ssid",
+        "value": "Hidden",
+        "if": "ctx?.network?.wireless?.ssid_cloaked != null && ctx?.network?.wireless?.ssid_cloaked == 1"
+      }
+    },
+    {
+      "rename": {
+        "field": "message2.dot11_device.dot11_device_last_beaconed_ssid_record.dot11_advertisedssid_dot11e_channel_utilization_perc",
+        "target_field": "network.wireless.channel_utilization",
+        "if": "ctx?.message2?.dot11_device?.dot11_device_last_beaconed_ssid_record?.dot11_advertisedssid_dot11e_channel_utilization_perc != null"
+      }
+    },
+    {
+      "rename": {
+        "field": "message2.dot11_device.dot11_device_last_bssid",
+        "target_field": "network.wireless.bssid"
+      }
+    },
+    {
+      "foreach": {
+        "field": "message2.dot11_device.dot11_device_associated_client_map",
+        "processor": {
+          "append": {
+            "field": "network.wireless.associated_clients",
+            "value": "{{_ingest._key}}"
+          }
+        },
+        "if": "ctx?.message2?.dot11_device?.dot11_device_associated_client_map != null"
+      }
+    }
+  ]
+}

salt/elasticsearch/files/ingest/kismet.bridged

@@ -0,0 +1,16 @@
+{
+  "processors": [
+    {
+      "rename": {
+        "field": "message2.kismet_device_base_macaddr",
+        "target_field": "client.mac"
+      }
+    },
+    {
+      "rename": {
+        "field": "message2.dot11_device.dot11_device_last_bssid",
+        "target_field": "network.wireless.bssid"
+      }
+    }
+  ]
+}

salt/elasticsearch/files/ingest/kismet.client

@@ -0,0 +1,29 @@
+{
+  "processors": [
+    {
+      "rename": {
+        "field": "message2.kismet_device_base_macaddr",
+        "target_field": "client.mac"
+      }
+    },
+    {
+      "rename": {
+        "field": "message2.dot11_device.dot11_device_last_bssid",
+        "target_field": "network.wireless.last_connected_bssid",
+        "if": "ctx?.message2?.dot11_device?.dot11_device_last_bssid != null"
+      }
+    },
+    {
+      "foreach": {
+        "field": "message2.dot11_device.dot11_device_client_map",
+        "processor": {
+          "append": {
+            "field": "network.wireless.known_connected_bssid",
+            "value": "{{_ingest._key}}"
+          }
+        },
+        "if": "ctx?.message2?.dot11_device?.dot11_device_client_map != null"
+      }
+    }
+  ]
+}

salt/elasticsearch/files/ingest/kismet.common

@@ -0,0 +1,159 @@
+{
+  "processors": [
+    {
+      "json": {
+        "field": "message",
+        "target_field": "message2"
+      }
+    },
+    {
+      "date": {
+        "field": "message2.kismet_device_base_mod_time",
+        "formats": [
+          "epoch_second"
+        ],
+        "target_field": "@timestamp"
+      }
+    },
+    {
+      "set": {
+        "field": "event.category",
+        "value": "network"
+      }
+    },
+    {
+      "dissect": {
+        "field": "message2.kismet_device_base_type",
+        "pattern": "%{wifi} %{device_type}"
+      }
+    },
+    {
+      "lowercase": {
+        "field": "device_type"
+      }
+    },
+    {
+      "set": {
+        "field": "event.dataset",
+        "value": "kismet.{{device_type}}"
+      }
+    },
+    {
+      "set": {
+        "field": "event.dataset",
+        "value": "kismet.wds_ap",
+        "if": "ctx?.device_type == 'wds ap'"
+      }
+    },
+    {
+      "set": {
+        "field": "event.dataset",
+        "value": "kismet.ad_hoc",
+        "if": "ctx?.device_type == 'ad-hoc'"
+      }
+    },
+    {
+      "set": {
+        "field": "event.module",
+        "value": "kismet"
+      }
+    },
+    {
+      "rename": {
+        "field": "message2.kismet_device_base_packets_tx_total",
+        "target_field": "source.packets"
+      }
+    },
+    {
+      "rename": {
+        "field": "message2.kismet_device_base_num_alerts",
+        "target_field": "kismet.alerts.count"
+      }
+    },
+    {
+      "rename": {
+        "field": "message2.kismet_device_base_channel",
+        "target_field": "network.wireless.channel",
+        "if": "ctx?.message2?.kismet_device_base_channel != ''"
+      }
+    },
+    {
+      "rename": {
+        "field": "message2.kismet_device_base_frequency",
+        "target_field": "network.wireless.frequency",
+        "if": "ctx?.message2?.kismet_device_base_frequency != 0"
+      }
+    },
+    {
+      "rename": {
+        "field": "message2.kismet_device_base_last_time",
+        "target_field": "kismet.last_seen"
+      }
+    },
+    {
+      "date": {
+        "field": "kismet.last_seen",
+        "formats": [
+          "epoch_second"
+        ],
+        "target_field": "kismet.last_seen"
+      }
+    },
+    {
+      "rename": {
+        "field": "message2.kismet_device_base_first_time",
+        "target_field": "kismet.first_seen"
+      }
+    },
+    {
+      "date": {
+        "field": "kismet.first_seen",
+        "formats": [
+          "epoch_second"
+        ],
+        "target_field": "kismet.first_seen"
+      }
+    },
+    {
+      "rename": {
+        "field": "message2.kismet_device_base_seenby",
+        "target_field": "kismet.seenby"
+      }
+    },
+    {
+      "foreach": {
+        "field": "kismet.seenby",
+        "processor": {
+          "pipeline": {
+            "name": "kismet.seenby"
+          }
+        }
+      }
+    },
+    {
+      "rename": {
+        "field": "message2.kismet_device_base_manuf",
+        "target_field": "device.manufacturer"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "{{event.dataset}}"
+      }
+    },
+    {
+      "remove": {
+        "field": [
+          "message2",
+          "message",
+          "device_type",
+          "wifi",
+          "agent",
+          "host",
+          "event.created"
+        ],
+        "ignore_failure": true
+      }
+    }
+  ]
+}

salt/elasticsearch/files/ingest/kismet.device

@@ -0,0 +1,9 @@
+{
+  "processors": [
+    {
+      "pipeline": {
+        "name": "kismet.client"
+      }
+    }
+  ]
+}

salt/elasticsearch/files/ingest/kismet.seenby

@@ -0,0 +1,52 @@
+{
+  "processors": [
+    {
+      "rename": {
+        "field": "_ingest._value.kismet_common_seenby_num_packets",
+        "target_field": "_ingest._value.packets_seen",
+        "ignore_missing": true
+      }
+    },
+    {
+      "rename": {
+        "field": "_ingest._value.kismet_common_seenby_uuid",
+        "target_field": "_ingest._value.serial_number",
+        "ignore_missing": true
+      }
+    },
+    {
+      "rename": {
+        "field": "_ingest._value.kismet_common_seenby_first_time",
+        "target_field": "_ingest._value.first_seen",
+        "ignore_missing": true
+      }
+    },
+    {
+      "rename": {
+        "field": "_ingest._value.kismet_common_seenby_last_time",
+        "target_field": "_ingest._value.last_seen",
+        "ignore_missing": true
+      }
+    },
+    {
+      "date": {
+        "field": "_ingest._value.first_seen",
+        "formats": [
+          "epoch_second"
+        ],
+        "target_field": "_ingest._value.first_seen",
+        "ignore_failure": true
+      }
+    },
+    {
+      "date": {
+        "field": "_ingest._value.last_seen",
+        "formats": [
+          "epoch_second"
+        ],
+        "target_field": "_ingest._value.last_seen",
+        "ignore_failure": true
+      }
+    }
+  ]
+}

salt/elasticsearch/files/ingest/kismet.wds

@@ -0,0 +1,10 @@
+{
+  "processors": [
+    {
+      "rename": {
+        "field": "message2.kismet_device_base_macaddr",
+        "target_field": "client.mac"
+      }
+    }
+  ]
+}

salt/elasticsearch/files/ingest/kismet.wds_ap

@@ -0,0 +1,22 @@
+{
+  "processors": [
+    {
+      "rename": {
+        "field": "message2.kismet_device_base_commonname",
+        "target_field": "network.wireless.bssid"
+      }
+    },
+    {
+      "foreach": {
+        "field": "message2.dot11_device.dot11_device_associated_client_map",
+        "processor": {
+          "append": {
+            "field": "network.wireless.associated_clients",
+            "value": "{{_ingest._key}}"
+          }
+        },
+        "if": "ctx?.message2?.dot11_device?.dot11_device_associated_client_map != null"
+      }
+    }
+  ]
+}

salt/elasticsearch/files/ingest/strelka.file

@@ -56,6 +56,7 @@
     { "set":         { "if": "ctx.exiftool?.Subsystem != null", "field": "host.subsystem",        "value": "{{exiftool.Subsystem}}", "ignore_failure": true }},
     { "set":         { "if": "ctx.scan?.yara?.matches instanceof List", "field": "rule.name",        "value": "{{scan.yara.matches.0}}" }},
     { "set":         { "if": "ctx.rule?.name != null", "field": "event.dataset",        "value": "alert", "override": true }},
+    { "set":         { "if": "ctx.rule?.name != null", "field": "rule.uuid",        "value": "{{rule.name}}", "override": true }},
     { "rename":         { "field": "file.flavors.mime", "target_field": "file.mime_type",        "ignore_missing": true }},
     { "set":         { "if": "ctx.rule?.name != null && ctx.rule?.score == null",   "field": "event.severity", "value": 3, "override": true }  },
     { "convert" :    { "if": "ctx.rule?.score != null", "field" : "rule.score","type": "integer"}},

salt/elasticsearch/files/ingest/suricata.alert

@@ -1,6 +1,7 @@
 {
   "description" : "suricata.alert",
   "processors" : [
+    { "set":   { "field": "_index", "value": "logs-suricata.alerts-so" } },
     { "set":   { "field": "tags","value": "alert" }},
     { "rename":{ "field": "message2.alert",	"target_field": "rule",	"ignore_failure": true 	} },
     { "rename":{ "field": "rule.signature",     "target_field": "rule.name", "ignore_failure": true  } },

salt/elasticsearch/roles/analyst.json

@@ -27,7 +27,8 @@
         "monitor",
         "read",
         "read_cross_cluster",
-        "view_index_metadata"
+        "view_index_metadata",
+        "write"
       ]
     }
   ],

salt/elasticsearch/roles/limited-analyst.json

@@ -13,7 +13,8 @@
         "monitor",
         "read",
         "read_cross_cluster",
-        "view_index_metadata"
+        "view_index_metadata",
+        "write"
       ]
     }
   ],

salt/elasticsearch/soc_elasticsearch.yaml

@@ -5,6 +5,10 @@ elasticsearch:
   esheap:
     description: Specify the memory heap size in (m)egabytes for Elasticsearch.
     helpLink: elasticsearch.html
+  index_clean:
+    description: Determines if indices should be considered for deletion by available disk space in the cluster. Otherwise, indices will only be deleted by the age defined in the ILM settings.
+    forcedType: bool
+    helpLink: elasticsearch.html
   retention: 
     retention_pct:
       decription: Total percentage of space used by Elasticsearch for multi node clusters
@@ -98,10 +102,6 @@ elasticsearch:
       policy:
         phases:
           hot:
-            max_age:
-              description: Maximum age of index. ex. 7d - This determines when the index should be moved out of the hot tier.
-              global: True
-              helpLink: elasticsearch.html
             actions:
               set_priority:
                 priority:
@@ -120,7 +120,9 @@ elasticsearch:
                   helpLink: elasticsearch.html
           cold:
             min_age:
-              description: Minimum age of index. ex. 30d - This determines when the index should be moved to the cold tier.  While still searchable, this tier is typically optimized for lower storage costs rather than search speed.
+              description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier.  While still searchable, this tier is typically optimized for lower storage costs rather than search speed.
+              regex: ^[0-9]{1,5}d$
+              forcedType: string
               global: True
               helpLink: elasticsearch.html
             actions:
@@ -131,8 +133,8 @@ elasticsearch:
                   helpLink: elasticsearch.html
           warm:
             min_age: 
-              description: Minimum age of index. ex. 30d - This determines when the index should be moved to the cold tier.  While still searchable, this tier is typically optimized for lower storage costs rather than search speed.
-              regex: ^\[0-9\]{1,5}d$
+              description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier.
+              regex: ^[0-9]{1,5}d$
               forcedType: string
               global: True
             actions:
@@ -145,6 +147,8 @@ elasticsearch:
           delete:
             min_age:
               description: Minimum age of index. ex. 90d - This determines when the index should be deleted.
+              regex: ^[0-9]{1,5}d$
+              forcedType: string
               global: True
               helpLink: elasticsearch.html
     so-logs: &indexSettings
@@ -271,7 +275,9 @@ elasticsearch:
                   helpLink: elasticsearch.html
           warm:
             min_age:
-              description: Minimum age of index. This determines when the index should be moved to the hot tier.
+              description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier.
+              regex: ^[0-9]{1,5}d$
+              forcedType: string
               global: True
               advanced: True
               helpLink: elasticsearch.html
@@ -296,7 +302,9 @@ elasticsearch:
                   helpLink: elasticsearch.html
           cold:
             min_age:
-              description: Minimum age of index.  This determines when the index should be moved to the cold tier.  While still searchable, this tier is typically optimized for lower storage costs rather than search speed.
+              description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier.  While still searchable, this tier is typically optimized for lower storage costs rather than search speed.
+              regex: ^[0-9]{1,5}d$
+              forcedType: string
               global: True
               advanced: True
               helpLink: elasticsearch.html
@@ -311,6 +319,8 @@ elasticsearch:
           delete:
             min_age:
               description: Minimum age of index. This determines when the index should be deleted.
+              regex: ^[0-9]{1,5}d$
+              forcedType: string
               global: True
               advanced: True
               helpLink: elasticsearch.html
@@ -384,6 +394,7 @@ elasticsearch:
     so-logs-darktrace_x_ai_analyst_alert: *indexSettings
     so-logs-darktrace_x_model_breach_alert: *indexSettings
     so-logs-darktrace_x_system_status_alert: *indexSettings
+    so-logs-detections_x_alerts: *indexSettings
     so-logs-f5_bigip_x_log: *indexSettings
     so-logs-fim_x_event: *indexSettings
     so-logs-fortinet_x_clientendpoint: *indexSettings
@@ -510,8 +521,10 @@ elasticsearch:
     so-endgame: *indexSettings
     so-idh: *indexSettings
     so-suricata: *indexSettings
+    so-suricata_x_alerts: *indexSettings
     so-import: *indexSettings
     so-kratos: *indexSettings
+    so-kismet: *indexSettings
     so-logstash: *indexSettings
     so-redis: *indexSettings
     so-strelka: *indexSettings

salt/elasticsearch/template.map.jinja

@@ -2,11 +2,9 @@
 {% set DEFAULT_GLOBAL_OVERRIDES = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings.pop('global_overrides') %}
 
 {% set PILLAR_GLOBAL_OVERRIDES = {} %}
-{% if salt['pillar.get']('elasticsearch:index_settings') is defined %}
-{%   set ES_INDEX_PILLAR = salt['pillar.get']('elasticsearch:index_settings') %}
-{%   if ES_INDEX_PILLAR.global_overrides is defined %}
-{%     set PILLAR_GLOBAL_OVERRIDES = ES_INDEX_PILLAR.pop('global_overrides') %}
-{%   endif %}
+{% set ES_INDEX_PILLAR = salt['pillar.get']('elasticsearch:index_settings', {}) %}
+{% if ES_INDEX_PILLAR.global_overrides is defined %}
+{%   set PILLAR_GLOBAL_OVERRIDES = ES_INDEX_PILLAR.pop('global_overrides') %}
 {% endif %}
 
 {% set ES_INDEX_SETTINGS_ORIG = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings %}
@@ -19,6 +17,12 @@
 {% set ES_INDEX_SETTINGS = {} %}
 {% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update(salt['defaults.merge'](ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %}
 {% for index, settings in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.items() %}
+{#   if policy isn't defined in the original index settings, then dont merge policy from the global_overrides #}
+{#   this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #}
+{%   if not ES_INDEX_SETTINGS_ORIG[index].policy is defined and ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %}
+{%     do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %}
+{%   endif %}
+
 {%   if settings.index_template is defined %}
 {%     if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %}
 {%       do settings.index_template.template.settings.index.pop('sort') %}

salt/elasticsearch/templates/component/ecs/device.json

@@ -0,0 +1,36 @@
+{
+  "_meta": {
+    "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-device.html",
+    "ecs_version": "1.12.2"
+  },
+  "template": {
+    "mappings": {
+      "properties": {
+        "device": {
+          "properties": {
+            "id": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "manufacturer": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "model": {
+              "properties": {
+                "identifier": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "name": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/templates/component/ecs/kismet.json

@@ -0,0 +1,32 @@
+{
+  "_meta": {
+    "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-base.html",
+    "ecs_version": "1.12.2"
+  },
+  "template": {
+    "mappings": {
+      "properties": {
+        "kismet": {
+          "properties": {
+            "alerts": {
+              "properties": {
+                "count": {
+                  "type": "long"
+                }
+              }
+            },
+            "first_seen": {
+              "type": "date"
+            },
+            "last_seen": {
+              "type": "date"
+            },
+            "seenby": {
+              "type": "nested"
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/templates/component/ecs/network.json

@@ -77,6 +77,43 @@
                   "type": "keyword"
                 }
               }
+            },
+            "wireless": {
+              "properties": {
+                "associated_clients": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "bssid": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "channel": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "channel_utilization": {
+                  "type": "float"
+                },
+                "frequency": {
+                  "type": "double"
+                },
+                "ssid": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "ssid_cloaked": {
+                  "type": "integer"
+                },
+                "known_connected_bssid": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "last_connected_bssid": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                }
+              }
             }
           }
         }

salt/elasticsearch/templates/component/elastic-agent/so-items-mappings.json

@@ -0,0 +1,112 @@
+{
+  "template": {
+    "mappings": {
+      "dynamic": "strict",
+      "properties": {
+        "binary": {
+          "type": "binary"
+        },
+        "boolean": {
+          "type": "boolean"
+        },
+        "byte": {
+          "type": "byte"
+        },
+        "created_at": {
+          "type": "date"
+        },
+        "created_by": {
+          "type": "keyword"
+        },
+        "date": {
+          "type": "date"
+        },
+        "date_nanos": {
+          "type": "date_nanos"
+        },
+        "date_range": {
+          "type": "date_range"
+        },
+        "deserializer": {
+          "type": "keyword"
+        },
+        "double": {
+          "type": "double"
+        },
+        "double_range": {
+          "type": "double_range"
+        },
+        "float": {
+          "type": "float"
+        },
+        "float_range": {
+          "type": "float_range"
+        },
+        "geo_point": {
+          "type": "geo_point"
+        },
+        "geo_shape": {
+          "type": "geo_shape"
+        },
+        "half_float": {
+          "type": "half_float"
+        },
+        "integer": {
+          "type": "integer"
+        },
+        "integer_range": {
+          "type": "integer_range"
+        },
+        "ip": {
+          "type": "ip"
+        },
+        "ip_range": {
+          "type": "ip_range"
+        },
+        "keyword": {
+          "type": "keyword"
+        },
+        "list_id": {
+          "type": "keyword"
+        },
+        "long": {
+          "type": "long"
+        },
+        "long_range": {
+          "type": "long_range"
+        },
+        "meta": {
+          "type": "object",
+          "enabled": false
+        },
+        "serializer": {
+          "type": "keyword"
+        },
+        "shape": {
+          "type": "shape"
+        },
+        "short": {
+          "type": "short"
+        },
+        "text": {
+          "type": "text"
+        },
+        "tie_breaker_id": {
+          "type": "keyword"
+        },
+        "updated_at": {
+          "type": "date"
+        },
+        "updated_by": {
+          "type": "keyword"
+        }
+      }
+    },
+    "aliases": {}
+  },
+  "version": 2,
+    "_meta": {
+      "managed": true,
+      "description": "default mappings for the .items index template installed by Kibana/Security"
+    }
+}

salt/elasticsearch/templates/component/elastic-agent/so-lists-mappings.json

@@ -0,0 +1,55 @@
+{
+  "template": {
+  "mappings": {
+    "dynamic": "strict",
+    "properties": {
+      "created_at": {
+        "type": "date"
+      },
+      "created_by": {
+        "type": "keyword"
+      },
+      "description": {
+        "type": "keyword"
+      },
+      "deserializer": {
+        "type": "keyword"
+      },
+      "immutable": {
+        "type": "boolean"
+      },
+      "meta": {
+        "type": "object",
+        "enabled": false
+      },
+      "name": {
+        "type": "keyword"
+      },
+      "serializer": {
+        "type": "keyword"
+      },
+      "tie_breaker_id": {
+        "type": "keyword"
+      },
+      "type": {
+        "type": "keyword"
+      },
+      "updated_at": {
+        "type": "date"
+      },
+      "updated_by": {
+        "type": "keyword"
+      },
+      "version": {
+        "type": "keyword"
+      }
+    }
+  },
+    "aliases": {}
+  },
+  "version": 2,
+    "_meta": {
+      "managed": true,
+      "description": "default mappings for the .lists index template installed by Kibana/Security"
+    }
+}

salt/elasticsearch/templates/component/so/detection-mappings.json

@@ -20,10 +20,12 @@
         "so_detection": {
           "properties": {
             "publicId": {
-              "type": "text"
+              "ignore_above": 1024,
+	      "type": "keyword"
             },
             "title": {
-              "type": "text"
+	      "ignore_above": 1024,
+              "type": "keyword"
             },
             "severity": {
               "ignore_above": 1024,
@@ -36,6 +38,18 @@
             "description": {
               "type": "text"
             },
+	    "category": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+	    "product": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+	    "service": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
             "content": {
               "type": "text"
             },
@@ -49,7 +63,8 @@
               "type": "boolean"
             },
             "tags": {
-              "type": "text"
+              "ignore_above": 1024,
+	      "type": "keyword"
             },
             "ruleset": {
               "ignore_above": 1024,
@@ -136,4 +151,4 @@
   "_meta": {
     "ecs_version": "1.12.2"
   }
-}
+}

salt/elasticsearch/tools/sbin/so-index-list

@@ -5,6 +5,6 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
+. /usr/sbin/so-common
 
-
-curl -K /opt/so/conf/elasticsearch/curl.config-X GET -k -L "https://localhost:9200/_cat/indices?v&s=index"
+curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L "https://localhost:9200/_cat/indices?pretty&v&s=index"

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-total

@@ -40,7 +40,7 @@ fi
 
 # Iterate through the output of _cat/allocation for each node in the cluster to determine the total available space
 {% if GLOBALS.role == 'so-manager' %}
-for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | grep -v {{ GLOBALS.manager }} | awk '{print $5}'); do
+for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | grep -v "{{ GLOBALS.manager }}$" | awk '{print $5}'); do
 {% else %}
 for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | awk '{print $5}'); do
 {% endif %}

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-used

@@ -13,7 +13,7 @@ TOTAL_USED_SPACE=0
 # Iterate through the output of _cat/allocation for each node in the cluster to determine the total used space
 {% if GLOBALS.role == 'so-manager' %}
 # Get total disk space - disk.total
-for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | grep -v {{ GLOBALS.manager }} | awk '{print $3}'); do
+for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | grep -v "{{ GLOBALS.manager }}$" | awk '{print $3}'); do
 {% else %}
 # Get disk space taken up by indices - disk.indices
 for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | awk '{print $2}'); do

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete

@@ -27,6 +27,7 @@ overlimit() {
 # 2. Check if the maximum number of iterations - MAX_ITERATIONS - has been exceeded. If so, exit.
 # Closed indices will be deleted first. If we are able to bring disk space under LOG_SIZE_LIMIT, or the number of iterations has exceeded the maximum allowed number of iterations, we will break out of the loop.
 
+
 while overlimit && [[ $ITERATION -lt $MAX_ITERATIONS ]]; do
 
   # If we can't query Elasticsearch, then immediately return false.
@@ -34,28 +35,36 @@ while overlimit && [[ $ITERATION -lt $MAX_ITERATIONS ]]; do
   [ $? -eq 1 ] && echo "$(date) - Could not query Elasticsearch." >> ${LOG} && exit
 
   # We iterate through the closed and open indices
-  CLOSED_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'close$' | awk '{print $1}' | grep -vE "playbook|so-case" | grep -E "(logstash-|so-|.ds-logs-)" | sort -t- -k3)
-  OPEN_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'open$' | awk '{print $1}' | grep -vE "playbook|so-case" | grep -E "(logstash-|so-|.ds-logs-)" | sort -t- -k3)
+  CLOSED_SO_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'close$' | awk '{print $1}' | grep -E "(^logstash-.*|^so-.*)" | grep -vE "so-case|so-detection" | sort -t- -k3)
+  CLOSED_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'close$' | awk '{print $1}' | grep -E "^.ds-logs-.*" | grep -v "suricata" | sort -t- -k4)
+  OPEN_SO_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'open$' | awk '{print $1}' | grep -E "(^logstash-.*|^so-.*)" | grep -vE "so-case|so-detection" | sort -t- -k3)
+  OPEN_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'open$' | awk '{print $1}' | grep -E "^.ds-logs-.*" | grep -v "suricata" | sort -t- -k4)
 
-  for INDEX in ${CLOSED_INDICES} ${OPEN_INDICES}; do
-    # Now that we've sorted the indices from oldest to newest, we need to check each index to see if it is assigned as the current write index for a data stream
-    # To do so, we need to identify to which data stream this index is associated
-    # We extract the data stream name using the pattern below
-    DATASTREAM_PATTERN="logs-[a-zA-Z_.]+-[a-zA-Z_.]+"
-    DATASTREAM=$(echo "${INDEX}" | grep -oE "$DATASTREAM_PATTERN")
-    # We look up the data stream, and determine the write index. If there is only one backing index, we delete the entire data stream
-    BACKING_INDICES=$(/usr/sbin/so-elasticsearch-query _data_stream/${DATASTREAM} | jq -r '.data_streams[0].indices | length')
-    if [ "$BACKING_INDICES" -gt 1 ]; then
-      CURRENT_WRITE_INDEX=$(/usr/sbin/so-elasticsearch-query _data_stream/$DATASTREAM | jq -r .data_streams[0].indices[-1].index_name)
-      # We make sure we are not trying to delete a write index
-      if [ "${INDEX}" != "${CURRENT_WRITE_INDEX}" ]; then
-        # This should not be a write index, so we should be allowed to delete it
-        printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - Deleting ${INDEX} index...\n" >> ${LOG}
-	/usr/sbin/so-elasticsearch-query ${INDEX} -XDELETE >> ${LOG} 2>&1
-      fi
+  for INDEX in ${CLOSED_SO_INDICES} ${OPEN_SO_INDICES} ${CLOSED_INDICES} ${OPEN_INDICES}; do
+    # Check if index is an older index. If it is an older index, delete it before moving on to newer indices.
+    if [[ "$INDEX" =~ "^logstash-.*|so-.*" ]]; then
+      printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - Deleting ${INDEX} index...\n" >> ${LOG}
+      /usr/sbin/so-elasticsearch-query ${INDEX} -XDELETE >> ${LOG} 2>&1
     else
-	    printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - There is only one backing index (${INDEX}).  Deleting ${DATASTREAM} data stream...\n" >> ${LOG}
+      # Now that we've sorted the indices from oldest to newest, we need to check each index to see if it is assigned as the current write index for a data stream
+      # To do so, we need to identify to which data stream this index is associated
+      # We extract the data stream name using the pattern below
+      DATASTREAM_PATTERN="logs-[a-zA-Z_.]+-[a-zA-Z_.]+"
+      DATASTREAM=$(echo "${INDEX}" | grep -oE "$DATASTREAM_PATTERN")
+      # We look up the data stream, and determine the write index. If there is only one backing index, we delete the entire data stream
+      BACKING_INDICES=$(/usr/sbin/so-elasticsearch-query _data_stream/${DATASTREAM} | jq -r '.data_streams[0].indices | length')
+      if [ "$BACKING_INDICES" -gt 1 ]; then
+        CURRENT_WRITE_INDEX=$(/usr/sbin/so-elasticsearch-query _data_stream/$DATASTREAM | jq -r .data_streams[0].indices[-1].index_name)
+        # We make sure we are not trying to delete a write index
+        if [ "${INDEX}" != "${CURRENT_WRITE_INDEX}" ]; then
+          # This should not be a write index, so we should be allowed to delete it
+          printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - Deleting ${INDEX} index...\n" >> ${LOG}
+          /usr/sbin/so-elasticsearch-query ${INDEX} -XDELETE >> ${LOG} 2>&1
+        fi
+      else
+            printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - There is only one backing index (${INDEX}).  Deleting ${DATASTREAM} data stream...\n" >> ${LOG}
       /usr/sbin/so-elasticsearch-query _data_stream/$DATASTREAM -XDELETE >> ${LOG} 2>&1
+      fi
     fi
     if ! overlimit ; then
       exit

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load

@@ -133,7 +133,7 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then
     for i in $pattern; do
       TEMPLATE=${i::-14}
       COMPONENT_PATTERN=${TEMPLATE:3}
-      MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -v osquery)
+      MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -vE "detections|osquery")
       if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" ]]; then
         load_failures=$((load_failures+1))
         echo "Component template does not exist for $COMPONENT_PATTERN. The index template will not be loaded. Load failures: $load_failures"

salt/firewall/defaults.yaml

@@ -90,17 +90,20 @@ firewall:
       tcp:
         - 8086
       udp: []
-    kafka:
+    kafka_controller:
+      tcp:
+        - 9093
+      udp: []
+    kafka_data:
       tcp:
         - 9092
-        - 9093
       udp: []
     kibana:
       tcp:
         - 5601
       udp: []
     localrules:
-      tcp: 
+      tcp:
         - 7788
       udp: []
     nginx:
@@ -369,7 +372,6 @@ firewall:
                 - elastic_agent_update
                 - localrules
                 - sensoroni
-                - kafka
             fleet:
               portgroups:
                 - elasticsearch_rest
@@ -405,7 +407,6 @@ firewall:
                 - docker_registry
                 - influxdb
                 - sensoroni
-                - kafka
             searchnode:
               portgroups:
                 - redis
@@ -419,7 +420,6 @@ firewall:
                 - elastic_agent_data
                 - elastic_agent_update
                 - sensoroni
-                - kafka
             heavynode:
               portgroups:
                 - redis
@@ -761,7 +761,6 @@ firewall:
                 - beats_5044
                 - beats_5644
                 - beats_5056
-                - redis
                 - elasticsearch_node
                 - elastic_agent_control
                 - elastic_agent_data
@@ -1275,38 +1274,51 @@ firewall:
       chain:
         DOCKER-USER:
           hostgroups:
+            desktop:
+              portgroups:
+                - elastic_agent_data
             fleet:
               portgroups:
-                - beats_5056
+                - elastic_agent_data
+            idh:
+              portgroups:
+                - elastic_agent_data
             sensor:
               portgroups:
-                - beats_5044
-                - beats_5644
                 - elastic_agent_data
-                - kafka
             searchnode:
               portgroups:
                 - redis
-                - beats_5644
-                - kafka
+                - elastic_agent_data
+            standalone:
+              portgroups:
+                - redis
+                - elastic_agent_data
+            manager:
+              portgroups:
+                - elastic_agent_data
             managersearch:
               portgroups:
                 - redis
-                - beats_5644
-                - kafka
+                - elastic_agent_data
             self:
               portgroups:
                 - redis
-                - beats_5644
+                - elastic_agent_data
             beats_endpoint:
               portgroups:
                 - beats_5044
             beats_endpoint_ssl:
               portgroups:
                 - beats_5644
+            elastic_agent_endpoint:
+              portgroups:
+                - elastic_agent_data
             endgame:
               portgroups:
                 - endgame
+            receiver:
+              portgroups: []
             customhostgroup0:
               portgroups: []
             customhostgroup1:

salt/firewall/map.jinja

@@ -18,4 +18,28 @@
 {%   endfor %}
 {% endif %}
 
+{# Only add Kafka firewall items when Kafka enabled #}
+{% set role = GLOBALS.role.split('-')[1] %}
+
+{% if GLOBALS.pipeline == 'KAFKA' and role in ['manager', 'managersearch', 'standalone'] %}
+{%   do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[role].portgroups.append('kafka_controller') %}
+{%   do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.receiver.portgroups.append('kafka_controller') %}
+{% endif %}
+
+{% if GLOBALS.pipeline == 'KAFKA' and role == 'receiver' %}
+{%   do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.self.portgroups.append('kafka_controller') %}
+{%   do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.standalone.portgroups.append('kafka_controller') %}
+{%   do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.manager.portgroups.append('kafka_controller') %}
+{%   do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.managersearch.portgroups.append('kafka_controller') %}
+{%   do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.receiver.portgroups.append('kafka_controller') %}
+{% endif %}
+
+{% if GLOBALS.pipeline == 'KAFKA' and role in ['manager', 'managersearch', 'standalone', 'receiver'] %}
+{%   for r in ['manager', 'managersearch', 'standalone', 'receiver', 'fleet', 'idh', 'sensor', 'searchnode','heavynode', 'elastic_agent_endpoint', 'desktop'] %}
+{%     if FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[r] is defined %}
+{%       do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[r].portgroups.append('kafka_data') %}
+{%     endif %}
+{%   endfor %}
+{% endif %}
+
 {% set FIREWALL_MERGED = salt['pillar.get']('firewall', FIREWALL_DEFAULT.firewall, merge=True) %}

salt/firewall/soc_firewall.yaml

@@ -7,6 +7,7 @@ firewall:
       multiline: True
       regex: ^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?)?$
       regexFailureMessage: You must enter a valid IP address or CIDR.
+      duplicates: True
     anywhere: &hostgroupsettingsadv
       description: List of IP or CIDR blocks to allow access to this hostgroup.
       forcedType: "[]string"
@@ -15,6 +16,7 @@ firewall:
       advanced: True
       regex: ^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?)?$
       regexFailureMessage: You must enter a valid IP address or CIDR.
+      duplicates: True
     beats_endpoint: *hostgroupsettings
     beats_endpoint_ssl: *hostgroupsettings
     dockernet: &ROhostgroupsettingsadv
@@ -53,6 +55,7 @@ firewall:
       multiline: True
       regex: ^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?)?$
       regexFailureMessage: You must enter a valid IP address or CIDR.
+      duplicates: True
     customhostgroup1: *customhostgroupsettings
     customhostgroup2: *customhostgroupsettings
     customhostgroup3: *customhostgroupsettings
@@ -70,12 +73,14 @@ firewall:
         helpLink: firewall.html
         advanced: True
         multiline: True
+        duplicates: True
       udp: &udpsettings
         description: List of UDP ports for this port group.
         forcedType: "[]string"
         helpLink: firewall.html
         advanced: True
         multiline: True
+        duplicates: True
     agrules:
       tcp: *tcpsettings
       udp: *udpsettings
@@ -190,6 +195,7 @@ firewall:
                 multiline: True
                 forcedType: "[]string"
                 helpLink: firewall.html
+                duplicates: True
             sensor:
               portgroups: *portgroupsdocker
             searchnode:
@@ -243,6 +249,7 @@ firewall:
                 multiline: True
                 forcedType: "[]string"
                 helpLink: firewall.html
+                duplicates: True
             dockernet: 
               portgroups: *portgroupshost
             localhost:

salt/idh/openssh/config.sls

@@ -11,6 +11,8 @@ idh_sshd_selinux:
     - sel_type: ssh_port_t
     - prereq:
       - file: openssh_config
+    - require:
+      - pkg: python_selinux_mgmt_tools
 {% endif %}
 
 openssh_config:

salt/idh/openssh/init.sls

@@ -15,3 +15,9 @@ openssh:
     - enable: False
     - name: {{ openssh_map.service }}
   {% endif %}
+
+{% if grains.os_family == 'RedHat' %}
+python_selinux_mgmt_tools:
+  pkg.installed:
+    - name: policycoreutils-python-utils
+{% endif %}

salt/idstools/config.sls

@@ -33,6 +33,19 @@ idstools_sbin_jinja:
     - file_mode: 755
     - template: jinja
 
+suricatacustomdirsfile:
+  file.directory:
+    - name: /nsm/rules/detect-suricata/custom_file
+    - user: 939
+    - group: 939
+    - makedirs: True
+
+suricatacustomdirsurl:
+  file.directory:
+    - name: /nsm/rules/detect-suricata/custom_temp
+    - user: 939
+    - group: 939
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/idstools/etc/rulecat.conf

@@ -1,6 +1,8 @@
 {%- from 'vars/globals.map.jinja' import GLOBALS -%}
-{%- from 'idstools/map.jinja' import IDSTOOLSMERGED -%}
+{%- from 'soc/merged.map.jinja' import SOCMERGED -%}
+--suricata-version=6.0
 --merged=/opt/so/rules/nids/suri/all.rules
+--output=/nsm/rules/detect-suricata/custom_temp
 --local=/opt/so/rules/nids/suri/local.rules
 {%-   if GLOBALS.md_engine == "SURICATA" %}
 --local=/opt/so/rules/nids/suri/extraction.rules
@@ -10,8 +12,12 @@
 --disable=/opt/so/idstools/etc/disable.conf
 --enable=/opt/so/idstools/etc/enable.conf
 --modify=/opt/so/idstools/etc/modify.conf
-{%- if IDSTOOLSMERGED.config.urls | length > 0 %}
-{%-   for URL in IDSTOOLSMERGED.config.urls %}
---url={{ URL }}
-{%-   endfor %}
-{%- endif %}
+{%- if SOCMERGED.config.server.modules.suricataengine.customRulesets %}
+    {%- for ruleset in SOCMERGED.config.server.modules.suricataengine.customRulesets %}
+        {%- if 'url' in ruleset %}
+--url={{ ruleset.url }}
+        {%- elif 'file' in ruleset %}
+--local={{ ruleset.file }}
+        {%- endif %}
+    {%- endfor %}
+{%- endif %}

salt/idstools/soc_idstools.yaml

@@ -9,43 +9,53 @@ idstools:
       forcedType: string
       helpLink: rules.html
     ruleset:
-      description: 'Defines the ruleset you want to run. Options are ETOPEN or ETPRO. WARNING! Changing the ruleset will remove all existing Suricata rules of the previous ruleset and their associated overrides. This removal cannot be undone.'
+      description: 'Defines the ruleset you want to run. Options are ETOPEN or ETPRO. Once you have changed the ruleset here, you will need to wait for the rule update to take place (every 24 hours), or you can force the update by nagivating to Detections -->  Options dropdown menu --> Suricata --> Full Update. WARNING! Changing the ruleset will remove all existing non-overlapping Suricata rules of the previous ruleset and their associated overrides. This removal cannot be undone.'
       global: True
       regex:  ETPRO\b|ETOPEN\b
       helpLink: rules.html
     urls:
-      description: This is a list of additional rule download locations.
+      description: This is a list of additional rule download locations. This feature is currently disabled. 
       global: True
+      multiline: True
+      forcedType: "[]string"
+      readonly: True
       helpLink: rules.html
   sids:
     disabled:
-      description: Contains the list of NIDS rules manually disabled across the grid. To disable a rule, add its Signature ID (SID) to the Current Grid Value box, one entry per line. To disable multiple rules, you can use regular expressions.
+      description: Contains the list of NIDS rules (or regex patterns) disabled across the grid. This setting is readonly; Use the Detections screen to disable rules.
       global: True
       multiline: True
       forcedType: "[]string"
       regex: \d*|re:.*
       helpLink: managing-alerts.html
+      readonlyUi: True
+      advanced: true
     enabled:
-      description: Contains the list of NIDS rules manually enabled across the grid. To enable a rule, add its Signature ID (SID) to the Current Grid Value box, one entry per line. To enable multiple rules, you can use regular expressions.
+      description: Contains the list of NIDS rules (or regex patterns) enabled across the grid. This setting is readonly; Use the Detections screen to enable rules.
       global: True
       multiline: True
       forcedType: "[]string"
       regex: \d*|re:.*
       helpLink: managing-alerts.html
+      readonlyUi: True
+      advanced: true
     modify:
-      description: Contains the list of NIDS rules that were modified from their default values. Entries must adhere to the following format - SID "REGEX_SEARCH_TERM" "REGEX_REPLACE_TERM"
+      description: Contains the list of NIDS rules (SID "REGEX_SEARCH_TERM" "REGEX_REPLACE_TERM"). This setting is readonly; Use the Detections screen to modify rules.
       global: True
       multiline: True
       forcedType: "[]string"
       helpLink: managing-alerts.html
+      readonlyUi: True
+      advanced: true
   rules:
     local__rules:
-      description: Contains the list of custom NIDS rules applied to the grid. To add custom NIDS rules to the grid, enter one rule per line in the Current Grid Value box.
+      description: Contains the list of custom NIDS rules applied to the grid. This setting is readonly; Use the Detections screen to adjust rules.
       file: True
       global: True
       advanced: True
       title: Local Rules
       helpLink: local-rules.html
+      readonlyUi: True
     filters__rules:
       description: If you are using Suricata for metadata, then you can set custom filters for that metadata here.
       file: True

salt/idstools/tools/sbin_jinja/so-rule-update

@@ -26,8 +26,6 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
     docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force
 {%-   elif IDSTOOLSMERGED.config.ruleset == 'ETPRO' %}
     docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force --etpro={{ IDSTOOLSMERGED.config.oinkcode }}
-{%-   elif IDSTOOLSMERGED.config.ruleset == 'TALOS' %}
-    docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force --url=https://www.snort.org/rules/snortrules-snapshot-2983.tar.gz?oinkcode={{ IDSTOOLSMERGED.config.oinkcode }}
 {%-   endif %}
 {%- endif %}
 

salt/kafka/config.map.jinja

@@ -0,0 +1,91 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+{% from 'kafka/map.jinja' import KAFKAMERGED %}
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+
+{% set KAFKA_NODES_PILLAR = salt['pillar.get']('kafka:nodes') %}
+{% set KAFKA_PASSWORD = salt['pillar.get']('kafka:password') %}
+
+{# Create list of KRaft controllers #}
+{% set controllers = [] %}
+
+{# Check for Kafka nodes with controller in process_x_roles #}
+{% for node in KAFKA_NODES_PILLAR %}
+{%   if 'controller' in KAFKA_NODES_PILLAR[node].role %}
+{%     do controllers.append(KAFKA_NODES_PILLAR[node].nodeid ~ "@" ~ node ~ ":9093") %}
+{%   endif %}
+{% endfor %}
+
+{% set kafka_controller_quorum_voters = ','.join(controllers) %}
+
+{# By default all Kafka eligible nodes are given the role of broker, except for
+     grid MANAGER (broker,controller) until overridden through SOC UI #}
+{% set node_type = salt['pillar.get']('kafka:nodes:'+ GLOBALS.hostname + ':role') %}
+
+{# Generate server.properties for 'broker' , 'controller', 'broker,controller' node types
+    anything above this line is a configuration needed for ALL Kafka nodes #}
+{% if node_type == 'broker' %}
+{%   do KAFKAMERGED.config.broker.update({'advertised_x_listeners': 'BROKER://'+ GLOBALS.node_ip +':9092' }) %}
+{%   do KAFKAMERGED.config.broker.update({'controller_x_quorum_x_voters': kafka_controller_quorum_voters }) %}
+{%   do KAFKAMERGED.config.broker.update({'node_x_id': salt['pillar.get']('kafka:nodes:'+ GLOBALS.hostname +':nodeid') }) %}
+{%   do KAFKAMERGED.config.broker.update({'ssl_x_keystore_x_password': KAFKA_PASSWORD }) %}
+
+{# Nodes with only the 'broker' role need to have the below settings for communicating with controller nodes #}
+{%   do KAFKAMERGED.config.broker.update({'controller_x_listener_x_names': KAFKAMERGED.config.controller.controller_x_listener_x_names }) %}
+{%   do KAFKAMERGED.config.broker.update({
+       'listener_x_security_x_protocol_x_map': KAFKAMERGED.config.broker.listener_x_security_x_protocol_x_map
+        + ',' + KAFKAMERGED.config.controller.listener_x_security_x_protocol_x_map })
+  %}
+{% endif %}
+
+{% if node_type == 'controller' %}
+{%   do KAFKAMERGED.config.controller.update({'controller_x_quorum_x_voters': kafka_controller_quorum_voters }) %}
+{%   do KAFKAMERGED.config.controller.update({'node_x_id': salt['pillar.get']('kafka:nodes:'+ GLOBALS.hostname +':nodeid') }) %}
+{%   do KAFKAMERGED.config.controller.update({'ssl_x_keystore_x_password': KAFKA_PASSWORD }) %}
+
+{% endif %}
+
+{# Kafka nodes of this type are not recommended for use outside of development / testing. #}
+{% if node_type == 'broker,controller' %}
+{%   do KAFKAMERGED.config.broker.update({'advertised_x_listeners': 'BROKER://'+ GLOBALS.node_ip +':9092' }) %}
+{%   do KAFKAMERGED.config.broker.update({'controller_x_listener_x_names': KAFKAMERGED.config.controller.controller_x_listener_x_names }) %}
+{%   do KAFKAMERGED.config.broker.update({'controller_x_quorum_x_voters': kafka_controller_quorum_voters }) %}
+{%   do KAFKAMERGED.config.broker.update({'node_x_id': salt['pillar.get']('kafka:nodes:'+ GLOBALS.hostname +':nodeid') }) %}
+{%   do KAFKAMERGED.config.broker.update({'process_x_roles': 'broker,controller' }) %}
+{%   do KAFKAMERGED.config.broker.update({'ssl_x_keystore_x_password': KAFKA_PASSWORD }) %}
+
+{%   do KAFKAMERGED.config.broker.update({
+       'listeners': KAFKAMERGED.config.broker.listeners + ',' + KAFKAMERGED.config.controller.listeners })
+  %}
+
+{%   do KAFKAMERGED.config.broker.update({
+       'listener_x_security_x_protocol_x_map': KAFKAMERGED.config.broker.listener_x_security_x_protocol_x_map
+        + ',' + KAFKAMERGED.config.controller.listener_x_security_x_protocol_x_map })
+  %}
+
+{% endif %}
+
+{# If a password other than PLACEHOLDER isn't set remove it from the server.properties #}
+{% if KAFKAMERGED.config.broker.ssl_x_truststore_x_password == 'PLACEHOLDER' %}
+{%   do KAFKAMERGED.config.broker.pop('ssl_x_truststore_x_password') %}
+{% endif %}
+
+{% if KAFKAMERGED.config.controller.ssl_x_truststore_x_password == 'PLACEHOLDER' %}
+{%   do KAFKAMERGED.config.controller.pop('ssl_x_truststore_x_password') %}
+{% endif %}
+
+{# Client properties stuff #}
+{% if KAFKAMERGED.config.client.ssl_x_truststore_x_password == 'PLACEHOLDER' %}
+{%   do KAFKAMERGED.config.client.pop('ssl_x_truststore_x_password') %}
+{% endif %}
+{% do KAFKAMERGED.config.client.update({'ssl_x_keystore_x_password': KAFKA_PASSWORD }) %}
+
+{% if 'broker' in node_type %}
+{%   set KAFKACONFIG = KAFKAMERGED.config.broker %}
+{% else %}
+{%   set KAFKACONFIG = KAFKAMERGED.config.controller %}
+{% endif %}
+
+{% set KAFKACLIENT = KAFKAMERGED.config.client %}

salt/kafka/config.sls

@@ -7,23 +7,6 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 
-{% set kafka_ips_logstash = [] %}
-{% set kafka_ips_kraft = [] %}
-{% set kafkanodes =  salt['pillar.get']('kafka:nodes', {}) %}
-{% set kafka_ip = GLOBALS.node_ip %}
-
-{# Create list for kafka <-> logstash/searchnode communcations #}
-{% for node, node_data in kafkanodes.items() %}
-{%   do kafka_ips_logstash.append(node_data['ip'] + ":9092") %}
-{% endfor %}
-{% set kafka_server_list = "','".join(kafka_ips_logstash) %}
-
-{# Create a list for kraft controller <-> kraft controller communications. Used for Kafka metadata management #}
-{% for node, node_data in kafkanodes.items() %}
-{%   do kafka_ips_kraft.append(node_data['nodeid'] ~ "@" ~ node_data['ip'] ~ ":9093") %}
-{% endfor %}
-{% set kraft_server_list = "','".join(kafka_ips_kraft) %}
-
 include:
   - ssl
 
@@ -37,27 +20,15 @@ kafka:
     - uid: 960
     - gid: 960
 
-{# Future tools to query kafka directly / show consumer groups
 kafka_sbin_tools:
   file.recurse:
     - name: /usr/sbin
     - source: salt://kafka/tools/sbin
     - user: 960
     - group: 960
-    - file_mode: 755 #}
-
-kafka_sbin_jinja_tools:
-  file.recurse:
-    - name: /usr/sbin
-    - source: salt://kafka/tools/sbin_jinja
-    - user: 960
-    - group: 960
     - file_mode: 755
-    - template: jinja
-    - defaults:
-        GLOBALS: {{ GLOBALS }}
 
-kakfa_log_dir:
+kafka_log_dir:
   file.directory:
     - name: /opt/so/log/kafka
     - user: 960
@@ -71,20 +42,6 @@ kafka_data_dir:
     - group: 960
     - makedirs: True
 
-kafka_generate_keystore:
-  cmd.run:
-    - name: "/usr/sbin/so-kafka-generate-keystore"
-    - onchanges:
-      - x509: /etc/pki/kafka.key
-
-kafka_keystore_perms:
-  file.managed:
-    - replace: False
-    - name: /etc/pki/kafka.jks
-    - mode: 640
-    - user: 960
-    - group: 939
-
 {%   for sc in ['server', 'client'] %}
 kafka_kraft_{{sc}}_properties:
   file.managed:
@@ -97,6 +54,12 @@ kafka_kraft_{{sc}}_properties:
     - show_changes: False
 {%   endfor %}
 
+reset_quorum_on_changes:
+  cmd.run:
+    - name: rm -f /nsm/kafka/data/__cluster_metadata-0/quorum-state
+    - onchanges:
+      - file: /opt/so/conf/kafka/server.properties
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/kafka/defaults.yaml

@@ -1,14 +1,18 @@
 kafka:
   enabled: False
+  cluster_id:
+  password:
+  controllers:
+  reset:
   config:
-    server:
+    broker:
       advertised_x_listeners:
       auto_x_create_x_topics_x_enable: true
-      controller_x_listener_x_names: CONTROLLER
       controller_x_quorum_x_voters:
+      default_x_replication_x_factor: 1
       inter_x_broker_x_listener_x_name: BROKER
-      listeners: BROKER://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093
-      listener_x_security_x_protocol_x_map: CONTROLLER:SSL,BROKER:SSL
+      listeners: BROKER://0.0.0.0:9092
+      listener_x_security_x_protocol_x_map: BROKER:SSL
       log_x_dirs: /nsm/kafka/data
       log_x_retention_x_check_x_interval_x_ms: 300000
       log_x_retention_x_hours: 168
@@ -16,24 +20,43 @@ kafka:
       node_x_id:
       num_x_io_x_threads: 8
       num_x_network_x_threads: 3
-      num_x_partitions: 1
+      num_x_partitions: 3
       num_x_recovery_x_threads_x_per_x_data_x_dir: 1
       offsets_x_topic_x_replication_x_factor: 1
       process_x_roles: broker
       socket_x_receive_x_buffer_x_bytes: 102400
       socket_x_request_x_max_x_bytes: 104857600
       socket_x_send_x_buffer_x_bytes: 102400
-      ssl_x_keystore_x_location: /etc/pki/kafka.jks
-      ssl_x_keystore_x_password: changeit
-      ssl_x_keystore_x_type: JKS
+      ssl_x_keystore_x_location: /etc/pki/kafka.p12
+      ssl_x_keystore_x_type: PKCS12
+      ssl_x_keystore_x_password:
       ssl_x_truststore_x_location: /etc/pki/java/sos/cacerts
-      ssl_x_truststore_x_password: changeit
+      ssl_x_truststore_x_password: PLACEHOLDER
+      ssl_x_truststore_x_type: PEM
       transaction_x_state_x_log_x_min_x_isr: 1
       transaction_x_state_x_log_x_replication_x_factor: 1
     client:
       security_x_protocol: SSL
       ssl_x_truststore_x_location: /etc/pki/java/sos/cacerts
-      ssl_x_truststore_x_password: changeit
-      ssl_x_keystore_x_location: /etc/pki/kafka.jks
-      ssl_x_keystore_x_type: JKS
-      ssl_x_keystore_x_password: changeit
+      ssl_x_truststore_x_password: PLACEHOLDER
+      ssl_x_truststore_x_type: PEM
+      ssl_x_keystore_x_location: /etc/pki/kafka.p12
+      ssl_x_keystore_x_type: PKCS12
+      ssl_x_keystore_x_password:
+    controller:
+      controller_x_listener_x_names: CONTROLLER
+      controller_x_quorum_x_voters:
+      listeners: CONTROLLER://0.0.0.0:9093
+      listener_x_security_x_protocol_x_map: CONTROLLER:SSL
+      log_x_dirs: /nsm/kafka/data
+      log_x_retention_x_check_x_interval_x_ms: 300000
+      log_x_retention_x_hours: 168
+      log_x_segment_x_bytes: 1073741824
+      node_x_id:
+      process_x_roles: controller
+      ssl_x_keystore_x_location: /etc/pki/kafka.p12
+      ssl_x_keystore_x_type: PKCS12
+      ssl_x_keystore_x_password:
+      ssl_x_truststore_x_location: /etc/pki/java/sos/cacerts
+      ssl_x_truststore_x_password: PLACEHOLDER
+      ssl_x_truststore_x_type: PEM

salt/kafka/disabled.sls

@@ -3,8 +3,8 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-include:
-  - kafka.sostatus
+{% from 'kafka/map.jinja' import KAFKAMERGED %}
+{% from 'vars/globals.map.jinja' import GLOBALS %}
 
 so-kafka:
   docker_container.absent:
@@ -14,3 +14,12 @@ so-kafka_so-status.disabled:
   file.comment:
     - name: /opt/so/conf/so-status/so-status.conf
     - regex: ^so-kafka$
+    - onlyif: grep -q '^so-kafka$' /opt/so/conf/so-status/so-status.conf
+
+{% if GLOBALS.is_manager and KAFKAMERGED.enabled or GLOBALS.pipeline == "KAFKA" %}
+ensure_default_pipeline:
+  cmd.run:
+    - name: |
+        /usr/sbin/so-yaml.py replace /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.enabled False;
+        /usr/sbin/so-yaml.py replace /opt/so/saltstack/local/pillar/global/soc_global.sls global.pipeline REDIS
+{% endif %}

salt/kafka/enabled.sls

@@ -1,13 +1,20 @@
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
+#
+# Note: Per the Elastic License 2.0, the second limitation states:
+#
+#   "You may not move, change, disable, or circumvent the license key functionality
+#    in the software, and you may not remove or obscure any functionality in the
+#    software that is protected by the license key."
 
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKER %}
-{%   set KAFKANODES = salt['pillar.get']('kafka:nodes', {}) %}
+{%   set KAFKANODES = salt['pillar.get']('kafka:nodes') %}
+{%   if 'gmd' in salt['pillar.get']('features', []) %}
 
 include:
   - elasticsearch.ca
@@ -25,7 +32,8 @@ so-kafka:
         - ipv4_address: {{ DOCKER.containers['so-kafka'].ip }}
     - user: kafka
     - environment:
-      - KAFKA_HEAP_OPTS=-Xmx2G -Xms1G
+        KAFKA_HEAP_OPTS: -Xmx2G -Xms1G
+        KAFKA_OPTS: -javaagent:/opt/jolokia/agents/jolokia-agent-jvm-javaagent.jar=port=8778,host={{ DOCKER.containers['so-kafka'].ip }},policyLocation=file:/opt/jolokia/jolokia.xml
     - extra_hosts:
       {% for node in KAFKANODES %}
       - {{ node }}:{{ KAFKANODES[node].ip }}
@@ -40,11 +48,12 @@ so-kafka:
       - {{ BINDING }}
       {% endfor %}
     - binds:
-      - /etc/pki/kafka.jks:/etc/pki/kafka.jks
-      - /opt/so/conf/ca/cacerts:/etc/pki/java/sos/cacerts
+      - /etc/pki/kafka.p12:/etc/pki/kafka.p12:ro
+      - /etc/pki/tls/certs/intca.crt:/etc/pki/java/sos/cacerts:ro
       - /nsm/kafka/data/:/nsm/kafka/data/:rw
-      - /opt/so/conf/kafka/server.properties:/kafka/config/kraft/server.properties
-      - /opt/so/conf/kafka/client.properties:/kafka/config/kraft/client.properties
+      - /opt/so/log/kafka:/opt/kafka/logs/:rw
+      - /opt/so/conf/kafka/server.properties:/opt/kafka/config/kraft/server.properties:ro
+      - /opt/so/conf/kafka/client.properties:/opt/kafka/config/kraft/client.properties
     - watch:
       {% for sc in ['server', 'client'] %}
       - file: kafka_kraft_{{sc}}_properties
@@ -55,6 +64,19 @@ delete_so-kafka_so-status.disabled:
     - name: /opt/so/conf/so-status/so-status.conf
     - regex: ^so-kafka$
 
+{%   else %}
+
+{{sls}}_no_license_detected:
+  test.fail_without_changes:
+    - name: {{sls}}_no_license_detected
+    - comment:
+      - "Kafka for Guaranteed Message Delivery is a feature supported only for customers with a valid license.
+      Contact Security Onion Solutions, LLC via our website at https://securityonionsolutions.com
+      for more information about purchasing a license to enable this feature."
+include:
+  - kafka.disabled
+{%   endif %}
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/kafka/etc/client.properties.jinja

@@ -3,5 +3,5 @@
    https://securityonion.net/license; you may not use this file except in compliance with the
    Elastic License 2.0. #}
 
-{% from 'kafka/map.jinja' import KAFKAMERGED -%}
-{{ KAFKAMERGED.config.client | yaml(False) | replace("_x_", ".") }}
+{% from 'kafka/config.map.jinja' import KAFKACLIENT -%}
+{{ KAFKACLIENT | yaml(False) | replace("_x_", ".") }}

salt/kafka/etc/server.properties.jinja

@@ -3,5 +3,5 @@
    https://securityonion.net/license; you may not use this file except in compliance with the
    Elastic License 2.0. #}
 
-{% from 'kafka/map.jinja' import KAFKAMERGED -%}
-{{ KAFKAMERGED.config.server | yaml(False) | replace("_x_", ".") }}
+{% from 'kafka/config.map.jinja' import KAFKACONFIG -%}
+{{ KAFKACONFIG | yaml(False) | replace("_x_", ".") }}

salt/kafka/files/managed_node_pillar.jinja

@@ -0,0 +1,10 @@
+kafka:
+  nodes:
+{% for node, values in COMBINED_KAFKANODES.items() %}
+    {{ node }}:
+      ip: {{ values['ip'] }}
+      nodeid: {{ values['nodeid'] }}
+{%-   if values['role'] != none %}
+      role: {{ values['role'] }}
+{%-   endif %}
+{% endfor %}

salt/kafka/init.sls

@@ -1,12 +1,22 @@
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
+#
+# Note: Per the Elastic License 2.0, the second limitation states:
+#
+#   "You may not move, change, disable, or circumvent the license key functionality
+#    in the software, and you may not remove or obscure any functionality in the
+#    software that is protected by the license key."
 
 {% from 'kafka/map.jinja' import KAFKAMERGED %}
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 
 include:
+{# Run kafka/nodes.sls before Kafka is enabled, so kafka nodes pillar is setup #}
+{% if grains.role in ['so-manager','so-managersearch', 'so-standalone'] %}
+  - kafka.nodes
+{% endif %}
 {% if GLOBALS.pipeline == "KAFKA" and KAFKAMERGED.enabled %}
   - kafka.enabled
 {% else %}

salt/kafka/map.jinja

@@ -3,18 +3,8 @@
    https://securityonion.net/license; you may not use this file except in compliance with the
    Elastic License 2.0. #}
 
+{# This is only used to determine if Kafka is enabled / disabled. Configuration is found in kafka/config.map.jinja #}
+{# kafka/config.map.jinja depends on there being a kafka nodes pillar being populated #}
+
 {% import_yaml 'kafka/defaults.yaml' as KAFKADEFAULTS %}
 {% set KAFKAMERGED = salt['pillar.get']('kafka', KAFKADEFAULTS.kafka, merge=True) %}
-{% from 'vars/globals.map.jinja' import GLOBALS %}
-
-{% do KAFKAMERGED.config.server.update({ 'node_x_id': salt['pillar.get']('kafka:nodes:' ~ GLOBALS.hostname ~ ':nodeid')}) %}
-{% do KAFKAMERGED.config.server.update({'advertised_x_listeners': 'BROKER://' ~ GLOBALS.node_ip ~ ':9092'}) %}
-
-{% set nodes = salt['pillar.get']('kafka:nodes', {}) %}
-{% set combined = [] %}
-{% for hostname, data in nodes.items() %}
-  {% do combined.append(data.nodeid ~ "@" ~ hostname ~ ":9093") %}
-{% endfor %}
-{% set kraft_controller_quorum_voters = ','.join(combined) %}
-
-{% do KAFKAMERGED.config.server.update({'controller_x_quorum_x_voters': kraft_controller_quorum_voters}) %}

salt/kafka/nodes.map.jinja

@@ -0,0 +1,88 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+
+{# USED TO GENERATE PILLAR/KAFKA/NODES.SLS. #}
+{% import_yaml 'kafka/defaults.yaml' as KAFKADEFAULTS %}
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+
+{% set process_x_roles = KAFKADEFAULTS.kafka.config.broker.process_x_roles %}
+
+{% set current_kafkanodes = salt.saltutil.runner(
+        'mine.get',
+         tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-receiver',
+         fun='network.ip_addrs',
+         tgt_type='compound') %}
+
+{% set STORED_KAFKANODES = salt['pillar.get']('kafka:nodes', default=None) %}
+{% set KAFKA_CONTROLLERS_PILLAR = salt['pillar.get']('kafka:controllers', default=None) %}
+
+{% set existing_ids = [] %}
+
+{# Check STORED_KAFKANODES for existing kafka nodes and pull their IDs so they are not reused across the grid #}
+{% if STORED_KAFKANODES != none %}
+{%   for node, values in STORED_KAFKANODES.items() %}
+{%     if values.get('nodeid') %}
+{%       do existing_ids.append(values['nodeid']) %}
+{%     endif %}
+{%   endfor %}
+{% endif %}
+
+{# Create list of possible node ids #}
+{% set all_possible_ids = range(1, 2000)|list %}
+
+{# Create list of available node ids by looping through all_possible_ids and ensuring it isn't in existing_ids #}
+{% set available_ids = [] %}
+{% for id in all_possible_ids %}
+{%   if id not in existing_ids %}
+{%     do available_ids.append(id) %}
+{%   endif %}
+{% endfor %}
+
+{# Collect kafka eligible nodes and check if they're already in STORED_KAFKANODES to avoid potentially reassigning a nodeid #}
+{% set NEW_KAFKANODES = {} %}
+{% for minionid, ip in current_kafkanodes.items() %}
+{%   set hostname = minionid.split('_')[0] %}
+{%   if not STORED_KAFKANODES or hostname not in STORED_KAFKANODES %}
+{%     set new_id = available_ids.pop(0) %}
+{%     do NEW_KAFKANODES.update({hostname: {'nodeid': new_id, 'ip': ip[0], 'role': process_x_roles }}) %}
+{%   endif %}
+{% endfor %}
+
+{# Combine STORED_KAFKANODES and NEW_KAFKANODES for writing to the pillar/kafka/nodes.sls #}
+{% set COMBINED_KAFKANODES = {} %}
+{% for node, details in NEW_KAFKANODES.items() %}
+{%   do COMBINED_KAFKANODES.update({node: details}) %}
+{% endfor %}
+{% if STORED_KAFKANODES != none %}
+{%  for node, details in STORED_KAFKANODES.items() %}
+{%    do COMBINED_KAFKANODES.update({node: details}) %}
+{%  endfor %}
+{% endif %}
+
+{# Update the process_x_roles value for any host in the kafka_controllers_pillar configured from SOC UI #}
+{% set ns = namespace(has_controller=false) %}
+{% if KAFKA_CONTROLLERS_PILLAR != none %}
+{%   set KAFKA_CONTROLLERS_PILLAR_LIST = KAFKA_CONTROLLERS_PILLAR.split(',') %}
+{%   for hostname in KAFKA_CONTROLLERS_PILLAR_LIST %}
+{%     if hostname in COMBINED_KAFKANODES %}
+{%       do COMBINED_KAFKANODES[hostname].update({'role': 'controller'}) %}
+{%       set ns.has_controller = true %}
+{%     endif %}
+{%   endfor %}
+{%   for hostname in COMBINED_KAFKANODES %}
+{%     if hostname not in KAFKA_CONTROLLERS_PILLAR_LIST %}
+{%       do COMBINED_KAFKANODES[hostname].update({'role': 'broker'}) %}
+{%     endif %}
+{%   endfor %}
+{# If the kafka_controllers_pillar is NOT empty check that atleast one node contains the controller role.
+     otherwise default to GLOBALS.manager having broker,controller role #}
+{%   if not ns.has_controller %}
+{%     do COMBINED_KAFKANODES[GLOBALS.manager].update({'role': 'broker,controller'}) %}
+{%   endif %}
+{# If kafka_controllers_pillar is empty, default to having grid manager as 'broker,controller'
+   so there is always atleast 1 controller in the cluster #}
+{% else %}
+{%   do COMBINED_KAFKANODES[GLOBALS.manager].update({'role': 'broker,controller'}) %}
+{% endif %}

salt/kafka/nodes.sls

@@ -0,0 +1,18 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'kafka/nodes.map.jinja' import COMBINED_KAFKANODES %}
+{% set kafka_cluster_id = salt['pillar.get']('kafka:cluster_id', default=None) %}
+
+{# Write Kafka pillar, so all grid members have access to nodeid of other kafka nodes and their roles #}
+write_kafka_pillar_yaml:
+  file.managed:
+    - name: /opt/so/saltstack/local/pillar/kafka/nodes.sls
+    - mode: 644
+    - user: socore
+    - source: salt://kafka/files/managed_node_pillar.jinja
+    - template: jinja
+    - context:
+        COMBINED_KAFKANODES: {{ COMBINED_KAFKANODES }}

salt/kafka/reset.sls

@@ -0,0 +1,9 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+wipe_kafka_data:
+  file.absent:
+    - name: /nsm/kafka/data/
+    - force: True

salt/kafka/soc_kafka.yaml

@@ -1,6 +1,6 @@
 kafka:
   enabled:
-    description: Enable or disable Kafka.
+    description: Set to True to enable Kafka. To avoid grid problems, do not enable Kafka until the related configuration is in place. Requires a valid Security Onion license key.
     helpLink: kafka.html
   cluster_id:
     description: The ID of the Kafka cluster.
@@ -8,8 +8,20 @@ kafka:
     advanced: True
     sensitive: True
     helpLink: kafka.html
+  password:
+    description: The password to use for the Kafka certificates.
+    sensitive: True
+    helpLink: kafka.html
+  controllers:
+    description: A comma-separated list of hostnames that will act as Kafka controllers. These hosts will be responsible for managing the Kafka cluster. Note that only manager and receiver nodes are eligible to run Kafka. This configuration needs to be set before enabling Kafka. Failure to do so may result in Kafka topics becoming unavailable requiring manual intervention to restore functionality or reset Kafka, either of which can result in data loss.
+    forcedType: "string"
+    helpLink: kafka.html
+  reset:
+    description: Disable and reset the Kafka cluster. This will remove all Kafka data including logs that may have not yet been ingested into Elasticsearch and reverts the grid to using REDIS as the global pipeline. This is useful when testing different Kafka configurations such as rearranging Kafka brokers / controllers allowing you to reset the cluster rather than manually fixing any issues arising from attempting to reassign a Kafka broker into a controller. Enter 'YES_RESET_KAFKA' and submit to disable and reset Kafka. Make any configuration changes required and re-enable Kafka when ready. This action CANNOT be reversed.
+    advanced: True
+    helpLink: kafka.html
   config:
-    server:
+    broker:
       advertised_x_listeners:
         description: Specify the list of listeners (hostname and port) that Kafka brokers provide to clients for communication.
         title: advertised.listeners
@@ -19,13 +31,10 @@ kafka:
         title: auto.create.topics.enable
         forcedType: bool
         helpLink: kafka.html
-      controller_x_listener_x_names:
-        description: Set listeners used by the controller in a comma-seperated list. 
-        title: controller.listener.names
-        helpLink: kafka.html
-      controller_x_quorum_x_voters:
-        description: A comma-seperated list of ID and endpoint information mapped for a set of voters.
-        title: controller.quorum.voters
+      default_x_replication_x_factor:
+        description: The default replication factor for automatically created topics. This value must be less than the amount of brokers in the cluster. Hosts specified in controllers should not be counted towards total broker count.
+        title: default.replication.factor
+        forcedType: int
         helpLink: kafka.html
       inter_x_broker_x_listener_x_name:
         description: The name of the listener used for inter-broker communication.
@@ -56,12 +65,6 @@ kafka:
         title: log.segment.bytes
         forcedType: int
         helpLink: kafka.html
-      node_x_id:
-        description: The node ID corresponds to the roles performed by this process whenever process.roles is populated.
-        title: node.id
-        forcedType: int
-        readonly: True
-        helpLink: kafka.html
       num_x_io_x_threads:
         description: The number of threads used by Kafka.
         title: num.io.threads
@@ -88,8 +91,9 @@ kafka:
         forcedType: int
         helpLink: kafka.html
       process_x_roles:
-        description: The roles the process performs. Use a comma-seperated list is multiple.
+        description: The role performed by Kafka brokers.
         title: process.roles
+        readonly: True
         helpLink: kafka.html
       socket_x_receive_x_buffer_x_bytes:
         description: Size, in bytes of the SO_RCVBUF buffer. A value of -1 will use the OS default.
@@ -168,3 +172,38 @@ kafka:
         title: ssl.truststore.password
         sensitive: True
         helpLink: kafka.html
+    controller:
+      controller_x_listener_x_names:
+        description: Set listeners used by the controller in a comma-seperated list.
+        title: controller.listener.names
+        helpLink: kafka.html
+      listeners:
+        description: Set of URIs that is listened on and the listener names in a comma-seperated list.
+        helpLink: kafka.html
+      listener_x_security_x_protocol_x_map:
+        description: Comma-seperated mapping of listener name and security protocols.
+        title: listener.security.protocol.map
+        helpLink: kafka.html
+      log_x_dirs:
+        description: Where Kafka logs are stored within the Docker container.
+        title: log.dirs
+        helpLink: kafka.html
+      log_x_retention_x_check_x_interval_x_ms:
+        description: Frequency at which log files are checked if they are qualified for deletion.
+        title: log.retention.check.interval.ms
+        helpLink: kafka.html
+      log_x_retention_x_hours:
+        description: How long, in hours, a log file is kept.
+        title: log.retention.hours
+        forcedType: int
+        helpLink: kafka.html
+      log_x_segment_x_bytes:
+        description: The maximum allowable size for a log file.
+        title: log.segment.bytes
+        forcedType: int
+        helpLink: kafka.html
+      process_x_roles:
+        description: The role performed by controller node.
+        title: process.roles
+        readonly: True
+        helpLink: kafka.html

salt/kafka/storage.sls

@@ -6,22 +6,14 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
-{%   set kafka_cluster_id = salt['pillar.get']('kafka:cluster_id', default=None) %}
-
-{%   if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone'] %}
-{%     if kafka_cluster_id is none %}
-generate_kafka_cluster_id:
-  cmd.run:
-    - name: /usr/sbin/so-kafka-clusterid
-{%     endif %}
-{%   endif %}
+{%   set kafka_cluster_id = salt['pillar.get']('kafka:cluster_id') %}
 
 {# Initialize kafka storage if it doesn't already exist. Just looking for meta.properties in /nsm/kafka/data #}
 {%   if not salt['file.file_exists']('/nsm/kafka/data/meta.properties') %}
 kafka_storage_init:
   cmd.run:
     - name: |
-        docker run -v /nsm/kafka/data:/nsm/kafka/data -v /opt/so/conf/kafka/server.properties:/kafka/config/kraft/newserver.properties --name so-kafkainit --user root --entrypoint /kafka/bin/kafka-storage.sh {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kafka:{{ GLOBALS.so_version }} format -t {{ kafka_cluster_id }} -c /kafka/config/kraft/newserver.properties
+        docker run -v /nsm/kafka/data:/nsm/kafka/data -v /opt/so/conf/kafka/server.properties:/opt/kafka/config/kraft/newserver.properties --name so-kafkainit --user root --entrypoint /opt/kafka/bin/kafka-storage.sh {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kafka:{{ GLOBALS.so_version }} format -t {{ kafka_cluster_id }} -c /opt/kafka/config/kraft/newserver.properties
 kafka_rm_kafkainit:
   cmd.run:
     - name: |

salt/kafka/tools/sbin/so-kafka-cli

@@ -0,0 +1,47 @@
+#! /bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+if [ -z "$NOROOT" ]; then
+	# Check for prerequisites
+	if [ "$(id -u)" -ne 0 ]; then
+		echo "This script must be run using sudo!"
+		exit 1
+	fi
+fi
+
+function usage() {
+  echo -e "\nUsage: $0 <script> [options]"
+  echo ""
+  echo "Available scripts:"
+  show_available_kafka_cli_tools
+}
+
+function show_available_kafka_cli_tools(){
+  docker exec so-kafka ls /opt/kafka/bin | grep kafka
+}
+
+if [ -z $1 ]; then
+  usage
+  exit 1
+fi
+
+available_tools=$(show_available_kafka_cli_tools)
+script_exists=false
+
+for script in $available_tools; do
+  if [ "$script" == "$1" ]; then
+    script_exists=true
+    break
+  fi
+done
+
+if [ "$script_exists" == true ]; then
+  docker exec so-kafka /opt/kafka/bin/$1 "${@:2}"
+else
+  echo -e  "\nInvalid script: $1"
+  usage
+  exit 1
+fi

salt/kafka/tools/sbin/so-kafka-config-update

@@ -0,0 +1,87 @@
+#! /bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+if [ -z "$NOROOT" ]; then
+	# Check for prerequisites
+	if [ "$(id -u)" -ne 0 ]; then
+		echo "This script must be run using sudo!"
+		exit 1
+	fi
+fi
+
+usage() {
+  cat <<USAGE_EOF
+
+  Usage: $0 <operation> [parameters]
+
+  Where <operation> is one of the following:
+
+    topic-partitions: Increase the number of partitions for a Kafka topic
+      Required arguments: topic-partitions <topic name> <# partitions>
+      Example: $0 topic-partitions suricata-topic 6
+
+	list-topics: List of Kafka topics
+	  Example: $0 list-topics
+
+USAGE_EOF
+  exit 1
+}
+
+if [[ $# -lt 1 || $1 == --help || $1 == -h ]]; then
+  usage
+fi
+
+kafka_client_config="/opt/kafka/config/kraft/client.properties"
+
+too_few_arguments() {
+  echo -e "\nMissing one or more required arguments!\n"
+  usage
+}
+
+get_kafka_brokers() {
+	brokers_cache="/opt/so/state/kafka_brokers"
+  broker_port="9092"
+  if [[ ! -f "$brokers_cache" ]] || [[ $(find "/$brokers_cache" -mmin +120) ]]; then
+    echo "Refreshing Kafka brokers list"
+	  salt-call pillar.get kafka:nodes --out=json | jq -r --arg broker_port "$broker_port" '.local | to_entries[] | select(.value.role | contains("broker")) | "\(.value.ip):\($broker_port)"' | paste -sd "," - > "$brokers_cache"
+  else
+    echo "Using cached Kafka brokers list"
+  fi
+  brokers=$(cat "$brokers_cache")
+}
+
+increase_topic_partitions() {
+  get_kafka_brokers
+	command=$(so-kafka-cli kafka-topics.sh --bootstrap-server $brokers --command-config $kafka_client_config --alter --topic $topic --partitions $partition_count)
+  if $command; then
+    echo -e "Successfully increased the number of partitions for topic $topic to $partition_count\n"
+    so-kafka-cli kafka-topics.sh --bootstrap-server $brokers --command-config $kafka_client_config --describe --topic $topic
+  fi
+}
+
+get_kafka_topics_list() {
+	get_kafka_brokers
+	so-kafka-cli kafka-topics.sh --bootstrap-server $brokers --command-config $kafka_client_config --exclude-internal --list | sort
+}
+
+operation=$1
+case "${operation}" in
+  "topic-partitions")
+	if [[ $# -lt 3 ]]; then
+	  too_few_arguments
+	fi
+	topic=$2
+	partition_count=$3
+	increase_topic_partitions
+	;;
+  "list-topics")
+	get_kafka_topics_list
+	;;
+  *)
+	usage
+	;;
+esac
+

salt/kafka/tools/sbin_jinja/so-kafka-generate-keystore

@@ -1,13 +0,0 @@
-#!/bin/bash
-#
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-. /usr/sbin/so-common
-
-# Generate a new keystore
-docker run -v /etc/pki/kafka.p12:/etc/pki/kafka.p12 --name so-kafka-keystore --user root --entrypoint keytool {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kafka:{{ GLOBALS.so_version }} -importkeystore -srckeystore /etc/pki/kafka.p12 -srcstoretype PKCS12 -srcstorepass changeit -destkeystore /etc/pki/kafka.jks -deststoretype JKS -deststorepass changeit -noprompt
-docker cp so-kafka-keystore:/etc/pki/kafka.jks /etc/pki/kafka.jks
-docker rm so-kafka-keystore

salt/logstash/defaults.yaml

@@ -37,6 +37,7 @@ logstash:
       - so/0900_input_redis.conf.jinja
       - so/9805_output_elastic_agent.conf.jinja
       - so/9900_output_endgame.conf.jinja
+      - so/0800_input_kafka.conf.jinja
     custom0: []
     custom1: []
     custom2: []

salt/logstash/download.sls

@@ -0,0 +1,20 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+{%   from 'vars/globals.map.jinja' import GLOBALS %}
+
+so-logstash_image:
+  docker_image.present:
+    - name: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-logstash:{{ GLOBALS.so_version }}
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}