Merge pull request #13255 from Security-Onion-Solutions/2.4/dev

2.4.80

.github/.gitleaks.toml

@@ -536,7 +536,7 @@ secretGroup = 4
 
 [allowlist]
 description = "global allow lists"
-regexes = ['''219-09-9999''', '''078-05-1120''', '''(9[0-9]{2}|666)-\d{2}-\d{4}''', '''RPM-GPG-KEY.*''', '''.*:.*StrelkaHexDump.*''', '''.*:.*PLACEHOLDER.*''']
+regexes = ['''219-09-9999''', '''078-05-1120''', '''(9[0-9]{2}|666)-\d{2}-\d{4}''', '''RPM-GPG-KEY.*''', '''.*:.*StrelkaHexDump.*''', '''.*:.*PLACEHOLDER.*''', '''ssl_.*password''']
 paths = [
     '''gitleaks.toml''',
     '''(.*?)(jpg|gif|doc|pdf|bin|svg|socket)$''',

DOWNLOAD_AND_VERIFY_ISO.md

@@ -1,17 +1,17 @@
-### 2.4.70-20240529 ISO image released on 2024/05/29
+### 2.4.80-20240624 ISO image released on 2024/06/25
 
 
 ### Download and Verify
 
-2.4.70-20240529 ISO image:  
+2.4.80-20240624 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.4.70-20240529.iso
+https://download.securityonion.net/file/securityonion/securityonion-2.4.80-20240624.iso
  
-MD5: 8FCCF31C2470D1ABA380AF196B611DEC  
+MD5: 139F9762E926F9CB3C4A9528A3752C31  
-SHA1: EE5E8F8C14819E7A1FE423E6920531A97F39600B  
+SHA1: BC6CA2C5F4ABC1A04E83A5CF8FFA6A53B1583CC9  
-SHA256: EF5E781D50D50660F452ADC54FD4911296ECBECED7879FA8E04687337CA89BEC  
+SHA256: 70E90845C84FFA30AD6CF21504634F57C273E7996CA72F7250428DDBAAC5B1BD  
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.70-20240529.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.80-20240624.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS  
@@ -25,27 +25,29 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.70-20240529.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.80-20240624.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.4.70-20240529.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.4.80-20240624.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.4.70-20240529.iso.sig securityonion-2.4.70-20240529.iso
+gpg --verify securityonion-2.4.80-20240624.iso.sig securityonion-2.4.80-20240624.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Wed 29 May 2024 11:40:59 AM EDT using RSA key ID FE507013
+gpg: Signature made Mon 24 Jun 2024 02:42:03 PM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.
 Primary key fingerprint: C804 A93D 36BE 0C73 3EA1  9644 7C10 60B7 FE50 7013
 ```
 
+If it fails to verify, try downloading again. If it still fails to verify, try downloading from another computer or another network.
+
 Once you've verified the ISO image, you're ready to proceed to our Installation guide:  
 https://docs.securityonion.net/en/2.4/installation.html

VERSION

@@ -1 +1 @@
-2.4.70
+2.4.80

pillar/kafka/nodes.sls

@@ -0,0 +1,2 @@
+kafka:
+  nodes:

pillar/top.sls

@@ -61,6 +61,9 @@ base:
     - backup.adv_backup
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
+    - kafka.nodes
+    - kafka.soc_kafka
+    - kafka.adv_kafka
     - stig.soc_stig
 
   '*_sensor':
@@ -176,6 +179,9 @@ base:
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
     - stig.soc_stig
+    - kafka.nodes
+    - kafka.soc_kafka
+    - kafka.adv_kafka
 
   '*_heavynode':
     - elasticsearch.auth
@@ -220,6 +226,7 @@ base:
     - minions.adv_{{ grains.id }}
     - stig.soc_stig
     - soc.license
+    - kafka.nodes
 
   '*_receiver':
     - logstash.nodes
@@ -232,6 +239,10 @@ base:
     - redis.adv_redis
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
+    - kafka.nodes
+    - kafka.soc_kafka
+    - kafka.adv_kafka
+    - soc.license
 
   '*_import':
     - secrets

pyci.sh

@@ -15,12 +15,16 @@ TARGET_DIR=${1:-.}
 
 PATH=$PATH:/usr/local/bin
 
-if ! which pytest &> /dev/null || ! which flake8 &> /dev/null ; then
+if [ ! -d .venv ]; then
-    echo "Missing dependencies. Consider running the following command:"
+    python -m venv .venv
-    echo "  python -m pip install flake8 pytest pytest-cov"
+fi
+
+source .venv/bin/activate
+
+if ! pip install flake8 pytest pytest-cov pyyaml; then
+    echo "Unable to install dependencies."
     exit 1
 fi
 
-pip install pytest pytest-cov
 flake8 "$TARGET_DIR" "--config=${HOME_DIR}/pytest.ini"
 python3 -m pytest "--cov-config=${HOME_DIR}/pytest.ini" "--cov=$TARGET_DIR" --doctest-modules --cov-report=term --cov-fail-under=100  "$TARGET_DIR" 

salt/allowed_states.map.jinja

@@ -103,7 +103,8 @@
           'utility',
           'schedule',
           'docker_clean',
-          'stig'
+          'stig',
+          'kafka'
           ],
       'so-managersearch': [
           'salt.master',
@@ -125,7 +126,8 @@
           'utility',
           'schedule',
           'docker_clean',
-          'stig'
+          'stig',
+          'kafka'
           ],
       'so-searchnode': [
           'ssl',
@@ -159,7 +161,8 @@
           'schedule',
           'tcpreplay',
           'docker_clean',
-          'stig'
+          'stig',
+          'kafka'
           ],
       'so-sensor': [
           'ssl',
@@ -190,7 +193,10 @@
           'telegraf',
           'firewall',
           'schedule',
-          'docker_clean'
+          'docker_clean',
+          'kafka',
+          'elasticsearch.ca',
+          'stig'
           ],
       'so-desktop': [
           'ssl',

salt/ca/files/signing_policies.conf

@@ -1,6 +1,3 @@
-mine_functions:
-  x509.get_pem_entries: [/etc/pki/ca.crt]
-
 x509_signing_policies:
   filebeat:
     - minions: '*'
@@ -70,3 +67,17 @@ x509_signing_policies:
     - authorityKeyIdentifier: keyid,issuer:always
     - days_valid: 820
     - copypath: /etc/pki/issued_certs/
+  kafka:
+    - minions: '*'
+    - signing_private_key: /etc/pki/ca.key
+    - signing_cert: /etc/pki/ca.crt
+    - C: US
+    - ST: Utah
+    - L: Salt Lake City
+    - basicConstraints: "critical CA:false"
+    - keyUsage: "digitalSignature, keyEncipherment"
+    - subjectKeyIdentifier: hash
+    - authorityKeyIdentifier: keyid,issuer:always
+    - extendedKeyUsage: "serverAuth, clientAuth"
+    - days_valid: 820
+    - copypath: /etc/pki/issued_certs/

salt/common/soup_scripts.sls

@@ -57,6 +57,12 @@ copy_so-yaml_manager_tools_sbin:
     - force: True
     - preserve: True
 
+copy_so-repo-sync_manager_tools_sbin:
+  file.copy:
+    - name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-repo-sync
+    - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-repo-sync
+    - preserve: True
+
 # This section is used to put the new script in place so that it can be called during soup.
 # It is faster than calling the states that normally manage them to put them in place.
 copy_so-common_sbin:
@@ -94,6 +100,13 @@ copy_so-yaml_sbin:
     - force: True
     - preserve: True
 
+copy_so-repo-sync_sbin:
+  file.copy:
+    - name: /usr/sbin/so-repo-sync
+    - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-repo-sync
+    - force: True
+    - preserve: True
+
 {% else %}
 fix_23_soup_sbin:
   cmd.run:

salt/common/tools/sbin/so-common

@@ -31,6 +31,11 @@ if ! echo "$PATH" | grep -q "/usr/sbin"; then
   export PATH="$PATH:/usr/sbin"
 fi
 
+# See if a proxy is set. If so use it.
+if [ -f /etc/profile.d/so-proxy.sh ]; then
+  . /etc/profile.d/so-proxy.sh
+fi
+
 # Define a banner to separate sections
 banner="========================================================================="
 

salt/common/tools/sbin/so-image-common

@@ -50,6 +50,7 @@ container_list() {
       "so-idh"
       "so-idstools"
       "so-influxdb"
+      "so-kafka"
       "so-kibana"
       "so-kratos"
       "so-logstash"

salt/common/tools/sbin_jinja/so-tcpreplay

@@ -10,7 +10,7 @@
 . /usr/sbin/so-common
 . /usr/sbin/so-image-common
 
-REPLAYIFACE=${REPLAYIFACE:-$(lookup_pillar interface sensor)}
+REPLAYIFACE=${REPLAYIFACE:-"{{salt['pillar.get']('sensor:interface', '')}}"}
 REPLAYSPEED=${REPLAYSPEED:-10}
 
 mkdir -p /opt/so/samples

salt/docker/defaults.yaml

@@ -187,3 +187,12 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+    'so-kafka':
+      final_octet: 88
+      port_bindings:
+        - 0.0.0.0:9092:9092
+        - 0.0.0.0:9093:9093
+        - 0.0.0.0:8778:8778
+      custom_bind_mounts: []
+      extra_hosts: []
+      extra_env: []

salt/docker/init.sls

@@ -20,30 +20,30 @@ dockergroup:
 dockerheldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 1.6.21-1
+      - containerd.io: 1.6.33-1
-      - docker-ce: 5:24.0.3-1~debian.12~bookworm
+      - docker-ce: 5:26.1.4-1~debian.12~bookworm
-      - docker-ce-cli: 5:24.0.3-1~debian.12~bookworm
+      - docker-ce-cli: 5:26.1.4-1~debian.12~bookworm
-      - docker-ce-rootless-extras: 5:24.0.3-1~debian.12~bookworm
+      - docker-ce-rootless-extras: 5:26.1.4-1~debian.12~bookworm
     - hold: True
     - update_holds: True
 {%    elif grains.oscodename == 'jammy' %}
 dockerheldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 1.6.21-1
+      - containerd.io: 1.6.33-1
-      - docker-ce: 5:24.0.2-1~ubuntu.22.04~jammy
+      - docker-ce: 5:26.1.4-1~ubuntu.22.04~jammy
-      - docker-ce-cli: 5:24.0.2-1~ubuntu.22.04~jammy
+      - docker-ce-cli: 5:26.1.4-1~ubuntu.22.04~jammy
-      - docker-ce-rootless-extras: 5:24.0.2-1~ubuntu.22.04~jammy
+      - docker-ce-rootless-extras: 5:26.1.4-1~ubuntu.22.04~jammy
     - hold: True
     - update_holds: True
 {%    else %}
 dockerheldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 1.4.9-1
+      - containerd.io: 1.6.33-1
-      - docker-ce: 5:20.10.8~3-0~ubuntu-focal
+      - docker-ce: 5:26.1.4-1~ubuntu.20.04~focal
-      - docker-ce-cli: 5:20.10.5~3-0~ubuntu-focal
+      - docker-ce-cli: 5:26.1.4-1~ubuntu.20.04~focal
-      - docker-ce-rootless-extras: 5:20.10.5~3-0~ubuntu-focal
+      - docker-ce-rootless-extras: 5:26.1.4-1~ubuntu.20.04~focal
     - hold: True
     - update_holds: True
 {% endif %}
@@ -51,10 +51,10 @@ dockerheldpackages:
 dockerheldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 1.6.21-3.1.el9
+      - containerd.io: 1.6.33-3.1.el9
-      - docker-ce: 24.0.4-1.el9
+      - docker-ce: 3:26.1.4-1.el9
-      - docker-ce-cli: 24.0.4-1.el9
+      - docker-ce-cli: 1:26.1.4-1.el9
-      - docker-ce-rootless-extras: 24.0.4-1.el9
+      - docker-ce-rootless-extras: 26.1.4-1.el9
     - hold: True
     - update_holds: True
 {% endif %}

salt/docker/soc_docker.yaml

@@ -101,3 +101,4 @@ docker:
         multiline: True
         forcedType: "[]string"
     so-zeek: *dockerOptions
+    so-kafka: *dockerOptions

salt/elasticfleet/enabled.sls

@@ -27,7 +27,9 @@ wait_for_elasticsearch_elasticfleet:
 so-elastic-fleet-auto-configure-logstash-outputs:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-outputs-update
-    - retry: True
+    - retry:
+        attempts: 4
+        interval: 30
 {% endif %}
 
 # If enabled, automatically update Fleet Server URLs & ES Connection
@@ -35,7 +37,9 @@ so-elastic-fleet-auto-configure-logstash-outputs:
 so-elastic-fleet-auto-configure-server-urls:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-urls-update
-    - retry: True
+    - retry:
+        attempts: 4
+        interval: 30
 {% endif %}
 
 # Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs
@@ -43,12 +47,16 @@ so-elastic-fleet-auto-configure-server-urls:
 so-elastic-fleet-auto-configure-elasticsearch-urls:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-es-url-update
-    - retry: True
+    - retry:
+        attempts: 4
+        interval: 30
 
 so-elastic-fleet-auto-configure-artifact-urls:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-artifacts-url-update
-    - retry: True 
+    - retry:
+        attempts: 4
+        interval: 30
 
 {% endif %}
 

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers

@@ -19,7 +19,7 @@ NUM_RUNNING=$(pgrep -cf "/bin/bash /sbin/so-elastic-agent-gen-installers")
 
 for i in {1..30}
 do
-   ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key')
+   ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys?perPage=100" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key')
    FLEETHOST=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/fleet_server_hosts/grid-default' | jq -r  '.item.host_urls[]' | paste -sd ',')
 if [[ $FLEETHOST ]] && [[  $ENROLLMENTOKEN ]]; then break; else sleep 10; fi
 done

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update

@@ -21,64 +21,104 @@ function update_logstash_outputs() {
     # Update Logstash Outputs
     curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_logstash" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" | jq
 }
+function update_kafka_outputs() {
+  # Make sure SSL configuration is included in policy updates for Kafka output. SSL is configured in so-elastic-fleet-setup
+  SSL_CONFIG=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs/so-manager_kafka" | jq -r '.item.ssl')
 
-# Get current list of Logstash Outputs
+  JSON_STRING=$(jq -n \
-RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_logstash')
+    --arg UPDATEDLIST "$NEW_LIST_JSON" \
+    --argjson SSL_CONFIG "$SSL_CONFIG" \
+    '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": $SSL_CONFIG}')
+  # Update Kafka outputs
+  curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_kafka" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" | jq
+}
 
-# Check to make sure that the server responded with good data - else, bail from script
+{% if GLOBALS.pipeline == "KAFKA" %}
-CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON")
+  # Get current list of Kafka Outputs
-if [ "$CHECKSUM" != "so-manager_logstash" ]; then
+  RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_kafka')
- printf "Failed to query for current Logstash Outputs..."
- exit 1
-fi
 
-# Get the current list of Logstash outputs & hash them
+  # Check to make sure that the server responded with good data - else, bail from script
-CURRENT_LIST=$(jq -c -r '.item.hosts' <<<  "$RAW_JSON")
+  CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON")
-CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}')
+  if [ "$CHECKSUM" != "so-manager_kafka" ]; then
+   printf "Failed to query for current Kafka Outputs..."
+   exit 1
+  fi
 
-declare -a NEW_LIST=()
+  # Get the current list of kafka outputs & hash them
+  CURRENT_LIST=$(jq -c -r '.item.hosts' <<<  "$RAW_JSON")
+  CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}')
+
+  declare -a NEW_LIST=()
+
+  # Query for the current Grid Nodes that are running kafka
+  KAFKANODES=$(salt-call --out=json pillar.get kafka:nodes | jq '.local')
+
+  # Query for Kafka nodes with Broker role and add hostname to list
+  while IFS= read -r line; do
+    NEW_LIST+=("$line")
+  done < <(jq -r 'to_entries | .[] | select(.value.role | contains("broker")) | .key + ":9092"' <<< $KAFKANODES)
+
+  {# If global pipeline isn't set to KAFKA then assume default of REDIS / logstash #}
+{% else %}
+  # Get current list of Logstash Outputs
+  RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_logstash')
+
+  # Check to make sure that the server responded with good data - else, bail from script
+  CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON")
+  if [ "$CHECKSUM" != "so-manager_logstash" ]; then
+   printf "Failed to query for current Logstash Outputs..."
+   exit 1
+  fi
+
+  # Get the current list of Logstash outputs & hash them
+  CURRENT_LIST=$(jq -c -r '.item.hosts' <<<  "$RAW_JSON")
+  CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}')
+
+  declare -a NEW_LIST=()
+
+  {# If we select to not send to manager via SOC, then omit the code that adds manager to NEW_LIST #}
+  {% if ELASTICFLEETMERGED.enable_manager_output %}
+  # Create array & add initial elements
+  if [ "{{ GLOBALS.hostname }}" = "{{ GLOBALS.url_base }}" ]; then
+      NEW_LIST+=("{{ GLOBALS.url_base }}:5055")
+  else
+      NEW_LIST+=("{{ GLOBALS.url_base }}:5055" "{{ GLOBALS.hostname }}:5055")
+  fi
+  {% endif %}
+
+  # Query for FQDN entries & add them to the list
+  {% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %}
+  CUSTOMFQDNLIST=('{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(' ') }}')
+  readarray -t -d ' ' CUSTOMFQDN < <(printf '%s' "$CUSTOMFQDNLIST")
+  for CUSTOMNAME in "${CUSTOMFQDN[@]}"
+  do
+   NEW_LIST+=("$CUSTOMNAME:5055")
+  done
+  {% endif %}
+
+  # Query for the current Grid Nodes that are running Logstash
+  LOGSTASHNODES=$(salt-call --out=json pillar.get logstash:nodes | jq '.local')
+
+  # Query for Receiver Nodes & add them to the list
+  if grep -q "receiver" <<< $LOGSTASHNODES; then
+     readarray -t RECEIVERNODES < <(jq -r ' .receiver | keys_unsorted[]'  <<< $LOGSTASHNODES)
+     for NODE in "${RECEIVERNODES[@]}"
+     do
+      NEW_LIST+=("$NODE:5055")
+     done
+  fi
+
+  # Query for Fleet Nodes & add them to the list
+  if grep -q "fleet" <<< $LOGSTASHNODES; then
+     readarray -t FLEETNODES < <(jq -r ' .fleet | keys_unsorted[]'  <<< $LOGSTASHNODES)
+     for NODE in "${FLEETNODES[@]}"
+     do
+      NEW_LIST+=("$NODE:5055")
+     done
+  fi
 
-{# If we select to not send to manager via SOC, then omit the code that adds manager to NEW_LIST #}
-{% if ELASTICFLEETMERGED.enable_manager_output %}
-# Create array & add initial elements
-if [ "{{ GLOBALS.hostname }}" = "{{ GLOBALS.url_base }}" ]; then
-    NEW_LIST+=("{{ GLOBALS.url_base }}:5055")
-else
-    NEW_LIST+=("{{ GLOBALS.url_base }}:5055" "{{ GLOBALS.hostname }}:5055")
-fi
 {% endif %}
 
-# Query for FQDN entries & add them to the list
-{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %}
-CUSTOMFQDNLIST=('{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(' ') }}')
-readarray -t -d ' ' CUSTOMFQDN < <(printf '%s' "$CUSTOMFQDNLIST")
-for CUSTOMNAME in "${CUSTOMFQDN[@]}"
-do
- NEW_LIST+=("$CUSTOMNAME:5055")
-done
-{% endif %}
-
-# Query for the current Grid Nodes that are running Logstash
-LOGSTASHNODES=$(salt-call --out=json pillar.get logstash:nodes | jq '.local')
-
-# Query for Receiver Nodes & add them to the list
-if grep -q "receiver" <<< $LOGSTASHNODES; then
-   readarray -t RECEIVERNODES < <(jq -r ' .receiver | keys_unsorted[]'  <<< $LOGSTASHNODES)
-   for NODE in "${RECEIVERNODES[@]}"
-   do
-    NEW_LIST+=("$NODE:5055")
-   done
-fi
-
-# Query for Fleet Nodes & add them to the list
-if grep -q "fleet" <<< $LOGSTASHNODES; then
-   readarray -t FLEETNODES < <(jq -r ' .fleet | keys_unsorted[]'  <<< $LOGSTASHNODES)
-   for NODE in "${FLEETNODES[@]}"
-   do
-    NEW_LIST+=("$NODE:5055")
-   done
-fi
-
 # Sort & hash the new list of Logstash Outputs
 NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${NEW_LIST[@]}")
 NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}')
@@ -87,9 +127,28 @@ NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}')
 if [ "$NEW_HASH" = "$CURRENT_HASH" ]; then
     printf "\nHashes match - no update needed.\n"
     printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n"
+
+    # Since output can be KAFKA or LOGSTASH, we need to check if the policy set as default matches the value set in GLOBALS.pipeline and update if needed
+    printf "Checking if the correct output policy is set as default\n"
+    OUTPUT_DEFAULT=$(jq -r '.item.is_default' <<< $RAW_JSON)
+    OUTPUT_DEFAULT_MONITORING=$(jq -r '.item.is_default_monitoring' <<< $RAW_JSON)
+    if [[ "$OUTPUT_DEFAULT" = "false" || "$OUTPUT_DEFAULT_MONITORING" = "false" ]]; then
+      printf "Default output policy needs to be updated.\n"
+    {%- if GLOBALS.pipeline == "KAFKA" and 'gmd' in salt['pillar.get']('features', []) %}
+      update_kafka_outputs
+    {%- else %}
+      update_logstash_outputs
+    {%- endif %}
+    else
+      printf "Default output policy is set - no update needed.\n"
+    fi
     exit 0
 else
     printf "\nHashes don't match - update needed.\n"
     printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n"
+    {%- if GLOBALS.pipeline == "KAFKA" and 'gmd' in salt['pillar.get']('features', []) %}
+    update_kafka_outputs
+    {%- else %}
     update_logstash_outputs
+    {%- endif %}
 fi

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup

@@ -77,6 +77,11 @@ curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fl
 printf "\n\n"
 {%- endif %}
 
+printf "\nCreate Kafka Output Config if node is not an Import or Eval install\n"
+{% if grains.role not in ['so-import', 'so-eval'] %}
+/usr/sbin/so-kafka-fleet-output-policy
+{% endif %}
+
 # Add Manager Hostname & URL Base to Fleet Host URLs
 printf "\nAdd SO-Manager Fleet URL\n"
 if [ "{{ GLOBALS.hostname }}" = "{{ GLOBALS.url_base }}" ]; then

salt/elasticfleet/tools/sbin_jinja/so-kafka-fleet-output-policy

@@ -0,0 +1,52 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+{% if GLOBALS.role in ['so-manager', 'so-standalone', 'so-managersearch'] %}
+
+. /usr/sbin/so-common
+
+# Check to make sure that Kibana API is up & ready
+RETURN_CODE=0
+wait_for_web_response "http://localhost:5601/api/fleet/settings" "fleet" 300 "curl -K /opt/so/conf/elasticsearch/curl.config"
+RETURN_CODE=$?
+
+if [[ "$RETURN_CODE" != "0" ]]; then
+    printf "Kibana API not accessible, can't setup Elastic Fleet output policy for Kafka..."
+    exit 1
+fi
+
+output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs" | jq -r .items[].id)
+
+if ! echo "$output" | grep -q "so-manager_kafka"; then
+  KAFKACRT=$(openssl x509 -in /etc/pki/elasticfleet-kafka.crt)
+  KAFKAKEY=$(openssl rsa -in  /etc/pki/elasticfleet-kafka.key)
+  KAFKACA=$(openssl x509 -in  /etc/pki/tls/certs/intca.crt)
+  KAFKA_OUTPUT_VERSION="2.6.0"
+  JSON_STRING=$( jq -n \
+                --arg KAFKACRT "$KAFKACRT" \
+                --arg KAFKAKEY "$KAFKAKEY" \
+                --arg KAFKACA "$KAFKACA" \
+                --arg MANAGER_IP "{{ GLOBALS.manager_ip }}:9092" \
+                --arg KAFKA_OUTPUT_VERSION "$KAFKA_OUTPUT_VERSION" \
+                    '{ "name": "grid-kafka", "id": "so-manager_kafka", "type": "kafka", "hosts": [ $MANAGER_IP ], "is_default": false, "is_default_monitoring": false, "config_yaml": "", "ssl": { "certificate_authorities": [ $KAFKACA ], "certificate": $KAFKACRT, "key": $KAFKAKEY, "verification_mode": "full" }, "proxy_id": null, "client_id": "Elastic", "version": $KAFKA_OUTPUT_VERSION, "compression": "none", "auth_type": "ssl", "partition": "round_robin", "round_robin": { "group_events": 1 }, "topics":[{"topic":"%{[event.module]}-securityonion","when":{"type":"regexp","condition":"event.module:.+"}},{"topic":"default-securityonion"}], "headers": [ { "key": "", "value": "" } ], "timeout": 30, "broker_timeout": 30, "required_acks": 1 }'
+                )
+  curl -sK /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" -o /dev/null
+  refresh_output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs" | jq -r .items[].id)
+
+  if ! echo "$refresh_output" | grep -q "so-manager_kafka"; then
+    echo -e "\nFailed to setup Elastic Fleet output policy for Kafka...\n"
+    exit 1
+  elif echo "$refresh_output" | grep -q "so-manager_kafka"; then
+    echo -e "\nSuccessfully setup Elastic Fleet output policy for Kafka...\n"
+  fi
+
+elif echo "$output" | grep -q "so-manager_kafka"; then
+  echo -e "\nElastic Fleet output policy for Kafka already exists...\n"
+fi
+{% else %}
+echo -e "\nNo update required...\n"
+{% endif %}

salt/elasticsearch/ca.sls

@@ -4,7 +4,7 @@
 # Elastic License 2.0.
 
 {% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls.split('.')[0] in allowed_states %}
+{% if sls.split('.')[0] in allowed_states or sls in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 
 # Move our new CA over so Elastic and Logstash can use SSL with the internal CA

salt/elasticsearch/defaults.yaml

@@ -170,6 +170,78 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
+    so-items:
+      index_sorting: false
+      index_template:
+        composed_of:
+        - so-items-mappings
+        index_patterns:
+        - .items-default-**
+        priority: 500
+        template:
+          mappings:
+            date_detection: false
+          settings:
+            index:
+              lifecycle:
+                name: so-items-logs
+                rollover_alias: ".items-default"
+              routing:
+                allocation:
+                  include:
+                    _tier_preference: "data_content"
+              mapping:
+                total_fields:
+                  limit: 10000
+              number_of_replicas: 0
+              number_of_shards: 1
+              refresh_interval: 30s
+              sort:
+                field: '@timestamp'
+                order: desc
+      policy:
+        phases:
+          hot:
+            actions:
+              rollover:
+                max_size: 50gb
+            min_age: 0ms
+    so-lists:
+      index_sorting: false
+      index_template:
+        composed_of:
+        - so-lists-mappings
+        index_patterns:
+        - .lists-default-**
+        priority: 500
+        template:
+          mappings:
+            date_detection: false
+          settings:
+            index:
+              lifecycle:
+                name: so-lists-logs
+                rollover_alias: ".lists-default"
+              routing:
+                allocation:
+                  include:
+                    _tier_preference: "data_content"
+              mapping:
+                total_fields:
+                  limit: 10000
+              number_of_replicas: 0
+              number_of_shards: 1
+              refresh_interval: 30s
+              sort:
+                field: '@timestamp'
+                order: desc
+      policy:
+        phases:
+          hot:
+            actions:
+              rollover:
+                max_size: 50gb
+            min_age: 0ms
     so-case:
       index_sorting: false
       index_template:
@@ -11088,6 +11160,117 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
+    so-suricata_x_alerts:
+      index_sorting: false
+      index_template:
+        composed_of:
+        - agent-mappings
+        - dtc-agent-mappings
+        - base-mappings
+        - dtc-base-mappings
+        - client-mappings
+        - dtc-client-mappings
+        - cloud-mappings
+        - container-mappings
+        - data_stream-mappings
+        - destination-mappings
+        - dtc-destination-mappings
+        - pb-override-destination-mappings
+        - dll-mappings
+        - dns-mappings
+        - dtc-dns-mappings
+        - ecs-mappings
+        - dtc-ecs-mappings
+        - error-mappings
+        - event-mappings
+        - dtc-event-mappings
+        - file-mappings
+        - dtc-file-mappings
+        - group-mappings
+        - host-mappings
+        - dtc-host-mappings
+        - http-mappings
+        - dtc-http-mappings
+        - log-mappings
+        - network-mappings
+        - dtc-network-mappings
+        - observer-mappings
+        - dtc-observer-mappings
+        - orchestrator-mappings
+        - organization-mappings
+        - package-mappings
+        - process-mappings
+        - dtc-process-mappings
+        - registry-mappings
+        - related-mappings
+        - rule-mappings
+        - dtc-rule-mappings
+        - server-mappings
+        - service-mappings
+        - dtc-service-mappings
+        - source-mappings
+        - dtc-source-mappings
+        - pb-override-source-mappings
+        - suricata-mappings
+        - threat-mappings
+        - tls-mappings
+        - tracing-mappings
+        - url-mappings
+        - user_agent-mappings
+        - dtc-user_agent-mappings
+        - vulnerability-mappings
+        - common-settings
+        - common-dynamic-mappings
+        data_stream: {}
+        index_patterns:
+        - logs-suricata.alerts-*
+        priority: 500
+        template:
+          mappings:
+            date_detection: false
+            dynamic_templates:
+            - strings_as_keyword:
+                mapping:
+                  ignore_above: 1024
+                  type: keyword
+                match_mapping_type: string
+          settings:
+            index:
+              lifecycle:
+                name: so-suricata.alerts-logs
+              mapping:
+                total_fields:
+                  limit: 5000
+              number_of_replicas: 0
+              number_of_shards: 1
+              refresh_interval: 30s
+              sort:
+                field: '@timestamp'
+                order: desc
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 1d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
     so-syslog:
       index_sorting: false
       index_template:

salt/elasticsearch/files/ingest/.fleet_final_pipeline-1

@@ -84,6 +84,7 @@
     { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true }  },
     { "set":        { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } },
     { "rename":        { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } },
+    { "set":        { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } },
     { "remove":     { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp" ], "ignore_missing": true, "ignore_failure": true } } 
   ],
   "on_failure": [

salt/elasticsearch/files/ingest/suricata.alert

@@ -1,6 +1,7 @@
 {
   "description" : "suricata.alert",
   "processors" : [
+    { "set":   { "field": "_index", "value": "logs-suricata.alerts-so" } },
     { "set":   { "field": "tags","value": "alert" }},
     { "rename":{ "field": "message2.alert",	"target_field": "rule",	"ignore_failure": true 	} },
     { "rename":{ "field": "rule.signature",     "target_field": "rule.name", "ignore_failure": true  } },

salt/elasticsearch/soc_elasticsearch.yaml

@@ -521,6 +521,7 @@ elasticsearch:
     so-endgame: *indexSettings
     so-idh: *indexSettings
     so-suricata: *indexSettings
+    so-suricata_x_alerts: *indexSettings
     so-import: *indexSettings
     so-kratos: *indexSettings
     so-kismet: *indexSettings

salt/elasticsearch/templates/component/elastic-agent/so-items-mappings.json

@@ -0,0 +1,112 @@
+{
+  "template": {
+    "mappings": {
+      "dynamic": "strict",
+      "properties": {
+        "binary": {
+          "type": "binary"
+        },
+        "boolean": {
+          "type": "boolean"
+        },
+        "byte": {
+          "type": "byte"
+        },
+        "created_at": {
+          "type": "date"
+        },
+        "created_by": {
+          "type": "keyword"
+        },
+        "date": {
+          "type": "date"
+        },
+        "date_nanos": {
+          "type": "date_nanos"
+        },
+        "date_range": {
+          "type": "date_range"
+        },
+        "deserializer": {
+          "type": "keyword"
+        },
+        "double": {
+          "type": "double"
+        },
+        "double_range": {
+          "type": "double_range"
+        },
+        "float": {
+          "type": "float"
+        },
+        "float_range": {
+          "type": "float_range"
+        },
+        "geo_point": {
+          "type": "geo_point"
+        },
+        "geo_shape": {
+          "type": "geo_shape"
+        },
+        "half_float": {
+          "type": "half_float"
+        },
+        "integer": {
+          "type": "integer"
+        },
+        "integer_range": {
+          "type": "integer_range"
+        },
+        "ip": {
+          "type": "ip"
+        },
+        "ip_range": {
+          "type": "ip_range"
+        },
+        "keyword": {
+          "type": "keyword"
+        },
+        "list_id": {
+          "type": "keyword"
+        },
+        "long": {
+          "type": "long"
+        },
+        "long_range": {
+          "type": "long_range"
+        },
+        "meta": {
+          "type": "object",
+          "enabled": false
+        },
+        "serializer": {
+          "type": "keyword"
+        },
+        "shape": {
+          "type": "shape"
+        },
+        "short": {
+          "type": "short"
+        },
+        "text": {
+          "type": "text"
+        },
+        "tie_breaker_id": {
+          "type": "keyword"
+        },
+        "updated_at": {
+          "type": "date"
+        },
+        "updated_by": {
+          "type": "keyword"
+        }
+      }
+    },
+    "aliases": {}
+  },
+  "version": 2,
+    "_meta": {
+      "managed": true,
+      "description": "default mappings for the .items index template installed by Kibana/Security"
+    }
+}

salt/elasticsearch/templates/component/elastic-agent/so-lists-mappings.json

@@ -0,0 +1,55 @@
+{
+  "template": {
+  "mappings": {
+    "dynamic": "strict",
+    "properties": {
+      "created_at": {
+        "type": "date"
+      },
+      "created_by": {
+        "type": "keyword"
+      },
+      "description": {
+        "type": "keyword"
+      },
+      "deserializer": {
+        "type": "keyword"
+      },
+      "immutable": {
+        "type": "boolean"
+      },
+      "meta": {
+        "type": "object",
+        "enabled": false
+      },
+      "name": {
+        "type": "keyword"
+      },
+      "serializer": {
+        "type": "keyword"
+      },
+      "tie_breaker_id": {
+        "type": "keyword"
+      },
+      "type": {
+        "type": "keyword"
+      },
+      "updated_at": {
+        "type": "date"
+      },
+      "updated_by": {
+        "type": "keyword"
+      },
+      "version": {
+        "type": "keyword"
+      }
+    }
+  },
+    "aliases": {}
+  },
+  "version": 2,
+    "_meta": {
+      "managed": true,
+      "description": "default mappings for the .lists index template installed by Kibana/Security"
+    }
+}

salt/firewall/containers.map.jinja

@@ -27,6 +27,7 @@
      'so-elastic-fleet',
      'so-elastic-fleet-package-registry',
      'so-influxdb',
+     'so-kafka',
      'so-kibana',
      'so-kratos',
      'so-logstash',
@@ -80,6 +81,7 @@
 {% set NODE_CONTAINERS = [
      'so-logstash',
      'so-redis',
+     'so-kafka'
 ] %}
 
 {% elif GLOBALS.role == 'so-idh' %}

salt/firewall/defaults.yaml

@@ -90,6 +90,14 @@ firewall:
       tcp:
         - 8086
       udp: []
+    kafka_controller:
+      tcp:
+        - 9093
+      udp: []
+    kafka_data:
+      tcp:
+        - 9092
+      udp: []
     kibana:
       tcp:
         - 5601
@@ -753,7 +761,6 @@ firewall:
                 - beats_5044
                 - beats_5644
                 - beats_5056
-                - redis
                 - elasticsearch_node
                 - elastic_agent_control
                 - elastic_agent_data
@@ -1267,35 +1274,51 @@ firewall:
       chain:
         DOCKER-USER:
           hostgroups:
+            desktop:
+              portgroups:
+                - elastic_agent_data
             fleet:
               portgroups:
-                - beats_5056
+                - elastic_agent_data
+            idh:
+              portgroups:
+                - elastic_agent_data
             sensor:
               portgroups:
-                - beats_5044
-                - beats_5644
                 - elastic_agent_data
             searchnode:
               portgroups:
                 - redis
-                - beats_5644
+                - elastic_agent_data
+            standalone:
+              portgroups:
+                - redis
+                - elastic_agent_data
+            manager:
+              portgroups:
+                - elastic_agent_data
             managersearch:
               portgroups:
                 - redis
-                - beats_5644
+                - elastic_agent_data
             self:
               portgroups:
                 - redis
-                - beats_5644
+                - elastic_agent_data
             beats_endpoint:
               portgroups:
                 - beats_5044
             beats_endpoint_ssl:
               portgroups:
                 - beats_5644
+            elastic_agent_endpoint:
+              portgroups:
+                - elastic_agent_data
             endgame:
               portgroups:
                 - endgame
+            receiver:
+              portgroups: []
             customhostgroup0:
               portgroups: []
             customhostgroup1:

salt/firewall/map.jinja

@@ -18,4 +18,28 @@
 {%   endfor %}
 {% endif %}
 
+{# Only add Kafka firewall items when Kafka enabled #}
+{% set role = GLOBALS.role.split('-')[1] %}
+
+{% if GLOBALS.pipeline == 'KAFKA' and role in ['manager', 'managersearch', 'standalone'] %}
+{%   do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[role].portgroups.append('kafka_controller') %}
+{%   do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.receiver.portgroups.append('kafka_controller') %}
+{% endif %}
+
+{% if GLOBALS.pipeline == 'KAFKA' and role == 'receiver' %}
+{%   do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.self.portgroups.append('kafka_controller') %}
+{%   do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.standalone.portgroups.append('kafka_controller') %}
+{%   do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.manager.portgroups.append('kafka_controller') %}
+{%   do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.managersearch.portgroups.append('kafka_controller') %}
+{%   do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.receiver.portgroups.append('kafka_controller') %}
+{% endif %}
+
+{% if GLOBALS.pipeline == 'KAFKA' and role in ['manager', 'managersearch', 'standalone', 'receiver'] %}
+{%   for r in ['manager', 'managersearch', 'standalone', 'receiver', 'fleet', 'idh', 'sensor', 'searchnode','heavynode', 'elastic_agent_endpoint', 'desktop'] %}
+{%     if FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[r] is defined %}
+{%       do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[r].portgroups.append('kafka_data') %}
+{%     endif %}
+{%   endfor %}
+{% endif %}
+
 {% set FIREWALL_MERGED = salt['pillar.get']('firewall', FIREWALL_DEFAULT.firewall, merge=True) %}

salt/firewall/soc_firewall.yaml

@@ -120,6 +120,9 @@ firewall:
     influxdb:
       tcp: *tcpsettings
       udp: *udpsettings
+    kafka:
+      tcp: *tcpsettings
+      udp: *udpsettings
     kibana:
       tcp: *tcpsettings
       udp: *udpsettings
@@ -939,7 +942,6 @@ firewall:
               portgroups: *portgroupshost
             customhostgroup9: 
               portgroups: *portgroupshost
-
     idh:
       chain:
         DOCKER-USER:

salt/global/defaults.yaml

@@ -1,2 +1,3 @@
 global:
   pcapengine: STENO
+  pipeline: REDIS

salt/global/soc_global.yaml

@@ -36,9 +36,10 @@ global:
     global: True
     advanced: True
   pipeline:
-    description: Sets which pipeline technology for events to use. Currently only Redis is supported.
+    description: Sets which pipeline technology for events to use. Currently only Redis is fully supported. Kafka is experimental and requires a Security Onion Pro license.
+    regex: ^(REDIS|KAFKA)$
+    regexFailureMessage: You must enter either REDIS or KAFKA.
     global: True
-    readonly: True
     advanced: True
   repo_host:
     description: Specify the host where operating system packages will be served from.

salt/idstools/config.sls

@@ -33,6 +33,19 @@ idstools_sbin_jinja:
     - file_mode: 755
     - template: jinja
 
+suricatacustomdirsfile:
+  file.directory:
+    - name: /nsm/rules/detect-suricata/custom_file
+    - user: 939
+    - group: 939
+    - makedirs: True
+
+suricatacustomdirsurl:
+  file.directory:
+    - name: /nsm/rules/detect-suricata/custom_temp
+    - user: 939
+    - group: 939
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/idstools/etc/rulecat.conf

@@ -1,6 +1,8 @@
 {%- from 'vars/globals.map.jinja' import GLOBALS -%}
-{%- from 'idstools/map.jinja' import IDSTOOLSMERGED -%}
+{%- from 'soc/merged.map.jinja' import SOCMERGED -%}
+--suricata-version=6.0
 --merged=/opt/so/rules/nids/suri/all.rules
+--output=/nsm/rules/detect-suricata/custom_temp
 --local=/opt/so/rules/nids/suri/local.rules
 {%-   if GLOBALS.md_engine == "SURICATA" %}
 --local=/opt/so/rules/nids/suri/extraction.rules
@@ -10,8 +12,12 @@
 --disable=/opt/so/idstools/etc/disable.conf
 --enable=/opt/so/idstools/etc/enable.conf
 --modify=/opt/so/idstools/etc/modify.conf
-{%- if IDSTOOLSMERGED.config.urls | length > 0 %}
+{%- if SOCMERGED.config.server.modules.suricataengine.customRulesets %}
-{%-   for URL in IDSTOOLSMERGED.config.urls %}
+    {%- for ruleset in SOCMERGED.config.server.modules.suricataengine.customRulesets %}
---url={{ URL }}
+        {%- if 'url' in ruleset %}
-{%-   endfor %}
+--url={{ ruleset.url }}
+        {%- elif 'file' in ruleset %}
+--local={{ ruleset.file }}
+        {%- endif %}
+    {%- endfor %}
 {%- endif %}

salt/idstools/tools/sbin_jinja/so-rule-update

@@ -26,8 +26,6 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
     docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force
 {%-   elif IDSTOOLSMERGED.config.ruleset == 'ETPRO' %}
     docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force --etpro={{ IDSTOOLSMERGED.config.oinkcode }}
-{%-   elif IDSTOOLSMERGED.config.ruleset == 'TALOS' %}
-    docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force --url=https://www.snort.org/rules/snortrules-snapshot-2983.tar.gz?oinkcode={{ IDSTOOLSMERGED.config.oinkcode }}
 {%-   endif %}
 {%- endif %}
 

salt/kafka/config.map.jinja

@@ -0,0 +1,91 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+{% from 'kafka/map.jinja' import KAFKAMERGED %}
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+
+{% set KAFKA_NODES_PILLAR = salt['pillar.get']('kafka:nodes') %}
+{% set KAFKA_PASSWORD = salt['pillar.get']('kafka:password') %}
+
+{# Create list of KRaft controllers #}
+{% set controllers = [] %}
+
+{# Check for Kafka nodes with controller in process_x_roles #}
+{% for node in KAFKA_NODES_PILLAR %}
+{%   if 'controller' in KAFKA_NODES_PILLAR[node].role %}
+{%     do controllers.append(KAFKA_NODES_PILLAR[node].nodeid ~ "@" ~ node ~ ":9093") %}
+{%   endif %}
+{% endfor %}
+
+{% set kafka_controller_quorum_voters = ','.join(controllers) %}
+
+{# By default all Kafka eligible nodes are given the role of broker, except for
+     grid MANAGER (broker,controller) until overridden through SOC UI #}
+{% set node_type = salt['pillar.get']('kafka:nodes:'+ GLOBALS.hostname + ':role') %}
+
+{# Generate server.properties for 'broker' , 'controller', 'broker,controller' node types
+    anything above this line is a configuration needed for ALL Kafka nodes #}
+{% if node_type == 'broker' %}
+{%   do KAFKAMERGED.config.broker.update({'advertised_x_listeners': 'BROKER://'+ GLOBALS.node_ip +':9092' }) %}
+{%   do KAFKAMERGED.config.broker.update({'controller_x_quorum_x_voters': kafka_controller_quorum_voters }) %}
+{%   do KAFKAMERGED.config.broker.update({'node_x_id': salt['pillar.get']('kafka:nodes:'+ GLOBALS.hostname +':nodeid') }) %}
+{%   do KAFKAMERGED.config.broker.update({'ssl_x_keystore_x_password': KAFKA_PASSWORD }) %}
+
+{# Nodes with only the 'broker' role need to have the below settings for communicating with controller nodes #}
+{%   do KAFKAMERGED.config.broker.update({'controller_x_listener_x_names': KAFKAMERGED.config.controller.controller_x_listener_x_names }) %}
+{%   do KAFKAMERGED.config.broker.update({
+       'listener_x_security_x_protocol_x_map': KAFKAMERGED.config.broker.listener_x_security_x_protocol_x_map
+        + ',' + KAFKAMERGED.config.controller.listener_x_security_x_protocol_x_map })
+  %}
+{% endif %}
+
+{% if node_type == 'controller' %}
+{%   do KAFKAMERGED.config.controller.update({'controller_x_quorum_x_voters': kafka_controller_quorum_voters }) %}
+{%   do KAFKAMERGED.config.controller.update({'node_x_id': salt['pillar.get']('kafka:nodes:'+ GLOBALS.hostname +':nodeid') }) %}
+{%   do KAFKAMERGED.config.controller.update({'ssl_x_keystore_x_password': KAFKA_PASSWORD }) %}
+
+{% endif %}
+
+{# Kafka nodes of this type are not recommended for use outside of development / testing. #}
+{% if node_type == 'broker,controller' %}
+{%   do KAFKAMERGED.config.broker.update({'advertised_x_listeners': 'BROKER://'+ GLOBALS.node_ip +':9092' }) %}
+{%   do KAFKAMERGED.config.broker.update({'controller_x_listener_x_names': KAFKAMERGED.config.controller.controller_x_listener_x_names }) %}
+{%   do KAFKAMERGED.config.broker.update({'controller_x_quorum_x_voters': kafka_controller_quorum_voters }) %}
+{%   do KAFKAMERGED.config.broker.update({'node_x_id': salt['pillar.get']('kafka:nodes:'+ GLOBALS.hostname +':nodeid') }) %}
+{%   do KAFKAMERGED.config.broker.update({'process_x_roles': 'broker,controller' }) %}
+{%   do KAFKAMERGED.config.broker.update({'ssl_x_keystore_x_password': KAFKA_PASSWORD }) %}
+
+{%   do KAFKAMERGED.config.broker.update({
+       'listeners': KAFKAMERGED.config.broker.listeners + ',' + KAFKAMERGED.config.controller.listeners })
+  %}
+
+{%   do KAFKAMERGED.config.broker.update({
+       'listener_x_security_x_protocol_x_map': KAFKAMERGED.config.broker.listener_x_security_x_protocol_x_map
+        + ',' + KAFKAMERGED.config.controller.listener_x_security_x_protocol_x_map })
+  %}
+
+{% endif %}
+
+{# If a password other than PLACEHOLDER isn't set remove it from the server.properties #}
+{% if KAFKAMERGED.config.broker.ssl_x_truststore_x_password == 'PLACEHOLDER' %}
+{%   do KAFKAMERGED.config.broker.pop('ssl_x_truststore_x_password') %}
+{% endif %}
+
+{% if KAFKAMERGED.config.controller.ssl_x_truststore_x_password == 'PLACEHOLDER' %}
+{%   do KAFKAMERGED.config.controller.pop('ssl_x_truststore_x_password') %}
+{% endif %}
+
+{# Client properties stuff #}
+{% if KAFKAMERGED.config.client.ssl_x_truststore_x_password == 'PLACEHOLDER' %}
+{%   do KAFKAMERGED.config.client.pop('ssl_x_truststore_x_password') %}
+{% endif %}
+{% do KAFKAMERGED.config.client.update({'ssl_x_keystore_x_password': KAFKA_PASSWORD }) %}
+
+{% if 'broker' in node_type %}
+{%   set KAFKACONFIG = KAFKAMERGED.config.broker %}
+{% else %}
+{%   set KAFKACONFIG = KAFKAMERGED.config.controller %}
+{% endif %}
+
+{% set KAFKACLIENT = KAFKAMERGED.config.client %}

salt/kafka/config.sls

@@ -0,0 +1,69 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+{%   from 'vars/globals.map.jinja' import GLOBALS %}
+
+include:
+  - ssl
+
+kafka_group:
+  group.present:
+    - name: kafka
+    - gid: 960
+
+kafka:
+  user.present:
+    - uid: 960
+    - gid: 960
+
+kafka_sbin_tools:
+  file.recurse:
+    - name: /usr/sbin
+    - source: salt://kafka/tools/sbin
+    - user: 960
+    - group: 960
+    - file_mode: 755
+
+kafka_log_dir:
+  file.directory:
+    - name: /opt/so/log/kafka
+    - user: 960
+    - group: 960
+    - makedirs: True
+
+kafka_data_dir:
+  file.directory:
+    - name: /nsm/kafka/data
+    - user: 960
+    - group: 960
+    - makedirs: True
+
+{%   for sc in ['server', 'client'] %}
+kafka_kraft_{{sc}}_properties:
+  file.managed:
+    - source: salt://kafka/etc/{{sc}}.properties.jinja
+    - name: /opt/so/conf/kafka/{{sc}}.properties
+    - template: jinja
+    - user: 960
+    - group: 960
+    - makedirs: True
+    - show_changes: False
+{%   endfor %}
+
+reset_quorum_on_changes:
+  cmd.run:
+    - name: rm -f /nsm/kafka/data/__cluster_metadata-0/quorum-state
+    - onchanges:
+      - file: /opt/so/conf/kafka/server.properties
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/kafka/defaults.yaml

@@ -0,0 +1,62 @@
+kafka:
+  enabled: False
+  cluster_id:
+  password:
+  controllers:
+  reset:
+  config:
+    broker:
+      advertised_x_listeners:
+      auto_x_create_x_topics_x_enable: true
+      controller_x_quorum_x_voters:
+      default_x_replication_x_factor: 1
+      inter_x_broker_x_listener_x_name: BROKER
+      listeners: BROKER://0.0.0.0:9092
+      listener_x_security_x_protocol_x_map: BROKER:SSL
+      log_x_dirs: /nsm/kafka/data
+      log_x_retention_x_check_x_interval_x_ms: 300000
+      log_x_retention_x_hours: 168
+      log_x_segment_x_bytes: 1073741824
+      node_x_id:
+      num_x_io_x_threads: 8
+      num_x_network_x_threads: 3
+      num_x_partitions: 3
+      num_x_recovery_x_threads_x_per_x_data_x_dir: 1
+      offsets_x_topic_x_replication_x_factor: 1
+      process_x_roles: broker
+      socket_x_receive_x_buffer_x_bytes: 102400
+      socket_x_request_x_max_x_bytes: 104857600
+      socket_x_send_x_buffer_x_bytes: 102400
+      ssl_x_keystore_x_location: /etc/pki/kafka.p12
+      ssl_x_keystore_x_type: PKCS12
+      ssl_x_keystore_x_password:
+      ssl_x_truststore_x_location: /etc/pki/java/sos/cacerts
+      ssl_x_truststore_x_password: PLACEHOLDER
+      ssl_x_truststore_x_type: PEM
+      transaction_x_state_x_log_x_min_x_isr: 1
+      transaction_x_state_x_log_x_replication_x_factor: 1
+    client:
+      security_x_protocol: SSL
+      ssl_x_truststore_x_location: /etc/pki/java/sos/cacerts
+      ssl_x_truststore_x_password: PLACEHOLDER
+      ssl_x_truststore_x_type: PEM
+      ssl_x_keystore_x_location: /etc/pki/kafka.p12
+      ssl_x_keystore_x_type: PKCS12
+      ssl_x_keystore_x_password:
+    controller:
+      controller_x_listener_x_names: CONTROLLER
+      controller_x_quorum_x_voters:
+      listeners: CONTROLLER://0.0.0.0:9093
+      listener_x_security_x_protocol_x_map: CONTROLLER:SSL
+      log_x_dirs: /nsm/kafka/data
+      log_x_retention_x_check_x_interval_x_ms: 300000
+      log_x_retention_x_hours: 168
+      log_x_segment_x_bytes: 1073741824
+      node_x_id:
+      process_x_roles: controller
+      ssl_x_keystore_x_location: /etc/pki/kafka.p12
+      ssl_x_keystore_x_type: PKCS12
+      ssl_x_keystore_x_password:
+      ssl_x_truststore_x_location: /etc/pki/java/sos/cacerts
+      ssl_x_truststore_x_password: PLACEHOLDER
+      ssl_x_truststore_x_type: PEM

salt/kafka/disabled.sls

@@ -0,0 +1,25 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'kafka/map.jinja' import KAFKAMERGED %}
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+
+so-kafka:
+  docker_container.absent:
+    - force: True
+
+so-kafka_so-status.disabled:
+  file.comment:
+    - name: /opt/so/conf/so-status/so-status.conf
+    - regex: ^so-kafka$
+    - onlyif: grep -q '^so-kafka$' /opt/so/conf/so-status/so-status.conf
+
+{% if GLOBALS.is_manager and KAFKAMERGED.enabled or GLOBALS.pipeline == "KAFKA" %}
+ensure_default_pipeline:
+  cmd.run:
+    - name: |
+        /usr/sbin/so-yaml.py replace /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.enabled False;
+        /usr/sbin/so-yaml.py replace /opt/so/saltstack/local/pillar/global/soc_global.sls global.pipeline REDIS
+{% endif %}

salt/kafka/enabled.sls

@@ -0,0 +1,86 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+#
+# Note: Per the Elastic License 2.0, the second limitation states:
+#
+#   "You may not move, change, disable, or circumvent the license key functionality
+#    in the software, and you may not remove or obscure any functionality in the
+#    software that is protected by the license key."
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+{%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/docker.map.jinja' import DOCKER %}
+{%   set KAFKANODES = salt['pillar.get']('kafka:nodes') %}
+{%   if 'gmd' in salt['pillar.get']('features', []) %}
+
+include:
+  - elasticsearch.ca
+  - kafka.sostatus
+  - kafka.config
+  - kafka.storage
+
+so-kafka:
+  docker_container.running:
+    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kafka:{{ GLOBALS.so_version }}
+    - hostname: so-kafka
+    - name: so-kafka
+    - networks:
+      - sobridge:
+        - ipv4_address: {{ DOCKER.containers['so-kafka'].ip }}
+    - user: kafka
+    - environment:
+        KAFKA_HEAP_OPTS: -Xmx2G -Xms1G
+        KAFKA_OPTS: -javaagent:/opt/jolokia/agents/jolokia-agent-jvm-javaagent.jar=port=8778,host={{ DOCKER.containers['so-kafka'].ip }},policyLocation=file:/opt/jolokia/jolokia.xml
+    - extra_hosts:
+      {% for node in KAFKANODES %}
+      - {{ node }}:{{ KAFKANODES[node].ip }}
+      {% endfor %}
+      {% if DOCKER.containers['so-kafka'].extra_hosts %}
+      {%   for XTRAHOST in DOCKER.containers['so-kafka'].extra_hosts %}
+      - {{ XTRAHOST }}
+      {%   endfor %}
+      {% endif %}
+    - port_bindings:
+      {% for BINDING in DOCKER.containers['so-kafka'].port_bindings %}
+      - {{ BINDING }}
+      {% endfor %}
+    - binds:
+      - /etc/pki/kafka.p12:/etc/pki/kafka.p12:ro
+      - /etc/pki/tls/certs/intca.crt:/etc/pki/java/sos/cacerts:ro
+      - /nsm/kafka/data/:/nsm/kafka/data/:rw
+      - /opt/so/log/kafka:/opt/kafka/logs/:rw
+      - /opt/so/conf/kafka/server.properties:/opt/kafka/config/kraft/server.properties:ro
+      - /opt/so/conf/kafka/client.properties:/opt/kafka/config/kraft/client.properties
+    - watch:
+      {% for sc in ['server', 'client'] %}
+      - file: kafka_kraft_{{sc}}_properties
+      {% endfor %}
+
+delete_so-kafka_so-status.disabled:
+  file.uncomment:
+    - name: /opt/so/conf/so-status/so-status.conf
+    - regex: ^so-kafka$
+
+{%   else %}
+
+{{sls}}_no_license_detected:
+  test.fail_without_changes:
+    - name: {{sls}}_no_license_detected
+    - comment:
+      - "Kafka for Guaranteed Message Delivery is a feature supported only for customers with a valid license.
+      Contact Security Onion Solutions, LLC via our website at https://securityonionsolutions.com
+      for more information about purchasing a license to enable this feature."
+include:
+  - kafka.disabled
+{%   endif %}
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/kafka/etc/client.properties.jinja

@@ -0,0 +1,7 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+
+{% from 'kafka/config.map.jinja' import KAFKACLIENT -%}
+{{ KAFKACLIENT | yaml(False) | replace("_x_", ".") }}

salt/kafka/etc/server.properties.jinja

@@ -0,0 +1,7 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+
+{% from 'kafka/config.map.jinja' import KAFKACONFIG -%}
+{{ KAFKACONFIG | yaml(False) | replace("_x_", ".") }}

salt/kafka/files/managed_node_pillar.jinja

@@ -0,0 +1,10 @@
+kafka:
+  nodes:
+{% for node, values in COMBINED_KAFKANODES.items() %}
+    {{ node }}:
+      ip: {{ values['ip'] }}
+      nodeid: {{ values['nodeid'] }}
+{%-   if values['role'] != none %}
+      role: {{ values['role'] }}
+{%-   endif %}
+{% endfor %}

salt/kafka/init.sls

@@ -0,0 +1,24 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+#
+# Note: Per the Elastic License 2.0, the second limitation states:
+#
+#   "You may not move, change, disable, or circumvent the license key functionality
+#    in the software, and you may not remove or obscure any functionality in the
+#    software that is protected by the license key."
+
+{% from 'kafka/map.jinja' import KAFKAMERGED %}
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+
+include:
+{# Run kafka/nodes.sls before Kafka is enabled, so kafka nodes pillar is setup #}
+{% if grains.role in ['so-manager','so-managersearch', 'so-standalone'] %}
+  - kafka.nodes
+{% endif %}
+{% if GLOBALS.pipeline == "KAFKA" and KAFKAMERGED.enabled %}
+  - kafka.enabled
+{% else %}
+  - kafka.disabled
+{% endif %}

salt/kafka/map.jinja

@@ -0,0 +1,10 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+
+{# This is only used to determine if Kafka is enabled / disabled. Configuration is found in kafka/config.map.jinja #}
+{# kafka/config.map.jinja depends on there being a kafka nodes pillar being populated #}
+
+{% import_yaml 'kafka/defaults.yaml' as KAFKADEFAULTS %}
+{% set KAFKAMERGED = salt['pillar.get']('kafka', KAFKADEFAULTS.kafka, merge=True) %}

salt/kafka/nodes.map.jinja

@@ -0,0 +1,88 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+
+{# USED TO GENERATE PILLAR/KAFKA/NODES.SLS. #}
+{% import_yaml 'kafka/defaults.yaml' as KAFKADEFAULTS %}
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+
+{% set process_x_roles = KAFKADEFAULTS.kafka.config.broker.process_x_roles %}
+
+{% set current_kafkanodes = salt.saltutil.runner(
+        'mine.get',
+         tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-receiver',
+         fun='network.ip_addrs',
+         tgt_type='compound') %}
+
+{% set STORED_KAFKANODES = salt['pillar.get']('kafka:nodes', default=None) %}
+{% set KAFKA_CONTROLLERS_PILLAR = salt['pillar.get']('kafka:controllers', default=None) %}
+
+{% set existing_ids = [] %}
+
+{# Check STORED_KAFKANODES for existing kafka nodes and pull their IDs so they are not reused across the grid #}
+{% if STORED_KAFKANODES != none %}
+{%   for node, values in STORED_KAFKANODES.items() %}
+{%     if values.get('nodeid') %}
+{%       do existing_ids.append(values['nodeid']) %}
+{%     endif %}
+{%   endfor %}
+{% endif %}
+
+{# Create list of possible node ids #}
+{% set all_possible_ids = range(1, 2000)|list %}
+
+{# Create list of available node ids by looping through all_possible_ids and ensuring it isn't in existing_ids #}
+{% set available_ids = [] %}
+{% for id in all_possible_ids %}
+{%   if id not in existing_ids %}
+{%     do available_ids.append(id) %}
+{%   endif %}
+{% endfor %}
+
+{# Collect kafka eligible nodes and check if they're already in STORED_KAFKANODES to avoid potentially reassigning a nodeid #}
+{% set NEW_KAFKANODES = {} %}
+{% for minionid, ip in current_kafkanodes.items() %}
+{%   set hostname = minionid.split('_')[0] %}
+{%   if not STORED_KAFKANODES or hostname not in STORED_KAFKANODES %}
+{%     set new_id = available_ids.pop(0) %}
+{%     do NEW_KAFKANODES.update({hostname: {'nodeid': new_id, 'ip': ip[0], 'role': process_x_roles }}) %}
+{%   endif %}
+{% endfor %}
+
+{# Combine STORED_KAFKANODES and NEW_KAFKANODES for writing to the pillar/kafka/nodes.sls #}
+{% set COMBINED_KAFKANODES = {} %}
+{% for node, details in NEW_KAFKANODES.items() %}
+{%   do COMBINED_KAFKANODES.update({node: details}) %}
+{% endfor %}
+{% if STORED_KAFKANODES != none %}
+{%  for node, details in STORED_KAFKANODES.items() %}
+{%    do COMBINED_KAFKANODES.update({node: details}) %}
+{%  endfor %}
+{% endif %}
+
+{# Update the process_x_roles value for any host in the kafka_controllers_pillar configured from SOC UI #}
+{% set ns = namespace(has_controller=false) %}
+{% if KAFKA_CONTROLLERS_PILLAR != none %}
+{%   set KAFKA_CONTROLLERS_PILLAR_LIST = KAFKA_CONTROLLERS_PILLAR.split(',') %}
+{%   for hostname in KAFKA_CONTROLLERS_PILLAR_LIST %}
+{%     if hostname in COMBINED_KAFKANODES %}
+{%       do COMBINED_KAFKANODES[hostname].update({'role': 'controller'}) %}
+{%       set ns.has_controller = true %}
+{%     endif %}
+{%   endfor %}
+{%   for hostname in COMBINED_KAFKANODES %}
+{%     if hostname not in KAFKA_CONTROLLERS_PILLAR_LIST %}
+{%       do COMBINED_KAFKANODES[hostname].update({'role': 'broker'}) %}
+{%     endif %}
+{%   endfor %}
+{# If the kafka_controllers_pillar is NOT empty check that atleast one node contains the controller role.
+     otherwise default to GLOBALS.manager having broker,controller role #}
+{%   if not ns.has_controller %}
+{%     do COMBINED_KAFKANODES[GLOBALS.manager].update({'role': 'broker,controller'}) %}
+{%   endif %}
+{# If kafka_controllers_pillar is empty, default to having grid manager as 'broker,controller'
+   so there is always atleast 1 controller in the cluster #}
+{% else %}
+{%   do COMBINED_KAFKANODES[GLOBALS.manager].update({'role': 'broker,controller'}) %}
+{% endif %}

salt/kafka/nodes.sls

@@ -0,0 +1,18 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'kafka/nodes.map.jinja' import COMBINED_KAFKANODES %}
+{% set kafka_cluster_id = salt['pillar.get']('kafka:cluster_id', default=None) %}
+
+{# Write Kafka pillar, so all grid members have access to nodeid of other kafka nodes and their roles #}
+write_kafka_pillar_yaml:
+  file.managed:
+    - name: /opt/so/saltstack/local/pillar/kafka/nodes.sls
+    - mode: 644
+    - user: socore
+    - source: salt://kafka/files/managed_node_pillar.jinja
+    - template: jinja
+    - context:
+        COMBINED_KAFKANODES: {{ COMBINED_KAFKANODES }}

salt/kafka/reset.sls

@@ -0,0 +1,9 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+wipe_kafka_data:
+  file.absent:
+    - name: /nsm/kafka/data/
+    - force: True

salt/kafka/soc_kafka.yaml

@@ -0,0 +1,209 @@
+kafka:
+  enabled:
+    description: Set to True to enable Kafka. To avoid grid problems, do not enable Kafka until the related configuration is in place. Requires a valid Security Onion license key.
+    helpLink: kafka.html
+  cluster_id:
+    description: The ID of the Kafka cluster.
+    readonly: True
+    advanced: True
+    sensitive: True
+    helpLink: kafka.html
+  password:
+    description: The password to use for the Kafka certificates.
+    sensitive: True
+    helpLink: kafka.html
+  controllers:
+    description: A comma-separated list of hostnames that will act as Kafka controllers. These hosts will be responsible for managing the Kafka cluster. Note that only manager and receiver nodes are eligible to run Kafka. This configuration needs to be set before enabling Kafka. Failure to do so may result in Kafka topics becoming unavailable requiring manual intervention to restore functionality or reset Kafka, either of which can result in data loss.
+    forcedType: "string"
+    helpLink: kafka.html
+  reset:
+    description: Disable and reset the Kafka cluster. This will remove all Kafka data including logs that may have not yet been ingested into Elasticsearch and reverts the grid to using REDIS as the global pipeline. This is useful when testing different Kafka configurations such as rearranging Kafka brokers / controllers allowing you to reset the cluster rather than manually fixing any issues arising from attempting to reassign a Kafka broker into a controller. Enter 'YES_RESET_KAFKA' and submit to disable and reset Kafka. Make any configuration changes required and re-enable Kafka when ready. This action CANNOT be reversed.
+    advanced: True
+    helpLink: kafka.html
+  config:
+    broker:
+      advertised_x_listeners:
+        description: Specify the list of listeners (hostname and port) that Kafka brokers provide to clients for communication.
+        title: advertised.listeners
+        helpLink: kafka.html
+      auto_x_create_x_topics_x_enable:
+        description: Enable the auto creation of topics.
+        title: auto.create.topics.enable
+        forcedType: bool
+        helpLink: kafka.html
+      default_x_replication_x_factor:
+        description: The default replication factor for automatically created topics. This value must be less than the amount of brokers in the cluster. Hosts specified in controllers should not be counted towards total broker count.
+        title: default.replication.factor
+        forcedType: int
+        helpLink: kafka.html
+      inter_x_broker_x_listener_x_name:
+        description: The name of the listener used for inter-broker communication.
+        title: inter.broker.listener.name
+        helpLink: kafka.html
+      listeners:
+        description: Set of URIs that is listened on and the listener names in a comma-seperated list.
+        helpLink: kafka.html
+      listener_x_security_x_protocol_x_map:
+        description: Comma-seperated mapping of listener name and security protocols.
+        title: listener.security.protocol.map
+        helpLink: kafka.html
+      log_x_dirs:
+        description: Where Kafka logs are stored within the Docker container.
+        title: log.dirs
+        helpLink: kafka.html
+      log_x_retention_x_check_x_interval_x_ms:
+        description: Frequency at which log files are checked if they are qualified for deletion.
+        title: log.retention.check.interval.ms
+        helpLink: kafka.html
+      log_x_retention_x_hours:
+        description: How long, in hours, a log file is kept.
+        title: log.retention.hours
+        forcedType: int
+        helpLink: kafka.html
+      log_x_segment_x_bytes:
+        description: The maximum allowable size for a log file.
+        title: log.segment.bytes
+        forcedType: int
+        helpLink: kafka.html
+      num_x_io_x_threads:
+        description: The number of threads used by Kafka.
+        title: num.io.threads
+        forcedType: int
+        helpLink: kafka.html
+      num_x_network_x_threads:
+        description: The number of threads used for network communication.
+        title: num.network.threads
+        forcedType: int
+        helpLink: kafka.html
+      num_x_partitions:
+        description: The number of log partitions assigned per topic.
+        title: num.partitions
+        forcedType: int
+        helpLink: kafka.html
+      num_x_recovery_x_threads_x_per_x_data_x_dir:
+        description: The number of threads used for log recuperation at startup and purging at shutdown. This ammount of threads is used per data directory.
+        title: num.recovery.threads.per.data.dir
+        forcedType: int
+        helpLink: kafka.html
+      offsets_x_topic_x_replication_x_factor:
+        description: The offsets topic replication factor.
+        title: offsets.topic.replication.factor
+        forcedType: int
+        helpLink: kafka.html
+      process_x_roles:
+        description: The role performed by Kafka brokers.
+        title: process.roles
+        readonly: True
+        helpLink: kafka.html
+      socket_x_receive_x_buffer_x_bytes:
+        description: Size, in bytes of the SO_RCVBUF buffer. A value of -1 will use the OS default.
+        title: socket.receive.buffer.bytes
+        #forcedType: int - soc needs to allow -1 as an int before we can use this
+        helpLink: kafka.html
+      socket_x_request_x_max_x_bytes:
+        description: The maximum bytes allowed for a request to the socket.
+        title: socket.request.max.bytes
+        forcedType: int
+        helpLink: kafka.html
+      socket_x_send_x_buffer_x_bytes:
+        description: Size, in bytes of the SO_SNDBUF buffer. A value of -1 will use the OS default.
+        title: socket.send.buffer.byte
+        #forcedType: int - soc needs to allow -1 as an int before we can use this
+        helpLink: kafka.html
+      ssl_x_keystore_x_location:
+        description: The key store file location within the Docker container.
+        title: ssl.keystore.location
+        helpLink: kafka.html
+      ssl_x_keystore_x_password:
+        description: The key store file password. Invalid for PEM format.
+        title: ssl.keystore.password
+        sensitive: True
+        helpLink: kafka.html
+      ssl_x_keystore_x_type: 
+        description: The key store file format.
+        title: ssl.keystore.type
+        regex: ^(JKS|PKCS12|PEM)$
+        helpLink: kafka.html
+      ssl_x_truststore_x_location:
+        description: The trust store file location within the Docker container.
+        title: ssl.truststore.location
+        helpLink: kafka.html
+      ssl_x_truststore_x_password:
+        description: The trust store file password. If null, the trust store file is still use, but integrity checking is disabled. Invalid for PEM format.
+        title: ssl.truststore.password
+        sensitive: True
+        helpLink: kafka.html
+      transaction_x_state_x_log_x_min_x_isr:
+        description: Overrides min.insync.replicas for the transaction topic. When a producer configures acks to "all" (or "-1"), this setting determines the minimum number of replicas required to acknowledge a write as successful. Failure to meet this minimum triggers an exception (either NotEnoughReplicas or NotEnoughReplicasAfterAppend). When used in conjunction, min.insync.replicas and acks enable stronger durability guarantees. For instance, creating a topic with a replication factor of 3, setting min.insync.replicas to 2, and using acks of "all" ensures that the producer raises an exception if a majority of replicas fail to receive a write.
+        title: transaction.state.log.min.isr
+        forcedType: int
+        helpLink: kafka.html
+      transaction_x_state_x_log_x_replication_x_factor:
+        description: Set the replication factor higher for the transaction topic to ensure availability. Internal topic creation will not proceed until the cluster size satisfies this replication factor prerequisite.
+        title: transaction.state.log.replication.factor
+        forcedType: int
+        helpLink: kafka.html
+    client:
+      security_x_protocol:
+        description: 'Broker communication protocol. Options are: SASL_SSL, PLAINTEXT, SSL, SASL_PLAINTEXT'
+        title: security.protocol
+        regex: ^(SASL_SSL|PLAINTEXT|SSL|SASL_PLAINTEXT)
+        helpLink: kafka.html
+      ssl_x_keystore_x_location:
+        description: The key store file location within the Docker container.
+        title: ssl.keystore.location
+        helpLink: kafka.html
+      ssl_x_keystore_x_password:
+        description: The key store file password. Invalid for PEM format.
+        title: ssl.keystore.password
+        sensitive: True
+        helpLink: kafka.html
+      ssl_x_keystore_x_type:
+        description: The key store file format.
+        title: ssl.keystore.type
+        regex: ^(JKS|PKCS12|PEM)$
+        helpLink: kafka.html
+      ssl_x_truststore_x_location:
+        description: The trust store file location within the Docker container.
+        title: ssl.truststore.location
+        helpLink: kafka.html
+      ssl_x_truststore_x_password:
+        description: The trust store file password. If null, the trust store file is still use, but integrity checking is disabled. Invalid for PEM format.
+        title: ssl.truststore.password
+        sensitive: True
+        helpLink: kafka.html
+    controller:
+      controller_x_listener_x_names:
+        description: Set listeners used by the controller in a comma-seperated list.
+        title: controller.listener.names
+        helpLink: kafka.html
+      listeners:
+        description: Set of URIs that is listened on and the listener names in a comma-seperated list.
+        helpLink: kafka.html
+      listener_x_security_x_protocol_x_map:
+        description: Comma-seperated mapping of listener name and security protocols.
+        title: listener.security.protocol.map
+        helpLink: kafka.html
+      log_x_dirs:
+        description: Where Kafka logs are stored within the Docker container.
+        title: log.dirs
+        helpLink: kafka.html
+      log_x_retention_x_check_x_interval_x_ms:
+        description: Frequency at which log files are checked if they are qualified for deletion.
+        title: log.retention.check.interval.ms
+        helpLink: kafka.html
+      log_x_retention_x_hours:
+        description: How long, in hours, a log file is kept.
+        title: log.retention.hours
+        forcedType: int
+        helpLink: kafka.html
+      log_x_segment_x_bytes:
+        description: The maximum allowable size for a log file.
+        title: log.segment.bytes
+        forcedType: int
+        helpLink: kafka.html
+      process_x_roles:
+        description: The role performed by controller node.
+        title: process.roles
+        readonly: True
+        helpLink: kafka.html

salt/kafka/sostatus.sls

@@ -0,0 +1,21 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+
+append_so-kafka_so-status.conf:
+  file.append:
+    - name: /opt/so/conf/so-status/so-status.conf
+    - text: so-kafka
+    - unless: grep -q so-kafka /opt/so/conf/so-status/so-status.conf
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/kafka/storage.sls

@@ -0,0 +1,30 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+{%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   set kafka_cluster_id = salt['pillar.get']('kafka:cluster_id') %}
+
+{# Initialize kafka storage if it doesn't already exist. Just looking for meta.properties in /nsm/kafka/data #}
+{%   if not salt['file.file_exists']('/nsm/kafka/data/meta.properties') %}
+kafka_storage_init:
+  cmd.run:
+    - name: |
+        docker run -v /nsm/kafka/data:/nsm/kafka/data -v /opt/so/conf/kafka/server.properties:/opt/kafka/config/kraft/newserver.properties --name so-kafkainit --user root --entrypoint /opt/kafka/bin/kafka-storage.sh {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kafka:{{ GLOBALS.so_version }} format -t {{ kafka_cluster_id }} -c /opt/kafka/config/kraft/newserver.properties
+kafka_rm_kafkainit:
+  cmd.run:
+    - name: |
+        docker rm so-kafkainit
+{%   endif %}
+
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/kafka/tools/sbin/so-kafka-cli

@@ -0,0 +1,47 @@
+#! /bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+if [ -z "$NOROOT" ]; then
+	# Check for prerequisites
+	if [ "$(id -u)" -ne 0 ]; then
+		echo "This script must be run using sudo!"
+		exit 1
+	fi
+fi
+
+function usage() {
+  echo -e "\nUsage: $0 <script> [options]"
+  echo ""
+  echo "Available scripts:"
+  show_available_kafka_cli_tools
+}
+
+function show_available_kafka_cli_tools(){
+  docker exec so-kafka ls /opt/kafka/bin | grep kafka
+}
+
+if [ -z $1 ]; then
+  usage
+  exit 1
+fi
+
+available_tools=$(show_available_kafka_cli_tools)
+script_exists=false
+
+for script in $available_tools; do
+  if [ "$script" == "$1" ]; then
+    script_exists=true
+    break
+  fi
+done
+
+if [ "$script_exists" == true ]; then
+  docker exec so-kafka /opt/kafka/bin/$1 "${@:2}"
+else
+  echo -e  "\nInvalid script: $1"
+  usage
+  exit 1
+fi

salt/kafka/tools/sbin/so-kafka-config-update

@@ -0,0 +1,87 @@
+#! /bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+if [ -z "$NOROOT" ]; then
+	# Check for prerequisites
+	if [ "$(id -u)" -ne 0 ]; then
+		echo "This script must be run using sudo!"
+		exit 1
+	fi
+fi
+
+usage() {
+  cat <<USAGE_EOF
+
+  Usage: $0 <operation> [parameters]
+
+  Where <operation> is one of the following:
+
+    topic-partitions: Increase the number of partitions for a Kafka topic
+      Required arguments: topic-partitions <topic name> <# partitions>
+      Example: $0 topic-partitions suricata-topic 6
+
+	list-topics: List of Kafka topics
+	  Example: $0 list-topics
+
+USAGE_EOF
+  exit 1
+}
+
+if [[ $# -lt 1 || $1 == --help || $1 == -h ]]; then
+  usage
+fi
+
+kafka_client_config="/opt/kafka/config/kraft/client.properties"
+
+too_few_arguments() {
+  echo -e "\nMissing one or more required arguments!\n"
+  usage
+}
+
+get_kafka_brokers() {
+	brokers_cache="/opt/so/state/kafka_brokers"
+  broker_port="9092"
+  if [[ ! -f "$brokers_cache" ]] || [[ $(find "/$brokers_cache" -mmin +120) ]]; then
+    echo "Refreshing Kafka brokers list"
+	  salt-call pillar.get kafka:nodes --out=json | jq -r --arg broker_port "$broker_port" '.local | to_entries[] | select(.value.role | contains("broker")) | "\(.value.ip):\($broker_port)"' | paste -sd "," - > "$brokers_cache"
+  else
+    echo "Using cached Kafka brokers list"
+  fi
+  brokers=$(cat "$brokers_cache")
+}
+
+increase_topic_partitions() {
+  get_kafka_brokers
+	command=$(so-kafka-cli kafka-topics.sh --bootstrap-server $brokers --command-config $kafka_client_config --alter --topic $topic --partitions $partition_count)
+  if $command; then
+    echo -e "Successfully increased the number of partitions for topic $topic to $partition_count\n"
+    so-kafka-cli kafka-topics.sh --bootstrap-server $brokers --command-config $kafka_client_config --describe --topic $topic
+  fi
+}
+
+get_kafka_topics_list() {
+	get_kafka_brokers
+	so-kafka-cli kafka-topics.sh --bootstrap-server $brokers --command-config $kafka_client_config --exclude-internal --list | sort
+}
+
+operation=$1
+case "${operation}" in
+  "topic-partitions")
+	if [[ $# -lt 3 ]]; then
+	  too_few_arguments
+	fi
+	topic=$2
+	partition_count=$3
+	increase_topic_partitions
+	;;
+  "list-topics")
+	get_kafka_topics_list
+	;;
+  *)
+	usage
+	;;
+esac
+

salt/logstash/defaults.yaml

@@ -37,6 +37,7 @@ logstash:
       - so/0900_input_redis.conf.jinja
       - so/9805_output_elastic_agent.conf.jinja
       - so/9900_output_endgame.conf.jinja
+      - so/0800_input_kafka.conf.jinja
     custom0: []
     custom1: []
     custom2: []

salt/logstash/enabled.sls

@@ -75,10 +75,13 @@ so-logstash:
       {% else %}
       - /etc/pki/tls/certs/intca.crt:/usr/share/filebeat/ca.crt:ro
       {% endif %}
-      {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-searchnode'] %}
+      {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-searchnode' ] %}
       - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro
       - /opt/so/conf/ca/tls-ca-bundle.pem:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem:ro
       {% endif %}
+      {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-searchnode'] %}
+      - /etc/pki/kafka-logstash.p12:/usr/share/logstash/kafka-logstash.p12:ro
+      {% endif %}
       {% if GLOBALS.role == 'so-eval' %}
       - /nsm/zeek:/nsm/zeek:ro
       - /nsm/suricata:/suricata:ro

salt/logstash/init.sls

@@ -4,9 +4,13 @@
 # Elastic License 2.0.
 
 {% from 'logstash/map.jinja' import LOGSTASH_MERGED %}
+{% from 'kafka/map.jinja' import KAFKAMERGED %}
 
 include:
-{% if LOGSTASH_MERGED.enabled %}
+{# Disable logstash when Kafka is enabled except when the role is standalone #}
+{% if LOGSTASH_MERGED.enabled and grains.role == 'so-standalone' %}
+  - logstash.enabled
+{% elif LOGSTASH_MERGED.enabled and not KAFKAMERGED.enabled %}
   - logstash.enabled
 {% else %}
   - logstash.disabled

salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja

@@ -0,0 +1,38 @@
+{%- set kafka_password = salt['pillar.get']('kafka:password') %}
+{%- set kafka_brokers = salt['pillar.get']('kafka:nodes', {}) %}
+{%- set brokers = [] %}
+
+{%- if kafka_brokers %}
+{%-   for key, values in kafka_brokers.items() %}
+{%-     if 'broker' in values['role'] %}
+{%-       do brokers.append(key ~ ':9092') %}
+{%-     endif %}
+{%-   endfor %}
+{%-   set bootstrap_servers = ','.join(brokers) %}
+
+input {
+    kafka {
+        codec => json
+        topics_pattern => '.*-securityonion$'
+        group_id => 'searchnodes'
+        consumer_threads => 3
+        client_id => '{{ GLOBALS.hostname }}'
+        security_protocol => 'SSL'
+        bootstrap_servers => '{{ bootstrap_servers }}'
+        ssl_keystore_location => '/usr/share/logstash/kafka-logstash.p12'
+        ssl_keystore_password => '{{ kafka_password }}'
+        ssl_keystore_type => 'PKCS12'
+        ssl_truststore_location => '/etc/pki/ca-trust/extracted/java/cacerts'
+        ssl_truststore_password => 'changeit'
+        decorate_events => true
+        tags => [ "elastic-agent", "input-{{ GLOBALS.hostname}}", "kafka" ]
+    }
+}
+filter {
+  if ![metadata] {
+    mutate {
+      rename => { "@metadata" => "metadata" }
+    }
+  }
+}
+{% endif %}

salt/manager/defaults.yaml

@@ -3,3 +3,5 @@ manager:
     enabled: True
     hour: 3
     minute: 0
+  additionalCA: ''
+  insecureSkipVerify: False

salt/manager/init.sls

@@ -73,6 +73,15 @@ manager_sbin:
     - exclude_pat:
       - "*_test.py"
 
+manager_sbin_jinja:
+  file.recurse:
+    - name: /usr/sbin/
+    - source: salt://manager/tools/sbin_jinja/
+    - user: socore
+    - group: socore
+    - file_mode: 755
+    - template: jinja
+
 so-repo-file:
   file.managed:
     - name: /opt/so/conf/reposync/repodownload.conf

salt/manager/map.jinja

@@ -0,0 +1,7 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+
+{% import_yaml 'manager/defaults.yaml' as MANAGERDEFAULTS %}
+{% set MANAGERMERGED = salt['pillar.get']('manager', MANAGERDEFAULTS.manager, merge=True) %}

salt/manager/soc_manager.yaml

@@ -24,3 +24,16 @@ manager:
     description: Proxy server to use for updates.
     global: True
     helpLink: proxy.html
+  additionalCA:
+    description: Additional CA certificates to trust in PEM format.
+    global: True
+    advanced: True
+    multiline: True
+    forcedType: string
+    helpLink: proxy.html
+  insecureSkipVerify:
+    description: Disable TLS verification for outgoing requests. This will make your installation less secure to MITM attacks. Recommended only for debugging purposes.
+    advanced: True
+    forcedType: bool
+    global: True
+    helpLink: proxy.html

salt/manager/tools/sbin/so-minion

@@ -113,7 +113,7 @@ function testMinion() {
 
 	# If this so-minion script is not running on the given minion ID, run so-test remotely on the sensor as well
 	local_id=$(lookup_grain id)	
-	if [[ ! "$local_id" =~ "${MINION_ID}_" ]]; then
+	if [[ ! "$local_id" =~ "${MINION_ID}_" && "$local_id" != "${MINION_ID}" ]]; then
 		salt "$MINION_ID" cmd.run 'so-test'
 		result=$?
 	fi
@@ -604,6 +604,10 @@ function updateMineAndApplyStates() {
 	if [[ "$NODETYPE" == "SEARCHNODE" || "$NODETYPE" == "HEAVYNODE" ]]; then
 		salt-run state.orch orch.container_download pillar="{'setup': {'newnode': $MINION_ID }}" > /dev/null 2>&1 &
 	fi
+	if [[ "$NODETYPE" == "RECEIVER" ]]; then
+		# Setup nodeid for Kafka
+		salt-call state.apply kafka.nodes queue=True
+	fi
 	# $MINIONID is the minion id of the manager and $MINION_ID is the target node or the node being configured
 	salt-run state.orch orch.deploy_newnode pillar="{'setup': {'manager': $MINIONID, 'newnode': $MINION_ID }}" > /dev/null 2>&1 &
 }

salt/manager/tools/sbin/so-yaml.py

@@ -14,19 +14,20 @@ lockFile = "/tmp/so-yaml.lock"
 
 
 def showUsage(args):
-    print('Usage: {} <COMMAND> <YAML_FILE> [ARGS...]'.format(sys.argv[0]))
+    print('Usage: {} <COMMAND> <YAML_FILE> [ARGS...]'.format(sys.argv[0]), file=sys.stderr)
-    print('  General commands:')
+    print('  General commands:', file=sys.stderr)
-    print('    append         - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.')
+    print('    append         - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr)
-    print('    add            - Add a new key and set its value. Fails if key already exists. Requires KEY and VALUE args.')
+    print('    add            - Add a new key and set its value. Fails if key already exists. Requires KEY and VALUE args.', file=sys.stderr)
-    print('    remove         - Removes a yaml key, if it exists. Requires KEY arg.')
+    print('    get            - Displays (to stdout) the value stored in the given key. Requires KEY arg.', file=sys.stderr)
-    print('    replace        - Replaces (or adds) a new key and set its value. Requires KEY and VALUE args.')
+    print('    remove         - Removes a yaml key, if it exists. Requires KEY arg.', file=sys.stderr)
-    print('    help           - Prints this usage information.')
+    print('    replace        - Replaces (or adds) a new key and set its value. Requires KEY and VALUE args.', file=sys.stderr)
-    print('')
+    print('    help           - Prints this usage information.', file=sys.stderr)
-    print('  Where:')
+    print('', file=sys.stderr)
-    print('   YAML_FILE       - Path to the file that will be modified. Ex: /opt/so/conf/service/conf.yaml')
+    print('  Where:', file=sys.stderr)
-    print('   KEY             - YAML key, does not support \' or " characters at this time. Ex: level1.level2')
+    print('   YAML_FILE       - Path to the file that will be modified. Ex: /opt/so/conf/service/conf.yaml', file=sys.stderr)
-    print('   VALUE           - Value to set for a given key')
+    print('   KEY             - YAML key, does not support \' or " characters at this time. Ex: level1.level2', file=sys.stderr)
-    print('   LISTITEM        - Item to append to a given key\'s list value')
+    print('   VALUE           - Value to set for a given key', file=sys.stderr)
+    print('   LISTITEM        - Item to append to a given key\'s list value', file=sys.stderr)
     sys.exit(1)
 
 
@@ -38,7 +39,7 @@ def loadYaml(filename):
 
 def writeYaml(filename, content):
     file = open(filename, "w")
-    return yaml.dump(content, file)
+    return yaml.safe_dump(content, file)
 
 
 def appendItem(content, key, listItem):
@@ -49,15 +50,15 @@ def appendItem(content, key, listItem):
         try:
             content[key].append(listItem)
         except AttributeError:
-            print("The existing value for the given key is not a list. No action was taken on the file.")
+            print("The existing value for the given key is not a list. No action was taken on the file.", file=sys.stderr)
             return 1
         except KeyError:
-            print("The key provided does not exist. No action was taken on the file.")
+            print("The key provided does not exist. No action was taken on the file.", file=sys.stderr)
             return 1
 
 
 def convertType(value):
-    if len(value) > 0 and (not value.startswith("0") or len(value) == 1):
+    if isinstance(value, str) and len(value) > 0 and (not value.startswith("0") or len(value) == 1):
         if "." in value:
             try:
                 value = float(value)
@@ -83,7 +84,7 @@ def append(args):
     if len(args) != 3:
         print('Missing filename, key arg, or list item to append', file=sys.stderr)
         showUsage(None)
-        return
+        return 1
 
     filename = args[0]
     key = args[1]
@@ -112,7 +113,7 @@ def add(args):
     if len(args) != 3:
         print('Missing filename, key arg, and/or value', file=sys.stderr)
         showUsage(None)
-        return
+        return 1
 
     filename = args[0]
     key = args[1]
@@ -137,7 +138,7 @@ def remove(args):
     if len(args) != 2:
         print('Missing filename or key arg', file=sys.stderr)
         showUsage(None)
-        return
+        return 1
 
     filename = args[0]
     key = args[1]
@@ -153,7 +154,7 @@ def replace(args):
     if len(args) != 3:
         print('Missing filename, key arg, and/or value', file=sys.stderr)
         showUsage(None)
-        return
+        return 1
 
     filename = args[0]
     key = args[1]
@@ -167,6 +168,32 @@ def replace(args):
     return 0
 
 
+def getKeyValue(content, key):
+    pieces = key.split(".", 1)
+    if len(pieces) > 1 and pieces[0] in content:
+        return getKeyValue(content[pieces[0]], pieces[1])
+    return content.get(key, None)
+
+
+def get(args):
+    if len(args) != 2:
+        print('Missing filename or key arg', file=sys.stderr)
+        showUsage(None)
+        return 1
+
+    filename = args[0]
+    key = args[1]
+
+    content = loadYaml(filename)
+    output = getKeyValue(content, key)
+    if output is None:
+        print("Not found", file=sys.stderr)
+        return 2
+
+    print(yaml.safe_dump(output))
+    return 0
+
+
 def main():
     args = sys.argv[1:]
 
@@ -178,6 +205,7 @@ def main():
         "help": showUsage,
         "add": add,
         "append": append,
+        "get": get,
         "remove": remove,
         "replace": replace,
     }
@@ -195,11 +223,11 @@ def main():
                 break
             except Exception:
                 if lockAttempts == 1:
-                    print("Waiting for lock file to be released from another process...")
+                    print("Waiting for lock file to be released from another process...", file=sys.stderr)
                 time.sleep(2)
 
         if lockAttempts == maxAttempts:
-            print("Lock file (" + lockFile + ") could not be created; proceeding without lock.")
+            print("Lock file (" + lockFile + ") could not be created; proceeding without lock.", file=sys.stderr)
 
         cmd = commands.get(args[0], showUsage)
         code = cmd(args[1:])

salt/manager/tools/sbin/so-yaml_test.py

@@ -15,40 +15,40 @@ class TestRemove(unittest.TestCase):
 
     def test_main_missing_input(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stderr', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 sys.argv = ["cmd"]
                 soyaml.main()
                 sysmock.assert_called_once_with(1)
-                self.assertIn(mock_stdout.getvalue(), "Usage:")
+                self.assertIn("Usage:", mock_stderr.getvalue())
 
     def test_main_help_locked(self):
         filename = "/tmp/so-yaml.lock"
         file = open(filename, "w")
         file.write = "fake lock file"
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stderr', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 with patch('time.sleep', new=MagicMock()) as mock_sleep:
                     sys.argv = ["cmd", "help"]
                     soyaml.main()
                     sysmock.assert_called()
                     mock_sleep.assert_called_with(2)
-                    self.assertIn(mock_stdout.getvalue(), "Usage:")
+                    self.assertIn("Usage:", mock_stderr.getvalue())
 
     def test_main_help(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stderr', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 sys.argv = ["cmd", "help"]
                 soyaml.main()
                 sysmock.assert_called()
-                self.assertIn(mock_stdout.getvalue(), "Usage:")
+                self.assertIn("Usage:", mock_stderr.getvalue())
 
     def test_remove_missing_arg(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stderr', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 sys.argv = ["cmd", "help"]
                 soyaml.remove(["file"])
                 sysmock.assert_called()
-                self.assertIn(mock_stdout.getvalue(), "Missing filename or key arg\n")
+                self.assertIn("Missing filename or key arg\n", mock_stderr.getvalue())
 
     def test_remove(self):
         filename = "/tmp/so-yaml_test-remove.yaml"
@@ -97,7 +97,7 @@ class TestRemove(unittest.TestCase):
 
     def test_remove_missing_args(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stderr', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 filename = "/tmp/so-yaml_test-remove.yaml"
                 file = open(filename, "w")
                 file.write("{key1: { child1: 123, child2: abc }, key2: false}")
@@ -112,15 +112,15 @@ class TestRemove(unittest.TestCase):
                 expected = "{key1: { child1: 123, child2: abc }, key2: false}"
                 self.assertEqual(actual, expected)
                 sysmock.assert_called_once_with(1)
-                self.assertIn(mock_stdout.getvalue(), "Missing filename or key arg\n")
+                self.assertIn("Missing filename or key arg\n", mock_stderr.getvalue())
 
     def test_append_missing_arg(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stderr', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 sys.argv = ["cmd", "help"]
                 soyaml.append(["file", "key"])
                 sysmock.assert_called()
-                self.assertIn(mock_stdout.getvalue(), "Missing filename, key arg, or list item to append\n")
+                self.assertIn("Missing filename, key arg, or list item to append\n", mock_stderr.getvalue())
 
     def test_append(self):
         filename = "/tmp/so-yaml_test-remove.yaml"
@@ -173,11 +173,11 @@ class TestRemove(unittest.TestCase):
         file.close()
 
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 sys.argv = ["cmd", "append", filename, "key4", "h"]
                 soyaml.main()
                 sysmock.assert_called()
-                self.assertEqual(mock_stdout.getvalue(), "The key provided does not exist. No action was taken on the file.\n")
+                self.assertEqual("The key provided does not exist. No action was taken on the file.\n", mock_stderr.getvalue())
 
     def test_append_key_noexist_deep(self):
         filename = "/tmp/so-yaml_test-append.yaml"
@@ -186,11 +186,11 @@ class TestRemove(unittest.TestCase):
         file.close()
 
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 sys.argv = ["cmd", "append", filename, "key1.child2.deep3", "h"]
                 soyaml.main()
                 sysmock.assert_called()
-                self.assertEqual(mock_stdout.getvalue(), "The key provided does not exist. No action was taken on the file.\n")
+                self.assertEqual("The key provided does not exist. No action was taken on the file.\n", mock_stderr.getvalue())
 
     def test_append_key_nonlist(self):
         filename = "/tmp/so-yaml_test-append.yaml"
@@ -199,11 +199,11 @@ class TestRemove(unittest.TestCase):
         file.close()
 
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 sys.argv = ["cmd", "append", filename, "key1", "h"]
                 soyaml.main()
                 sysmock.assert_called()
-                self.assertEqual(mock_stdout.getvalue(), "The existing value for the given key is not a list. No action was taken on the file.\n")
+                self.assertEqual("The existing value for the given key is not a list. No action was taken on the file.\n", mock_stderr.getvalue())
 
     def test_append_key_nonlist_deep(self):
         filename = "/tmp/so-yaml_test-append.yaml"
@@ -212,11 +212,11 @@ class TestRemove(unittest.TestCase):
         file.close()
 
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 sys.argv = ["cmd", "append", filename, "key1.child2.deep1", "h"]
                 soyaml.main()
                 sysmock.assert_called()
-                self.assertEqual(mock_stdout.getvalue(), "The existing value for the given key is not a list. No action was taken on the file.\n")
+                self.assertEqual("The existing value for the given key is not a list. No action was taken on the file.\n", mock_stderr.getvalue())
 
     def test_add_key(self):
         content = {}
@@ -244,11 +244,11 @@ class TestRemove(unittest.TestCase):
 
     def test_add_missing_arg(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stderr', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 sys.argv = ["cmd", "help"]
                 soyaml.add(["file", "key"])
                 sysmock.assert_called()
-                self.assertIn(mock_stdout.getvalue(), "Missing filename, key arg, and/or value\n")
+                self.assertIn("Missing filename, key arg, and/or value\n", mock_stderr.getvalue())
 
     def test_add(self):
         filename = "/tmp/so-yaml_test-add.yaml"
@@ -296,11 +296,11 @@ class TestRemove(unittest.TestCase):
 
     def test_replace_missing_arg(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stderr', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 sys.argv = ["cmd", "help"]
                 soyaml.replace(["file", "key"])
                 sysmock.assert_called()
-                self.assertIn(mock_stdout.getvalue(), "Missing filename, key arg, and/or value\n")
+                self.assertIn("Missing filename, key arg, and/or value\n", mock_stderr.getvalue())
 
     def test_replace(self):
         filename = "/tmp/so-yaml_test-add.yaml"
@@ -360,3 +360,77 @@ class TestRemove(unittest.TestCase):
         self.assertEqual(soyaml.convertType("false"), False)
         self.assertEqual(soyaml.convertType("FALSE"), False)
         self.assertEqual(soyaml.convertType(""), "")
+
+    def test_get_int(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            filename = "/tmp/so-yaml_test-get.yaml"
+            file = open(filename, "w")
+            file.write("{key1: { child1: 123, child2: { deep1: 45 } }, key2: false, key3: [e,f,g]}")
+            file.close()
+
+            result = soyaml.get([filename, "key1.child2.deep1"])
+            self.assertEqual(result, 0)
+            self.assertIn("45\n...", mock_stdout.getvalue())
+
+    def test_get_str(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            filename = "/tmp/so-yaml_test-get.yaml"
+            file = open(filename, "w")
+            file.write("{key1: { child1: 123, child2: { deep1: \"hello\" } }, key2: false, key3: [e,f,g]}")
+            file.close()
+
+            result = soyaml.get([filename, "key1.child2.deep1"])
+            self.assertEqual(result, 0)
+            self.assertIn("hello\n...", mock_stdout.getvalue())
+
+    def test_get_list(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            filename = "/tmp/so-yaml_test-get.yaml"
+            file = open(filename, "w")
+            file.write("{key1: { child1: 123, child2: { deep1: \"hello\" } }, key2: false, key3: [e,f,g]}")
+            file.close()
+
+            result = soyaml.get([filename, "key3"])
+            self.assertEqual(result, 0)
+            self.assertIn("- e\n- f\n- g\n", mock_stdout.getvalue())
+
+    def test_get_dict(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            filename = "/tmp/so-yaml_test-get.yaml"
+            file = open(filename, "w")
+            file.write("{key1: { child1: 123, child2: { deep1: \"hello\" } }, key2: false, key3: [e,f,g]}")
+            file.close()
+
+            result = soyaml.get([filename, "key1"])
+            self.assertEqual(result, 0)
+            self.assertIn("child1: 123\nchild2:\n  deep1: hello\n", mock_stdout.getvalue())
+
+    def test_get_missing(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            filename = "/tmp/so-yaml_test-get.yaml"
+            file = open(filename, "w")
+            file.write("{key1: { child1: 123, child2: { deep1: 45 } }, key2: false, key3: [e,f,g]}")
+            file.close()
+
+            result = soyaml.get([filename, "key1.child2.deep3"])
+            self.assertEqual(result, 2)
+            self.assertEqual("", mock_stdout.getvalue())
+
+    def test_get_missing_parent(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            filename = "/tmp/so-yaml_test-get.yaml"
+            file = open(filename, "w")
+            file.write("{key1: { child1: 123, child2: { deep1: 45 } }, key2: false, key3: [e,f,g]}")
+            file.close()
+
+            result = soyaml.get([filename, "key1.child3.deep3"])
+            self.assertEqual(result, 2)
+            self.assertEqual("", mock_stdout.getvalue())
+
+    def test_get_usage(self):
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
+                result = soyaml.get([])
+                self.assertEqual(result, 1)
+                self.assertIn("Missing filename or key arg", mock_stderr.getvalue())
+                sysmock.assert_called_once_with(1)

salt/manager/tools/sbin/soup

@@ -19,6 +19,9 @@ SOUP_LOG=/root/soup.log
 WHATWOULDYOUSAYYAHDOHERE=soup
 whiptail_title='Security Onion UPdater'
 NOTIFYCUSTOMELASTICCONFIG=false
+# used to display messages to the user at the end of soup
+declare -a FINAL_MESSAGE_QUEUE=()
+
 
 check_err() {
   local exit_code=$1
@@ -344,6 +347,29 @@ masterunlock() {
   mv -v $BACKUPTOPFILE $TOPFILE
 }
 
+phases_pillar_2_4_80() {
+  echo "Checking if pillar value: elasticsearch.index_settings.global_overrides.index_template.phases exists"
+  set +e
+  PHASES=$(so-yaml.py get /opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls elasticsearch.index_settings.global_overrides.index_template.phases)
+  case $? in
+    0)
+    so-yaml.py remove /opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls elasticsearch.index_settings.global_overrides.index_template.phases
+    read -r -d '' msg <<- EOF
+Found elasticsearch.index_settings.global_overrides.index_template.phases was set to:
+${PHASES}
+
+Removed unused pillar value: elasticsearch.index_settings.global_overrides.index_template.phases
+To set policies, navigate to the SOC Grid Configuration UI at elasticsearch.index_settings.global_overrides.policy.phases
+A backup of all pillar files was saved to /nsm/backup/
+EOF
+    FINAL_MESSAGE_QUEUE+=("$msg")
+    ;;
+    2) echo "Pillar elasticsearch.index_settings.global_overrides.index_template.phases does not exist. No action taken." ;;
+    *) echo "so-yaml.py returned something other than 0 or 2 exit code" ;; # we shouldn't see this
+  esac
+  set -e
+}
+
 preupgrade_changes() {
     # This function is to add any new pillar items if needed.
     echo "Checking to see if changes are needed."
@@ -358,6 +384,7 @@ preupgrade_changes() {
     [[ "$INSTALLEDVERSION" == 2.4.40 ]] && up_to_2.4.50
     [[ "$INSTALLEDVERSION" == 2.4.50 ]] && up_to_2.4.60
     [[ "$INSTALLEDVERSION" == 2.4.60 ]] && up_to_2.4.70
+    [[ "$INSTALLEDVERSION" == 2.4.70 ]] && up_to_2.4.80
     true
 }
 
@@ -375,6 +402,7 @@ postupgrade_changes() {
     [[ "$POSTVERSION"  == 2.4.40 ]] && post_to_2.4.50
     [[ "$POSTVERSION"  == 2.4.50 ]] && post_to_2.4.60
     [[ "$POSTVERSION"  == 2.4.60 ]] && post_to_2.4.70
+    [[ "$POSTVERSION"  == 2.4.70 ]] && post_to_2.4.80
     true
 }
 
@@ -448,6 +476,12 @@ post_to_2.4.70() {
   POSTVERSION=2.4.70
 }
 
+post_to_2.4.80() {
+  echo -e "\nChecking if update to Elastic Fleet output policy is required\n"
+  so-kafka-fleet-output-policy
+  POSTVERSION=2.4.80
+}
+
 repo_sync() {
   echo "Sync the local repo."
   su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync."
@@ -592,9 +626,30 @@ up_to_2.4.70() {
   suricata_idstools_migration
   toggle_telemetry
   add_detection_test_pillars
+
   INSTALLEDVERSION=2.4.70
 }
 
+up_to_2.4.80() {
+  phases_pillar_2_4_80
+  # Kafka configuration changes
+
+  # Global pipeline changes to REDIS or KAFKA
+  echo "Removing global.pipeline pillar configuration"
+  sed -i '/pipeline:/d' /opt/so/saltstack/local/pillar/global/soc_global.sls
+  # Kafka pillars
+  mkdir -p /opt/so/saltstack/local/pillar/kafka
+  touch /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls
+  touch /opt/so/saltstack/local/pillar/kafka/adv_kafka.sls
+  echo 'kafka: ' > /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls
+  kafka_cluster_id=$(get_random_value 22)
+  echo '  cluster_id: '$kafka_cluster_id >> /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls
+  kafkapass=$(get_random_value)
+  echo '  password: '$kafkapass >> /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls
+
+  INSTALLEDVERSION=2.4.80
+}
+
 add_detection_test_pillars() {
   if [[ -n "$SOUP_INTERNAL_TESTING" ]]; then
     echo "Adding detection pillar values for automated testing"
@@ -1255,6 +1310,15 @@ EOF
 
   fi
 
+# check if the FINAL_MESSAGE_QUEUE is not empty
+if (( ${#FINAL_MESSAGE_QUEUE[@]} != 0 )); then
+  echo "The following additional information applies specifically to your grid:"
+  for m in "${FINAL_MESSAGE_QUEUE[@]}"; do
+    echo "$m"
+    echo
+  done
+fi
+
   echo "### soup has been served at $(date) ###"
 }
 

salt/redis/init.sls

@@ -4,9 +4,10 @@
 # Elastic License 2.0.
 
 {% from 'redis/map.jinja' import REDISMERGED %}
+{% from 'vars/globals.map.jinja' import GLOBALS %}
 
 include:
-{% if REDISMERGED.enabled %}
+{% if GLOBALS.pipeline == "REDIS" and REDISMERGED.enabled %}
   - redis.enabled
 {% else %}
   - redis.disabled

salt/salt/etc/minion.d/mine_functions.conf.jinja

@@ -2,7 +2,7 @@ mine_interval: 25
 mine_functions:
   network.ip_addrs:
     - interface: {{ pillar.host.mainint }}
-{% if grains.role in ['so-eval','so-import','so-manager','so-managersearch','so-standalone'] -%}
+{%- if grains.role in ['so-eval','so-import','so-manager','so-managersearch','so-standalone'] %}
   x509.get_pem_entries:
     - glob_path: '/etc/pki/ca.crt'
 {% endif -%}

salt/salt/files/engines.conf

@@ -30,4 +30,65 @@ engines:
                   '*':
                   - cmd.run:
                       cmd: /usr/sbin/so-rule-update
+        - files:
+            - /opt/so/saltstack/local/pillar/global/soc_global.sls
+            - /opt/so/saltstack/local/pillar/global/adv_global.sls
+          pillar: global.pipeline
+          default: REDIS
+          actions:
+            from:
+              '*':
+                to:
+                  'KAFKA':
+                  - cmd.run:
+                      cmd: /usr/sbin/so-yaml.py replace /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.enabled True
+                  - cmd.run:
+                      cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver' saltutil.kill_all_jobs
+                  - cmd.run:
+                      cmd: salt-call state.apply kafka.nodes
+                  - cmd.run:
+                      cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver' state.highstate
+              'KAFKA':
+                to:
+                  'REDIS':
+                  - cmd.run:
+                      cmd: /usr/sbin/so-yaml.py replace /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.enabled False
+                  - cmd.run:
+                      cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver' saltutil.kill_all_jobs
+                  - cmd.run:
+                      cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver' state.highstate
+        - files:
+            - /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls
+            - /opt/so/saltstack/local/pillar/kafka/adv_kafka.sls
+          pillar: kafka.controllers
+          default: ''
+          actions:
+            from:
+              '*':
+                to:
+                  '*':
+                  - cmd.run:
+                      cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver' saltutil.kill_all_jobs
+                  - cmd.run:
+                      cmd: salt-call state.apply kafka.nodes
+                  - cmd.run:
+                      cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver' state.apply kafka
+                  - cmd.run:
+                      cmd: salt-call state.apply elasticfleet
+        - files:
+            - /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls
+            - /opt/so/saltstack/local/pillar/kafka/adv_kafka.sls
+          pillar: kafka.reset
+          default: ''
+          actions:
+            from:
+              '*':
+                to:
+                  'YES_RESET_KAFKA':
+                  - cmd.run:
+                      cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver' saltutil.kill_all_jobs
+                  - cmd.run:
+                      cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver' state.apply kafka.disabled,kafka.reset
+                  - cmd.run:
+                      cmd: /usr/sbin/so-yaml.py remove /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.reset
       interval: 10

salt/salt/minion.sls

@@ -13,6 +13,9 @@ include:
   - systemd.reload
   - repo.client
   - salt.mine_functions
+{% if GLOBALS.role in GLOBALS.manager_roles %}
+  - ca
+{% endif %}
 
 {% if INSTALLEDSALTVERSION|string != SALTVERSION|string %}
 
@@ -98,5 +101,8 @@ salt_minion_service:
       - file: mine_functions
 {% if INSTALLEDSALTVERSION|string == SALTVERSION|string %}
       - file: set_log_levels
+{% endif %}
+{% if GLOBALS.role in GLOBALS.manager_roles %}
+      - file: /etc/salt/minion.d/signing_policies.conf
 {% endif %}
     - order: last

salt/soc/defaults.yaml

@@ -72,6 +72,18 @@ soc:
         target: ''
         links:
           - '/#/hunt?q=(process.entity_id:"{:process.entity_id}") | groupby event.dataset | groupby -sankey event.dataset event.action | groupby event.action | groupby process.name | groupby process.command_line | groupby host.name user.name | groupby source.ip source.port destination.ip destination.port | groupby dns.question.name | groupby dns.answers.data | groupby file.path | groupby registry.path | groupby dll.path'
+      - name: actionProcessChildInfo
+        description: actionProcessChildInfoHelp
+        icon: fa-users-line
+        target: ''
+        links:
+          - '/#/hunt?q=(process.entity_id:"{:process.entity_id}" OR process.parent.entity_id:"{:process.entity_id}") | groupby event.dataset | groupby -sankey event.dataset event.action | groupby event.action | groupby process.name | groupby process.command_line | groupby host.name user.name | groupby source.ip source.port destination.ip destination.port | groupby dns.question.name | groupby dns.answers.data | groupby file.path | groupby registry.path | groupby dll.path'
+      - name: actionProcessAllInfo
+        description: actionProcessAllInfoHelp
+        icon: fa-users-between-lines
+        target: ''
+        links:
+          - '/#/hunt?q=({:process.entity_id}) | groupby event.dataset | groupby -sankey event.dataset event.action | groupby event.action | groupby process.name | groupby process.command_line | groupby host.name user.name | groupby source.ip source.port destination.ip destination.port | groupby dns.question.name | groupby dns.answers.data | groupby file.path | groupby registry.path | groupby dll.path'
       - name: actionProcessAncestors
         description: actionProcessAncestorsHelp
         icon: fa-people-roof
@@ -1314,7 +1326,7 @@ soc:
           reposFolder: /opt/sensoroni/sigma/repos
           rulesFingerprintFile: /opt/sensoroni/fingerprints/sigma.fingerprint
           stateFilePath: /opt/sensoroni/fingerprints/elastalertengine.state
-          integrityCheckFrequencySeconds: 600
+          integrityCheckFrequencySeconds: 1200
           rulesRepos:
             default:
               - repo: https://github.com/Security-Onion-Solutions/securityonion-resources
@@ -1393,18 +1405,19 @@ soc:
                 community: true
           yaraRulesFolder: /opt/sensoroni/yara/rules
           stateFilePath: /opt/sensoroni/fingerprints/strelkaengine.state
-          integrityCheckFrequencySeconds: 600
+          integrityCheckFrequencySeconds: 1200
         suricataengine:
           allowRegex: ''
           autoUpdateEnabled: true
           communityRulesImportFrequencySeconds: 86400
           communityRulesImportErrorSeconds: 300
+          customRulesets:
           failAfterConsecutiveErrorCount: 10
           communityRulesFile: /nsm/rules/suricata/emerging-all.rules
           denyRegex: ''
           rulesFingerprintFile: /opt/sensoroni/fingerprints/emerging-all.fingerprint
           stateFilePath: /opt/sensoroni/fingerprints/suricataengine.state
-          integrityCheckFrequencySeconds: 600
+          integrityCheckFrequencySeconds: 1200
       client:
         enableReverseLookup: false
         docsUrl: /docs/
@@ -2250,6 +2263,78 @@ soc:
                   - suricata
                   - sigma
                   - yara
+            license:
+                customEnabled: true
+                labels:
+                  - None
+                  - Apache-2.0
+                  - AGPL-3.0-only
+                  - BSD-3-Clause
+                  - DRL-1.1
+                  - GPL-2.0-only
+                  - GPL-3.0-only
+                  - MIT
           severityTranslations:
             minor: low
             major: high
+          templateDetections:
+            suricata: |
+              # This is a Suricata rule template. Replace all template values with your own values.
+              # The rule identifier [sid] is pregenerated and known to be unique for this Security Onion installation.
+              # Docs: https://docs.suricata.io/en/latest/rules/intro.html
+              # Delete these comments before attempting to "Create" the rule
+
+              alert http $EXTERNAL_NET any -> $HOME_NET any (msg:"Example Rule Title - 'example' String Detected"; content:"example"; sid:[publicId]; rev:1;)
+            strelka: |
+              /* 
+              This is a YARA rule template. Replace all template values with your own values.
+              The YARA rule name is the unique identifier for the rule.
+              Docs: https://yara.readthedocs.io/en/stable/writingrules.html#writing-yara-rules
+              */ 
+
+              rule Example // This identifier _must_ be unique
+              {
+                  meta: 
+                      description = "Generic YARA Rule"
+                      author = "@SecurityOnion"
+                      date = "YYYY-MM-DD"
+                      reference = "https://local.invalid"
+                  strings:
+                      $my_text_string = "text here"
+                      $my_hex_string = { E2 34 A1 C8 23 FB }
+                  condition:
+                      filesize < 3MB and ($my_text_string or $my_hex_string)
+              }
+            elastalert: |
+              # This is a Sigma rule template, which uses YAML. Replace all template values with your own values.
+              # The id (UUIDv4) is pregenerated and can safely be used.
+              # Click "Convert" to convert the Sigma rule to use Security Onion field mappings within an EQL query
+              #
+              # Rule Creation Guide: https://github.com/SigmaHQ/sigma/wiki/Rule-Creation-Guide
+              # Logsources: https://sigmahq.io/docs/basics/log-sources.html
+
+              title: 'A Short Capitalized Title With Less Than 50 Characters'
+              id: [publicId]
+              status: 'experimental'
+              description: |
+                  This should be a detailed description of what this Detection focuses on: what we are trying to find and why we are trying to find it. 
+                  For example, from rule 97a80ec7-0e2f-4d05-9ef4-65760e634f6b: "Detects a whoami.exe executed with the /priv command line flag instructing the tool to show all current user privileges. This is often used after a privilege escalation attempt."
+              references:
+                - 'https://local.invalid'
+              author: '@SecurityOnion'
+              date: 'YYYY/MM/DD'
+              tags:
+                - detection.threat_hunting
+                - attack.technique_id
+              logsource: 
+                category: process_creation
+                product: windows
+              detection:
+                  selection_img:
+                      - Image|endswith: '\whoami.exe'
+                      - OriginalFileName: 'whoami.exe'
+                  selection_cli:
+                      CommandLine|contains|windash:
+                          - ' -priv'
+                  condition: all of selection_*
+              level: 'high'  # info | low | medium | high | critical

salt/soc/enabled.sls

@@ -27,6 +27,7 @@ so-soc:
       - /opt/so/conf/strelka:/opt/sensoroni/yara:rw
       - /opt/so/conf/sigma:/opt/sensoroni/sigma:rw
       - /opt/so/rules/elastalert/rules:/opt/sensoroni/elastalert:rw
+      - /opt/so/rules/nids/suri:/opt/sensoroni/nids:ro
       - /opt/so/conf/soc/fingerprints:/opt/sensoroni/fingerprints:rw
       - /nsm/soc/jobs:/opt/sensoroni/jobs:rw
       - /nsm/soc/uploads:/nsm/soc/uploads:rw

salt/soc/files/soc/motd.md

@@ -1,8 +1,20 @@
 ## Getting Started
 
-New to Security Onion 2? Click the menu in the upper-right corner and you'll find links for [Help](/docs/) and a [Cheat Sheet](/docs/cheatsheet.pdf) that will help you best utilize Security Onion to hunt for evil! In addition, check out our free Security Onion 2 Essentials online course, available on our [Training](https://securityonionsolutions.com/training) website.
+New to Security Onion? Click the menu in the upper-right corner and you'll find links for [Help](/docs/) and a [Cheat Sheet](/docs/cheatsheet.pdf) that will help you best utilize Security Onion to hunt for evil! In addition, check out our free Security Onion Essentials online course, available on our [Training](https://securityonionsolutions.com/training) website.
 
-If you're ready to dive in, take a look at the [Alerts](/#/alerts) interface to see what Security Onion has detected so far. Then go to the [Dashboards](/#/dashboards) interface for a general overview of all logs collected or go to the [Hunt](/#/hunt) interface for more focused threat hunting. Once you've found something of interest, escalate it to [Cases](/#/cases) to then collect evidence and analyze observables as you work towards closing the case.
+If you're ready to dive in, take a look at the [Alerts](/#/alerts) interface to see what Security Onion has detected so far. If you find any false positives, then you can tune those in [Detections](/#/detections).
+
+Next, go to the [Dashboards](/#/dashboards) interface for a general overview of all logs collected. Here are a few overview dashboards to get you started:
+
+[Overview Dashboard](/#/dashboards) | [Elastic Agent Overview](/#/dashboards?q=event.module%3Aendpoint%20%7C%20groupby%20event.dataset%20%7C%20groupby%20host.name%20%7C%20groupby%20-sankey%20host.name%20user.name%20%7C%20groupby%20user.name%20%7C%20groupby%20-sankey%20user.name%20process.name%20%7C%20groupby%20process.name) | [Network Connection Overview](/#/dashboards?q=tags%3Aconn%20%7C%20groupby%20source.ip%20%7C%20groupby%20destination.ip%20%7C%20groupby%20destination.port%20%7C%20groupby%20-sankey%20destination.port%20network.protocol%20%7C%20groupby%20network.protocol%20%7C%20groupby%20network.transport%20%7C%20groupby%20connection.history%20%7C%20groupby%20connection.state%20%7C%20groupby%20connection.state_description%20%7C%20groupby%20source.geo.country_name%20%7C%20groupby%20destination.geo.country_name%20%7C%20groupby%20client.ip_bytes%20%7C%20groupby%20server.ip_bytes%20%7C%20groupby%20client.oui) | [DNS](/#/dashboards?q=tags%3Adns%20%7C%20groupby%20dns.query.name%20%7C%20groupby%20source.ip%20%7C%20groupby%20-sankey%20source.ip%20destination.ip%20%7C%20groupby%20destination.ip%20%7C%20groupby%20destination.port%20%7C%20groupby%20dns.highest_registered_domain%20%7C%20groupby%20dns.parent_domain%20%7C%20groupby%20dns.query.type_name%20%7C%20groupby%20dns.response.code_name%20%7C%20groupby%20dns.answers.name%20%7C%20groupby%20destination_geo.organization_name) | [Files](/#/dashboards?q=tags%3Afile%20%7C%20groupby%20file.mime_type%20%7C%20groupby%20-sankey%20file.mime_type%20file.source%20%7C%20groupby%20file.source%20%7C%20groupby%20file.bytes.total%20%7C%20groupby%20source.ip%20%7C%20groupby%20destination.ip%20%7C%20groupby%20destination_geo.organization_name) | [HTTP](/#/dashboards?q=tags%3Ahttp%20%7C%20groupby%20http.method%20%7C%20groupby%20-sankey%20http.method%20http.virtual_host%20%7C%20groupby%20http.virtual_host%20%7C%20groupby%20http.uri%20%7C%20groupby%20http.useragent%20%7C%20groupby%20http.status_code%20%7C%20groupby%20http.status_message%20%7C%20groupby%20file.resp_mime_types%20%7C%20groupby%20source.ip%20%7C%20groupby%20destination.ip%20%7C%20groupby%20destination.port%20%7C%20groupby%20destination_geo.organization_name) | [SSL](/#/dashboards?q=tags%3Assl%20%7C%20groupby%20ssl.version%20%7C%20groupby%20-sankey%20ssl.version%20ssl.server_name%20%7C%20groupby%20ssl.server_name%20%7C%20groupby%20source.ip%20%7C%20groupby%20destination.ip%20%7C%20groupby%20destination.port%20%7C%20groupby%20destination_geo.organization_name)
+
+Click the drop-down menu in Dashboards to find many more dashboards. You might also want to explore the [Hunt](/#/hunt) interface for more focused threat hunting. 
+
+Once you've found something of interest, escalate it to [Cases](/#/cases) to then collect evidence and analyze observables as you work towards closing the case.
+
+If you want to check the health of your deployment, check out the [Grid](/#/grid) interface.
+
+For more coverage of your enterprise, you can deploy the Elastic Agent to endpoints by going to the [Downloads](/#/downloads) page.
 
 ## What's New 
 

salt/soc/merged.map.jinja

@@ -6,11 +6,16 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% from 'soc/defaults.map.jinja' import SOCDEFAULTS with context %}
 {% from 'logstash/map.jinja' import LOGSTASH_NODES %}
+{% from 'manager/map.jinja' import MANAGERMERGED %}
 {% set DOCKER_EXTRA_HOSTS = LOGSTASH_NODES %}
 {% do DOCKER_EXTRA_HOSTS.append({GLOBALS.influxdb_host:pillar.node_data[GLOBALS.influxdb_host].ip}) %}
 
 {% set SOCMERGED = salt['pillar.get']('soc', SOCDEFAULTS, merge=true) %}
 
+{% do SOCMERGED.config.server.update({'proxy': MANAGERMERGED.proxy}) %}
+{% do SOCMERGED.config.server.update({'additionalCA': MANAGERMERGED.additionalCA}) %}
+{% do SOCMERGED.config.server.update({'insecureSkipVerify': MANAGERMERGED.insecureSkipVerify}) %}
+
 {# if SOCMERGED.config.server.modules.cases == httpcase details come from the soc pillar #}
 {% if SOCMERGED.config.server.modules.cases != 'soc' %}
 {%   do SOCMERGED.config.server.modules.elastic.update({'casesEnabled': false}) %}

salt/soc/soc_soc.yaml

@@ -247,6 +247,12 @@ soc:
             description: 'How often the Suricata integrity checker runs (in seconds). This verifies the integrity of deployed rules.'
             global: True
             advanced: True
+          customRulesets:
+            description: 'URLs and/or Local File configurations for Suricata custom rulesets. Refer to the linked documentation for important specification and file placement information'
+            global: True
+            advanced: True
+            forcedType: "[]{}"
+            helpLink: suricata.html
       client:
         enableReverseLookup:
           description: Set to true to enable reverse DNS lookups for IP addresses in the SOC UI.
@@ -319,6 +325,17 @@ soc:
         cases: *appSettings
         dashboards: *appSettings
         detections: *appSettings
+        detection:
+          templateDetections:
+            suricata:
+              description: The template used when creating a new Suricata detection. [publicId] will be replaced with an unused Public Id.
+              multiline: True
+            strelka:
+              description: The template used when creating a new Strelka detection.
+              multiline: True
+            elastalert:
+              description: The template used when creating a new ElastAlert detection. [publicId] will be replaced with an unused Public Id.
+              multiline: True
         grid:
           maxUploadSize:
             description: The maximum number of bytes for an uploaded PCAP import file.

salt/ssl/init.sls

@@ -17,6 +17,8 @@
   {% set COMMONNAME = GLOBALS.manager %}
 {% endif %}
 
+{% set kafka_password = salt['pillar.get']('kafka:password') %}
+
 {% if grains.id.split('_')|last in ['manager', 'managersearch', 'eval', 'standalone', 'import'] %}
 include:
   - ca
@@ -661,9 +663,240 @@ elastickeyperms:
     - name: /etc/pki/elasticsearch.key
     - mode: 640
     - group: 930
-
 {%- endif %}
 
+{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone'] %}
+
+elasticfleet_kafka_key:
+  x509.private_key_managed:
+    - name: /etc/pki/elasticfleet-kafka.key
+    - keysize: 4096
+    - backup: True
+    - new: True
+    {% if salt['file.file_exists']('/etc/pki/elasticfleet-kafka.key') -%}
+    - prereq:
+      - x509: elasticfleet_kafka_crt
+    {%- endif %}
+    - retry:
+        attempts: 5
+        interval: 30
+
+elasticfleet_kafka_crt:
+  x509.certificate_managed:
+    - name: /etc/pki/elasticfleet-kafka.crt
+    - ca_server: {{ ca_server }}
+    - signing_policy: kafka
+    - private_key: /etc/pki/elasticfleet-kafka.key
+    - CN: {{ GLOBALS.hostname }}
+    - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
+    - days_remaining: 0
+    - days_valid: 820
+    - backup: True
+    - timeout: 30
+    - retry:
+        attempts: 5
+        interval: 30
+  cmd.run:
+  - name: "/usr/bin/openssl pkcs8 -in /etc/pki/elasticfleet-kafka.key -topk8 -out /etc/pki/elasticfleet-kafka.p8 -nocrypt"
+  - onchanges:
+    - x509: elasticfleet_kafka_key
+
+elasticfleet_kafka_cert_perms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/elasticfleet-kafka.crt
+    - mode: 640
+    - user: 960
+    - group: 939
+
+elasticfleet_kafka_key_perms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/elasticfleet-kafka.key
+    - mode: 640
+    - user: 960
+    - group: 939
+
+elasticfleet_kafka_pkcs8_perms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/elasticfleet-kafka.p8
+    - mode: 640
+    - user: 960
+    - group: 939
+
+kafka_client_key:
+  x509.private_key_managed:
+    - name: /etc/pki/kafka-client.key
+    - keysize: 4096
+    - backup: True
+    - new: True
+    {% if salt['file.file_exists']('/etc/pki/kafka-client.key') -%}
+    - prereq:
+      - x509: /etc/pki/kafka-client.crt
+    {%- endif %}
+    - retry:
+        attempts: 5
+        interval: 30
+
+kafka_client_crt:
+  x509.certificate_managed:
+    - name: /etc/pki/kafka-client.crt
+    - ca_server: {{ ca_server }}
+    - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
+    - signing_policy: kafka
+    - private_key: /etc/pki/kafka-client.key
+    - CN: {{ GLOBALS.hostname }}
+    - days_remaining: 0
+    - days_valid: 820
+    - backup: True
+    - timeout: 30
+    - retry:
+        attempts: 5
+        interval: 30
+
+kafka_client_key_perms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/kafka-client.key
+    - mode: 640
+    - user: 960
+    - group: 939
+
+kafka_client_crt_perms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/kafka-client.crt
+    - mode: 640
+    - user: 960
+    - group: 939
+
+{% endif %}
+
+{% if grains['role'] in ['so-manager', 'so-managersearch','so-receiver', 'so-standalone'] %}
+
+kafka_key:
+  x509.private_key_managed:
+    - name: /etc/pki/kafka.key
+    - keysize: 4096
+    - backup: True
+    - new: True
+    {% if salt['file.file_exists']('/etc/pki/kafka.key') -%}
+    - prereq:
+      - x509: /etc/pki/kafka.crt
+    {%- endif %}
+    - retry:
+        attempts: 5
+        interval: 30
+
+kafka_crt:
+  x509.certificate_managed:
+    - name: /etc/pki/kafka.crt
+    - ca_server: {{ ca_server }}
+    - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
+    - signing_policy: kafka
+    - private_key: /etc/pki/kafka.key
+    - CN: {{ GLOBALS.hostname }}
+    - days_remaining: 0
+    - days_valid: 820
+    - backup: True
+    - timeout: 30
+    - retry:
+        attempts: 5
+        interval: 30
+  cmd.run:
+    - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/kafka.key -in /etc/pki/kafka.crt -export -out /etc/pki/kafka.p12 -nodes -passout pass:{{ kafka_password }}"
+    - onchanges:
+      - x509: /etc/pki/kafka.key
+kafka_key_perms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/kafka.key
+    - mode: 640
+    - user: 960
+    - group: 939
+
+kafka_crt_perms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/kafka.crt
+    - mode: 640
+    - user: 960
+    - group: 939
+
+kafka_pkcs12_perms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/kafka.p12
+    - mode: 640
+    - user: 960
+    - group: 939
+
+{% endif %}
+
+# Standalone needs kafka-logstash for automated testing. Searchnode/manager search need it for logstash to consume from Kafka.
+# Manager will have cert, but be unused until a pipeline is created and logstash enabled.
+{% if grains['role'] in ['so-standalone', 'so-managersearch', 'so-searchnode', 'so-manager'] %}
+kafka_logstash_key:
+  x509.private_key_managed:
+    - name: /etc/pki/kafka-logstash.key
+    - keysize: 4096
+    - backup: True
+    - new: True
+    {% if salt['file.file_exists']('/etc/pki/kafka-logstash.key') -%}
+    - prereq:
+      - x509: /etc/pki/kafka-logstash.crt
+    {%- endif %}
+    - retry:
+        attempts: 5
+        interval: 30
+
+kafka_logstash_crt:
+  x509.certificate_managed:
+    - name: /etc/pki/kafka-logstash.crt
+    - ca_server: {{ ca_server }}
+    - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
+    - signing_policy: kafka
+    - private_key: /etc/pki/kafka-logstash.key
+    - CN: {{ GLOBALS.hostname }}
+    - days_remaining: 0
+    - days_valid: 820
+    - backup: True
+    - timeout: 30
+    - retry:
+        attempts: 5
+        interval: 30
+  cmd.run:
+    - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/kafka-logstash.key -in /etc/pki/kafka-logstash.crt -export -out /etc/pki/kafka-logstash.p12 -nodes -passout pass:{{ kafka_password }}"
+    - onchanges:
+      - x509: /etc/pki/kafka-logstash.key
+
+kafka_logstash_key_perms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/kafka-logstash.key
+    - mode: 640
+    - user: 960
+    - group: 939
+
+kafka_logstash_crt_perms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/kafka-logstash.crt
+    - mode: 640
+    - user: 960
+    - group: 939
+
+kafka_logstash_pkcs12_perms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/kafka-logstash.p12
+    - mode: 640
+    - user: 960
+    - group: 931
+
+{% endif %}
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/ssl/remove.sls

@@ -71,3 +71,20 @@ fleet_crt:
 fbcertdir:
   file.absent:
     - name: /opt/so/conf/filebeat/etc/pki
+
+kafka_crt:
+  file.absent:
+    - name: /etc/pki/kafka.crt
+kafka_key:
+  file.absent:
+    - name: /etc/pki/kafka.key
+
+kafka_logstash_crt:
+  file.absent:
+    - name: /etc/pki/kafka-logstash.crt
+kafka_logstash_key:
+  file.absent:
+    - name: /etc/pki/kafka-logstash.key
+kafka_logstash_keystore:
+  file.absent:
+    - name: /etc/pki/kafka-logstash.p12

salt/stig/files/sos-oscap.xml

@@ -951,7 +951,7 @@ the release. Additionally, the original security profile has been modified by Se
         <ns5:select idref="xccdf_org.ssgproject.content_rule_no_empty_passwords_etc_shadow" selected="true" />
         <ns5:select idref="xccdf_org.ssgproject.content_rule_no_files_unowned_by_user" selected="true" />
         <ns5:select idref="xccdf_org.ssgproject.content_rule_no_host_based_files" selected="true" />
-        <ns5:select idref="xccdf_org.ssgproject.content_rule_no_shelllogin_for_systemaccounts" selected="true" />
+        <ns5:select idref="xccdf_org.ssgproject.content_rule_no_shelllogin_for_systemaccounts" selected="false" />
         <ns5:select idref="xccdf_org.ssgproject.content_rule_no_tmux_in_shells" selected="true" />
         <ns5:select idref="xccdf_org.ssgproject.content_rule_no_user_host_based_files" selected="true" />
         <ns5:select idref="xccdf_org.ssgproject.content_rule_package_MFEhiplsm_installed" selected="true" />
@@ -962,7 +962,7 @@ the release. Additionally, the original security profile has been modified by Se
         <ns5:select idref="xccdf_org.ssgproject.content_rule_package_crypto-policies_installed" selected="true" />
         <ns5:select idref="xccdf_org.ssgproject.content_rule_package_fapolicyd_installed" selected="false" />
         <ns5:select idref="xccdf_org.ssgproject.content_rule_package_firewalld_installed" selected="false" />
-        <ns5:select idref="xccdf_org.ssgproject.content_rule_package_gnutls-utils_installed" selected="true" />
+        <ns5:select idref="xccdf_org.ssgproject.content_rule_package_gnutls-utils_installed" selected="false" />
         <ns5:select idref="xccdf_org.ssgproject.content_rule_package_gssproxy_removed" selected="true" />
         <ns5:select idref="xccdf_org.ssgproject.content_rule_package_iprutils_removed" selected="true" />
         <ns5:select idref="xccdf_org.ssgproject.content_rule_package_libreswan_installed" selected="true" />

salt/suricata/soc_suricata.yaml

@@ -150,13 +150,16 @@ suricata:
             helpLink: suricata.html
     vars:
       address-groups:
-        HOME_NET: &suriaddressgroup
+        HOME_NET: 
           description: Assign a list of hosts, or networks, using CIDR notation, to this Suricata variable. The variable can then be re-used within Suricata rules. This allows for a single adjustment to the variable that will then affect all rules referencing the variable.
           regex: ^(((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\/([0-9]|[1-2][0-9]|3[0-2]))?$|^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?))|:))|(([0-9A-Fa-f]{1,4}:){5}((:[0-9A-Fa-f]{1,4}){1,2}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){4}((:[0-9A-Fa-f]{1,4}){1,3}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){3}((:[0-9A-Fa-f]{1,4}){1,4}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){2}((:[0-9A-Fa-f]{1,4}){1,5}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){1}((:[0-9A-Fa-f]{1,4}){1,6}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(:((:[0-9A-Fa-f]{1,4}){1,7}|:)))(\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8]))?$
           regexFailureMessage: You must enter a valid IP address or CIDR.
           helpLink: suricata.html
           duplicates: True
-        EXTERNAL_NET: *suriaddressgroup
+        EXTERNAL_NET: &suriaddressgroup
+          description: Assign a list of hosts, or networks, or other customization, to this Suricata variable. The variable can then be re-used within Suricata rules. This allows for a single adjustment to the variable that will then affect all rules referencing the variable.
+          helpLink: suricata.html
+          duplicates: True
         HTTP_SERVERS: *suriaddressgroup
         SMTP_SERVERS: *suriaddressgroup
         SQL_SERVERS: *suriaddressgroup

salt/telegraf/etc/telegraf.conf

@@ -243,6 +243,40 @@
   password = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass') }}"
 {%- endif %}
 
+{% if grains.role in ['so-manager','so-managersearch','so-standalone','so-receiver'] and GLOBALS.pipeline == "KAFKA" -%}
+[[inputs.jolokia2_agent]]
+  name_prefix= "kafka_"
+  urls = ["http://localhost:8778/jolokia"]
+
+[[inputs.jolokia2_agent.metric]]
+  name         = "topics"
+  mbean        = "kafka.server:name=*,type=BrokerTopicMetrics"
+  field_prefix = "$1."
+
+[[inputs.jolokia2_agent.metric]]
+  name         = "topic"
+  mbean        = "kafka.server:name=*,topic=*,type=BrokerTopicMetrics"
+  field_prefix = "$1."
+  tag_keys     = ["topic"]
+
+[[inputs.jolokia2_agent.metric]]
+  name         = "controller"
+  mbean        = "kafka.controller:name=*,type=*"
+  field_prefix = "$1."
+
+[[inputs.jolokia2_agent.metric]]
+  name       = "partition"
+  mbean      = "kafka.log:name=*,partition=*,topic=*,type=Log"
+  field_name = "$1"
+  tag_keys   = ["topic", "partition"]
+
+[[inputs.jolokia2_agent.metric]]
+  name       = "partition"
+  mbean      = "kafka.cluster:name=UnderReplicated,partition=*,topic=*,type=Partition"
+  field_name = "UnderReplicatedPartitions"
+  tag_keys   = ["topic", "partition"]
+
+{%- endif %}
 # # Read metrics from one or more commands that can output to stdout
 {%- if 'sostatus.sh' in TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] %}
 {%-   do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('sostatus.sh') %}

salt/telegraf/map.jinja

@@ -22,3 +22,10 @@
 {%   endif %}
 
 {% endif %}
+
+{% if GLOBALS.pipeline != 'REDIS' %}
+{# When global pipeline is not REDIS remove redis.sh script. KAFKA metrics are collected via jolokia agent. Config in telegraf.conf #}
+{%   if GLOBALS.role in ['so-standalone', 'so-manager', 'so-managersearch', 'so-receiver', 'so-heavynode'] %}
+{%     do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('redis.sh') %}
+{%   endif %}
+{% endif %}

salt/top.sls

@@ -107,6 +107,7 @@ base:
     - utility
     - elasticfleet
     - stig
+    - kafka
 
   '*_standalone and G@saltversion:{{saltversion}}':
     - match: compound
@@ -141,6 +142,7 @@ base:
     - utility
     - elasticfleet
     - stig
+    - kafka
 
   '*_searchnode and G@saltversion:{{saltversion}}':
     - match: compound
@@ -238,6 +240,8 @@ base:
     - logstash
     - redis
     - elasticfleet.install_agent_grid
+    - kafka
+    - stig
 
   '*_idh and G@saltversion:{{saltversion}}':
     - match: compound

salt/vars/globals.map.jinja

@@ -23,7 +23,7 @@
     'manager_ip': INIT.PILLAR.global.managerip,
     'md_engine': INIT.PILLAR.global.mdengine,
     'pcap_engine': GLOBALMERGED.pcapengine,
-    'pipeline': INIT.PILLAR.global.pipeline,
+    'pipeline': GLOBALMERGED.pipeline,
     'so_version': INIT.PILLAR.global.soversion,
     'so_docker_gateway': DOCKER.gateway,
     'so_docker_range': DOCKER.range,

setup/so-functions

@@ -788,6 +788,7 @@ create_manager_pillars() {
 	patch_pillar
 	nginx_pillar
 	kibana_pillar
+	kafka_pillar
 }
 
 create_repo() {
@@ -1176,6 +1177,18 @@ kibana_pillar() {
 	logCmd "touch $kibana_pillar_file"
 }
 
+kafka_pillar() {
+	KAFKACLUSTERID=$(get_random_value 22)
+	KAFKAPASS=$(get_random_value)
+	logCmd "mkdir -p $local_salt_dir/pillar/kafka"
+	logCmd "touch $adv_kafka_pillar_file"
+	logCmd "touch $kafka_pillar_file"
+	printf '%s\n'\
+		"kafka:"\
+		"  cluster_id: $KAFKACLUSTERID"\
+		"  password: $KAFKAPASS" > $kafka_pillar_file
+}
+
 logrotate_pillar() {
 	logCmd "mkdir -p $local_salt_dir/pillar/logrotate"
 	logCmd "touch $adv_logrotate_pillar_file"
@@ -1332,7 +1345,6 @@ create_global() {
 
 	# Continue adding other details	
 	echo "  imagerepo: '$IMAGEREPO'" >> $global_pillar_file
-	echo "  pipeline: 'redis'" >> $global_pillar_file
 	echo "  repo_host: '$HOSTNAME'" >> $global_pillar_file
 	echo "  influxdb_host: '$HOSTNAME'" >> $global_pillar_file
 	echo "  registry_host: '$HOSTNAME'" >> $global_pillar_file
@@ -1402,7 +1414,7 @@ make_some_dirs() {
 	mkdir -p $local_salt_dir/salt/firewall/portgroups
 	mkdir -p $local_salt_dir/salt/firewall/ports
 
-	for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni soc docker zeek suricata nginx telegraf logstash soc manager kratos idstools idh elastalert stig global;do
+	for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni soc docker zeek suricata nginx telegraf logstash soc manager kratos idstools idh elastalert stig global kafka;do
 	mkdir -p $local_salt_dir/pillar/$THEDIR
 	touch $local_salt_dir/pillar/$THEDIR/adv_$THEDIR.sls
 	touch $local_salt_dir/pillar/$THEDIR/soc_$THEDIR.sls

setup/so-variables

@@ -178,6 +178,12 @@ export redis_pillar_file
 adv_redis_pillar_file="$local_salt_dir/pillar/redis/adv_redis.sls"
 export adv_redis_pillar_file
 
+kafka_pillar_file="$local_salt_dir/pillar/kafka/soc_kafka.sls"
+export kafka_pillar_file
+
+adv_kafka_pillar_file="$local_salt_dir/pillar/kafka/adv_kafka.sls"
+export kafka_pillar_file
+
 idh_pillar_file="$local_salt_dir/pillar/idh/soc_idh.sls"
 export idh_pillar_file