Merge pull request #13596 from Security-Onion-Solutions/hotfix/2.4.100

Hotfix 2.4.100

.github/.gitleaks.toml

@@ -536,7 +536,7 @@ secretGroup = 4
 
 [allowlist]
 description = "global allow lists"
-regexes = ['''219-09-9999''', '''078-05-1120''', '''(9[0-9]{2}|666)-\d{2}-\d{4}''', '''RPM-GPG-KEY.*''', '''.*:.*StrelkaHexDump.*''', '''.*:.*PLACEHOLDER.*''']
+regexes = ['''219-09-9999''', '''078-05-1120''', '''(9[0-9]{2}|666)-\d{2}-\d{4}''', '''RPM-GPG-KEY.*''', '''.*:.*StrelkaHexDump.*''', '''.*:.*PLACEHOLDER.*''', '''ssl_.*password''']
 paths = [
     '''gitleaks.toml''',
     '''(.*?)(jpg|gif|doc|pdf|bin|svg|socket)$''',

.github/workflows/close-threads.yml

@@ -15,6 +15,7 @@ concurrency:
 
 jobs:
   close-threads:
+    if: github.repository_owner == 'security-onion-solutions'
     runs-on: ubuntu-latest
     permissions:
       issues: write

.github/workflows/lock-threads.yml

@@ -15,6 +15,7 @@ concurrency:
 
 jobs:
   lock-threads:
+    if: github.repository_owner == 'security-onion-solutions'
     runs-on: ubuntu-latest
     steps:
       - uses: jertel/lock-threads@main

DOWNLOAD_AND_VERIFY_ISO.md

@@ -1,17 +1,17 @@
-### 2.4.60-20240320 ISO image released on 2024/03/20
+### 2.4.100-20240903 ISO image released on 2024/09/03
 
 
 ### Download and Verify
 
-2.4.60-20240320 ISO image:  
+2.4.100-20240903 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.4.60-20240320.iso
+https://download.securityonion.net/file/securityonion/securityonion-2.4.100-20240903.iso
  
-MD5: 178DD42D06B2F32F3870E0C27219821E  
+MD5: 856BBB4F0764C0A479D8949725FC096B  
-SHA1: 73EDCD50817A7F6003FE405CF1808A30D034F89D  
+SHA1: B3FCFB8F1031EB8AA833A90C6C5BB61328A73842  
-SHA256: DD334B8D7088A7B78160C253B680D645E25984BA5CCAB5CC5C327CA72137FC06  
+SHA256: 0103EB9D78970396BB47CBD18DA1FFE64524F5C1C559487A1B2D293E1882B265  
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.60-20240320.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.100-20240903.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS  
@@ -25,27 +25,29 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.60-20240320.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.100-20240903.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.4.60-20240320.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.4.100-20240903.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.4.60-20240320.iso.sig securityonion-2.4.60-20240320.iso
+gpg --verify securityonion-2.4.100-20240903.iso.sig securityonion-2.4.100-20240903.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Tue 19 Mar 2024 03:17:58 PM EDT using RSA key ID FE507013
+gpg: Signature made Sat 31 Aug 2024 05:05:05 PM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.
 Primary key fingerprint: C804 A93D 36BE 0C73 3EA1  9644 7C10 60B7 FE50 7013
 ```
 
+If it fails to verify, try downloading again. If it still fails to verify, try downloading from another computer or another network.
+
 Once you've verified the ISO image, you're ready to proceed to our Installation guide:  
 https://docs.securityonion.net/en/2.4/installation.html

HOTFIX

@@ -0,0 +1 @@
+20240903

README.md

@@ -8,19 +8,22 @@ Alerts
 ![Alerts](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/50_alerts.png)
 
 Dashboards
-![Dashboards](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/51_dashboards.png)
+![Dashboards](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/53_dashboards.png)
 
 Hunt
-![Hunt](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/52_hunt.png)
+![Hunt](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/56_hunt.png)
+
+Detections
+![Detections](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/57_detections.png)
 
 PCAP
-![PCAP](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/53_pcap.png)
+![PCAP](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/62_pcap.png)
 
 Grid
-![Grid](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/57_grid.png)
+![Grid](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/75_grid.png)
 
 Config
-![Config](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/61_config.png)
+![Config](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/87_config.png)
 
 ### Release Notes
 

SECURITY.md

@@ -5,9 +5,11 @@
 | Version | Supported          |
 | ------- | ------------------ |
 | 2.4.x   | :white_check_mark: |
-| 2.3.x   | :white_check_mark: |
+| 2.3.x   | :x:                |
 | 16.04.x | :x:                |
 
+Security Onion 2.3 has reached End Of Life and is no longer supported.
+
 Security Onion 16.04 has reached End Of Life and is no longer supported.
 
 ## Reporting a Vulnerability

VERSION

@@ -1 +1 @@
-2.4.70
+2.4.100

files/firewall/assigned_hostgroups.local.map.yaml

@@ -19,4 +19,4 @@ role:
   receiver:
   standalone:
   searchnode:
-  sensor:
+  sensor:

pillar/elasticsearch/nodes.sls

@@ -0,0 +1,34 @@
+{% set node_types = {} %}
+{% for minionid, ip in salt.saltutil.runner(
+    'mine.get',
+    tgt='elasticsearch:enabled:true',
+    fun='network.ip_addrs',
+    tgt_type='pillar') | dictsort()
+%}
+
+# only add a node to the pillar if it returned an ip from the mine
+{%   if ip | length > 0%}
+{%     set hostname = minionid.split('_') | first %}
+{%     set node_type = minionid.split('_') | last %}
+{%     if node_type not in node_types.keys() %}
+{%       do node_types.update({node_type: {hostname: ip[0]}}) %}
+{%     else %}
+{%       if hostname not in node_types[node_type] %}
+{%         do node_types[node_type].update({hostname: ip[0]}) %}
+{%       else %}
+{%         do node_types[node_type][hostname].update(ip[0]) %}
+{%       endif %}
+{%     endif %}
+{%   endif %}
+{% endfor %}
+
+
+elasticsearch:
+  nodes:
+{% for node_type, values in node_types.items() %}
+    {{node_type}}:
+{%   for hostname, ip in values.items() %}
+      {{hostname}}:
+        ip: {{ip}}
+{%   endfor %}
+{% endfor %}

pillar/kafka/nodes.sls

@@ -0,0 +1,2 @@
+kafka:
+  nodes:

pillar/logstash/nodes.sls

@@ -1,16 +1,15 @@
 {% set node_types = {} %}
-{% set cached_grains = salt.saltutil.runner('cache.grains', tgt='*') %}
 {% for minionid, ip in salt.saltutil.runner(
     'mine.get',
-    tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-searchnode or G@role:so-heavynode or G@role:so-receiver or G@role:so-fleet ',
+    tgt='logstash:enabled:true',
     fun='network.ip_addrs',
-    tgt_type='compound') | dictsort()
+    tgt_type='pillar') | dictsort()
 %}
 
 # only add a node to the pillar if it returned an ip from the mine
 {%   if ip | length > 0%}
-{%     set hostname = cached_grains[minionid]['host'] %}
+{%     set hostname = minionid.split('_') | first %}
-{%     set node_type = minionid.split('_')[1] %}
+{%     set node_type = minionid.split('_') | last %}
 {%     if node_type not in node_types.keys() %}
 {%       do node_types.update({node_type: {hostname: ip[0]}}) %}
 {%     else %}

pillar/redis/nodes.sls

@@ -0,0 +1,34 @@
+{% set node_types = {} %}
+{% for minionid, ip in salt.saltutil.runner(
+    'mine.get',
+    tgt='redis:enabled:true',
+    fun='network.ip_addrs',
+    tgt_type='pillar') | dictsort()
+%}
+
+# only add a node to the pillar if it returned an ip from the mine
+{%   if ip | length > 0%}
+{%     set hostname = minionid.split('_') | first %}
+{%     set node_type = minionid.split('_') | last %}
+{%     if node_type not in node_types.keys() %}
+{%       do node_types.update({node_type: {hostname: ip[0]}}) %}
+{%     else %}
+{%       if hostname not in node_types[node_type] %}
+{%         do node_types[node_type].update({hostname: ip[0]}) %}
+{%       else %}
+{%         do node_types[node_type][hostname].update(ip[0]) %}
+{%       endif %}
+{%     endif %}
+{%   endif %}
+{% endfor %}
+
+
+redis:
+  nodes:
+{% for node_type, values in node_types.items() %}
+    {{node_type}}:
+{%   for hostname, ip in values.items() %}
+      {{hostname}}:
+        ip: {{ip}}
+{%   endfor %}
+{% endfor %}

pillar/top.sls

@@ -47,10 +47,12 @@ base:
     - kibana.adv_kibana
     - kratos.soc_kratos
     - kratos.adv_kratos
+    - redis.nodes
     - redis.soc_redis
     - redis.adv_redis
     - influxdb.soc_influxdb
     - influxdb.adv_influxdb
+    - elasticsearch.nodes
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
     - elasticfleet.soc_elasticfleet
@@ -61,6 +63,9 @@ base:
     - backup.adv_backup
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
+    - kafka.nodes
+    - kafka.soc_kafka
+    - kafka.adv_kafka
     - stig.soc_stig
 
   '*_sensor':
@@ -144,10 +149,12 @@ base:
     - idstools.adv_idstools
     - kratos.soc_kratos
     - kratos.adv_kratos
+    - redis.nodes
     - redis.soc_redis
     - redis.adv_redis
     - influxdb.soc_influxdb
     - influxdb.adv_influxdb
+    - elasticsearch.nodes
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
     - elasticfleet.soc_elasticfleet
@@ -176,6 +183,9 @@ base:
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
     - stig.soc_stig
+    - kafka.nodes
+    - kafka.soc_kafka
+    - kafka.adv_kafka
 
   '*_heavynode':
     - elasticsearch.auth
@@ -209,17 +219,22 @@ base:
     - logstash.nodes
     - logstash.soc_logstash
     - logstash.adv_logstash
+    - elasticsearch.nodes
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
     {% endif %}
+    - redis.nodes
     - redis.soc_redis
     - redis.adv_redis
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
     - stig.soc_stig
     - soc.license
+    - kafka.nodes
+    - kafka.soc_kafka
+    - kafka.adv_kafka
 
   '*_receiver':
     - logstash.nodes
@@ -232,6 +247,10 @@ base:
     - redis.adv_redis
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
+    - kafka.nodes
+    - kafka.soc_kafka
+    - kafka.adv_kafka
+    - soc.license
 
   '*_import':
     - secrets

pyci.sh

@@ -15,12 +15,16 @@ TARGET_DIR=${1:-.}
 
 PATH=$PATH:/usr/local/bin
 
-if ! which pytest &> /dev/null || ! which flake8 &> /dev/null ; then
+if [ ! -d .venv ]; then
-    echo "Missing dependencies. Consider running the following command:"
+    python -m venv .venv
-    echo "  python -m pip install flake8 pytest pytest-cov"
+fi
+
+source .venv/bin/activate
+
+if ! pip install flake8 pytest pytest-cov pyyaml; then
+    echo "Unable to install dependencies."
     exit 1
 fi
 
-pip install pytest pytest-cov
 flake8 "$TARGET_DIR" "--config=${HOME_DIR}/pytest.ini"
-python3 -m pytest "--cov-config=${HOME_DIR}/pytest.ini" "--cov=$TARGET_DIR" --doctest-modules --cov-report=term --cov-fail-under=100  "$TARGET_DIR" 
+python3 -m pytest "--cov-config=${HOME_DIR}/pytest.ini" "--cov=$TARGET_DIR" --doctest-modules --cov-report=term --cov-fail-under=100  "$TARGET_DIR" 

salt/allowed_states.map.jinja

@@ -65,6 +65,7 @@
           'registry',
           'manager',
           'nginx',
+          'strelka.manager',
           'soc',
           'kratos',
           'influxdb',
@@ -91,6 +92,7 @@
           'nginx',
           'telegraf',
           'influxdb',
+          'strelka.manager',
           'soc',
           'kratos',
           'elasticfleet',
@@ -101,7 +103,8 @@
           'utility',
           'schedule',
           'docker_clean',
-          'stig'
+          'stig',
+          'kafka'
           ],
       'so-managersearch': [
           'salt.master',
@@ -111,6 +114,7 @@
           'nginx',
           'telegraf',
           'influxdb',
+          'strelka.manager',
           'soc',
           'kratos',
           'elastic-fleet-package-registry',
@@ -122,7 +126,8 @@
           'utility',
           'schedule',
           'docker_clean',
-          'stig'
+          'stig',
+          'kafka'
           ],
       'so-searchnode': [
           'ssl',
@@ -131,7 +136,9 @@
           'firewall',
           'schedule',
           'docker_clean',
-          'stig'
+          'stig',
+          'kafka.ca',
+          'kafka.ssl'
           ],
       'so-standalone': [
           'salt.master',
@@ -156,7 +163,8 @@
           'schedule',
           'tcpreplay',
           'docker_clean',
-          'stig'
+          'stig',
+          'kafka'
           ],
       'so-sensor': [
           'ssl',
@@ -187,7 +195,9 @@
           'telegraf',
           'firewall',
           'schedule',
-          'docker_clean'
+          'docker_clean',
+          'kafka',
+          'stig'
           ],
       'so-desktop': [
           'ssl',

salt/ca/files/signing_policies.conf

@@ -1,6 +1,3 @@
-mine_functions:
-  x509.get_pem_entries: [/etc/pki/ca.crt]
-
 x509_signing_policies:
   filebeat:
     - minions: '*'
@@ -70,3 +67,17 @@ x509_signing_policies:
     - authorityKeyIdentifier: keyid,issuer:always
     - days_valid: 820
     - copypath: /etc/pki/issued_certs/
+  kafka:
+    - minions: '*'
+    - signing_private_key: /etc/pki/ca.key
+    - signing_cert: /etc/pki/ca.crt
+    - C: US
+    - ST: Utah
+    - L: Salt Lake City
+    - basicConstraints: "critical CA:false"
+    - keyUsage: "digitalSignature, keyEncipherment"
+    - subjectKeyIdentifier: hash
+    - authorityKeyIdentifier: keyid,issuer:always
+    - extendedKeyUsage: "serverAuth, clientAuth"
+    - days_valid: 820
+    - copypath: /etc/pki/issued_certs/

salt/common/init.sls

@@ -14,6 +14,11 @@ net.core.wmem_default:
   sysctl.present:
     - value: 26214400
 
+# Users are not a fan of console messages
+kernel.printk:
+  sysctl.present:
+    - value: "3 4 1 3"
+
 # Remove variables.txt from /tmp - This is temp
 rmvariablesfile:
   file.absent:

salt/common/soup_scripts.sls

@@ -1,9 +1,16 @@
-{% import_yaml '/opt/so/saltstack/local/pillar/global/soc_global.sls' as SOC_GLOBAL %}
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-{% if SOC_GLOBAL.global.airgap %}
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-{%   set UPDATE_DIR='/tmp/soagupdate/SecurityOnion' %}
+# https://securityonion.net/license; you may not use this file except in compliance with the
-{% else %}
+# Elastic License 2.0.
-{%   set UPDATE_DIR='/tmp/sogh/securityonion' %}
+
-{% endif %}
+{% if '2.4' in salt['cp.get_file_str']('/etc/soversion') %}
+
+{%   import_yaml '/opt/so/saltstack/local/pillar/global/soc_global.sls' as SOC_GLOBAL %}
+{%   if SOC_GLOBAL.global.airgap %}
+{%     set UPDATE_DIR='/tmp/soagupdate/SecurityOnion' %}
+{%   else %}
+{%     set UPDATE_DIR='/tmp/sogh/securityonion' %}
+{%   endif %}
 
 remove_common_soup:
   file.absent:
@@ -13,6 +20,8 @@ remove_common_so-firewall:
   file.absent:
     - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-firewall
 
+# This section is used to put the scripts in place in the Salt file system
+# in case a state run tries to overwrite what we do in the next section.
 copy_so-common_common_tools_sbin:
   file.copy:
     - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-common
@@ -41,6 +50,21 @@ copy_so-firewall_manager_tools_sbin:
     - force: True
     - preserve: True
 
+copy_so-yaml_manager_tools_sbin:
+  file.copy:
+    - name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-yaml.py
+    - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-yaml.py
+    - force: True
+    - preserve: True
+
+copy_so-repo-sync_manager_tools_sbin:
+  file.copy:
+    - name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-repo-sync
+    - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-repo-sync
+    - preserve: True
+
+# This section is used to put the new script in place so that it can be called during soup.
+# It is faster than calling the states that normally manage them to put them in place.
 copy_so-common_sbin:
   file.copy:
     - name: /usr/sbin/so-common
@@ -68,3 +92,26 @@ copy_so-firewall_sbin:
     - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-firewall
     - force: True
     - preserve: True
+
+copy_so-yaml_sbin:
+  file.copy:
+    - name: /usr/sbin/so-yaml.py
+    - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-yaml.py
+    - force: True
+    - preserve: True
+
+copy_so-repo-sync_sbin:
+  file.copy:
+    - name: /usr/sbin/so-repo-sync
+    - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-repo-sync
+    - force: True
+    - preserve: True
+
+{% else %}
+fix_23_soup_sbin:
+  cmd.run:
+    - name: curl -s -f -o /usr/sbin/soup https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.3/main/salt/common/tools/sbin/soup
+fix_23_soup_salt:
+  cmd.run:
+    - name: curl -s -f -o /opt/so/saltstack/defalt/salt/common/tools/sbin/soup https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.3/main/salt/common/tools/sbin/soup
+{% endif %}

salt/common/tools/sbin/so-checkin

@@ -5,8 +5,13 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-
-
 . /usr/sbin/so-common
 
-salt-call state.highstate -l info 
+cat << EOF
+
+so-checkin will run a full salt highstate to apply all salt states. If a highstate is already running, this request will be queued and so it may pause for a few minutes before you see any more output. For more information about so-checkin and salt, please see:
+https://docs.securityonion.net/en/2.4/salt.html
+
+EOF
+
+salt-call state.highstate -l info queue=True

salt/common/tools/sbin/so-common

@@ -8,7 +8,7 @@
 # Elastic agent is not managed by salt. Because of this we must store this base information in a 
 # script that accompanies the soup system. Since so-common is one of those special soup files, 
 # and since this same logic is required during installation, it's included in this file.
-ELASTIC_AGENT_TARBALL_VERSION="8.10.4"
+ELASTIC_AGENT_TARBALL_VERSION="8.14.3"
 ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
 ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
 ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
@@ -31,6 +31,11 @@ if ! echo "$PATH" | grep -q "/usr/sbin"; then
   export PATH="$PATH:/usr/sbin"
 fi
 
+# See if a proxy is set. If so use it.
+if [ -f /etc/profile.d/so-proxy.sh ]; then
+  . /etc/profile.d/so-proxy.sh
+fi
+
 # Define a banner to separate sections
 banner="========================================================================="
 
@@ -179,6 +184,21 @@ copy_new_files() {
   cd /tmp
 }
 
+create_local_directories() {
+	echo "Creating local pillar and salt directories if needed"
+	PILLARSALTDIR=$1
+	local_salt_dir="/opt/so/saltstack/local"
+	for i in "pillar" "salt"; do
+		for d in $(find $PILLARSALTDIR/$i -type d); do
+			suffixdir=${d//$PILLARSALTDIR/}
+			if [ ! -d "$local_salt_dir/$suffixdir" ]; then
+				mkdir -pv $local_salt_dir$suffixdir
+			fi
+		done
+		chown -R socore:socore $local_salt_dir/$i
+	done
+}
+
 disable_fastestmirror() {
 	sed -i 's/enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf
 }
@@ -248,6 +268,14 @@ get_random_value() {
 	head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1
 }
 
+get_agent_count() {
+	if [ -f /opt/so/log/agents/agentstatus.log ]; then
+		AGENTCOUNT=$(cat /opt/so/log/agents/agentstatus.log | grep -wF active | awk '{print $2}')
+	else
+		AGENTCOUNT=0
+	fi
+}
+
 gpg_rpm_import() {
 	if [[ $is_oracle ]]; then
 	    if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then
@@ -329,7 +357,7 @@ lookup_salt_value() {
 		local=""
 	fi
 
-	salt-call --no-color  ${kind}.get ${group}${key} --out=${output} ${local}
+	salt-call -lerror --no-color  ${kind}.get ${group}${key} --out=${output} ${local}
 }
 
 lookup_pillar() {
@@ -570,8 +598,9 @@ sync_options() {
 	set_version
 	set_os
 	salt_minion_count
+	get_agent_count
 	
-	echo "$VERSION/$OS/$(uname -r)/$MINIONCOUNT/$(read_feat)"
+	echo "$VERSION/$OS/$(uname -r)/$MINIONCOUNT:$AGENTCOUNT/$(read_feat)"
 }
 
 systemctl_func() {

salt/common/tools/sbin/so-image-common

@@ -50,6 +50,7 @@ container_list() {
       "so-idh"
       "so-idstools"
       "so-influxdb"
+      "so-kafka"
       "so-kibana"
       "so-kratos"
       "so-logstash"
@@ -64,7 +65,7 @@ container_list() {
       "so-strelka-manager"
       "so-suricata"
       "so-telegraf"
-      "so-zeek" 
+      "so-zeek"
     )
   else
     TRUSTED_CONTAINERS=(

salt/common/tools/sbin/so-log-check

@@ -95,6 +95,8 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|shutdown process"             # server not yet ready (logstash waiting on elastic)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|contain valid certificates"   # server not yet ready (logstash waiting on elastic)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failedaction"                 # server not yet ready (logstash waiting on elastic)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|block in start_workers"       # server not yet ready (logstash waiting on elastic)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|block in buffer_initialize"   # server not yet ready (logstash waiting on elastic)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|no route to host"             # server not yet ready
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|not running"                  # server not yet ready
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|unavailable"                  # server not yet ready
@@ -147,6 +149,7 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|status 200"                   # false positive (request successful, contained error string in content)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|app_layer.error"              # false positive (suricata 7) in stats.log e.g. app_layer.error.imap.parser | Total | 0
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|is not an ip string literal"  # false positive (Open Canary logging out blank IP addresses)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|syncing rule"                 # false positive (rule sync log line includes rule name which can contain 'error') 
 fi
 
 if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
@@ -170,6 +173,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|cannot join on an empty table" # InfluxDB flux query, import nodes
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|exhausting result iterator"   # InfluxDB flux query mismatched table results (temporary data issue)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to finish run"         # InfluxDB rare error, self-recoverable
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unable to gather disk name"   # InfluxDB known error, can't read disks because the container doesn't have them mounted
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|iteration"
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|communication packets"
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|use of closed"
@@ -198,7 +202,14 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|req.LocalMeta.host.ip"        # known issue in GH
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|sendmail"                     # zeek
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|stats.log"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unknown column"               # Elastalert errors from running EQL queries
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|parsing_exception"            # Elastalert EQL parsing issue. Temp.
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|context deadline exceeded"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error running query:"         # Specific issues with detection rules
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|detect-parse"                 # Suricata encountering a malformed rule
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|integrity check failed"       # Detections: Exclude false positive due to automated testing
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|syncErrors"                   # Detections: Not an actual error
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Initialized license manager"  # SOC log: before fields.status was changed to fields.licenseStatus
 fi
 
 RESULT=0
@@ -207,6 +218,9 @@ RESULT=0
 CONTAINER_IDS=$(docker ps -q)
 exclude_container so-kibana   # kibana error logs are too verbose with large varieties of errors most of which are temporary
 exclude_container so-idstools # ignore due to known issues and noisy logging
+exclude_container so-playbook # Playbook is removed as of 2.4.70, disregard output in stopped containers
+exclude_container so-mysql    # MySQL is removed as of 2.4.70, disregard output in stopped containers
+exclude_container so-soctopus # Soctopus is removed as of 2.4.70, disregard output in stopped containers
 
 for container_id in $CONTAINER_IDS; do
     container_name=$(docker ps --format json | jq ". | select(.ID==\"$container_id\")|.Names")
@@ -224,10 +238,15 @@ exclude_log "kibana.log"   # kibana error logs are too verbose with large variet
 exclude_log "spool"        # disregard zeek analyze logs as this is data specific
 exclude_log "import"       # disregard imported test data the contains error strings
 exclude_log "update.log"   # ignore playbook updates due to several known issues
-exclude_log "playbook.log" # ignore due to several playbook known issues
 exclude_log "cron-cluster-delete.log" # ignore since Curator has been removed
 exclude_log "cron-close.log" # ignore since Curator has been removed
-exclude_log "curator.log" # ignore since Curator has been removed
+exclude_log "curator.log"  # ignore since Curator has been removed
+exclude_log "playbook.log" # Playbook is removed as of 2.4.70, logs may still be on disk
+exclude_log "mysqld.log"   # MySQL is removed as of 2.4.70, logs may still be on disk
+exclude_log "soctopus.log" # Soctopus is removed as of 2.4.70, logs may still be on disk
+exclude_log "agentstatus.log" # ignore this log since it tracks agents in error state
+exclude_log "detections_runtime-status_yara.log" # temporarily ignore this log until Detections is more stable
+exclude_log "/nsm/kafka/data/" # ignore Kafka data directory from log check.
 
 for log_file in $(cat /tmp/log_check_files); do
     status "Checking log file $log_file"

salt/common/tools/sbin/so-luks-tpm-regen

@@ -0,0 +1,98 @@
+#!/bin/bash
+#
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0."
+
+set -e
+# This script is intended to be used in the case the ISO install did not properly setup TPM decrypt for LUKS partitions at boot.
+if [ -z $NOROOT ]; then
+	# Check for prerequisites
+	if [ "$(id -u)" -ne 0 ]; then
+		echo "This script must be run using sudo!"
+		exit 1
+	fi
+fi
+ENROLL_TPM=N
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --enroll-tpm)
+      ENROLL_TPM=Y
+      ;;
+    *)
+      echo "Usage: $0 [options]"
+      echo ""
+      echo "where options are:"
+      echo "  --enroll-tpm            for when TPM enrollment was not selected during ISO install."
+      echo ""
+      exit 1
+      ;;
+  esac
+  shift
+done
+
+check_for_tpm() {
+  echo -n "Checking for TPM: "
+  if [ -d /sys/class/tpm/tpm0 ]; then
+    echo -e "tpm0 found."
+    TPM="yes"
+    # Check if TPM is using sha1 or sha256
+    if [ -d /sys/class/tpm/tpm0/pcr-sha1 ]; then
+      echo -e "TPM is using sha1.\n"
+      TPM_PCR="sha1"
+    elif [ -d /sys/class/tpm/tpm0/pcr-sha256 ]; then
+      echo -e "TPM is using sha256.\n"
+      TPM_PCR="sha256"
+    fi
+  else
+    echo -e "No TPM found.\n"
+    exit 1
+  fi
+}
+
+check_for_luks_partitions() {
+  echo "Checking for LUKS partitions"
+  for part in $(lsblk -o NAME,FSTYPE -ln | grep crypto_LUKS | awk '{print $1}'); do
+    echo "Found LUKS partition: $part"
+    LUKS_PARTITIONS+=("$part")
+  done
+  if [ ${#LUKS_PARTITIONS[@]} -eq 0 ]; then
+    echo -e "No LUKS partitions found.\n"
+    exit 1
+  fi
+  echo ""
+}
+
+enroll_tpm_in_luks() {
+  read -s -p "Enter the LUKS passphrase used during ISO install: " LUKS_PASSPHRASE
+  echo ""
+  for part in "${LUKS_PARTITIONS[@]}"; do
+    echo "Enrolling TPM for LUKS device: /dev/$part"
+    if [ "$TPM_PCR" == "sha1" ]; then
+      clevis luks bind -d /dev/$part tpm2 '{"pcr_bank":"sha1","pcr_ids":"7"}' <<< $LUKS_PASSPHRASE
+    elif [ "$TPM_PCR" == "sha256" ]; then
+      clevis luks bind -d /dev/$part tpm2 '{"pcr_bank":"sha256","pcr_ids":"7"}' <<< $LUKS_PASSPHRASE
+    fi
+  done
+ }
+
+regenerate_tpm_enrollment_token() {
+  for part in "${LUKS_PARTITIONS[@]}"; do
+    clevis luks regen -d /dev/$part -s 1 -q
+  done
+}
+
+check_for_tpm
+check_for_luks_partitions
+
+if [[ $ENROLL_TPM == "Y" ]]; then
+  enroll_tpm_in_luks
+else
+  regenerate_tpm_enrollment_token
+fi
+
+echo "Running dracut"
+dracut -fv
+echo -e "\nTPM configuration complete. Reboot the system to verify the TPM is correctly decrypting the LUKS partition(s) at boot.\n"

salt/common/tools/sbin_jinja/so-import-pcap

@@ -89,6 +89,7 @@ function suricata() {
     -v ${LOG_PATH}:/var/log/suricata/:rw \
     -v ${NSM_PATH}/:/nsm/:rw \
     -v "$PCAP:/input.pcap:ro" \
+    -v /dev/null:/nsm/suripcap:rw \
     -v /opt/so/conf/suricata/bpf:/etc/suricata/bpf:ro \
     {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-suricata:{{ VERSION }} \
     --runmode single -k none -r /input.pcap > $LOG_PATH/console.log 2>&1
@@ -247,7 +248,7 @@ fi
 START_OLDEST_SLASH=$(echo $START_OLDEST | sed -e 's/-/%2F/g')
 END_NEWEST_SLASH=$(echo $END_NEWEST | sed -e 's/-/%2F/g')
 if [[ $VALID_PCAPS_COUNT -gt 0 ]] || [[ $SKIPPED_PCAPS_COUNT -gt 0 ]]; then
-  URL="https://{{ URLBASE }}/#/dashboards?q=$HASH_FILTERS%20%7C%20groupby%20-sankey%20event.dataset%20event.category%2a%20%7C%20groupby%20-pie%20event.category%20%7C%20groupby%20-bar%20event.module%20%7C%20groupby%20event.dataset%20%7C%20groupby%20event.module%20%7C%20groupby%20event.category%20%7C%20groupby%20observer.name%20%7C%20groupby%20source.ip%20%7C%20groupby%20destination.ip%20%7C%20groupby%20destination.port&t=${START_OLDEST_SLASH}%2000%3A00%3A00%20AM%20-%20${END_NEWEST_SLASH}%2000%3A00%3A00%20AM&z=UTC"
+  URL="https://{{ URLBASE }}/#/dashboards?q=$HASH_FILTERS%20%7C%20groupby%20event.module*%20%7C%20groupby%20-sankey%20event.module*%20event.dataset%20%7C%20groupby%20event.dataset%20%7C%20groupby%20source.ip%20%7C%20groupby%20destination.ip%20%7C%20groupby%20destination.port%20%7C%20groupby%20network.protocol%20%7C%20groupby%20rule.name%20rule.category%20event.severity_label%20%7C%20groupby%20dns.query.name%20%7C%20groupby%20file.mime_type%20%7C%20groupby%20http.virtual_host%20http.uri%20%7C%20groupby%20notice.note%20notice.message%20notice.sub_message%20%7C%20groupby%20ssl.server_name%20%7C%20groupby%20source_geo.organization_name%20source.geo.country_name%20%7C%20groupby%20destination_geo.organization_name%20destination.geo.country_name&t=${START_OLDEST_SLASH}%2000%3A00%3A00%20AM%20-%20${END_NEWEST_SLASH}%2000%3A00%3A00%20AM&z=UTC"
 
   status "Import complete!"
   status

salt/common/tools/sbin_jinja/so-raid-status

@@ -9,6 +9,9 @@
 
 . /usr/sbin/so-common
 
+software_raid=("SOSMN" "SOSMN-DE02" "SOSSNNV" "SOSSNNV-DE02" "SOS10k-DE02" "SOS10KNV" "SOS10KNV-DE02" "SOS10KNV-DE02" "SOS2000-DE02" "SOS-GOFAST-LT-DE02" "SOS-GOFAST-MD-DE02" "SOS-GOFAST-HV-DE02")
+hardware_raid=("SOS1000" "SOS1000F" "SOSSN7200" "SOS5000" "SOS4000")
+
 {%- if salt['grains.get']('sosmodel', '') %}
 {%- set model = salt['grains.get']('sosmodel') %}
 model={{ model }}
@@ -16,33 +19,42 @@ model={{ model }}
 if [[ $model =~ ^(SO2AMI01|SO2AZI01|SO2GCI01)$ ]]; then
     exit 0
 fi
+
+for i in "${software_raid[@]}"; do
+  if [[ "$model" == $i ]]; then
+    is_softwareraid=true
+    is_hwraid=false
+    break
+  fi
+done
+
+for i in "${hardware_raid[@]}"; do
+  if [[ "$model" == $i ]]; then
+    is_softwareraid=false
+    is_hwraid=true
+    break
+  fi
+done
+
 {%- else %}
 echo "This is not an appliance"
 exit 0
 {%- endif %}
-if [[ $model =~ ^(SOS10K|SOS500|SOS1000|SOS1000F|SOS4000|SOSSN7200|SOSSNNV|SOSMN)$ ]]; then
-	is_bossraid=true
-fi
-if [[ $model =~ ^(SOSSNNV|SOSMN)$ ]]; then
-	is_swraid=true
-fi
-if [[ $model =~ ^(SOS10K|SOS500|SOS1000|SOS1000F|SOS4000|SOSSN7200)$ ]]; then
-	is_hwraid=true
-fi
 
 check_nsm_raid() {
   PERCCLI=$(/opt/raidtools/perccli/perccli64 /c0/v0 show|grep RAID|grep Optl)
   MEGACTL=$(/opt/raidtools/megasasctl |grep optimal)
-
+  if [[ "$model" == "SOS500" || "$model" == "SOS500-DE02" ]]; then
-  if [[ $APPLIANCE == '1' ]]; then
+    #This doesn't have raid
+    HWRAID=0
+  else
     if [[ -n $PERCCLI ]]; then
       HWRAID=0
     elif [[ -n $MEGACTL ]]; then
       HWRAID=0
     else
       HWRAID=1
-    fi
+    fi   
-
   fi
 
 }
@@ -50,17 +62,27 @@ check_nsm_raid() {
 check_boss_raid() {
   MVCLI=$(/usr/local/bin/mvcli info -o vd |grep status |grep functional)
   MVTEST=$(/usr/local/bin/mvcli info -o vd | grep "No adapter")
+  BOSSNVMECLI=$(/usr/local/bin/mnv_cli info -o vd -i 0 | grep Functional)
 
-  # Check to see if this is a SM based system
+  # Is this NVMe Boss Raid?
-  if [[ -z $MVTEST ]]; then
+  if [[ "$model" =~ "-DE02" ]]; then
-    if [[ -n $MVCLI ]]; then
+    if [[ -n $BOSSNVMECLI ]]; then
       BOSSRAID=0
     else
       BOSSRAID=1
     fi
   else
-    # This doesn't have boss raid so lets make it 0
+    # Check to see if this is a SM based system
-    BOSSRAID=0
+    if [[ -z $MVTEST ]]; then
+      if [[ -n $MVCLI ]]; then
+        BOSSRAID=0
+      else
+        BOSSRAID=1
+      fi
+    else
+      # This doesn't have boss raid so lets make it 0
+      BOSSRAID=0
+    fi
   fi
 }
 
@@ -79,14 +101,13 @@ SWRAID=0
 BOSSRAID=0
 HWRAID=0
 
-if [[ $is_hwraid ]]; then
+if [[ "$is_hwraid" == "true" ]]; then
 	check_nsm_raid
+  check_boss_raid
 fi
-if [[ $is_bossraid ]]; then
+if [[ "$is_softwareraid" == "true" ]]; then
-	check_boss_raid
-fi
-if [[ $is_swraid ]]; then
 	check_software_raid
+  check_boss_raid
 fi
 
 sum=$(($SWRAID + $BOSSRAID + $HWRAID))

salt/common/tools/sbin_jinja/so-tcpreplay

@@ -10,7 +10,7 @@
 . /usr/sbin/so-common
 . /usr/sbin/so-image-common
 
-REPLAYIFACE=${REPLAYIFACE:-$(lookup_pillar interface sensor)}
+REPLAYIFACE=${REPLAYIFACE:-"{{salt['pillar.get']('sensor:interface', '')}}"}
 REPLAYSPEED=${REPLAYSPEED:-10}
 
 mkdir -p /opt/so/samples

salt/docker/defaults.yaml

@@ -180,8 +180,19 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+      ulimits:
+        - memlock=524288000
     'so-zeek':
       final_octet: 99
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+    'so-kafka':
+      final_octet: 88
+      port_bindings:
+        - 0.0.0.0:9092:9092
+        - 0.0.0.0:9093:9093
+        - 0.0.0.0:8778:8778
+      custom_bind_mounts: []
+      extra_hosts: []
+      extra_env: []

salt/docker/init.sls

@@ -20,30 +20,30 @@ dockergroup:
 dockerheldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 1.6.21-1
+      - containerd.io: 1.6.33-1
-      - docker-ce: 5:24.0.3-1~debian.12~bookworm
+      - docker-ce: 5:26.1.4-1~debian.12~bookworm
-      - docker-ce-cli: 5:24.0.3-1~debian.12~bookworm
+      - docker-ce-cli: 5:26.1.4-1~debian.12~bookworm
-      - docker-ce-rootless-extras: 5:24.0.3-1~debian.12~bookworm
+      - docker-ce-rootless-extras: 5:26.1.4-1~debian.12~bookworm
     - hold: True
     - update_holds: True
 {%    elif grains.oscodename == 'jammy' %}
 dockerheldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 1.6.21-1
+      - containerd.io: 1.6.33-1
-      - docker-ce: 5:24.0.2-1~ubuntu.22.04~jammy
+      - docker-ce: 5:26.1.4-1~ubuntu.22.04~jammy
-      - docker-ce-cli: 5:24.0.2-1~ubuntu.22.04~jammy
+      - docker-ce-cli: 5:26.1.4-1~ubuntu.22.04~jammy
-      - docker-ce-rootless-extras: 5:24.0.2-1~ubuntu.22.04~jammy
+      - docker-ce-rootless-extras: 5:26.1.4-1~ubuntu.22.04~jammy
     - hold: True
     - update_holds: True
 {%    else %}
 dockerheldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 1.4.9-1
+      - containerd.io: 1.6.33-1
-      - docker-ce: 5:20.10.8~3-0~ubuntu-focal
+      - docker-ce: 5:26.1.4-1~ubuntu.20.04~focal
-      - docker-ce-cli: 5:20.10.5~3-0~ubuntu-focal
+      - docker-ce-cli: 5:26.1.4-1~ubuntu.20.04~focal
-      - docker-ce-rootless-extras: 5:20.10.5~3-0~ubuntu-focal
+      - docker-ce-rootless-extras: 5:26.1.4-1~ubuntu.20.04~focal
     - hold: True
     - update_holds: True
 {% endif %}
@@ -51,10 +51,10 @@ dockerheldpackages:
 dockerheldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 1.6.21-3.1.el9
+      - containerd.io: 1.6.33-3.1.el9
-      - docker-ce: 24.0.4-1.el9
+      - docker-ce: 3:26.1.4-1.el9
-      - docker-ce-cli: 24.0.4-1.el9
+      - docker-ce-cli: 1:26.1.4-1.el9
-      - docker-ce-rootless-extras: 24.0.4-1.el9
+      - docker-ce-rootless-extras: 26.1.4-1.el9
     - hold: True
     - update_holds: True
 {% endif %}

salt/docker/soc_docker.yaml

@@ -63,5 +63,42 @@ docker:
     so-elastic-agent: *dockerOptions
     so-telegraf: *dockerOptions
     so-steno: *dockerOptions
-    so-suricata: *dockerOptions
+    so-suricata:
+      final_octet:
+        description: Last octet of the container IP address.
+        helpLink: docker.html
+        readonly: True
+        advanced: True
+        global: True
+      port_bindings:
+        description: List of port bindings for the container.
+        helpLink: docker.html
+        advanced: True
+        multiline: True
+        forcedType: "[]string"
+      custom_bind_mounts:
+        description: List of custom local volume bindings.
+        advanced: True
+        helpLink: docker.html
+        multiline: True
+        forcedType: "[]string"
+      extra_hosts:
+        description: List of additional host entries for the container.
+        advanced: True
+        helpLink: docker.html
+        multiline: True
+        forcedType: "[]string"
+      extra_env:
+        description: List of additional ENV entries for the container.
+        advanced: True
+        helpLink: docker.html
+        multiline: True
+        forcedType: "[]string"
+      ulimits:
+        description: Ulimits for the container, in bytes.
+        advanced: True
+        helpLink: docker.html
+        multiline: True
+        forcedType: "[]string"
     so-zeek: *dockerOptions
+    so-kafka: *dockerOptions

salt/elastalert/config.sls

@@ -82,6 +82,36 @@ elastasomodulesync:
     - group: 933
     - makedirs: True
 
+elastacustomdir:
+  file.directory:
+    - name: /opt/so/conf/elastalert/custom
+    - user: 933
+    - group: 933
+    - makedirs: True
+
+elastacustomsync:
+  file.recurse:
+    - name: /opt/so/conf/elastalert/custom
+    - source: salt://elastalert/files/custom
+    - user: 933
+    - group: 933
+    - makedirs: True
+    - file_mode: 660
+    - show_changes: False
+
+elastapredefinedsync:
+  file.recurse:
+    - name: /opt/so/conf/elastalert/predefined
+    - source: salt://elastalert/files/predefined
+    - user: 933
+    - group: 933
+    - makedirs: True
+    - template: jinja
+    - file_mode: 660
+    - context:
+        elastalert: {{ ELASTALERTMERGED }}
+    - show_changes: False
+
 elastaconf:
   file.managed:
     - name: /opt/so/conf/elastalert/elastalert_config.yaml

salt/elastalert/defaults.yaml

@@ -1,5 +1,6 @@
 elastalert:
   enabled: False
+  alerter_parameters: ""
   config:
     rules_folder: /opt/elastalert/rules/
     scan_subdirectories: true

salt/elastalert/enabled.sls

@@ -30,6 +30,8 @@ so-elastalert:
       - /opt/so/rules/elastalert:/opt/elastalert/rules/:ro
       - /opt/so/log/elastalert:/var/log/elastalert:rw
       - /opt/so/conf/elastalert/modules/:/opt/elastalert/modules/:ro
+      - /opt/so/conf/elastalert/predefined/:/opt/elastalert/predefined/:ro
+      - /opt/so/conf/elastalert/custom/:/opt/elastalert/custom/:ro
       - /opt/so/conf/elastalert/elastalert_config.yaml:/opt/elastalert/config.yaml:ro
       {% if DOCKER.containers['so-elastalert'].custom_bind_mounts %}
         {% for BIND in DOCKER.containers['so-elastalert'].custom_bind_mounts %}

salt/elastalert/files/custom/placeholder

@@ -0,0 +1 @@
+THIS IS A PLACEHOLDER FILE

salt/elastalert/files/modules/so/playbook-es.py

@@ -1,38 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-from time import gmtime, strftime
-import requests,json
-from elastalert.alerts import Alerter
-
-import urllib3
-urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-
-class PlaybookESAlerter(Alerter):
-    """
-    Use matched data to create alerts in elasticsearch
-    """
-
-    required_options = set(['play_title','play_url','sigma_level'])
-
-    def alert(self, matches):
-       for match in matches:
-            today = strftime("%Y.%m.%d", gmtime())
-            timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S"'.000Z', gmtime())
-            headers = {"Content-Type": "application/json"}
-
-            creds = None
-            if 'es_username' in self.rule and 'es_password' in self.rule:
-                creds = (self.rule['es_username'], self.rule['es_password'])
-
-            payload = {"tags":"alert","rule": { "name": self.rule['play_title'],"case_template": self.rule['play_id'],"uuid": self.rule['play_id'],"category": self.rule['rule.category']},"event":{ "severity": self.rule['event.severity'],"module": self.rule['event.module'],"dataset": self.rule['event.dataset'],"severity_label": self.rule['sigma_level']},"kibana_pivot": self.rule['kibana_pivot'],"soc_pivot": self.rule['soc_pivot'],"play_url": self.rule['play_url'],"sigma_level": self.rule['sigma_level'],"event_data": match, "@timestamp": timestamp}
-            url = f"https://{self.rule['es_host']}:{self.rule['es_port']}/logs-playbook.alerts-so/_doc/"
-            requests.post(url, data=json.dumps(payload), headers=headers, verify=False, auth=creds)
-                            
-    def get_info(self):
-        return {'type': 'PlaybookESAlerter'} 

salt/elastalert/files/modules/so/securityonion-es.py

@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
+from time import gmtime, strftime
+import requests,json
+from elastalert.alerts import Alerter
+
+import urllib3
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+class SecurityOnionESAlerter(Alerter):
+    """
+    Use matched data to create alerts in Elasticsearch.
+    """
+
+    required_options = set(['detection_title', 'sigma_level'])
+    optional_fields = ['sigma_category', 'sigma_product', 'sigma_service']
+
+    def alert(self, matches):
+        for match in matches:
+            timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S"'.000Z', gmtime())
+            headers = {"Content-Type": "application/json"}
+
+            creds = None
+            if 'es_username' in self.rule and 'es_password' in self.rule:
+                creds = (self.rule['es_username'], self.rule['es_password'])
+
+            # Start building the rule dict
+            rule_info = {
+                "name": self.rule['detection_title'],
+                "uuid": self.rule['detection_public_id']
+            }
+
+            # Add optional fields if they are present in the rule
+            for field in self.optional_fields:
+                rule_key = field.split('_')[-1]  # Assumes field format "sigma_<key>"
+                if field in self.rule:
+                    rule_info[rule_key] = self.rule[field]
+
+            # Construct the payload with the conditional rule_info
+            payload = {
+                "tags": "alert",
+                "rule": rule_info,
+                "event": {
+                    "severity": self.rule['event.severity'],
+                    "module": self.rule['event.module'],
+                    "dataset": self.rule['event.dataset'],
+                    "severity_label": self.rule['sigma_level']
+                },
+                "sigma_level": self.rule['sigma_level'],
+                "event_data": match,
+                "@timestamp": timestamp
+            }
+            url = f"https://{self.rule['es_host']}:{self.rule['es_port']}/logs-detections.alerts-so/_doc/"
+            requests.post(url, data=json.dumps(payload), headers=headers, verify=False, auth=creds)
+
+    def get_info(self):
+        return {'type': 'SecurityOnionESAlerter'}

salt/elastalert/files/predefined/jira_auth.yaml

@@ -0,0 +1,6 @@
+{% if elastalert.get('jira_user', '') | length > 0 and elastalert.get('jira_pass', '') | length > 0 %}
+user: {{ elastalert.jira_user }}
+password: {{ elastalert.jira_pass }}
+{% else %}
+apikey: {{ elastalert.get('jira_api_key', '') }}
+{% endif %}

salt/elastalert/files/predefined/smtp_auth.yaml

@@ -0,0 +1,2 @@
+user: {{ elastalert.get('smtp_user', '') }}
+password: {{ elastalert.get('smtp_pass', '') }}

salt/elastalert/map.jinja

@@ -13,3 +13,19 @@
 {% do ELASTALERTDEFAULTS.elastalert.config.update({'es_password': pillar.elasticsearch.auth.users.so_elastic_user.pass}) %}
 
 {% set ELASTALERTMERGED = salt['pillar.get']('elastalert', ELASTALERTDEFAULTS.elastalert, merge=True) %}
+
+{% if 'ntf' in salt['pillar.get']('features', []) %}
+   {% set params = ELASTALERTMERGED.get('alerter_parameters', '') | load_yaml %}
+   {% if params != None and params | length > 0 %}
+      {% do ELASTALERTMERGED.config.update(params) %}
+   {% endif %}
+
+   {% if ELASTALERTMERGED.get('smtp_user', '') | length > 0 %}
+      {% do ELASTALERTMERGED.config.update({'smtp_auth_file': '/opt/elastalert/predefined/smtp_auth.yaml'}) %}
+   {% endif %}
+
+   {% if ELASTALERTMERGED.get('jira_user', '') | length > 0 or ELASTALERTMERGED.get('jira_key', '') | length > 0 %}
+      {% do ELASTALERTMERGED.config.update({'jira_account_file': '/opt/elastalert/predefined/jira_auth.yaml'}) %}
+   {% endif %}
+
+{% endif %}

salt/elastalert/soc_elastalert.yaml

@@ -2,6 +2,99 @@ elastalert:
   enabled:
     description: You can enable or disable Elastalert.
     helpLink: elastalert.html
+  alerter_parameters:
+    title: Custom Configuration Parameters
+    description: Optional configuration parameters made available as defaults for all rules and alerters. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available configuration parameters. Requires a valid Security Onion license key.
+    global: True
+    multiline: True
+    syntax: yaml
+    helpLink: elastalert.html
+    forcedType: string
+  jira_api_key:
+    title: Jira API Key
+    description: Optional configuration parameter for Jira API Key, used instead of the Jira username and password. Requires a valid Security Onion license key.
+    global: True
+    sensitive: True
+    helpLink: elastalert.html
+    forcedType: string
+  jira_pass:
+    title: Jira Password
+    description: Optional configuration parameter for Jira password. Requires a valid Security Onion license key.
+    global: True
+    sensitive: True
+    helpLink: elastalert.html
+    forcedType: string
+  jira_user:
+    title: Jira Username
+    description: Optional configuration parameter for Jira username. Requires a valid Security Onion license key.
+    global: True
+    helpLink: elastalert.html
+    forcedType: string
+  smtp_pass:
+    title: SMTP Password
+    description: Optional configuration parameter for SMTP password, required for authenticating email servers. Requires a valid Security Onion license key.
+    global: True
+    sensitive: True
+    helpLink: elastalert.html
+    forcedType: string
+  smtp_user:
+    title: SMTP Username
+    description: Optional configuration parameter for SMTP username, required for authenticating email servers. Requires a valid Security Onion license key.
+    global: True
+    helpLink: elastalert.html
+    forcedType: string
+  files:
+    custom:
+      alertmanager_ca__crt:
+        description: Optional custom Certificate Authority for connecting to an AlertManager server. To utilize this custom file, the alertmanager_ca_certs key must be set to /opt/elastalert/custom/alertmanager_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
+        global: True
+        file: True
+        helpLink: elastalert.html
+      gelf_ca__crt:
+        description: Optional custom Certificate Authority for connecting to a Graylog server. To utilize this custom file, the graylog_ca_certs key must be set to /opt/elastalert/custom/graylog_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
+        global: True
+        file: True
+        helpLink: elastalert.html
+      http_post_ca__crt:
+        description: Optional custom Certificate Authority for connecting to a generic HTTP server, via the legacy HTTP POST alerter. To utilize this custom file, the http_post_ca_certs key must be set to /opt/elastalert/custom/http_post2_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
+        global: True
+        file: True
+        helpLink: elastalert.html
+      http_post2_ca__crt:
+        description: Optional custom Certificate Authority for connecting to a generic HTTP server, via the newer HTTP POST 2 alerter. To utilize this custom file, the http_post2_ca_certs key must be set to /opt/elastalert/custom/http_post2_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
+        global: True
+        file: True
+        helpLink: elastalert.html
+      ms_teams_ca__crt:
+        description: Optional custom Certificate Authority for connecting to Microsoft Teams server. To utilize this custom file, the ms_teams_ca_certs key must be set to /opt/elastalert/custom/ms_teams_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
+        global: True
+        file: True
+        helpLink: elastalert.html
+      pagerduty_ca__crt:
+        description: Optional custom Certificate Authority for connecting to PagerDuty server. To utilize this custom file, the pagerduty_ca_certs key must be set to /opt/elastalert/custom/pagerduty_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
+        global: True
+        file: True
+        helpLink: elastalert.html
+      rocket_chat_ca__crt:
+        description: Optional custom Certificate Authority for connecting to PagerDuty server. To utilize this custom file, the rocket_chart_ca_certs key must be set to /opt/elastalert/custom/rocket_chat_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
+        global: True
+        file: True
+        helpLink: elastalert.html
+      smtp__crt:
+        description: Optional custom certificate for connecting to an SMTP server. To utilize this custom file, the smtp_cert_file key must be set to /opt/elastalert/custom/smtp.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
+        global: True
+        file: True
+        helpLink: elastalert.html
+      smtp__key:
+        description: Optional custom certificate key for connecting to an SMTP server. To utilize this custom file, the smtp_key_file key must be set to /opt/elastalert/custom/smtp.key in the Alerter Parameters setting. Requires a valid Security Onion license key.
+        global: True
+        file: True
+        helpLink: elastalert.html
+      slack_ca__crt:
+        description: Optional custom Certificate Authority for connecting to Slack. To utilize this custom file, the slack_ca_certs key must be set to /opt/elastalert/custom/slack_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
+        global: True
+        file: True
+        helpLink: elastalert.html
   config:
     disable_rules_on_error:
       description: Disable rules on failure.

salt/elasticfleet/defaults.yaml

@@ -37,6 +37,7 @@ elasticfleet:
     - azure
     - barracuda
     - carbonblack_edr
+    - cef
     - checkpoint
     - cisco_asa
     - cisco_duo
@@ -96,6 +97,7 @@ elasticfleet:
     - symantec_endpoint
     - system
     - tcp
+    - tenable_io
     - tenable_sc
     - ti_abusech
     - ti_anomali
@@ -118,3 +120,8 @@ elasticfleet:
       base_url: https://api.platform.sublimesecurity.com
       poll_interval: 5m
       limit: 100
+    kismet:
+      base_url: http://localhost:2501
+      poll_interval: 1m
+      api_key:
+      enabled_nodes: []

salt/elasticfleet/enabled.sls

@@ -27,7 +27,9 @@ wait_for_elasticsearch_elasticfleet:
 so-elastic-fleet-auto-configure-logstash-outputs:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-outputs-update
-    - retry: True
+    - retry:
+        attempts: 4
+        interval: 30
 {% endif %}
 
 # If enabled, automatically update Fleet Server URLs & ES Connection
@@ -35,7 +37,9 @@ so-elastic-fleet-auto-configure-logstash-outputs:
 so-elastic-fleet-auto-configure-server-urls:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-urls-update
-    - retry: True
+    - retry:
+        attempts: 4
+        interval: 30
 {% endif %}
 
 # Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs
@@ -43,12 +47,16 @@ so-elastic-fleet-auto-configure-server-urls:
 so-elastic-fleet-auto-configure-elasticsearch-urls:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-es-url-update
-    - retry: True
+    - retry:
+        attempts: 4
+        interval: 30
 
 so-elastic-fleet-auto-configure-artifact-urls:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-artifacts-url-update
-    - retry: True 
+    - retry:
+        attempts: 4
+        interval: 30
 
 {% endif %}
 

salt/elasticfleet/files/integrations-dynamic/fleet-server/fleet-server.json

@@ -0,0 +1,19 @@
+{
+  "package": {
+    "name": "fleet_server",
+    "version": ""
+  },
+  "name": "fleet_server-1",
+  "namespace": "default",
+  "policy_id": "FleetServer_hostname",
+  "vars": {},
+  "inputs": {
+    "fleet_server-fleet-server": {
+      "enabled": true,
+      "vars": {
+        "custom": "server.ssl.supported_protocols: [\"TLSv1.2\", \"TLSv1.3\"]\nserver.ssl.cipher_suites: [ \"ECDHE-RSA-AES-128-GCM-SHA256\", \"ECDHE-RSA-AES-256-GCM-SHA384\", \"ECDHE-RSA-AES-128-CBC-SHA\", \"ECDHE-RSA-AES-256-CBC-SHA\", \"RSA-AES-128-GCM-SHA256\", \"RSA-AES-256-GCM-SHA384\"]"
+      },
+      "streams": {}
+    }
+  }
+}

salt/elasticfleet/files/integrations-optional/kismet.json

@@ -0,0 +1,36 @@
+{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
+{% raw %}
+{
+  "package": {
+    "name": "httpjson",
+    "version": ""
+  },
+  "name": "kismet-logs",
+  "namespace": "so",
+  "description": "Kismet Logs",
+  "policy_id": "FleetServer_{% endraw %}{{ NAME }}{% raw %}",
+  "inputs": {
+    "generic-httpjson": {
+      "enabled": true,
+      "streams": {
+        "httpjson.generic": {
+          "enabled": true,
+          "vars": {
+            "data_stream.dataset": "kismet",
+            "request_url": "{% endraw %}{{ ELASTICFLEETMERGED.optional_integrations.kismet.base_url }}{% raw %}/devices/last-time/-600/devices.tjson",
+            "request_interval": "{% endraw %}{{ ELASTICFLEETMERGED.optional_integrations.kismet.poll_interval }}{% raw %}",
+            "request_method": "GET",
+            "request_transforms": "- set:\r\n    target: header.Cookie\r\n    value: 'KISMET={% endraw %}{{ ELASTICFLEETMERGED.optional_integrations.kismet.api_key }}{% raw %}'",
+            "request_redirect_headers_ban_list": [],
+            "oauth_scopes": [],
+            "processors": "",
+            "tags": [],
+            "pipeline": "kismet.common"
+          }
+        }
+      }
+    }
+  },
+  "force": true
+}
+{% endraw %}

salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json

@@ -5,7 +5,7 @@
 	"package": {
 		"name": "endpoint",
 		"title": "Elastic Defend",
-		"version": "8.10.2"
+		"version": "8.14.0"
 	},
 	"enabled": true,
 	"policy_id": "endpoints-initial",

salt/elasticfleet/files/integrations/endpoints-initial/windows-defender.json

@@ -0,0 +1,29 @@
+{
+  "package": {
+    "name": "winlog",
+    "version": ""
+  },
+  "name": "windows-defender",
+  "namespace": "default",
+  "description": "Windows Defender - Operational logs",
+  "policy_id": "endpoints-initial",
+  "inputs": {
+    "winlogs-winlog": {
+      "enabled": true,
+      "streams": {
+        "winlog.winlogs": {
+          "enabled": true,
+          "vars": {
+            "channel": "Microsoft-Windows-Windows Defender/Operational",
+            "data_stream.dataset": "winlog.winlog",
+            "preserve_original_event": false,
+            "providers": [],
+            "ignore_older": "72h",
+            "language": 0,
+            "tags": []          }
+        }
+      }
+    }
+  },
+  "force": true
+}

salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json

@@ -20,7 +20,7 @@
             ],
             "data_stream.dataset": "import",
             "custom": "",
-            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-1.43.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-1.38.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-1.43.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-1.43.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-1.38.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-1.59.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-1.45.1\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-1.59.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-1.59.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-1.45.1\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
             "tags": [
               "import"
             ]

salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json

@@ -0,0 +1,35 @@
+{
+  "policy_id": "so-grid-nodes_general",
+  "package": {
+    "name": "log",
+    "version": ""
+  },
+  "name": "soc-detections-logs",
+  "description": "Security Onion Console - Detections Logs",
+  "namespace": "so",
+  "inputs": {
+    "logs-logfile": {
+      "enabled": true,
+      "streams": {
+        "log.logs": {
+          "enabled": true,
+          "vars": {
+            "paths": [
+              "/opt/so/log/soc/detections_runtime-status_sigma.log",
+              "/opt/so/log/soc/detections_runtime-status_yara.log"
+            ],
+            "exclude_files": [],
+            "ignore_older": "72h",
+            "data_stream.dataset": "soc",
+            "tags": [
+              "so-soc"
+            ],
+            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"soc\"\n    process_array: true\n    max_depth: 2\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: detections\n- rename:\n    fields:\n      - from: \"soc.fields.sourceIp\"\n        to: \"source.ip\"\n      - from: \"soc.fields.status\"\n        to: \"http.response.status_code\"\n      - from: \"soc.fields.method\"\n        to: \"http.request.method\"\n      - from: \"soc.fields.path\"\n        to: \"url.path\"\n      - from: \"soc.message\"\n        to: \"event.action\"\n      - from: \"soc.level\"\n        to: \"log.level\"\n    ignore_missing: true",
+            "custom": "pipeline: common"
+          }
+        }
+      }
+    }
+  },
+  "force": true
+}

salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json

@@ -16,6 +16,9 @@
             "paths": [
               "/var/log/auth.log*",
               "/var/log/secure*"
+            ],
+            "tags": [
+              "so-grid-node"
             ]
           }
         },
@@ -25,6 +28,9 @@
             "paths": [
               "/var/log/messages*",
               "/var/log/syslog*"
+            ],
+            "tags": [
+              "so-grid-node"
             ]
           }
         }

salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json

@@ -16,6 +16,9 @@
             "paths": [
               "/var/log/auth.log*",
               "/var/log/secure*"
+            ],
+            "tags": [
+              "so-grid-node"
             ]
           }
         },
@@ -25,6 +28,9 @@
             "paths": [
               "/var/log/messages*",
               "/var/log/syslog*"
+            ],
+            "tags": [
+              "so-grid-node"
             ]
           }
         }

salt/elasticfleet/soc_elasticfleet.yaml

@@ -79,3 +79,29 @@ elasticfleet:
         helpLink: elastic-fleet.html
         advanced: True
         forcedType: int
+    kismet:
+      base_url:
+        description: Base URL for Kismet.
+        global: True
+        helpLink: elastic-fleet.html
+        advanced: True
+        forcedType: string
+      poll_interval:
+        description: Poll interval for wireless device data from Kismet. Integration is currently configured to return devices seen as active by any Kismet sensor within the last 10 minutes.
+        global: True
+        helpLink: elastic-fleet.html
+        advanced: True
+        forcedType: string
+      api_key:
+        description: API key for Kismet.
+        global: True
+        helpLink: elastic-fleet.html
+        advanced: True
+        forcedType: string
+        sensitive: True
+      enabled_nodes:
+        description: Fleet nodes with the Kismet integration enabled. Enter one per line.
+        global: True
+        helpLink: elastic-fleet.html
+        advanced: True
+        forcedType: "[]string"

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server

@@ -0,0 +1,29 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-elastic-fleet-common
+
+# Get all the fleet policies
+json_output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -L -X GET "localhost:5601/api/fleet/agent_policies" -H 'kbn-xsrf: true')
+
+# Extract the IDs that start with "FleetServer_"
+POLICY=$(echo "$json_output" | jq -r '.items[] | select(.id | startswith("FleetServer_")) | .id')
+
+# Iterate over each ID in the POLICY variable
+for POLICYNAME in $POLICY; do
+    printf "\nUpdating Policy: $POLICYNAME\n"
+
+    # First get the Integration ID
+    INTEGRATION_ID=$(/usr/sbin/so-elastic-fleet-agent-policy-view "$POLICYNAME" | jq -r '.item.package_policies[] | select(.package.name == "fleet_server") | .id')  
+
+    # Modify the default integration policy to update the policy_id and an with the correct naming
+	UPDATED_INTEGRATION_POLICY=$(jq --arg policy_id "$POLICYNAME" --arg name "fleet_server-$POLICYNAME" '
+    .policy_id = $policy_id |
+    .name = $name' /opt/so/conf/elastic-fleet/integrations/fleet-server/fleet-server.json)
+
+    # Now update the integration policy using the modified JSON
+    elastic_fleet_integration_update "$INTEGRATION_ID" "$UPDATED_INTEGRATION_POLICY"
+done

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load

@@ -12,7 +12,10 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
   # First, check for any package upgrades
   /usr/sbin/so-elastic-fleet-package-upgrade
 
-  # Second, configure Elastic Defend Integration seperately
+  # Second, update Fleet Server policies
+  /sbin/so-elastic-fleet-integration-policy-elastic-fleet-server
+
+  # Third, configure Elastic Defend Integration seperately
   /usr/sbin/so-elastic-fleet-integration-policy-elastic-defend
 
   # Initial Endpoints

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers

@@ -19,7 +19,7 @@ NUM_RUNNING=$(pgrep -cf "/bin/bash /sbin/so-elastic-agent-gen-installers")
 
 for i in {1..30}
 do
-   ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key')
+   ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys?perPage=100" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key')
    FLEETHOST=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/fleet_server_hosts/grid-default' | jq -r  '.item.host_urls[]' | paste -sd ',')
 if [[ $FLEETHOST ]] && [[  $ENROLLMENTOKEN ]]; then break; else sleep 10; fi
 done
@@ -72,5 +72,5 @@ do
     printf "\n### $GOOS/$GOARCH Installer Generated...\n"
 done
 
-printf "\n### Cleaning up temp files in /nsm/elastic-agent-workspace"
+printf "\n### Cleaning up temp files in /nsm/elastic-agent-workspace\n"
 rm -rf /nsm/elastic-agent-workspace

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update

@@ -21,64 +21,104 @@ function update_logstash_outputs() {
     # Update Logstash Outputs
     curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_logstash" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" | jq
 }
+function update_kafka_outputs() {
+  # Make sure SSL configuration is included in policy updates for Kafka output. SSL is configured in so-elastic-fleet-setup
+  SSL_CONFIG=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs/so-manager_kafka" | jq -r '.item.ssl')
 
-# Get current list of Logstash Outputs
+  JSON_STRING=$(jq -n \
-RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_logstash')
+    --arg UPDATEDLIST "$NEW_LIST_JSON" \
+    --argjson SSL_CONFIG "$SSL_CONFIG" \
+    '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": $SSL_CONFIG}')
+  # Update Kafka outputs
+  curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_kafka" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" | jq
+}
 
-# Check to make sure that the server responded with good data - else, bail from script
+{% if GLOBALS.pipeline == "KAFKA" %}
-CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON")
+  # Get current list of Kafka Outputs
-if [ "$CHECKSUM" != "so-manager_logstash" ]; then
+  RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_kafka')
- printf "Failed to query for current Logstash Outputs..."
- exit 1
-fi
 
-# Get the current list of Logstash outputs & hash them
+  # Check to make sure that the server responded with good data - else, bail from script
-CURRENT_LIST=$(jq -c -r '.item.hosts' <<<  "$RAW_JSON")
+  CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON")
-CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}')
+  if [ "$CHECKSUM" != "so-manager_kafka" ]; then
+   printf "Failed to query for current Kafka Outputs..."
+   exit 1
+  fi
 
-declare -a NEW_LIST=()
+  # Get the current list of kafka outputs & hash them
+  CURRENT_LIST=$(jq -c -r '.item.hosts' <<<  "$RAW_JSON")
+  CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}')
+
+  declare -a NEW_LIST=()
+
+  # Query for the current Grid Nodes that are running kafka
+  KAFKANODES=$(salt-call --out=json pillar.get kafka:nodes | jq '.local')
+
+  # Query for Kafka nodes with Broker role and add hostname to list
+  while IFS= read -r line; do
+    NEW_LIST+=("$line")
+  done < <(jq -r 'to_entries | .[] | select(.value.role | contains("broker")) | .key + ":9092"' <<< $KAFKANODES)
+
+  {# If global pipeline isn't set to KAFKA then assume default of REDIS / logstash #}
+{% else %}
+  # Get current list of Logstash Outputs
+  RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_logstash')
+
+  # Check to make sure that the server responded with good data - else, bail from script
+  CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON")
+  if [ "$CHECKSUM" != "so-manager_logstash" ]; then
+   printf "Failed to query for current Logstash Outputs..."
+   exit 1
+  fi
+
+  # Get the current list of Logstash outputs & hash them
+  CURRENT_LIST=$(jq -c -r '.item.hosts' <<<  "$RAW_JSON")
+  CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}')
+
+  declare -a NEW_LIST=()
+
+  {# If we select to not send to manager via SOC, then omit the code that adds manager to NEW_LIST #}
+  {% if ELASTICFLEETMERGED.enable_manager_output %}
+  # Create array & add initial elements
+  if [ "{{ GLOBALS.hostname }}" = "{{ GLOBALS.url_base }}" ]; then
+      NEW_LIST+=("{{ GLOBALS.url_base }}:5055")
+  else
+      NEW_LIST+=("{{ GLOBALS.url_base }}:5055" "{{ GLOBALS.hostname }}:5055")
+  fi
+  {% endif %}
+
+  # Query for FQDN entries & add them to the list
+  {% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %}
+  CUSTOMFQDNLIST=('{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(' ') }}')
+  readarray -t -d ' ' CUSTOMFQDN < <(printf '%s' "$CUSTOMFQDNLIST")
+  for CUSTOMNAME in "${CUSTOMFQDN[@]}"
+  do
+   NEW_LIST+=("$CUSTOMNAME:5055")
+  done
+  {% endif %}
+
+  # Query for the current Grid Nodes that are running Logstash
+  LOGSTASHNODES=$(salt-call --out=json pillar.get logstash:nodes | jq '.local')
+
+  # Query for Receiver Nodes & add them to the list
+  if grep -q "receiver" <<< $LOGSTASHNODES; then
+     readarray -t RECEIVERNODES < <(jq -r ' .receiver | keys_unsorted[]'  <<< $LOGSTASHNODES)
+     for NODE in "${RECEIVERNODES[@]}"
+     do
+      NEW_LIST+=("$NODE:5055")
+     done
+  fi
+
+  # Query for Fleet Nodes & add them to the list
+  if grep -q "fleet" <<< $LOGSTASHNODES; then
+     readarray -t FLEETNODES < <(jq -r ' .fleet | keys_unsorted[]'  <<< $LOGSTASHNODES)
+     for NODE in "${FLEETNODES[@]}"
+     do
+      NEW_LIST+=("$NODE:5055")
+     done
+  fi
 
-{# If we select to not send to manager via SOC, then omit the code that adds manager to NEW_LIST #}
-{% if ELASTICFLEETMERGED.enable_manager_output %}
-# Create array & add initial elements
-if [ "{{ GLOBALS.hostname }}" = "{{ GLOBALS.url_base }}" ]; then
-    NEW_LIST+=("{{ GLOBALS.url_base }}:5055")
-else
-    NEW_LIST+=("{{ GLOBALS.url_base }}:5055" "{{ GLOBALS.hostname }}:5055")
-fi
 {% endif %}
 
-# Query for FQDN entries & add them to the list
-{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %}
-CUSTOMFQDNLIST=('{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(' ') }}')
-readarray -t -d ' ' CUSTOMFQDN < <(printf '%s' "$CUSTOMFQDNLIST")
-for CUSTOMNAME in "${CUSTOMFQDN[@]}"
-do
- NEW_LIST+=("$CUSTOMNAME:5055")
-done
-{% endif %}
-
-# Query for the current Grid Nodes that are running Logstash
-LOGSTASHNODES=$(salt-call --out=json pillar.get logstash:nodes | jq '.local')
-
-# Query for Receiver Nodes & add them to the list
-if grep -q "receiver" <<< $LOGSTASHNODES; then
-   readarray -t RECEIVERNODES < <(jq -r ' .receiver | keys_unsorted[]'  <<< $LOGSTASHNODES)
-   for NODE in "${RECEIVERNODES[@]}"
-   do
-    NEW_LIST+=("$NODE:5055")
-   done
-fi
-
-# Query for Fleet Nodes & add them to the list
-if grep -q "fleet" <<< $LOGSTASHNODES; then
-   readarray -t FLEETNODES < <(jq -r ' .fleet | keys_unsorted[]'  <<< $LOGSTASHNODES)
-   for NODE in "${FLEETNODES[@]}"
-   do
-    NEW_LIST+=("$NODE:5055")
-   done
-fi
-
 # Sort & hash the new list of Logstash Outputs
 NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${NEW_LIST[@]}")
 NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}')
@@ -87,9 +127,28 @@ NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}')
 if [ "$NEW_HASH" = "$CURRENT_HASH" ]; then
     printf "\nHashes match - no update needed.\n"
     printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n"
+
+    # Since output can be KAFKA or LOGSTASH, we need to check if the policy set as default matches the value set in GLOBALS.pipeline and update if needed
+    printf "Checking if the correct output policy is set as default\n"
+    OUTPUT_DEFAULT=$(jq -r '.item.is_default' <<< $RAW_JSON)
+    OUTPUT_DEFAULT_MONITORING=$(jq -r '.item.is_default_monitoring' <<< $RAW_JSON)
+    if [[ "$OUTPUT_DEFAULT" = "false" || "$OUTPUT_DEFAULT_MONITORING" = "false" ]]; then
+      printf "Default output policy needs to be updated.\n"
+    {%- if GLOBALS.pipeline == "KAFKA" and 'gmd' in salt['pillar.get']('features', []) %}
+      update_kafka_outputs
+    {%- else %}
+      update_logstash_outputs
+    {%- endif %}
+    else
+      printf "Default output policy is set - no update needed.\n"
+    fi
     exit 0
 else
     printf "\nHashes don't match - update needed.\n"
     printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n"
+    {%- if GLOBALS.pipeline == "KAFKA" and 'gmd' in salt['pillar.get']('features', []) %}
+    update_kafka_outputs
+    {%- else %}
     update_logstash_outputs
+    {%- endif %}
 fi

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup

@@ -53,7 +53,8 @@ fi
 printf "\n### Create ES Token ###\n"
 ESTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/service_tokens" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' |  jq -r .value)
 
-### Create Outputs & Fleet URLs ###
+### Create Outputs, Fleet Policy and Fleet URLs ###
+# Create the Manager Elasticsearch Output first and set it as the default output
 printf "\nAdd Manager Elasticsearch Output...\n"
 ESCACRT=$(openssl x509 -in  $INTCA)
 JSON_STRING=$( jq -n \
@@ -62,7 +63,21 @@ JSON_STRING=$( jq -n \
 curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
 printf "\n\n"
 
-printf "\nCreate Logstash Output Config if node is not an Import or Eval install\n"
+# Create the Manager Fleet Server Host Agent Policy
+# This has to be done while the Elasticsearch Output is set to the default Output
+printf "Create Manager Fleet Server Policy...\n"
+elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "false" "120"
+
+# Modify the default integration policy to update the policy_id with the correct naming
+UPDATED_INTEGRATION_POLICY=$(jq --arg policy_id "FleetServer_{{ GLOBALS.hostname }}" --arg name "fleet_server-{{ GLOBALS.hostname }}" '
+.policy_id = $policy_id |
+.name = $name' /opt/so/conf/elastic-fleet/integrations/fleet-server/fleet-server.json)
+
+# Add the Fleet Server Integration to the new Fleet Policy
+elastic_fleet_integration_create "$UPDATED_INTEGRATION_POLICY"
+
+# Now we can create the Logstash Output and set it to to be the default Output
+printf "\n\nCreate Logstash Output Config if node is not an Import or Eval install\n"
 {% if grains.role not in ['so-import', 'so-eval'] %}
 LOGSTASHCRT=$(openssl x509 -in /etc/pki/elasticfleet-logstash.crt)
 LOGSTASHKEY=$(openssl rsa -in  /etc/pki/elasticfleet-logstash.key)
@@ -77,6 +92,11 @@ curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fl
 printf "\n\n"
 {%- endif %}
 
+printf "\nCreate Kafka Output Config if node is not an Import or Eval install\n"
+{% if grains.role not in ['so-import', 'so-eval'] %}
+/usr/sbin/so-kafka-fleet-output-policy
+{% endif %}
+
 # Add Manager Hostname & URL Base to Fleet Host URLs
 printf "\nAdd SO-Manager Fleet URL\n"
 if [ "{{ GLOBALS.hostname }}" = "{{ GLOBALS.url_base }}" ]; then
@@ -96,16 +116,6 @@ printf "\n\n"
 # Load Elasticsearch templates
 /usr/sbin/so-elasticsearch-templates-load
 
-# Manager Fleet Server Host
-elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "true" "120"
-
-#Temp Fixup for ES Output bug
-JSON_STRING=$( jq -n \
-                --arg NAME "FleetServer_{{ GLOBALS.hostname }}" \
-                '{"name": $NAME,"description": $NAME,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":120,"data_output_id":"so-manager_elasticsearch"}'
-                )
-curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/FleetServer_{{ GLOBALS.hostname }}" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
-
 # Initial Endpoints Policy
 elastic_fleet_policy_create "endpoints-initial" "Initial Endpoint Policy" "false" "1209600"
 
@@ -160,4 +170,4 @@ salt-call state.apply elasticfleet queue=True
 # Generate installers & install Elastic Agent on the node
 so-elastic-agent-gen-installers
 salt-call state.apply elasticfleet.install_agent_grid queue=True
-exit 0
+exit 0

salt/elasticfleet/tools/sbin_jinja/so-kafka-fleet-output-policy

@@ -0,0 +1,52 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+{% if GLOBALS.role in ['so-manager', 'so-standalone', 'so-managersearch'] %}
+
+. /usr/sbin/so-common
+
+# Check to make sure that Kibana API is up & ready
+RETURN_CODE=0
+wait_for_web_response "http://localhost:5601/api/fleet/settings" "fleet" 300 "curl -K /opt/so/conf/elasticsearch/curl.config"
+RETURN_CODE=$?
+
+if [[ "$RETURN_CODE" != "0" ]]; then
+    printf "Kibana API not accessible, can't setup Elastic Fleet output policy for Kafka..."
+    exit 1
+fi
+
+output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs" | jq -r .items[].id)
+
+if ! echo "$output" | grep -q "so-manager_kafka"; then
+  KAFKACRT=$(openssl x509 -in /etc/pki/elasticfleet-kafka.crt)
+  KAFKAKEY=$(openssl rsa -in  /etc/pki/elasticfleet-kafka.key)
+  KAFKACA=$(openssl x509 -in  /etc/pki/tls/certs/intca.crt)
+  KAFKA_OUTPUT_VERSION="2.6.0"
+  JSON_STRING=$( jq -n \
+                --arg KAFKACRT "$KAFKACRT" \
+                --arg KAFKAKEY "$KAFKAKEY" \
+                --arg KAFKACA "$KAFKACA" \
+                --arg MANAGER_IP "{{ GLOBALS.manager_ip }}:9092" \
+                --arg KAFKA_OUTPUT_VERSION "$KAFKA_OUTPUT_VERSION" \
+                    '{ "name": "grid-kafka", "id": "so-manager_kafka", "type": "kafka", "hosts": [ $MANAGER_IP ], "is_default": false, "is_default_monitoring": false, "config_yaml": "", "ssl": { "certificate_authorities": [ $KAFKACA ], "certificate": $KAFKACRT, "key": $KAFKAKEY, "verification_mode": "full" }, "proxy_id": null, "client_id": "Elastic", "version": $KAFKA_OUTPUT_VERSION, "compression": "none", "auth_type": "ssl", "partition": "round_robin", "round_robin": { "group_events": 1 }, "topics":[{"topic":"%{[event.module]}-securityonion","when":{"type":"regexp","condition":"event.module:.+"}},{"topic":"default-securityonion"}], "headers": [ { "key": "", "value": "" } ], "timeout": 30, "broker_timeout": 30, "required_acks": 1 }'
+                )
+  curl -sK /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" -o /dev/null
+  refresh_output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs" | jq -r .items[].id)
+
+  if ! echo "$refresh_output" | grep -q "so-manager_kafka"; then
+    echo -e "\nFailed to setup Elastic Fleet output policy for Kafka...\n"
+    exit 1
+  elif echo "$refresh_output" | grep -q "so-manager_kafka"; then
+    echo -e "\nSuccessfully setup Elastic Fleet output policy for Kafka...\n"
+  fi
+
+elif echo "$output" | grep -q "so-manager_kafka"; then
+  echo -e "\nElastic Fleet output policy for Kafka already exists...\n"
+fi
+{% else %}
+echo -e "\nNo update required...\n"
+{% endif %}

salt/elasticsearch/ca.sls

@@ -4,7 +4,7 @@
 # Elastic License 2.0.
 
 {% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls.split('.')[0] in allowed_states %}
+{% if sls.split('.')[0] in allowed_states or sls in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 
 # Move our new CA over so Elastic and Logstash can use SSL with the internal CA

salt/elasticsearch/config.map.jinja

@@ -1,23 +1,37 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS with context %}
 
 {% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %}
 
-{# ES_LOGSTASH_NODES is the same as LOGSTASH_NODES from logstash/map.jinja but heavynodes and fleet nodes are removed #}
+{# this is a list of dicts containing hostname:ip for elasticsearch nodes that need to know about each other for cluster #}
-{% set ES_LOGSTASH_NODES = [] %}
+{% set ELASTICSEARCH_SEED_HOSTS = [] %}
-{% set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
+{% set node_data = salt['pillar.get']('elasticsearch:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
 {% for node_type, node_details in node_data.items() | sort %}
-{%   if node_type not in ['heavynode', 'fleet'] %}
+{%   if node_type != 'heavynode' %}
 {%     for hostname in node_data[node_type].keys() %}
-{%       do ES_LOGSTASH_NODES.append({hostname:node_details[hostname].ip}) %}
+{%       do ELASTICSEARCH_SEED_HOSTS.append({hostname:node_details[hostname].ip}) %}
 {%     endfor %}
 {%   endif %}
 {% endfor %}
 
+{# this is a list of dicts containing hostname:ip of all nodes running elasticsearch #}
+{% set ELASTICSEARCH_NODES = [] %}
+{% set node_data = salt['pillar.get']('elasticsearch:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
+{% for node_type, node_details in node_data.items() %}
+{%   for hostname in node_data[node_type].keys() %}
+{%     do ELASTICSEARCH_NODES.append({hostname:node_details[hostname].ip}) %}
+{%   endfor %}
+{% endfor %}
+
 {% if grains.id.split('_') | last in ['manager','managersearch','standalone'] %}
-    {% if ES_LOGSTASH_NODES | length > 1 %}
+    {% if ELASTICSEARCH_SEED_HOSTS | length > 1 %}
       {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.update({'discovery': {'seed_hosts': []}}) %}
-      {% for NODE in ES_LOGSTASH_NODES %}
+      {% for NODE in ELASTICSEARCH_SEED_HOSTS %}
         {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.discovery.seed_hosts.append(NODE.keys()|first) %}
       {% endfor %}
     {% endif %}

salt/elasticsearch/config.sls

@@ -118,6 +118,11 @@ esingestconf:
     - user: 930
     - group: 939
 
+# Remove .fleet_final_pipeline-1 because we are using global@custom now
+so-fleet-final-pipeline-remove:
+  file.absent:
+    - name: /opt/so/conf/elasticsearch/ingest/.fleet_final_pipeline-1
+
 # Auto-generate Elasticsearch ingest node pipelines from pillar
 {% for pipeline, config in ELASTICSEARCHMERGED.pipelines.items() %}
 es_ingest_conf_{{pipeline}}:

salt/elasticsearch/download.sls

@@ -0,0 +1,20 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+{%   from 'vars/globals.map.jinja' import GLOBALS %}
+
+so-elasticsearch_image:
+  docker_image.present:
+    - name: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elasticsearch:{{ GLOBALS.so_version }}
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/elasticsearch/enabled.sls

@@ -7,8 +7,8 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKER %}
-{%   from 'logstash/map.jinja' import LOGSTASH_NODES %}
+{%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
-{%   from 'elasticsearch/config.map.jinja' import ES_LOGSTASH_NODES %}
+{%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
 {%   set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %}
 {%   from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %}
@@ -27,7 +27,7 @@ so-elasticsearch:
       - sobridge:
         - ipv4_address: {{ DOCKER.containers['so-elasticsearch'].ip }}
     - extra_hosts:
-    {% for node in LOGSTASH_NODES %}
+    {% for node in ELASTICSEARCH_NODES %}
     {%   for hostname, ip in node.items() %}
       - {{hostname}}:{{ip}}
     {%   endfor %}
@@ -38,7 +38,7 @@ so-elasticsearch:
       {% endfor %}
     {% endif %}
     - environment:
-      {% if ES_LOGSTASH_NODES | length == 1 or GLOBALS.role == 'so-heavynode' %}
+      {% if ELASTICSEARCH_SEED_HOSTS | length == 1 or GLOBALS.role == 'so-heavynode' %}
       - discovery.type=single-node
       {% endif %}
       - ES_JAVA_OPTS=-Xms{{ GLOBALS.elasticsearch.es_heap }} -Xmx{{ GLOBALS.elasticsearch.es_heap }} -Des.transport.cname_in_publish_address=true -Dlog4j2.formatMsgNoLookups=true
@@ -200,9 +200,15 @@ so-elasticsearch-roles-load:
     - require:
       - docker_container: so-elasticsearch
       - file: elasticsearch_sbin_jinja
-{% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
+
+{%     if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
+{%       if ELASTICSEARCHMERGED.index_clean %}
+{%         set ap = "present" %}
+{%       else %}
+{%         set ap = "absent" %}
+{%       endif %}
 so-elasticsearch-indices-delete:
-  cron.present:
+  cron.{{ap}}:
     - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1
     - identifier: so-elasticsearch-indices-delete
     - user: root
@@ -211,7 +217,8 @@ so-elasticsearch-indices-delete:
     - daymonth: '*'
     - month: '*'
     - dayweek: '*'
-{% endif %}
+{%     endif %}
+
 {%   endif %}
 
 {% else %}

salt/elasticsearch/files/ingest-dynamic/common

@@ -62,6 +62,7 @@
     { "split":           { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "dataset_tag_temp" } },
     { "append":          { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp.1}}"  } },
     { "grok":            { "if": "ctx.http?.response?.status_code != null", "field": "http.response.status_code", "patterns": ["%{NUMBER:http.response.status_code:long} %{GREEDYDATA}"]} },
+    { "set":             { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } },
     { "remove":          { "field": [ "message2", "type", "fields", "category", "module", "dataset", "dataset_tag_temp", "event.dataset_temp" ], "ignore_missing": true, "ignore_failure": true } }
 {%- endraw %}
 {%- if HIGHLANDER %}
@@ -72,7 +73,9 @@
       }
     }
 {%- endif %}
-{%- raw %} 
+{%- raw %}
+    ,
+    { "pipeline": { "name": "global@custom", "ignore_missing_pipeline": true, "description": "[Fleet] Global pipeline for all data streams" } }
   ]
 }
 {% endraw %}

salt/elasticsearch/files/ingest/.fleet_final_pipeline-1

@@ -1,105 +0,0 @@
-{
-  "version": 3,
-  "_meta": {
-    "managed_by": "fleet",
-    "managed": true
-  },
-  "description": "Final pipeline for processing all incoming Fleet Agent documents. \n",
-  "processors": [
-    {
-      "date": {
-        "description": "Add time when event was ingested (and remove sub-seconds to improve storage efficiency)",
-        "tag": "truncate-subseconds-event-ingested",
-        "field": "_ingest.timestamp",
-        "target_field": "event.ingested",
-        "formats": [
-          "ISO8601"
-        ],
-        "output_format": "date_time_no_millis",
-        "ignore_failure": true
-      }
-    },
-    {
-      "remove": {
-        "description": "Remove any pre-existing untrusted values.",
-        "field": [
-          "event.agent_id_status",
-          "_security"
-        ],
-        "ignore_missing": true
-      }
-    },
-    {
-      "set_security_user": {
-        "field": "_security",
-        "properties": [
-          "authentication_type",
-          "username",
-          "realm",
-          "api_key"
-        ]
-      }
-    },
-    {
-      "script": {
-        "description": "Add event.agent_id_status based on the API key metadata and the agent.id contained in the event.\n",
-        "tag": "agent-id-status",
-        "source": "boolean is_user_trusted(def ctx, def users) {\n  if (ctx?._security?.username == null) {\n    return false;\n  }\n\n  def user = null;\n  for (def item : users) {\n    if (item?.username == ctx._security.username) {\n      user = item;\n      break;\n    }\n  }\n\n  if (user == null || user?.realm == null || ctx?._security?.realm?.name == null) {\n    return false;\n  }\n\n  if (ctx._security.realm.name != user.realm) {\n    return false;\n  }\n\n  return true;\n}\n\nString verified(def ctx, def params) {\n  // No agent.id field to validate.\n  if (ctx?.agent?.id == null) {\n    return \"missing\";\n  }\n\n  // Check auth metadata from API key.\n  if (ctx?._security?.authentication_type == null\n      // Agents only use API keys.\n      || ctx._security.authentication_type != 'API_KEY'\n      // Verify the API key owner before trusting any metadata it contains.\n      || !is_user_trusted(ctx, params.trusted_users)\n      // Verify the API key has metadata indicating the assigned agent ID.\n      || ctx?._security?.api_key?.metadata?.agent_id == null) {\n    return \"auth_metadata_missing\";\n  }\n\n  // The API key can only be used represent the agent.id it was issued to.\n  if (ctx._security.api_key.metadata.agent_id != ctx.agent.id) {\n    // Potential masquerade attempt.\n    return \"mismatch\";\n  }\n\n  return \"verified\";\n}\n\nif (ctx?.event == null) {\n  ctx.event = [:];\n}\n\nctx.event.agent_id_status = verified(ctx, params);",
-        "params": {
-          "trusted_users": [
-            {
-              "username": "elastic/fleet-server",
-              "realm": "_service_account"
-            },
-            {
-              "username": "cloud-internal-agent-server",
-              "realm": "found"
-            },
-            {
-              "username": "elastic",
-              "realm": "reserved"
-            }
-          ]
-        }
-      }
-    },
-    {
-      "remove": {
-        "field": "_security",
-        "ignore_missing": true
-      }
-    }, 
-    { "set":        { "ignore_failure": true,  "field": "event.module", "value": "elastic_agent" } },
-    { "split":      { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "module_temp"  } },
-    { "set":        { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}" } },
-    { "gsub":       { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "pattern": "^[^.]*.", "replacement": "", "target_field": "dataset_tag_temp" } },
-    { "append":     { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp}}"  } },
-    { "set":        { "if": "ctx.network?.direction == 'egress'", "override": true, "field": "network.initiated", "value": "true" } },
-    { "set":        { "if": "ctx.network?.direction == 'ingress'", "override": true, "field": "network.initiated", "value": "false" } },
-    { "set":        { "if": "ctx.network?.type == 'ipv4'", "override": true, "field": "destination.ipv6", "value": "false" } },
-    { "set":        { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } },
-    { "set":        { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.dataset", "value": "import" } },
-    { "set":        { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.namespace", "value": "so" } },
-    { "date":       { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp", "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } },
-    { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true }  },
-    { "set":        { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } },
-    { "remove":     { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp" ], "ignore_missing": true, "ignore_failure": true } } 
-  ],
-  "on_failure": [
-    {
-      "remove": {
-        "field": "_security",
-        "ignore_missing": true,
-        "ignore_failure": true
-      }
-    },
-    {
-      "append": {
-        "field": "error.message",
-        "value": [
-          "failed in Fleet agent final_pipeline: {{ _ingest.on_failure_message }}"
-        ]
-      }
-    }
-  ]
-}

salt/elasticsearch/files/ingest/global@custom

@@ -0,0 +1,27 @@
+{
+  "version": 3,
+  "_meta": {
+    "managed_by": "securityonion",
+    "managed": true
+  },
+  "description": "Custom pipeline for processing all incoming Fleet Agent documents. \n",
+  "processors": [
+    { "set":        { "ignore_failure": true,  "field": "event.module", "value": "elastic_agent" } },
+    { "split":      { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "module_temp"  } },
+    { "set":        { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}" } },
+    { "gsub":       { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "pattern": "^[^.]*.", "replacement": "", "target_field": "dataset_tag_temp" } },
+    { "append":     { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp}}"  } },
+    { "set":        { "if": "ctx.network?.direction == 'egress'", "override": true, "field": "network.initiated", "value": "true" } },
+    { "set":        { "if": "ctx.network?.direction == 'ingress'", "override": true, "field": "network.initiated", "value": "false" } },
+    { "set":        { "if": "ctx.network?.type == 'ipv4'", "override": true, "field": "destination.ipv6", "value": "false" } },
+    { "set":        { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } },
+    { "set":        { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.dataset", "value": "import" } },
+    { "set":        { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.namespace", "value": "so" } },
+    { "date":       { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp","ignore_failure": true, "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSX","yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } },
+    { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true }  },
+    { "set":        { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } },
+    { "rename":        { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } },
+    { "set":        { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } },
+    { "remove":     { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp" ], "ignore_missing": true, "ignore_failure": true } } 
+  ]
+}

salt/elasticsearch/files/ingest/kismet.ad_hoc

@@ -0,0 +1,10 @@
+{
+  "processors": [
+    {
+      "rename": {
+        "field": "message2.kismet_device_base_macaddr",
+        "target_field": "network.wireless.bssid"
+      }
+    }
+  ]
+}

salt/elasticsearch/files/ingest/kismet.ap

@@ -0,0 +1,50 @@
+{
+  "processors": [
+    {
+      "rename": {
+        "field": "message2.dot11_device.dot11_device_last_beaconed_ssid_record.dot11_advertisedssid_cloaked",
+        "target_field": "network.wireless.ssid_cloaked",
+        "if": "ctx?.message2?.dot11_device?.dot11_device_last_beaconed_ssid_record?.dot11_advertisedssid_cloaked != null"
+      }
+    },
+    {
+      "rename": {
+        "field": "message2.dot11_device.dot11_device_last_beaconed_ssid_record.dot11_advertisedssid_ssid",
+        "target_field": "network.wireless.ssid",
+        "if": "ctx?.message2?.dot11_device?.dot11_device_last_beaconed_ssid_record?.dot11_advertisedssid_ssid != null"
+      }
+    },
+    {
+      "set": {
+        "field": "network.wireless.ssid",
+        "value": "Hidden",
+        "if": "ctx?.network?.wireless?.ssid_cloaked != null && ctx?.network?.wireless?.ssid_cloaked == 1"
+      }
+    },
+    {
+      "rename": {
+        "field": "message2.dot11_device.dot11_device_last_beaconed_ssid_record.dot11_advertisedssid_dot11e_channel_utilization_perc",
+        "target_field": "network.wireless.channel_utilization",
+        "if": "ctx?.message2?.dot11_device?.dot11_device_last_beaconed_ssid_record?.dot11_advertisedssid_dot11e_channel_utilization_perc != null"
+      }
+    },
+    {
+      "rename": {
+        "field": "message2.dot11_device.dot11_device_last_bssid",
+        "target_field": "network.wireless.bssid"
+      }
+    },
+    {
+      "foreach": {
+        "field": "message2.dot11_device.dot11_device_associated_client_map",
+        "processor": {
+          "append": {
+            "field": "network.wireless.associated_clients",
+            "value": "{{_ingest._key}}"
+          }
+        },
+        "if": "ctx?.message2?.dot11_device?.dot11_device_associated_client_map != null"
+      }
+    }
+  ]
+}

salt/elasticsearch/files/ingest/kismet.bridged

@@ -0,0 +1,16 @@
+{
+  "processors": [
+    {
+      "rename": {
+        "field": "message2.kismet_device_base_macaddr",
+        "target_field": "client.mac"
+      }
+    },
+    {
+      "rename": {
+        "field": "message2.dot11_device.dot11_device_last_bssid",
+        "target_field": "network.wireless.bssid"
+      }
+    }
+  ]
+}

salt/elasticsearch/files/ingest/kismet.client

@@ -0,0 +1,29 @@
+{
+  "processors": [
+    {
+      "rename": {
+        "field": "message2.kismet_device_base_macaddr",
+        "target_field": "client.mac"
+      }
+    },
+    {
+      "rename": {
+        "field": "message2.dot11_device.dot11_device_last_bssid",
+        "target_field": "network.wireless.last_connected_bssid",
+        "if": "ctx?.message2?.dot11_device?.dot11_device_last_bssid != null"
+      }
+    },
+    {
+      "foreach": {
+        "field": "message2.dot11_device.dot11_device_client_map",
+        "processor": {
+          "append": {
+            "field": "network.wireless.known_connected_bssid",
+            "value": "{{_ingest._key}}"
+          }
+        },
+        "if": "ctx?.message2?.dot11_device?.dot11_device_client_map != null"
+      }
+    }
+  ]
+}

salt/elasticsearch/files/ingest/kismet.common

@@ -0,0 +1,159 @@
+{
+  "processors": [
+    {
+      "json": {
+        "field": "message",
+        "target_field": "message2"
+      }
+    },
+    {
+      "date": {
+        "field": "message2.kismet_device_base_mod_time",
+        "formats": [
+          "epoch_second"
+        ],
+        "target_field": "@timestamp"
+      }
+    },
+    {
+      "set": {
+        "field": "event.category",
+        "value": "network"
+      }
+    },
+    {
+      "dissect": {
+        "field": "message2.kismet_device_base_type",
+        "pattern": "%{wifi} %{device_type}"
+      }
+    },
+    {
+      "lowercase": {
+        "field": "device_type"
+      }
+    },
+    {
+      "set": {
+        "field": "event.dataset",
+        "value": "kismet.{{device_type}}"
+      }
+    },
+    {
+      "set": {
+        "field": "event.dataset",
+        "value": "kismet.wds_ap",
+        "if": "ctx?.device_type == 'wds ap'"
+      }
+    },
+    {
+      "set": {
+        "field": "event.dataset",
+        "value": "kismet.ad_hoc",
+        "if": "ctx?.device_type == 'ad-hoc'"
+      }
+    },
+    {
+      "set": {
+        "field": "event.module",
+        "value": "kismet"
+      }
+    },
+    {
+      "rename": {
+        "field": "message2.kismet_device_base_packets_tx_total",
+        "target_field": "source.packets"
+      }
+    },
+    {
+      "rename": {
+        "field": "message2.kismet_device_base_num_alerts",
+        "target_field": "kismet.alerts.count"
+      }
+    },
+    {
+      "rename": {
+        "field": "message2.kismet_device_base_channel",
+        "target_field": "network.wireless.channel",
+        "if": "ctx?.message2?.kismet_device_base_channel != ''"
+      }
+    },
+    {
+      "rename": {
+        "field": "message2.kismet_device_base_frequency",
+        "target_field": "network.wireless.frequency",
+        "if": "ctx?.message2?.kismet_device_base_frequency != 0"
+      }
+    },
+    {
+      "rename": {
+        "field": "message2.kismet_device_base_last_time",
+        "target_field": "kismet.last_seen"
+      }
+    },
+    {
+      "date": {
+        "field": "kismet.last_seen",
+        "formats": [
+          "epoch_second"
+        ],
+        "target_field": "kismet.last_seen"
+      }
+    },
+    {
+      "rename": {
+        "field": "message2.kismet_device_base_first_time",
+        "target_field": "kismet.first_seen"
+      }
+    },
+    {
+      "date": {
+        "field": "kismet.first_seen",
+        "formats": [
+          "epoch_second"
+        ],
+        "target_field": "kismet.first_seen"
+      }
+    },
+    {
+      "rename": {
+        "field": "message2.kismet_device_base_seenby",
+        "target_field": "kismet.seenby"
+      }
+    },
+    {
+      "foreach": {
+        "field": "kismet.seenby",
+        "processor": {
+          "pipeline": {
+            "name": "kismet.seenby"
+          }
+        }
+      }
+    },
+    {
+      "rename": {
+        "field": "message2.kismet_device_base_manuf",
+        "target_field": "device.manufacturer"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "{{event.dataset}}"
+      }
+    },
+    {
+      "remove": {
+        "field": [
+          "message2",
+          "message",
+          "device_type",
+          "wifi",
+          "agent",
+          "host",
+          "event.created"
+        ],
+        "ignore_failure": true
+      }
+    }
+  ]
+}

salt/elasticsearch/files/ingest/kismet.device

@@ -0,0 +1,9 @@
+{
+  "processors": [
+    {
+      "pipeline": {
+        "name": "kismet.client"
+      }
+    }
+  ]
+}

salt/elasticsearch/files/ingest/kismet.seenby

@@ -0,0 +1,52 @@
+{
+  "processors": [
+    {
+      "rename": {
+        "field": "_ingest._value.kismet_common_seenby_num_packets",
+        "target_field": "_ingest._value.packets_seen",
+        "ignore_missing": true
+      }
+    },
+    {
+      "rename": {
+        "field": "_ingest._value.kismet_common_seenby_uuid",
+        "target_field": "_ingest._value.serial_number",
+        "ignore_missing": true
+      }
+    },
+    {
+      "rename": {
+        "field": "_ingest._value.kismet_common_seenby_first_time",
+        "target_field": "_ingest._value.first_seen",
+        "ignore_missing": true
+      }
+    },
+    {
+      "rename": {
+        "field": "_ingest._value.kismet_common_seenby_last_time",
+        "target_field": "_ingest._value.last_seen",
+        "ignore_missing": true
+      }
+    },
+    {
+      "date": {
+        "field": "_ingest._value.first_seen",
+        "formats": [
+          "epoch_second"
+        ],
+        "target_field": "_ingest._value.first_seen",
+        "ignore_failure": true
+      }
+    },
+    {
+      "date": {
+        "field": "_ingest._value.last_seen",
+        "formats": [
+          "epoch_second"
+        ],
+        "target_field": "_ingest._value.last_seen",
+        "ignore_failure": true
+      }
+    }
+  ]
+}

salt/elasticsearch/files/ingest/kismet.wds

@@ -0,0 +1,10 @@
+{
+  "processors": [
+    {
+      "rename": {
+        "field": "message2.kismet_device_base_macaddr",
+        "target_field": "client.mac"
+      }
+    }
+  ]
+}

salt/elasticsearch/files/ingest/kismet.wds_ap

@@ -0,0 +1,22 @@
+{
+  "processors": [
+    {
+      "rename": {
+        "field": "message2.kismet_device_base_commonname",
+        "target_field": "network.wireless.bssid"
+      }
+    },
+    {
+      "foreach": {
+        "field": "message2.dot11_device.dot11_device_associated_client_map",
+        "processor": {
+          "append": {
+            "field": "network.wireless.associated_clients",
+            "value": "{{_ingest._key}}"
+          }
+        },
+        "if": "ctx?.message2?.dot11_device?.dot11_device_associated_client_map != null"
+      }
+    }
+  ]
+}

salt/elasticsearch/files/ingest/strelka.file

@@ -56,6 +56,7 @@
     { "set":         { "if": "ctx.exiftool?.Subsystem != null", "field": "host.subsystem",        "value": "{{exiftool.Subsystem}}", "ignore_failure": true }},
     { "set":         { "if": "ctx.scan?.yara?.matches instanceof List", "field": "rule.name",        "value": "{{scan.yara.matches.0}}" }},
     { "set":         { "if": "ctx.rule?.name != null", "field": "event.dataset",        "value": "alert", "override": true }},
+    { "set":         { "if": "ctx.rule?.name != null", "field": "rule.uuid",        "value": "{{rule.name}}", "override": true }},
     { "rename":         { "field": "file.flavors.mime", "target_field": "file.mime_type",        "ignore_missing": true }},
     { "set":         { "if": "ctx.rule?.name != null && ctx.rule?.score == null",   "field": "event.severity", "value": 3, "override": true }  },
     { "convert" :    { "if": "ctx.rule?.score != null", "field" : "rule.score","type": "integer"}},

salt/elasticsearch/files/ingest/suricata.alert

@@ -1,6 +1,7 @@
 {
   "description" : "suricata.alert",
   "processors" : [
+    { "set":   { "field": "_index", "value": "logs-suricata.alerts-so" } },
     { "set":   { "field": "tags","value": "alert" }},
     { "rename":{ "field": "message2.alert",	"target_field": "rule",	"ignore_failure": true 	} },
     { "rename":{ "field": "rule.signature",     "target_field": "rule.name", "ignore_failure": true  } },

salt/elasticsearch/roles/analyst.json

@@ -27,7 +27,8 @@
         "monitor",
         "read",
         "read_cross_cluster",
-        "view_index_metadata"
+        "view_index_metadata",
+        "write"
       ]
     }
   ],

salt/elasticsearch/roles/limited-analyst.json

@@ -13,7 +13,8 @@
         "monitor",
         "read",
         "read_cross_cluster",
-        "view_index_metadata"
+        "view_index_metadata",
+        "write"
       ]
     }
   ],

salt/elasticsearch/soc_elasticsearch.yaml

@@ -5,6 +5,10 @@ elasticsearch:
   esheap:
     description: Specify the memory heap size in (m)egabytes for Elasticsearch.
     helpLink: elasticsearch.html
+  index_clean:
+    description: Determines if indices should be considered for deletion by available disk space in the cluster. Otherwise, indices will only be deleted by the age defined in the ILM settings.
+    forcedType: bool
+    helpLink: elasticsearch.html
   retention: 
     retention_pct:
       decription: Total percentage of space used by Elasticsearch for multi node clusters
@@ -98,10 +102,6 @@ elasticsearch:
       policy:
         phases:
           hot:
-            max_age:
-              description: Maximum age of index. ex. 7d - This determines when the index should be moved out of the hot tier.
-              global: True
-              helpLink: elasticsearch.html
             actions:
               set_priority:
                 priority:
@@ -120,7 +120,9 @@ elasticsearch:
                   helpLink: elasticsearch.html
           cold:
             min_age:
-              description: Minimum age of index. ex. 30d - This determines when the index should be moved to the cold tier.  While still searchable, this tier is typically optimized for lower storage costs rather than search speed.
+              description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier.  While still searchable, this tier is typically optimized for lower storage costs rather than search speed.
+              regex: ^[0-9]{1,5}d$
+              forcedType: string
               global: True
               helpLink: elasticsearch.html
             actions:
@@ -131,8 +133,8 @@ elasticsearch:
                   helpLink: elasticsearch.html
           warm:
             min_age: 
-              description: Minimum age of index. ex. 30d - This determines when the index should be moved to the cold tier.  While still searchable, this tier is typically optimized for lower storage costs rather than search speed.
+              description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier.
-              regex: ^\[0-9\]{1,5}d$
+              regex: ^[0-9]{1,5}d$
               forcedType: string
               global: True
             actions:
@@ -145,6 +147,8 @@ elasticsearch:
           delete:
             min_age:
               description: Minimum age of index. ex. 90d - This determines when the index should be deleted.
+              regex: ^[0-9]{1,5}d$
+              forcedType: string
               global: True
               helpLink: elasticsearch.html
     so-logs: &indexSettings
@@ -271,7 +275,9 @@ elasticsearch:
                   helpLink: elasticsearch.html
           warm:
             min_age:
-              description: Minimum age of index. This determines when the index should be moved to the hot tier.
+              description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier.
+              regex: ^[0-9]{1,5}d$
+              forcedType: string
               global: True
               advanced: True
               helpLink: elasticsearch.html
@@ -296,7 +302,9 @@ elasticsearch:
                   helpLink: elasticsearch.html
           cold:
             min_age:
-              description: Minimum age of index.  This determines when the index should be moved to the cold tier.  While still searchable, this tier is typically optimized for lower storage costs rather than search speed.
+              description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier.  While still searchable, this tier is typically optimized for lower storage costs rather than search speed.
+              regex: ^[0-9]{1,5}d$
+              forcedType: string
               global: True
               advanced: True
               helpLink: elasticsearch.html
@@ -311,6 +319,8 @@ elasticsearch:
           delete:
             min_age:
               description: Minimum age of index. This determines when the index should be deleted.
+              regex: ^[0-9]{1,5}d$
+              forcedType: string
               global: True
               advanced: True
               helpLink: elasticsearch.html
@@ -366,6 +376,7 @@ elasticsearch:
     so-logs-azure_x_signinlogs: *indexSettings
     so-logs-azure_x_springcloudlogs: *indexSettings
     so-logs-barracuda_x_waf: *indexSettings
+    so-logs-cef_x_log: *indexSettings
     so-logs-cisco_asa_x_log: *indexSettings
     so-logs-cisco_ftd_x_log: *indexSettings
     so-logs-cisco_ios_x_log: *indexSettings
@@ -383,6 +394,7 @@ elasticsearch:
     so-logs-darktrace_x_ai_analyst_alert: *indexSettings
     so-logs-darktrace_x_model_breach_alert: *indexSettings
     so-logs-darktrace_x_system_status_alert: *indexSettings
+    so-logs-detections_x_alerts: *indexSettings
     so-logs-f5_bigip_x_log: *indexSettings
     so-logs-fim_x_event: *indexSettings
     so-logs-fortinet_x_clientendpoint: *indexSettings
@@ -454,6 +466,13 @@ elasticsearch:
     so-logs-sonicwall_firewall_x_log: *indexSettings
     so-logs-snort_x_log: *indexSettings
     so-logs-symantec_endpoint_x_log: *indexSettings
+    so-logs-tenable_io_x_asset: *indexSettings
+    so-logs-tenable_io_x_plugin: *indexSettings
+    so-logs-tenable_io_x_scan: *indexSettings
+    so-logs-tenable_io_x_vulnerability: *indexSettings
+    so-logs-tenable_sc_x_asset: *indexSettings
+    so-logs-tenable_sc_x_plugin: *indexSettings
+    so-logs-tenable_sc_x_vulnerability: *indexSettings
     so-logs-ti_abusech_x_malware: *indexSettings
     so-logs-ti_abusech_x_malwarebazaar: *indexSettings
     so-logs-ti_abusech_x_threatfox: *indexSettings
@@ -509,13 +528,67 @@ elasticsearch:
     so-endgame: *indexSettings
     so-idh: *indexSettings
     so-suricata: *indexSettings
+    so-suricata_x_alerts: *indexSettings
     so-import: *indexSettings
     so-kratos: *indexSettings
+    so-kismet: *indexSettings
     so-logstash: *indexSettings
     so-redis: *indexSettings
     so-strelka: *indexSettings
     so-syslog: *indexSettings
     so-zeek: *indexSettings
+    so-metrics-fleet_server_x_agent_status: &fleetMetricsSettings
+      index_sorting:
+        description: Sorts the index by event time, at the cost of additional processing resource consumption.
+        advanced: True
+        readonly: True
+        helpLink: elasticsearch.html
+      index_template:
+        ignore_missing_component_templates:
+          description: Ignore component templates if they aren't in Elasticsearch.
+          advanced: True
+          readonly: True
+          helpLink: elasticsearch.html
+        index_patterns:
+          description: Patterns for matching multiple indices or tables.
+          advanced: True
+          readonly: True
+          helpLink: elasticsearch.html
+        template:
+          settings:
+            index:
+              mode:
+                description: Type of mode used for this index. Time series indices can be used for metrics to reduce necessary storage.
+                advanced: True
+                readonly: True
+                helpLink: elasticsearch.html
+              number_of_replicas:
+                description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs.
+                advanced: True
+                readonly: True
+                helpLink: elasticsearch.html
+        composed_of:
+          description: The index template is composed of these component templates.
+          advanced: True
+          readonly: True
+          helpLink: elasticsearch.html
+        priority:
+          description: The priority of the index template.
+          advanced: True
+          readonly: True
+          helpLink: elasticsearch.html
+        data_stream:
+          hidden:
+            description: Hide the data stream.
+            advanced: True
+            readonly: True
+            helpLink: elasticsearch.html
+          allow_custom_routing:
+            description: Allow custom routing for the data stream.
+            advanced: True
+            readonly: True
+            helpLink: elasticsearch.html
+    so-metrics-fleet_server_x_agent_versions: *fleetMetricsSettings
   so_roles:
     so-manager: &soroleSettings
       config:

salt/elasticsearch/template.map.jinja

@@ -1,12 +1,15 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+
 {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
 {% set DEFAULT_GLOBAL_OVERRIDES = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings.pop('global_overrides') %}
 
 {% set PILLAR_GLOBAL_OVERRIDES = {} %}
-{% if salt['pillar.get']('elasticsearch:index_settings') is defined %}
+{% set ES_INDEX_PILLAR = salt['pillar.get']('elasticsearch:index_settings', {}) %}
-{%   set ES_INDEX_PILLAR = salt['pillar.get']('elasticsearch:index_settings') %}
+{% if ES_INDEX_PILLAR.global_overrides is defined %}
-{%   if ES_INDEX_PILLAR.global_overrides is defined %}
+{%   set PILLAR_GLOBAL_OVERRIDES = ES_INDEX_PILLAR.pop('global_overrides') %}
-{%     set PILLAR_GLOBAL_OVERRIDES = ES_INDEX_PILLAR.pop('global_overrides') %}
-{%   endif %}
 {% endif %}
 
 {% set ES_INDEX_SETTINGS_ORIG = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings %}
@@ -19,6 +22,28 @@
 {% set ES_INDEX_SETTINGS = {} %}
 {% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update(salt['defaults.merge'](ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %}
 {% for index, settings in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.items() %}
+
+{#   prevent this action from being performed on custom defined indices. #}
+{#   the custom defined index is not present in either of the dictionaries and fails to reder. #}
+{%   if index in ES_INDEX_SETTINGS_ORIG and index in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES %}
+
+{#     dont merge policy from the global_overrides if policy isn't defined in the original index settingss #}
+{#     this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #}
+{%     if not ES_INDEX_SETTINGS_ORIG[index].policy is defined and ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %}
+{%       do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %}
+{%     endif %}
+
+{#     this prevents and index from inderiting a policy phase from global overrides if it wasnt defined in the defaults. #}
+{%     if ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %}
+{%       for phase in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.copy() %}
+{%         if ES_INDEX_SETTINGS_ORIG[index].policy.phases[phase] is not defined %}
+{%           do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.pop(phase) %}
+{%         endif %}
+{%       endfor %}
+{%     endif %}
+
+{%   endif %}
+
 {%   if settings.index_template is defined %}
 {%     if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %}
 {%       do settings.index_template.template.settings.index.pop('sort') %}

salt/elasticsearch/templates/component/ecs/device.json

@@ -0,0 +1,36 @@
+{
+  "_meta": {
+    "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-device.html",
+    "ecs_version": "1.12.2"
+  },
+  "template": {
+    "mappings": {
+      "properties": {
+        "device": {
+          "properties": {
+            "id": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "manufacturer": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "model": {
+              "properties": {
+                "identifier": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "name": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/templates/component/ecs/kismet.json

@@ -0,0 +1,32 @@
+{
+  "_meta": {
+    "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-base.html",
+    "ecs_version": "1.12.2"
+  },
+  "template": {
+    "mappings": {
+      "properties": {
+        "kismet": {
+          "properties": {
+            "alerts": {
+              "properties": {
+                "count": {
+                  "type": "long"
+                }
+              }
+            },
+            "first_seen": {
+              "type": "date"
+            },
+            "last_seen": {
+              "type": "date"
+            },
+            "seenby": {
+              "type": "nested"
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/templates/component/ecs/network.json

@@ -77,6 +77,43 @@
                   "type": "keyword"
                 }
               }
+            },
+            "wireless": {
+              "properties": {
+                "associated_clients": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "bssid": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "channel": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "channel_utilization": {
+                  "type": "float"
+                },
+                "frequency": {
+                  "type": "double"
+                },
+                "ssid": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "ssid_cloaked": {
+                  "type": "integer"
+                },
+                "known_connected_bssid": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "last_connected_bssid": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                }
+              }
             }
           }
         }

salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@package.json

@@ -6,7 +6,7 @@
           "name": "logs"
         },
         "codec": "best_compression",
-        "default_pipeline": "logs-elastic_agent-1.13.1",
+        "default_pipeline": "logs-elastic_agent-1.20.0",
         "mapping": {
           "total_fields": {
             "limit": "10000"

salt/elasticsearch/templates/component/elastic-agent/metrics-fleet_server.agent_status@package.json

@@ -0,0 +1,201 @@
+{
+  "template": {
+    "settings": {
+      "index": {
+        "lifecycle": {
+          "name": "metrics"
+        },
+        "default_pipeline": "metrics-fleet_server.agent_status-1.5.0",
+        "mapping": {
+          "total_fields": {
+            "limit": "1000"
+          }
+        }
+      }
+    },
+    "mappings": {
+      "dynamic": false,
+      "_source": {
+        "mode": "synthetic"
+      },
+      "properties": {
+        "cluster": {
+          "properties": {
+            "id": {
+              "time_series_dimension": true,
+              "type": "keyword"
+            }
+          }
+        },
+        "fleet": {
+          "properties": {
+            "agents": {
+              "properties": {
+                "offline": {
+                  "time_series_metric": "gauge",
+                  "meta": {},
+                  "type": "long"
+                },
+                "total": {
+                  "time_series_metric": "gauge",
+                  "meta": {},
+                  "type": "long"
+                },
+                "updating": {
+                  "time_series_metric": "gauge",
+                  "meta": {},
+                  "type": "long"
+                },
+                "inactive": {
+                  "time_series_metric": "gauge",
+                  "meta": {},
+                  "type": "long"
+                },
+                "healthy": {
+                  "time_series_metric": "gauge",
+                  "meta": {},
+                  "type": "long"
+                },
+                "unhealthy": {
+                  "time_series_metric": "gauge",
+                  "meta": {},
+                  "type": "long"
+                },
+                "unenrolled": {
+                  "time_series_metric": "gauge",
+                  "meta": {},
+                  "type": "long"
+                },
+                "enrolled": {
+                  "time_series_metric": "gauge",
+                  "meta": {},
+                  "type": "long"
+                },
+                "unhealthy_reason": {
+                  "properties": {
+                    "output": {
+                      "time_series_metric": "gauge",
+                      "meta": {},
+                      "type": "long"
+                    },
+                    "input": {
+                      "time_series_metric": "gauge",
+                      "meta": {},
+                      "type": "long"
+                    },
+                    "other": {
+                      "time_series_metric": "gauge",
+                      "meta": {},
+                      "type": "long"
+                    }
+                  }
+                },
+                "upgrading_step": {
+                  "properties": {
+                    "rollback": {
+                      "time_series_metric": "gauge",
+                      "meta": {},
+                      "type": "long"
+                    },
+                    "requested": {
+                      "time_series_metric": "gauge",
+                      "meta": {},
+                      "type": "long"
+                    },
+                    "restarting": {
+                      "time_series_metric": "gauge",
+                      "meta": {},
+                      "type": "long"
+                    },
+                    "downloading": {
+                      "time_series_metric": "gauge",
+                      "meta": {},
+                      "type": "long"
+                    },
+                    "scheduled": {
+                      "time_series_metric": "gauge",
+                      "meta": {},
+                      "type": "long"
+                    },
+                    "extracting": {
+                      "time_series_metric": "gauge",
+                      "meta": {},
+                      "type": "long"
+                    },
+                    "replacing": {
+                      "time_series_metric": "gauge",
+                      "meta": {},
+                      "type": "long"
+                    },
+                    "failed": {
+                      "time_series_metric": "gauge",
+                      "meta": {},
+                      "type": "long"
+                    },
+                    "watching": {
+                      "time_series_metric": "gauge",
+                      "meta": {},
+                      "type": "long"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
+        "agent": {
+          "properties": {
+            "id": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "type": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "version": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            }
+          }
+        },
+        "@timestamp": {
+          "ignore_malformed": false,
+          "type": "date"
+        },
+        "data_stream": {
+          "properties": {
+            "namespace": {
+              "type": "constant_keyword"
+            },
+            "type": {
+              "type": "constant_keyword"
+            },
+            "dataset": {
+              "type": "constant_keyword"
+            }
+          }
+        },
+        "kibana": {
+          "properties": {
+            "uuid": {
+              "path": "agent.id",
+              "type": "alias"
+            },
+            "version": {
+              "path": "agent.version",
+              "type": "alias"
+            }
+          }
+        }
+      }
+    }
+  },
+  "_meta": {
+    "package": {
+      "name": "fleet_server"
+    },
+    "managed_by": "fleet",
+    "managed": true
+  }
+}

salt/elasticsearch/templates/component/elastic-agent/metrics-fleet_server.agent_versions@package.json

@@ -0,0 +1,102 @@
+{
+  "template": {
+    "settings": {
+      "index": {
+        "lifecycle": {
+          "name": "metrics"
+        },
+        "default_pipeline": "metrics-fleet_server.agent_versions-1.5.0",
+        "mapping": {
+          "total_fields": {
+            "limit": "1000"
+          }
+        }
+      }
+    },
+    "mappings": {
+      "dynamic": false,
+      "_source": {
+        "mode": "synthetic"
+      },
+      "properties": {
+        "cluster": {
+          "properties": {
+            "id": {
+              "time_series_dimension": true,
+              "type": "keyword"
+            }
+          }
+        },
+        "fleet": {
+          "properties": {
+            "agent": {
+              "properties": {
+                "count": {
+                  "time_series_metric": "gauge",
+                  "meta": {},
+                  "type": "long"
+                },
+                "version": {
+                  "time_series_dimension": true,
+                  "type": "keyword"
+                }
+              }
+            }
+          }
+        },
+        "agent": {
+          "properties": {
+            "id": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "type": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "version": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            }
+          }
+        },
+        "@timestamp": {
+          "ignore_malformed": false,
+          "type": "date"
+        },
+        "data_stream": {
+          "properties": {
+            "namespace": {
+              "type": "constant_keyword"
+            },
+            "type": {
+              "type": "constant_keyword"
+            },
+            "dataset": {
+              "type": "constant_keyword"
+            }
+          }
+        },
+        "kibana": {
+          "properties": {
+            "uuid": {
+              "path": "agent.id",
+              "type": "alias"
+            },
+            "version": {
+              "path": "agent.version",
+              "type": "alias"
+            }
+          }
+        }
+      }
+    }
+  },
+  "_meta": {
+    "package": {
+      "name": "fleet_server"
+    },
+    "managed_by": "fleet",
+    "managed": true
+  }
+}

salt/elasticsearch/templates/component/elastic-agent/so-items-mappings.json

@@ -0,0 +1,112 @@
+{
+  "template": {
+    "mappings": {
+      "dynamic": "strict",
+      "properties": {
+        "binary": {
+          "type": "binary"
+        },
+        "boolean": {
+          "type": "boolean"
+        },
+        "byte": {
+          "type": "byte"
+        },
+        "created_at": {
+          "type": "date"
+        },
+        "created_by": {
+          "type": "keyword"
+        },
+        "date": {
+          "type": "date"
+        },
+        "date_nanos": {
+          "type": "date_nanos"
+        },
+        "date_range": {
+          "type": "date_range"
+        },
+        "deserializer": {
+          "type": "keyword"
+        },
+        "double": {
+          "type": "double"
+        },
+        "double_range": {
+          "type": "double_range"
+        },
+        "float": {
+          "type": "float"
+        },
+        "float_range": {
+          "type": "float_range"
+        },
+        "geo_point": {
+          "type": "geo_point"
+        },
+        "geo_shape": {
+          "type": "geo_shape"
+        },
+        "half_float": {
+          "type": "half_float"
+        },
+        "integer": {
+          "type": "integer"
+        },
+        "integer_range": {
+          "type": "integer_range"
+        },
+        "ip": {
+          "type": "ip"
+        },
+        "ip_range": {
+          "type": "ip_range"
+        },
+        "keyword": {
+          "type": "keyword"
+        },
+        "list_id": {
+          "type": "keyword"
+        },
+        "long": {
+          "type": "long"
+        },
+        "long_range": {
+          "type": "long_range"
+        },
+        "meta": {
+          "type": "object",
+          "enabled": false
+        },
+        "serializer": {
+          "type": "keyword"
+        },
+        "shape": {
+          "type": "shape"
+        },
+        "short": {
+          "type": "short"
+        },
+        "text": {
+          "type": "text"
+        },
+        "tie_breaker_id": {
+          "type": "keyword"
+        },
+        "updated_at": {
+          "type": "date"
+        },
+        "updated_by": {
+          "type": "keyword"
+        }
+      }
+    },
+    "aliases": {}
+  },
+  "version": 2,
+    "_meta": {
+      "managed": true,
+      "description": "default mappings for the .items index template installed by Kibana/Security"
+    }
+}

salt/elasticsearch/templates/component/elastic-agent/so-lists-mappings.json

@@ -0,0 +1,55 @@
+{
+  "template": {
+  "mappings": {
+    "dynamic": "strict",
+    "properties": {
+      "created_at": {
+        "type": "date"
+      },
+      "created_by": {
+        "type": "keyword"
+      },
+      "description": {
+        "type": "keyword"
+      },
+      "deserializer": {
+        "type": "keyword"
+      },
+      "immutable": {
+        "type": "boolean"
+      },
+      "meta": {
+        "type": "object",
+        "enabled": false
+      },
+      "name": {
+        "type": "keyword"
+      },
+      "serializer": {
+        "type": "keyword"
+      },
+      "tie_breaker_id": {
+        "type": "keyword"
+      },
+      "type": {
+        "type": "keyword"
+      },
+      "updated_at": {
+        "type": "date"
+      },
+      "updated_by": {
+        "type": "keyword"
+      },
+      "version": {
+        "type": "keyword"
+      }
+    }
+  },
+    "aliases": {}
+  },
+  "version": 2,
+    "_meta": {
+      "managed": true,
+      "description": "default mappings for the .lists index template installed by Kibana/Security"
+    }
+}

salt/elasticsearch/templates/component/so/detection-mappings.json

@@ -20,21 +20,36 @@
         "so_detection": {
           "properties": {
             "publicId": {
-              "type": "text"
+              "ignore_above": 1024,
+	      "type": "keyword"
             },
             "title": {
-              "type": "text"
+	      "ignore_above": 1024,
+              "type": "keyword"
             },
             "severity": {
               "ignore_above": 1024,
               "type": "keyword"
             },
             "author": {
-              "type": "text"
+              "ignore_above": 1024,
+              "type": "keyword"
             },
             "description": {
               "type": "text"
             },
+	    "category": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+	    "product": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+	    "service": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
             "content": {
               "type": "text"
             },
@@ -48,7 +63,8 @@
               "type": "boolean"
             },
             "tags": {
-              "type": "text"
+              "ignore_above": 1024,
+	      "type": "keyword"
             },
             "ruleset": {
               "ignore_above": 1024,
@@ -135,4 +151,4 @@
   "_meta": {
     "ecs_version": "1.12.2"
   }
-}
+}

salt/elasticsearch/templates/component/so/so-system-mappings.json

@@ -0,0 +1,29 @@
+{
+  "template": {
+    "mappings": {
+      "properties": {
+        "host": {
+	  "properties":{
+            "ip": {
+              "type": "ip"
+            }
+	  }
+	},
+	"related": {
+          "properties":{
+            "ip": {
+              "type": "ip"
+            }
+          }
+        },
+	"source": {
+          "properties":{
+            "ip": {
+              "type": "ip"
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines

@@ -20,7 +20,7 @@ if [ ! -f /opt/so/state/espipelines.txt ]; then
 
   cd ${ELASTICSEARCH_INGEST_PIPELINES}
   echo "Loading pipelines..."
-  for i in .[a-z]* *; 
+  for i in *; 
   do 
     echo $i;
     retry 5 5 "so-elasticsearch-query _ingest/pipeline/$i -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load pipeline: $i"

salt/elasticsearch/tools/sbin/so-index-list

@@ -5,6 +5,6 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
+. /usr/sbin/so-common
 
-
+curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L "https://localhost:9200/_cat/indices?pretty&v&s=index"
-curl -K /opt/so/conf/elasticsearch/curl.config-X GET -k -L "https://localhost:9200/_cat/indices?v&s=index"

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-settings

@@ -4,7 +4,7 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 {%- from 'vars/globals.map.jinja' import GLOBALS %}
-{%- set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
+{%- set node_data = salt['pillar.get']('elasticsearch:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
 
 . /usr/sbin/so-common
 

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-total

@@ -40,9 +40,9 @@ fi
 
 # Iterate through the output of _cat/allocation for each node in the cluster to determine the total available space
 {% if GLOBALS.role == 'so-manager' %}
-for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | grep -v {{ GLOBALS.manager }} | awk '{print $5}'); do
+for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | grep -v "{{ GLOBALS.manager }}$" | awk '{print $8}'); do
 {% else %}
-for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | awk '{print $5}'); do
+for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | awk '{print $8}'); do
 {% endif %}
   size=$(echo $i | grep -oE '[0-9].*' | awk '{print int($1+0.5)}')
   unit=$(echo $i | grep -oE '[A-Za-z]+')

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-used

@@ -13,10 +13,10 @@ TOTAL_USED_SPACE=0
 # Iterate through the output of _cat/allocation for each node in the cluster to determine the total used space
 {% if GLOBALS.role == 'so-manager' %}
 # Get total disk space - disk.total
-for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | grep -v {{ GLOBALS.manager }} | awk '{print $3}'); do
+for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | grep -v "{{ GLOBALS.manager }}$" | awk '{print $6}'); do
 {% else %}
 # Get disk space taken up by indices - disk.indices
-for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | awk '{print $2}'); do
+for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | awk '{print $5}'); do
 {% endif %}
   size=$(echo $i | grep -oE '[0-9].*' | awk '{print int($1+0.5)}')
   unit=$(echo $i | grep -oE '[A-Za-z]+')

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load

@@ -10,10 +10,26 @@
 
 {%- for index, settings in ES_INDEX_SETTINGS.items() %}
 {%-   if settings.policy is defined %}
-echo
+{%-     if index == 'so-logs-detections.alerts' %}
-echo "Setting up {{ index }}-logs policy..."
+  echo
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
+  echo "Setting up so-logs-detections.alerts-so policy..."
-echo
+  curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-so" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
+  echo
+{%-     elif index == 'so-logs-soc' %}
+  echo
+  echo "Setting up so-soc-logs policy..."
+  curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/so-soc-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
+  echo
+  echo
+  echo "Setting up {{ index }}-logs policy..."
+  curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
+  echo
+{%-     else %}
+  echo
+  echo "Setting up {{ index }}-logs policy..."
+  curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
+  echo
+{%-     endif %}
 {%-   endif %}
 {%- endfor %}
 echo

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete

@@ -27,6 +27,7 @@ overlimit() {
 # 2. Check if the maximum number of iterations - MAX_ITERATIONS - has been exceeded. If so, exit.
 # Closed indices will be deleted first. If we are able to bring disk space under LOG_SIZE_LIMIT, or the number of iterations has exceeded the maximum allowed number of iterations, we will break out of the loop.
 
+
 while overlimit && [[ $ITERATION -lt $MAX_ITERATIONS ]]; do
 
   # If we can't query Elasticsearch, then immediately return false.
@@ -34,28 +35,36 @@ while overlimit && [[ $ITERATION -lt $MAX_ITERATIONS ]]; do
   [ $? -eq 1 ] && echo "$(date) - Could not query Elasticsearch." >> ${LOG} && exit
 
   # We iterate through the closed and open indices
-  CLOSED_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'close$' | awk '{print $1}' | grep -vE "playbook|so-case" | grep -E "(logstash-|so-|.ds-logs-)" | sort -t- -k3)
+  CLOSED_SO_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'close$' | awk '{print $1}' | grep -E "(^logstash-.*|^so-.*)" | grep -vE "so-case|so-detection" | sort -t- -k3)
-  OPEN_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'open$' | awk '{print $1}' | grep -vE "playbook|so-case" | grep -E "(logstash-|so-|.ds-logs-)" | sort -t- -k3)
+  CLOSED_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'close$' | awk '{print $1}' | grep -E "^.ds-logs-.*" | grep -v "suricata" | sort -t- -k4)
+  OPEN_SO_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'open$' | awk '{print $1}' | grep -E "(^logstash-.*|^so-.*)" | grep -vE "so-case|so-detection" | sort -t- -k3)
+  OPEN_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'open$' | awk '{print $1}' | grep -E "^.ds-logs-.*" | grep -v "suricata" | sort -t- -k4)
 
-  for INDEX in ${CLOSED_INDICES} ${OPEN_INDICES}; do
+  for INDEX in ${CLOSED_SO_INDICES} ${OPEN_SO_INDICES} ${CLOSED_INDICES} ${OPEN_INDICES}; do
-    # Now that we've sorted the indices from oldest to newest, we need to check each index to see if it is assigned as the current write index for a data stream
+    # Check if index is an older index. If it is an older index, delete it before moving on to newer indices.
-    # To do so, we need to identify to which data stream this index is associated
+    if [[ "$INDEX" =~ "^logstash-.*|so-.*" ]]; then
-    # We extract the data stream name using the pattern below
+      printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - Deleting ${INDEX} index...\n" >> ${LOG}
-    DATASTREAM_PATTERN="logs-[a-zA-Z_.]+-[a-zA-Z_.]+"
+      /usr/sbin/so-elasticsearch-query ${INDEX} -XDELETE >> ${LOG} 2>&1
-    DATASTREAM=$(echo "${INDEX}" | grep -oE "$DATASTREAM_PATTERN")
-    # We look up the data stream, and determine the write index. If there is only one backing index, we delete the entire data stream
-    BACKING_INDICES=$(/usr/sbin/so-elasticsearch-query _data_stream/${DATASTREAM} | jq -r '.data_streams[0].indices | length')
-    if [ "$BACKING_INDICES" -gt 1 ]; then
-      CURRENT_WRITE_INDEX=$(/usr/sbin/so-elasticsearch-query _data_stream/$DATASTREAM | jq -r .data_streams[0].indices[-1].index_name)
-      # We make sure we are not trying to delete a write index
-      if [ "${INDEX}" != "${CURRENT_WRITE_INDEX}" ]; then
-        # This should not be a write index, so we should be allowed to delete it
-        printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - Deleting ${INDEX} index...\n" >> ${LOG}
-	/usr/sbin/so-elasticsearch-query ${INDEX} -XDELETE >> ${LOG} 2>&1
-      fi
     else
-	    printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - There is only one backing index (${INDEX}).  Deleting ${DATASTREAM} data stream...\n" >> ${LOG}
+      # Now that we've sorted the indices from oldest to newest, we need to check each index to see if it is assigned as the current write index for a data stream
+      # To do so, we need to identify to which data stream this index is associated
+      # We extract the data stream name using the pattern below
+      DATASTREAM_PATTERN="logs-[a-zA-Z_.]+-[a-zA-Z_.]+"
+      DATASTREAM=$(echo "${INDEX}" | grep -oE "$DATASTREAM_PATTERN")
+      # We look up the data stream, and determine the write index. If there is only one backing index, we delete the entire data stream
+      BACKING_INDICES=$(/usr/sbin/so-elasticsearch-query _data_stream/${DATASTREAM} | jq -r '.data_streams[0].indices | length')
+      if [ "$BACKING_INDICES" -gt 1 ]; then
+        CURRENT_WRITE_INDEX=$(/usr/sbin/so-elasticsearch-query _data_stream/$DATASTREAM | jq -r .data_streams[0].indices[-1].index_name)
+        # We make sure we are not trying to delete a write index
+        if [ "${INDEX}" != "${CURRENT_WRITE_INDEX}" ]; then
+          # This should not be a write index, so we should be allowed to delete it
+          printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - Deleting ${INDEX} index...\n" >> ${LOG}
+          /usr/sbin/so-elasticsearch-query ${INDEX} -XDELETE >> ${LOG} 2>&1
+        fi
+      else
+            printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - There is only one backing index (${INDEX}).  Deleting ${DATASTREAM} data stream...\n" >> ${LOG}
       /usr/sbin/so-elasticsearch-query _data_stream/$DATASTREAM -XDELETE >> ${LOG} 2>&1
+      fi
     fi
     if ! overlimit ; then
       exit

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load

@@ -5,7 +5,6 @@
 # Elastic License 2.0.
 {%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
 {%  from 'vars/globals.map.jinja' import GLOBALS %}
-{%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %}
 
 STATE_FILE_INITIAL=/opt/so/state/estemplates_initial_load_attempt.txt
 STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt
@@ -68,9 +67,9 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then
     echo -n "Waiting for ElasticSearch..."
     retry 240 1 "so-elasticsearch-query / -k --output /dev/null --silent --head --fail" || fail "Connection attempt timed out.  Unable to connect to ElasticSearch.  \nPlease try: \n  -checking log(s) in /var/log/elasticsearch/\n  -running 'sudo docker ps' \n  -running 'sudo so-elastic-restart'"
     {% if GLOBALS.role != 'so-heavynode' %}
-    SESSIONCOOKIE=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
+    TEMPLATE="logs-endpoint.alerts@package"
-    INSTALLED=$(elastic_fleet_package_is_installed {{ SUPPORTED_PACKAGES[0] }} )
+    INSTALLED=$(so-elasticsearch-query _component_template/$TEMPLATE | jq -r .component_templates[0].name)
-    if [ "$INSTALLED" != "installed" ]; then
+    if [ "$INSTALLED" != "$TEMPLATE" ]; then
       echo
       echo "Packages not yet installed."
       echo
@@ -133,8 +132,8 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then
     for i in $pattern; do
       TEMPLATE=${i::-14}
       COMPONENT_PATTERN=${TEMPLATE:3}
-      MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -v osquery)
+      MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -vE "detections|osquery")
-      if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" ]]; then
+      if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ logs-http_endpoint\.generic|logs-winlog\.winlog ]]; then
         load_failures=$((load_failures+1))
         echo "Component template does not exist for $COMPONENT_PATTERN. The index template will not be loaded. Load failures: $load_failures"
       else
@@ -153,7 +152,7 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then
   cd - >/dev/null
 
   if [[ $load_failures -eq 0 ]]; then
-    echo "All template loaded successfully"
+    echo "All templates loaded successfully"
     touch $STATE_FILE_SUCCESS
   else
     echo "Encountered $load_failures templates that were unable to load, likely due to missing dependencies that will be available later; will retry on next highstate"

salt/firewall/containers.map.jinja

@@ -27,6 +27,7 @@
      'so-elastic-fleet',
      'so-elastic-fleet-package-registry',
      'so-influxdb',
+     'so-kafka',
      'so-kibana',
      'so-kratos',
      'so-logstash',
@@ -80,6 +81,7 @@
 {% set NODE_CONTAINERS = [
      'so-logstash',
      'so-redis',
+     'so-kafka'
 ] %}
 
 {% elif GLOBALS.role == 'so-idh' %}