Merge branch '2.4/dev' of github.com:Security-Onion-Solutions/securityonion into 2.4/dev

DOWNLOAD_AND_VERIFY_ISO.md

@@ -1,17 +1,17 @@
-### 2.4.80-20240624 ISO image released on 2024/06/25
+### 2.4.90-20240729 ISO image released on 2024/07/29
 
 
 ### Download and Verify
 
-2.4.80-20240624 ISO image:  
+2.4.90-20240729 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.4.80-20240624.iso
+https://download.securityonion.net/file/securityonion/securityonion-2.4.90-20240729.iso
  
-MD5: 139F9762E926F9CB3C4A9528A3752C31  
+MD5: 9A7714F5922EE555F08675D25E6237D5  
-SHA1: BC6CA2C5F4ABC1A04E83A5CF8FFA6A53B1583CC9  
+SHA1: D3B331452627DB716906BA9F3922574DFA3852DC  
-SHA256: 70E90845C84FFA30AD6CF21504634F57C273E7996CA72F7250428DDBAAC5B1BD  
+SHA256: 5B0CE32543944DBC50C4E906857384211E1BE83EF409619778F18FC62017E0E0  
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.80-20240624.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.90-20240729.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS  
@@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.80-20240624.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.90-20240729.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.4.80-20240624.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.4.90-20240729.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.4.80-20240624.iso.sig securityonion-2.4.80-20240624.iso
+gpg --verify securityonion-2.4.90-20240729.iso.sig securityonion-2.4.90-20240729.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Mon 24 Jun 2024 02:42:03 PM EDT using RSA key ID FE507013
+gpg: Signature made Thu 25 Jul 2024 06:51:11 PM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

SECURITY.md

@@ -5,9 +5,11 @@
 | Version | Supported          |
 | ------- | ------------------ |
 | 2.4.x   | :white_check_mark: |
-| 2.3.x   | :white_check_mark: |
+| 2.3.x   | :x:                |
 | 16.04.x | :x:                |
 
+Security Onion 2.3 has reached End Of Life and is no longer supported.
+
 Security Onion 16.04 has reached End Of Life and is no longer supported.
 
 ## Reporting a Vulnerability

VERSION

@@ -1 +1 @@
-2.4.90
+2.4.100

pillar/elasticsearch/nodes.sls

@@ -0,0 +1,34 @@
+{% set node_types = {} %}
+{% for minionid, ip in salt.saltutil.runner(
+    'mine.get',
+    tgt='elasticsearch:enabled:true',
+    fun='network.ip_addrs',
+    tgt_type='pillar') | dictsort()
+%}
+
+# only add a node to the pillar if it returned an ip from the mine
+{%   if ip | length > 0%}
+{%     set hostname = minionid.split('_') | first %}
+{%     set node_type = minionid.split('_') | last %}
+{%     if node_type not in node_types.keys() %}
+{%       do node_types.update({node_type: {hostname: ip[0]}}) %}
+{%     else %}
+{%       if hostname not in node_types[node_type] %}
+{%         do node_types[node_type].update({hostname: ip[0]}) %}
+{%       else %}
+{%         do node_types[node_type][hostname].update(ip[0]) %}
+{%       endif %}
+{%     endif %}
+{%   endif %}
+{% endfor %}
+
+
+elasticsearch:
+  nodes:
+{% for node_type, values in node_types.items() %}
+    {{node_type}}:
+{%   for hostname, ip in values.items() %}
+      {{hostname}}:
+        ip: {{ip}}
+{%   endfor %}
+{% endfor %}

pillar/logstash/nodes.sls

@@ -1,16 +1,15 @@
 {% set node_types = {} %}
-{% set cached_grains = salt.saltutil.runner('cache.grains', tgt='*') %}
 {% for minionid, ip in salt.saltutil.runner(
     'mine.get',
-    tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-searchnode or G@role:so-heavynode or G@role:so-receiver or G@role:so-fleet ',
+    tgt='logstash:enabled:true',
     fun='network.ip_addrs',
-    tgt_type='compound') | dictsort()
+    tgt_type='pillar') | dictsort()
 %}
 
 # only add a node to the pillar if it returned an ip from the mine
 {%   if ip | length > 0%}
-{%     set hostname = cached_grains[minionid]['host'] %}
+{%     set hostname = minionid.split('_') | first %}
-{%     set node_type = minionid.split('_')[1] %}
+{%     set node_type = minionid.split('_') | last %}
 {%     if node_type not in node_types.keys() %}
 {%       do node_types.update({node_type: {hostname: ip[0]}}) %}
 {%     else %}

pillar/redis/nodes.sls

@@ -0,0 +1,34 @@
+{% set node_types = {} %}
+{% for minionid, ip in salt.saltutil.runner(
+    'mine.get',
+    tgt='redis:enabled:true',
+    fun='network.ip_addrs',
+    tgt_type='pillar') | dictsort()
+%}
+
+# only add a node to the pillar if it returned an ip from the mine
+{%   if ip | length > 0%}
+{%     set hostname = minionid.split('_') | first %}
+{%     set node_type = minionid.split('_') | last %}
+{%     if node_type not in node_types.keys() %}
+{%       do node_types.update({node_type: {hostname: ip[0]}}) %}
+{%     else %}
+{%       if hostname not in node_types[node_type] %}
+{%         do node_types[node_type].update({hostname: ip[0]}) %}
+{%       else %}
+{%         do node_types[node_type][hostname].update(ip[0]) %}
+{%       endif %}
+{%     endif %}
+{%   endif %}
+{% endfor %}
+
+
+redis:
+  nodes:
+{% for node_type, values in node_types.items() %}
+    {{node_type}}:
+{%   for hostname, ip in values.items() %}
+      {{hostname}}:
+        ip: {{ip}}
+{%   endfor %}
+{% endfor %}

pillar/top.sls

@@ -47,10 +47,12 @@ base:
     - kibana.adv_kibana
     - kratos.soc_kratos
     - kratos.adv_kratos
+    - redis.nodes
     - redis.soc_redis
     - redis.adv_redis
     - influxdb.soc_influxdb
     - influxdb.adv_influxdb
+    - elasticsearch.nodes
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
     - elasticfleet.soc_elasticfleet
@@ -147,10 +149,12 @@ base:
     - idstools.adv_idstools
     - kratos.soc_kratos
     - kratos.adv_kratos
+    - redis.nodes
     - redis.soc_redis
     - redis.adv_redis
     - influxdb.soc_influxdb
     - influxdb.adv_influxdb
+    - elasticsearch.nodes
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
     - elasticfleet.soc_elasticfleet
@@ -215,11 +219,13 @@ base:
     - logstash.nodes
     - logstash.soc_logstash
     - logstash.adv_logstash
+    - elasticsearch.nodes
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
     {% endif %}
+    - redis.nodes
     - redis.soc_redis
     - redis.adv_redis
     - minions.{{ grains.id }}
@@ -227,6 +233,8 @@ base:
     - stig.soc_stig
     - soc.license
     - kafka.nodes
+    - kafka.soc_kafka
+    - kafka.adv_kafka
 
   '*_receiver':
     - logstash.nodes

salt/allowed_states.map.jinja

@@ -136,7 +136,9 @@
           'firewall',
           'schedule',
           'docker_clean',
-          'stig'
+          'stig',
+          'kafka.ca',
+          'kafka.ssl'
           ],
       'so-standalone': [
           'salt.master',
@@ -195,7 +197,6 @@
           'schedule',
           'docker_clean',
           'kafka',
-          'elasticsearch.ca',
           'stig'
           ],
       'so-desktop': [

salt/common/init.sls

@@ -14,6 +14,11 @@ net.core.wmem_default:
   sysctl.present:
     - value: 26214400
 
+# Users are not a fan of console messages
+kernel.printk:
+  sysctl.present:
+    - value: "3 4 1 3"
+
 # Remove variables.txt from /tmp - This is temp
 rmvariablesfile:
   file.absent:

salt/common/tools/sbin/so-common

@@ -8,7 +8,7 @@
 # Elastic agent is not managed by salt. Because of this we must store this base information in a 
 # script that accompanies the soup system. Since so-common is one of those special soup files, 
 # and since this same logic is required during installation, it's included in this file.
-ELASTIC_AGENT_TARBALL_VERSION="8.10.4"
+ELASTIC_AGENT_TARBALL_VERSION="8.14.3"
 ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
 ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
 ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"

salt/common/tools/sbin/so-log-check

@@ -170,6 +170,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|cannot join on an empty table" # InfluxDB flux query, import nodes
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|exhausting result iterator"   # InfluxDB flux query mismatched table results (temporary data issue)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to finish run"         # InfluxDB rare error, self-recoverable
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unable to gather disk name"   # InfluxDB known error, can't read disks because the container doesn't have them mounted
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|iteration"
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|communication packets"
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|use of closed"
@@ -241,6 +242,7 @@ exclude_log "mysqld.log"   # MySQL is removed as of 2.4.70, logs may still be on
 exclude_log "soctopus.log" # Soctopus is removed as of 2.4.70, logs may still be on disk
 exclude_log "agentstatus.log" # ignore this log since it tracks agents in error state
 exclude_log "detections_runtime-status_yara.log" # temporarily ignore this log until Detections is more stable
+exclude_log "/nsm/kafka/data/" # ignore Kafka data directory from log check.
 
 for log_file in $(cat /tmp/log_check_files); do
     status "Checking log file $log_file"

salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json

@@ -5,7 +5,7 @@
 	"package": {
 		"name": "endpoint",
 		"title": "Elastic Defend",
-		"version": "8.10.2"
+		"version": "8.14.0"
 	},
 	"enabled": true,
 	"policy_id": "endpoints-initial",

salt/elasticfleet/files/integrations/endpoints-initial/windows-defender.json

@@ -11,7 +11,7 @@
     "winlogs-winlog": {
       "enabled": true,
       "streams": {
-        "winlog.winlog": {
+        "winlog.winlogs": {
           "enabled": true,
           "vars": {
             "channel": "Microsoft-Windows-Windows Defender/Operational",

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup

@@ -53,7 +53,8 @@ fi
 printf "\n### Create ES Token ###\n"
 ESTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/service_tokens" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' |  jq -r .value)
 
-### Create Outputs & Fleet URLs ###
+### Create Outputs, Fleet Policy and Fleet URLs ###
+# Create the Manager Elasticsearch Output first and set it as the default output
 printf "\nAdd Manager Elasticsearch Output...\n"
 ESCACRT=$(openssl x509 -in  $INTCA)
 JSON_STRING=$( jq -n \
@@ -62,7 +63,13 @@ JSON_STRING=$( jq -n \
 curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
 printf "\n\n"
 
-printf "\nCreate Logstash Output Config if node is not an Import or Eval install\n"
+# Create the Manager Fleet Server Host Agent Policy
+# This has to be done while the Elasticsearch Output is set to the default Output
+printf "Create Manager Fleet Server Policy...\n"
+elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "true" "120"
+
+# Now we can create the Logstash Output and set it to to be the default Output
+printf "\n\nCreate Logstash Output Config if node is not an Import or Eval install\n"
 {% if grains.role not in ['so-import', 'so-eval'] %}
 LOGSTASHCRT=$(openssl x509 -in /etc/pki/elasticfleet-logstash.crt)
 LOGSTASHKEY=$(openssl rsa -in  /etc/pki/elasticfleet-logstash.key)
@@ -101,16 +108,6 @@ printf "\n\n"
 # Load Elasticsearch templates
 /usr/sbin/so-elasticsearch-templates-load
 
-# Manager Fleet Server Host
-elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "true" "120"
-
-#Temp Fixup for ES Output bug
-JSON_STRING=$( jq -n \
-                --arg NAME "FleetServer_{{ GLOBALS.hostname }}" \
-                '{"name": $NAME,"description": $NAME,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":120,"data_output_id":"so-manager_elasticsearch"}'
-                )
-curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/FleetServer_{{ GLOBALS.hostname }}" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
-
 # Initial Endpoints Policy
 elastic_fleet_policy_create "endpoints-initial" "Initial Endpoint Policy" "false" "1209600"
 
@@ -165,4 +162,4 @@ salt-call state.apply elasticfleet queue=True
 # Generate installers & install Elastic Agent on the node
 so-elastic-agent-gen-installers
 salt-call state.apply elasticfleet.install_agent_grid queue=True
-exit 0
+exit 0

salt/elasticsearch/config.map.jinja

@@ -1,23 +1,37 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS with context %}
 
 {% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %}
 
-{# ES_LOGSTASH_NODES is the same as LOGSTASH_NODES from logstash/map.jinja but heavynodes and fleet nodes are removed #}
+{# this is a list of dicts containing hostname:ip for elasticsearch nodes that need to know about each other for cluster #}
-{% set ES_LOGSTASH_NODES = [] %}
+{% set ELASTICSEARCH_SEED_HOSTS = [] %}
-{% set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
+{% set node_data = salt['pillar.get']('elasticsearch:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
 {% for node_type, node_details in node_data.items() | sort %}
-{%   if node_type not in ['heavynode', 'fleet'] %}
+{%   if node_type != 'heavynode' %}
 {%     for hostname in node_data[node_type].keys() %}
-{%       do ES_LOGSTASH_NODES.append({hostname:node_details[hostname].ip}) %}
+{%       do ELASTICSEARCH_SEED_HOSTS.append({hostname:node_details[hostname].ip}) %}
 {%     endfor %}
 {%   endif %}
 {% endfor %}
 
+{# this is a list of dicts containing hostname:ip of all nodes running elasticsearch #}
+{% set ELASTICSEARCH_NODES = [] %}
+{% set node_data = salt['pillar.get']('elasticsearch:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
+{% for node_type, node_details in node_data.items() %}
+{%   for hostname in node_data[node_type].keys() %}
+{%     do ELASTICSEARCH_NODES.append({hostname:node_details[hostname].ip}) %}
+{%   endfor %}
+{% endfor %}
+
 {% if grains.id.split('_') | last in ['manager','managersearch','standalone'] %}
-    {% if ES_LOGSTASH_NODES | length > 1 %}
+    {% if ELASTICSEARCH_SEED_HOSTS | length > 1 %}
       {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.update({'discovery': {'seed_hosts': []}}) %}
-      {% for NODE in ES_LOGSTASH_NODES %}
+      {% for NODE in ELASTICSEARCH_SEED_HOSTS %}
         {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.discovery.seed_hosts.append(NODE.keys()|first) %}
       {% endfor %}
     {% endif %}

salt/elasticsearch/config.sls

@@ -118,6 +118,11 @@ esingestconf:
     - user: 930
     - group: 939
 
+# Remove .fleet_final_pipeline-1 because we are using global@custom now
+so-fleet-final-pipeline-remove:
+  file.absent:
+    - name: /opt/so/conf/elasticsearch/ingest/.fleet_final_pipeline-1
+
 # Auto-generate Elasticsearch ingest node pipelines from pillar
 {% for pipeline, config in ELASTICSEARCHMERGED.pipelines.items() %}
 es_ingest_conf_{{pipeline}}:

salt/elasticsearch/enabled.sls

@@ -7,8 +7,8 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKER %}
-{%   from 'logstash/map.jinja' import LOGSTASH_NODES %}
+{%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
-{%   from 'elasticsearch/config.map.jinja' import ES_LOGSTASH_NODES %}
+{%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
 {%   set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %}
 {%   from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %}
@@ -27,7 +27,7 @@ so-elasticsearch:
       - sobridge:
         - ipv4_address: {{ DOCKER.containers['so-elasticsearch'].ip }}
     - extra_hosts:
-    {% for node in LOGSTASH_NODES %}
+    {% for node in ELASTICSEARCH_NODES %}
     {%   for hostname, ip in node.items() %}
       - {{hostname}}:{{ip}}
     {%   endfor %}
@@ -38,7 +38,7 @@ so-elasticsearch:
       {% endfor %}
     {% endif %}
     - environment:
-      {% if ES_LOGSTASH_NODES | length == 1 or GLOBALS.role == 'so-heavynode' %}
+      {% if ELASTICSEARCH_SEED_HOSTS | length == 1 or GLOBALS.role == 'so-heavynode' %}
       - discovery.type=single-node
       {% endif %}
       - ES_JAVA_OPTS=-Xms{{ GLOBALS.elasticsearch.es_heap }} -Xmx{{ GLOBALS.elasticsearch.es_heap }} -Des.transport.cname_in_publish_address=true -Dlog4j2.formatMsgNoLookups=true

salt/elasticsearch/files/ingest/.fleet_final_pipeline-1

@@ -1,107 +0,0 @@
-{
-  "version": 3,
-  "_meta": {
-    "managed_by": "fleet",
-    "managed": true
-  },
-  "description": "Final pipeline for processing all incoming Fleet Agent documents. \n",
-  "processors": [
-    {
-      "date": {
-        "description": "Add time when event was ingested (and remove sub-seconds to improve storage efficiency)",
-        "tag": "truncate-subseconds-event-ingested",
-        "field": "_ingest.timestamp",
-        "target_field": "event.ingested",
-        "formats": [
-          "ISO8601"
-        ],
-        "output_format": "date_time_no_millis",
-        "ignore_failure": true
-      }
-    },
-    {
-      "remove": {
-        "description": "Remove any pre-existing untrusted values.",
-        "field": [
-          "event.agent_id_status",
-          "_security"
-        ],
-        "ignore_missing": true
-      }
-    },
-    {
-      "set_security_user": {
-        "field": "_security",
-        "properties": [
-          "authentication_type",
-          "username",
-          "realm",
-          "api_key"
-        ]
-      }
-    },
-    {
-      "script": {
-        "description": "Add event.agent_id_status based on the API key metadata and the agent.id contained in the event.\n",
-        "tag": "agent-id-status",
-        "source": "boolean is_user_trusted(def ctx, def users) {\n  if (ctx?._security?.username == null) {\n    return false;\n  }\n\n  def user = null;\n  for (def item : users) {\n    if (item?.username == ctx._security.username) {\n      user = item;\n      break;\n    }\n  }\n\n  if (user == null || user?.realm == null || ctx?._security?.realm?.name == null) {\n    return false;\n  }\n\n  if (ctx._security.realm.name != user.realm) {\n    return false;\n  }\n\n  return true;\n}\n\nString verified(def ctx, def params) {\n  // No agent.id field to validate.\n  if (ctx?.agent?.id == null) {\n    return \"missing\";\n  }\n\n  // Check auth metadata from API key.\n  if (ctx?._security?.authentication_type == null\n      // Agents only use API keys.\n      || ctx._security.authentication_type != 'API_KEY'\n      // Verify the API key owner before trusting any metadata it contains.\n      || !is_user_trusted(ctx, params.trusted_users)\n      // Verify the API key has metadata indicating the assigned agent ID.\n      || ctx?._security?.api_key?.metadata?.agent_id == null) {\n    return \"auth_metadata_missing\";\n  }\n\n  // The API key can only be used represent the agent.id it was issued to.\n  if (ctx._security.api_key.metadata.agent_id != ctx.agent.id) {\n    // Potential masquerade attempt.\n    return \"mismatch\";\n  }\n\n  return \"verified\";\n}\n\nif (ctx?.event == null) {\n  ctx.event = [:];\n}\n\nctx.event.agent_id_status = verified(ctx, params);",
-        "params": {
-          "trusted_users": [
-            {
-              "username": "elastic/fleet-server",
-              "realm": "_service_account"
-            },
-            {
-              "username": "cloud-internal-agent-server",
-              "realm": "found"
-            },
-            {
-              "username": "elastic",
-              "realm": "reserved"
-            }
-          ]
-        }
-      }
-    },
-    {
-      "remove": {
-        "field": "_security",
-        "ignore_missing": true
-      }
-    }, 
-    { "set":        { "ignore_failure": true,  "field": "event.module", "value": "elastic_agent" } },
-    { "split":      { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "module_temp"  } },
-    { "set":        { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}" } },
-    { "gsub":       { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "pattern": "^[^.]*.", "replacement": "", "target_field": "dataset_tag_temp" } },
-    { "append":     { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp}}"  } },
-    { "set":        { "if": "ctx.network?.direction == 'egress'", "override": true, "field": "network.initiated", "value": "true" } },
-    { "set":        { "if": "ctx.network?.direction == 'ingress'", "override": true, "field": "network.initiated", "value": "false" } },
-    { "set":        { "if": "ctx.network?.type == 'ipv4'", "override": true, "field": "destination.ipv6", "value": "false" } },
-    { "set":        { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } },
-    { "set":        { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.dataset", "value": "import" } },
-    { "set":        { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.namespace", "value": "so" } },
-    { "date":       { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp","ignore_failure": true, "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSX","yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } },
-    { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true }  },
-    { "set":        { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } },
-    { "rename":        { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } },
-    { "set":        { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } },
-    { "remove":     { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp" ], "ignore_missing": true, "ignore_failure": true } } 
-  ],
-  "on_failure": [
-    {
-      "remove": {
-        "field": "_security",
-        "ignore_missing": true,
-        "ignore_failure": true
-      }
-    },
-    {
-      "append": {
-        "field": "error.message",
-        "value": [
-          "failed in Fleet agent final_pipeline: {{ _ingest.on_failure_message }}"
-        ]
-      }
-    }
-  ]
-}

salt/elasticsearch/files/ingest/global@custom

@@ -0,0 +1,27 @@
+{
+  "version": 3,
+  "_meta": {
+    "managed_by": "securityonion",
+    "managed": true
+  },
+  "description": "Custom pipeline for processing all incoming Fleet Agent documents. \n",
+  "processors": [
+    { "set":        { "ignore_failure": true,  "field": "event.module", "value": "elastic_agent" } },
+    { "split":      { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "module_temp"  } },
+    { "set":        { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}" } },
+    { "gsub":       { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "pattern": "^[^.]*.", "replacement": "", "target_field": "dataset_tag_temp" } },
+    { "append":     { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp}}"  } },
+    { "set":        { "if": "ctx.network?.direction == 'egress'", "override": true, "field": "network.initiated", "value": "true" } },
+    { "set":        { "if": "ctx.network?.direction == 'ingress'", "override": true, "field": "network.initiated", "value": "false" } },
+    { "set":        { "if": "ctx.network?.type == 'ipv4'", "override": true, "field": "destination.ipv6", "value": "false" } },
+    { "set":        { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } },
+    { "set":        { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.dataset", "value": "import" } },
+    { "set":        { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.namespace", "value": "so" } },
+    { "date":       { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp","ignore_failure": true, "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSX","yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } },
+    { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true }  },
+    { "set":        { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } },
+    { "rename":        { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } },
+    { "set":        { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } },
+    { "remove":     { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp" ], "ignore_missing": true, "ignore_failure": true } } 
+  ]
+}

salt/elasticsearch/soc_elasticsearch.yaml

@@ -530,6 +530,58 @@ elasticsearch:
     so-strelka: *indexSettings
     so-syslog: *indexSettings
     so-zeek: *indexSettings
+    so-metrics-fleet_server_x_agent_status: &fleetMetricsSettings
+      index_sorting:
+        description: Sorts the index by event time, at the cost of additional processing resource consumption.
+        advanced: True
+        readonly: True
+        helpLink: elasticsearch.html
+      index_template:
+        ignore_missing_component_templates:
+          description: Ignore component templates if they aren't in Elasticsearch.
+          advanced: True
+          readonly: True
+          helpLink: elasticsearch.html
+        index_patterns:
+          description: Patterns for matching multiple indices or tables.
+          advanced: True
+          readonly: True
+          helpLink: elasticsearch.html
+        template:
+          settings:
+            index:
+              mode:
+                description: Type of mode used for this index. Time series indices can be used for metrics to reduce necessary storage.
+                advanced: True
+                readonly: True
+                helpLink: elasticsearch.html
+              number_of_replicas:
+                description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs.
+                advanced: True
+                readonly: True
+                helpLink: elasticsearch.html
+        composed_of:
+          description: The index template is composed of these component templates.
+          advanced: True
+          readonly: True
+          helpLink: elasticsearch.html
+        priority:
+          description: The priority of the index template.
+          advanced: True
+          readonly: True
+          helpLink: elasticsearch.html
+        data_stream:
+          hidden:
+            description: Hide the data stream.
+            advanced: True
+            readonly: True
+            helpLink: elasticsearch.html
+          allow_custom_routing:
+            description: Allow custom routing for the data stream.
+            advanced: True
+            readonly: True
+            helpLink: elasticsearch.html
+    so-metrics-fleet_server_x_agent_versions: *fleetMetricsSettings
   so_roles:
     so-manager: &soroleSettings
       config:

salt/elasticsearch/template.map.jinja

@@ -1,3 +1,8 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+
 {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
 {% set DEFAULT_GLOBAL_OVERRIDES = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings.pop('global_overrides') %}
 
@@ -17,10 +22,26 @@
 {% set ES_INDEX_SETTINGS = {} %}
 {% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update(salt['defaults.merge'](ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %}
 {% for index, settings in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.items() %}
-{#   if policy isn't defined in the original index settings, then dont merge policy from the global_overrides #}
+
-{#   this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #}
+{#   prevent this action from being performed on custom defined indices. #}
-{%   if not ES_INDEX_SETTINGS_ORIG[index].policy is defined and ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %}
+{#   the custom defined index is not present in either of the dictionaries and fails to reder. #}
-{%     do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %}
+{%   if index in ES_INDEX_SETTINGS_ORIG and index in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES %}
+
+{#     dont merge policy from the global_overrides if policy isn't defined in the original index settingss #}
+{#     this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #}
+{%     if not ES_INDEX_SETTINGS_ORIG[index].policy is defined and ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %}
+{%       do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %}
+{%     endif %}
+
+{#     this prevents and index from inderiting a policy phase from global overrides if it wasnt defined in the defaults. #}
+{%     if ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %}
+{%       for phase in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.copy() %}
+{%         if ES_INDEX_SETTINGS_ORIG[index].policy.phases[phase] is not defined %}
+{%           do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.pop(phase) %}
+{%         endif %}
+{%       endfor %}
+{%     endif %}
+
 {%   endif %}
 
 {%   if settings.index_template is defined %}

salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@package.json

@@ -6,7 +6,7 @@
           "name": "logs"
         },
         "codec": "best_compression",
-        "default_pipeline": "logs-elastic_agent-1.13.1",
+        "default_pipeline": "logs-elastic_agent-1.20.0",
         "mapping": {
           "total_fields": {
             "limit": "10000"

salt/elasticsearch/templates/component/elastic-agent/metrics-fleet_server.agent_status@package.json

@@ -0,0 +1,201 @@
+{
+  "template": {
+    "settings": {
+      "index": {
+        "lifecycle": {
+          "name": "metrics"
+        },
+        "default_pipeline": "metrics-fleet_server.agent_status-1.5.0",
+        "mapping": {
+          "total_fields": {
+            "limit": "1000"
+          }
+        }
+      }
+    },
+    "mappings": {
+      "dynamic": false,
+      "_source": {
+        "mode": "synthetic"
+      },
+      "properties": {
+        "cluster": {
+          "properties": {
+            "id": {
+              "time_series_dimension": true,
+              "type": "keyword"
+            }
+          }
+        },
+        "fleet": {
+          "properties": {
+            "agents": {
+              "properties": {
+                "offline": {
+                  "time_series_metric": "gauge",
+                  "meta": {},
+                  "type": "long"
+                },
+                "total": {
+                  "time_series_metric": "gauge",
+                  "meta": {},
+                  "type": "long"
+                },
+                "updating": {
+                  "time_series_metric": "gauge",
+                  "meta": {},
+                  "type": "long"
+                },
+                "inactive": {
+                  "time_series_metric": "gauge",
+                  "meta": {},
+                  "type": "long"
+                },
+                "healthy": {
+                  "time_series_metric": "gauge",
+                  "meta": {},
+                  "type": "long"
+                },
+                "unhealthy": {
+                  "time_series_metric": "gauge",
+                  "meta": {},
+                  "type": "long"
+                },
+                "unenrolled": {
+                  "time_series_metric": "gauge",
+                  "meta": {},
+                  "type": "long"
+                },
+                "enrolled": {
+                  "time_series_metric": "gauge",
+                  "meta": {},
+                  "type": "long"
+                },
+                "unhealthy_reason": {
+                  "properties": {
+                    "output": {
+                      "time_series_metric": "gauge",
+                      "meta": {},
+                      "type": "long"
+                    },
+                    "input": {
+                      "time_series_metric": "gauge",
+                      "meta": {},
+                      "type": "long"
+                    },
+                    "other": {
+                      "time_series_metric": "gauge",
+                      "meta": {},
+                      "type": "long"
+                    }
+                  }
+                },
+                "upgrading_step": {
+                  "properties": {
+                    "rollback": {
+                      "time_series_metric": "gauge",
+                      "meta": {},
+                      "type": "long"
+                    },
+                    "requested": {
+                      "time_series_metric": "gauge",
+                      "meta": {},
+                      "type": "long"
+                    },
+                    "restarting": {
+                      "time_series_metric": "gauge",
+                      "meta": {},
+                      "type": "long"
+                    },
+                    "downloading": {
+                      "time_series_metric": "gauge",
+                      "meta": {},
+                      "type": "long"
+                    },
+                    "scheduled": {
+                      "time_series_metric": "gauge",
+                      "meta": {},
+                      "type": "long"
+                    },
+                    "extracting": {
+                      "time_series_metric": "gauge",
+                      "meta": {},
+                      "type": "long"
+                    },
+                    "replacing": {
+                      "time_series_metric": "gauge",
+                      "meta": {},
+                      "type": "long"
+                    },
+                    "failed": {
+                      "time_series_metric": "gauge",
+                      "meta": {},
+                      "type": "long"
+                    },
+                    "watching": {
+                      "time_series_metric": "gauge",
+                      "meta": {},
+                      "type": "long"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
+        "agent": {
+          "properties": {
+            "id": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "type": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "version": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            }
+          }
+        },
+        "@timestamp": {
+          "ignore_malformed": false,
+          "type": "date"
+        },
+        "data_stream": {
+          "properties": {
+            "namespace": {
+              "type": "constant_keyword"
+            },
+            "type": {
+              "type": "constant_keyword"
+            },
+            "dataset": {
+              "type": "constant_keyword"
+            }
+          }
+        },
+        "kibana": {
+          "properties": {
+            "uuid": {
+              "path": "agent.id",
+              "type": "alias"
+            },
+            "version": {
+              "path": "agent.version",
+              "type": "alias"
+            }
+          }
+        }
+      }
+    }
+  },
+  "_meta": {
+    "package": {
+      "name": "fleet_server"
+    },
+    "managed_by": "fleet",
+    "managed": true
+  }
+}

salt/elasticsearch/templates/component/elastic-agent/metrics-fleet_server.agent_versions@package.json

@@ -0,0 +1,102 @@
+{
+  "template": {
+    "settings": {
+      "index": {
+        "lifecycle": {
+          "name": "metrics"
+        },
+        "default_pipeline": "metrics-fleet_server.agent_versions-1.5.0",
+        "mapping": {
+          "total_fields": {
+            "limit": "1000"
+          }
+        }
+      }
+    },
+    "mappings": {
+      "dynamic": false,
+      "_source": {
+        "mode": "synthetic"
+      },
+      "properties": {
+        "cluster": {
+          "properties": {
+            "id": {
+              "time_series_dimension": true,
+              "type": "keyword"
+            }
+          }
+        },
+        "fleet": {
+          "properties": {
+            "agent": {
+              "properties": {
+                "count": {
+                  "time_series_metric": "gauge",
+                  "meta": {},
+                  "type": "long"
+                },
+                "version": {
+                  "time_series_dimension": true,
+                  "type": "keyword"
+                }
+              }
+            }
+          }
+        },
+        "agent": {
+          "properties": {
+            "id": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "type": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "version": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            }
+          }
+        },
+        "@timestamp": {
+          "ignore_malformed": false,
+          "type": "date"
+        },
+        "data_stream": {
+          "properties": {
+            "namespace": {
+              "type": "constant_keyword"
+            },
+            "type": {
+              "type": "constant_keyword"
+            },
+            "dataset": {
+              "type": "constant_keyword"
+            }
+          }
+        },
+        "kibana": {
+          "properties": {
+            "uuid": {
+              "path": "agent.id",
+              "type": "alias"
+            },
+            "version": {
+              "path": "agent.version",
+              "type": "alias"
+            }
+          }
+        }
+      }
+    }
+  },
+  "_meta": {
+    "package": {
+      "name": "fleet_server"
+    },
+    "managed_by": "fleet",
+    "managed": true
+  }
+}

salt/elasticsearch/templates/component/so/so-system-mappings.json

@@ -0,0 +1,29 @@
+{
+  "template": {
+    "mappings": {
+      "properties": {
+        "host": {
+	  "properties":{
+            "ip": {
+              "type": "ip"
+            }
+	  }
+	},
+	"related": {
+          "properties":{
+            "ip": {
+              "type": "ip"
+            }
+          }
+        },
+	"source": {
+          "properties":{
+            "ip": {
+              "type": "ip"
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines

@@ -20,7 +20,7 @@ if [ ! -f /opt/so/state/espipelines.txt ]; then
 
   cd ${ELASTICSEARCH_INGEST_PIPELINES}
   echo "Loading pipelines..."
-  for i in .[a-z]* *; 
+  for i in *; 
   do 
     echo $i;
     retry 5 5 "so-elasticsearch-query _ingest/pipeline/$i -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load pipeline: $i"

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-settings

@@ -4,7 +4,7 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 {%- from 'vars/globals.map.jinja' import GLOBALS %}
-{%- set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
+{%- set node_data = salt['pillar.get']('elasticsearch:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
 
 . /usr/sbin/so-common
 

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-total

@@ -40,9 +40,9 @@ fi
 
 # Iterate through the output of _cat/allocation for each node in the cluster to determine the total available space
 {% if GLOBALS.role == 'so-manager' %}
-for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | grep -v "{{ GLOBALS.manager }}$" | awk '{print $5}'); do
+for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | grep -v "{{ GLOBALS.manager }}$" | awk '{print $8}'); do
 {% else %}
-for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | awk '{print $5}'); do
+for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | awk '{print $8}'); do
 {% endif %}
   size=$(echo $i | grep -oE '[0-9].*' | awk '{print int($1+0.5)}')
   unit=$(echo $i | grep -oE '[A-Za-z]+')

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-used

@@ -13,10 +13,10 @@ TOTAL_USED_SPACE=0
 # Iterate through the output of _cat/allocation for each node in the cluster to determine the total used space
 {% if GLOBALS.role == 'so-manager' %}
 # Get total disk space - disk.total
-for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | grep -v "{{ GLOBALS.manager }}$" | awk '{print $3}'); do
+for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | grep -v "{{ GLOBALS.manager }}$" | awk '{print $6}'); do
 {% else %}
 # Get disk space taken up by indices - disk.indices
-for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | awk '{print $2}'); do
+for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | awk '{print $5}'); do
 {% endif %}
   size=$(echo $i | grep -oE '[0-9].*' | awk '{print int($1+0.5)}')
   unit=$(echo $i | grep -oE '[A-Za-z]+')

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load

@@ -134,7 +134,7 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then
       TEMPLATE=${i::-14}
       COMPONENT_PATTERN=${TEMPLATE:3}
       MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -vE "detections|osquery")
-      if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" ]]; then
+      if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ logs-http_endpoint\.generic|logs-winlog\.winlog ]]; then
         load_failures=$((load_failures+1))
         echo "Component template does not exist for $COMPONENT_PATTERN. The index template will not be loaded. Load failures: $load_failures"
       else
@@ -153,7 +153,7 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then
   cd - >/dev/null
 
   if [[ $load_failures -eq 0 ]]; then
-    echo "All template loaded successfully"
+    echo "All templates loaded successfully"
     touch $STATE_FILE_SUCCESS
   else
     echo "Encountered $load_failures templates that were unable to load, likely due to missing dependencies that will be available later; will retry on next highstate"

salt/firewall/soc_firewall.yaml

@@ -120,7 +120,10 @@ firewall:
     influxdb:
       tcp: *tcpsettings
       udp: *udpsettings
-    kafka:
+    kafka_controller:
+      tcp: *tcpsettings
+      udp: *udpsettings
+    kafka_data:
       tcp: *tcpsettings
       udp: *udpsettings
     kibana:

salt/idstools/etc/rulecat.conf

@@ -1,6 +1,6 @@
 {%- from 'vars/globals.map.jinja' import GLOBALS -%}
 {%- from 'soc/merged.map.jinja' import SOCMERGED -%}
---suricata-version=6.0
+--suricata-version=7.0.3
 --merged=/opt/so/rules/nids/suri/all.rules
 --output=/nsm/rules/detect-suricata/custom_temp
 --local=/opt/so/rules/nids/suri/local.rules
@@ -20,4 +20,4 @@
 --local={{ ruleset.file }}
         {%- endif %}
     {%- endfor %}
-{%- endif %}
+{%- endif %}

salt/idstools/tools/sbin_jinja/so-rule-update

@@ -11,8 +11,8 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
 {%- set proxy = salt['pillar.get']('manager:proxy') %}
 {%- set noproxy = salt['pillar.get']('manager:no_proxy', '') %}
 
-# Download the rules from the internet
 {%- if proxy %}
+# Download the rules from the internet
     export http_proxy={{ proxy }}
     export https_proxy={{ proxy }}
     export no_proxy="{{ noproxy }}"
@@ -20,12 +20,12 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
 
     mkdir -p /nsm/rules/suricata
     chown -R socore:socore /nsm/rules/suricata
+{%- if not GLOBALS.airgap %}
 # Download the rules from the internet
-{%- if GLOBALS.airgap != 'True' %}
 {%-   if IDSTOOLSMERGED.config.ruleset == 'ETOPEN' %}
-    docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force
+    docker exec so-idstools idstools-rulecat -v --suricata-version 7.0.3 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force
 {%-   elif IDSTOOLSMERGED.config.ruleset == 'ETPRO' %}
-    docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force --etpro={{ IDSTOOLSMERGED.config.oinkcode }}
+    docker exec so-idstools idstools-rulecat -v --suricata-version 7.0.3 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force --etpro={{ IDSTOOLSMERGED.config.oinkcode }}
 {%-   endif %}
 {%- endif %}
 

salt/kafka/ca.sls

@@ -0,0 +1,37 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states or sls in allowed_states %}
+{%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   set KAFKATRUST = salt['pillar.get']('kafka:truststore') %}
+
+kafkaconfdir:
+  file.directory:
+    - name: /opt/so/conf/kafka
+    - user: 960
+    - group: 960
+    - makedirs: True
+
+{% if GLOBALS.is_manager %}
+# Manager runs so-kafka-trust to create truststore for Kafka ssl communication
+kafka_truststore:
+  cmd.script:
+    - source: salt://kafka/tools/sbin_jinja/so-kafka-trust
+    - template: jinja
+    - cwd: /opt/so
+    - defaults:
+        GLOBALS: {{ GLOBALS }}
+        KAFKATRUST: {{ KAFKATRUST }}
+{% endif %}
+
+kafkacertz:
+  file.managed:
+    - name: /opt/so/conf/kafka/kafka-truststore.jks
+    - source: salt://kafka/files/kafka-truststore
+    - user: 960
+    - group: 931
+
+{% endif %}

salt/kafka/config.map.jinja

@@ -6,7 +6,8 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 
 {% set KAFKA_NODES_PILLAR = salt['pillar.get']('kafka:nodes') %}
-{% set KAFKA_PASSWORD = salt['pillar.get']('kafka:password') %}
+{% set KAFKA_PASSWORD = salt['pillar.get']('kafka:config:password') %}
+{% set KAFKA_TRUSTPASS = salt['pillar.get']('kafka:config:trustpass') %}
 
 {# Create list of KRaft controllers #}
 {% set controllers = [] %}
@@ -67,19 +68,12 @@
 
 {% endif %}
 
-{# If a password other than PLACEHOLDER isn't set remove it from the server.properties #}
+{# Truststore config #}
-{% if KAFKAMERGED.config.broker.ssl_x_truststore_x_password == 'PLACEHOLDER' %}
+{% do KAFKAMERGED.config.broker.update({'ssl_x_truststore_x_password': KAFKA_TRUSTPASS }) %}
-{%   do KAFKAMERGED.config.broker.pop('ssl_x_truststore_x_password') %}
+{% do KAFKAMERGED.config.controller.update({'ssl_x_truststore_x_password': KAFKA_TRUSTPASS }) %}
-{% endif %}
+{% do KAFKAMERGED.config.client.update({'ssl_x_truststore_x_password': KAFKA_TRUSTPASS }) %}
-
-{% if KAFKAMERGED.config.controller.ssl_x_truststore_x_password == 'PLACEHOLDER' %}
-{%   do KAFKAMERGED.config.controller.pop('ssl_x_truststore_x_password') %}
-{% endif %}
 
 {# Client properties stuff #}
-{% if KAFKAMERGED.config.client.ssl_x_truststore_x_password == 'PLACEHOLDER' %}
-{%   do KAFKAMERGED.config.client.pop('ssl_x_truststore_x_password') %}
-{% endif %}
 {% do KAFKAMERGED.config.client.update({'ssl_x_keystore_x_password': KAFKA_PASSWORD }) %}
 
 {% if 'broker' in node_type %}

salt/kafka/config.sls

@@ -7,18 +7,22 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 
-include:
-  - ssl
-
 kafka_group:
   group.present:
     - name: kafka
     - gid: 960
 
-kafka:
+kafka_user:
   user.present:
+    - name: kafka
     - uid: 960
     - gid: 960
+    - home: /opt/so/conf/kafka
+    - createhome: False
+
+kafka_home_dir:
+  file.absent:
+    - name: /home/kafka
 
 kafka_sbin_tools:
   file.recurse:
@@ -28,6 +32,17 @@ kafka_sbin_tools:
     - group: 960
     - file_mode: 755
 
+kafka_sbin_jinja_tools:
+  file.recurse:
+    - name: /usr/sbin
+    - source: salt://kafka/tools/sbin_jinja
+    - user: 960
+    - group: 960
+    - file_mode: 755
+    - template: jinja
+    - defaults:
+        GLOBALS: {{ GLOBALS }}
+
 kafka_log_dir:
   file.directory:
     - name: /opt/so/log/kafka

salt/kafka/defaults.yaml

@@ -1,10 +1,12 @@
 kafka:
   enabled: False
   cluster_id:
-  password:
   controllers:
   reset:
+  logstash: []
   config:
+    password:
+    trustpass:
     broker:
       advertised_x_listeners:
       auto_x_create_x_topics_x_enable: true
@@ -30,16 +32,16 @@ kafka:
       ssl_x_keystore_x_location: /etc/pki/kafka.p12
       ssl_x_keystore_x_type: PKCS12
       ssl_x_keystore_x_password:
-      ssl_x_truststore_x_location: /etc/pki/java/sos/cacerts
+      ssl_x_truststore_x_location: /etc/pki/kafka-truststore.jks
-      ssl_x_truststore_x_password: PLACEHOLDER
+      ssl_x_truststore_x_type: JKS
-      ssl_x_truststore_x_type: PEM
+      ssl_x_truststore_x_password:
       transaction_x_state_x_log_x_min_x_isr: 1
       transaction_x_state_x_log_x_replication_x_factor: 1
     client:
       security_x_protocol: SSL
-      ssl_x_truststore_x_location: /etc/pki/java/sos/cacerts
+      ssl_x_truststore_x_location: /etc/pki/kafka-truststore.jks
-      ssl_x_truststore_x_password: PLACEHOLDER
+      ssl_x_truststore_x_type: JKS
-      ssl_x_truststore_x_type: PEM
+      ssl_x_truststore_x_password:
       ssl_x_keystore_x_location: /etc/pki/kafka.p12
       ssl_x_keystore_x_type: PKCS12
       ssl_x_keystore_x_password:
@@ -57,6 +59,6 @@ kafka:
       ssl_x_keystore_x_location: /etc/pki/kafka.p12
       ssl_x_keystore_x_type: PKCS12
       ssl_x_keystore_x_password:
-      ssl_x_truststore_x_location: /etc/pki/java/sos/cacerts
+      ssl_x_truststore_x_location: /etc/pki/kafka-truststore.jks
-      ssl_x_truststore_x_password: PLACEHOLDER
+      ssl_x_truststore_x_type: JKS
-      ssl_x_truststore_x_type: PEM
+      ssl_x_truststore_x_password:

salt/kafka/disabled.sls

@@ -22,4 +22,13 @@ ensure_default_pipeline:
     - name: |
         /usr/sbin/so-yaml.py replace /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.enabled False;
         /usr/sbin/so-yaml.py replace /opt/so/saltstack/local/pillar/global/soc_global.sls global.pipeline REDIS
-{% endif %}
+{% endif %}
+
+{# If Kafka has never been manually enabled, the 'Kafka' user does not exist. In this case certs for Kafka should not exist since they'll be owned by uid 960 #}
+{% for cert in ['kafka-client.crt','kafka-client.key','kafka.crt','kafka.key','kafka-logstash.crt','kafka-logstash.key','kafka-logstash.p12','kafka.p12','elasticfleet-kafka.p8'] %}
+check_kafka_cert_{{cert}}:
+  file.absent:
+    - name: /etc/pki/{{cert}}
+    - onlyif: stat -c %U /etc/pki/{{cert}} | grep -q UNKNOWN
+    - show_changes: False
+{% endfor %}

salt/kafka/enabled.sls

@@ -17,10 +17,11 @@
 {%   if 'gmd' in salt['pillar.get']('features', []) %}
 
 include:
-  - elasticsearch.ca
+  - kafka.ca
-  - kafka.sostatus
   - kafka.config
+  - kafka.ssl
   - kafka.storage
+  - kafka.sostatus
 
 so-kafka:
   docker_container.running:
@@ -49,7 +50,7 @@ so-kafka:
       {% endfor %}
     - binds:
       - /etc/pki/kafka.p12:/etc/pki/kafka.p12:ro
-      - /etc/pki/tls/certs/intca.crt:/etc/pki/java/sos/cacerts:ro
+      - /opt/so/conf/kafka/kafka-truststore.jks:/etc/pki/kafka-truststore.jks:ro
       - /nsm/kafka/data/:/nsm/kafka/data/:rw
       - /opt/so/log/kafka:/opt/kafka/logs/:rw
       - /opt/so/conf/kafka/server.properties:/opt/kafka/config/kraft/server.properties:ro
@@ -58,6 +59,9 @@ so-kafka:
       {% for sc in ['server', 'client'] %}
       - file: kafka_kraft_{{sc}}_properties
       {% endfor %}
+      - file: kafkacertz
+    - require:
+      - file: kafkacertz
 
 delete_so-kafka_so-status.disabled:
   file.uncomment:

salt/kafka/init.sls

@@ -18,7 +18,12 @@ include:
   - kafka.nodes
 {% endif %}
 {% if GLOBALS.pipeline == "KAFKA" and KAFKAMERGED.enabled %}
+{%   if grains.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-receiver'] %}
   - kafka.enabled
+{# Searchnodes only run kafka.ssl state when Kafka is enabled #}
+{%   elif grains.role == "so-searchnode" %}
+  - kafka.ssl
+{%   endif %}
 {% else %}
   - kafka.disabled
 {% endif %}

salt/kafka/soc_kafka.yaml

@@ -8,19 +8,31 @@ kafka:
     advanced: True
     sensitive: True
     helpLink: kafka.html
-  password:
-    description: The password to use for the Kafka certificates.
-    sensitive: True
-    helpLink: kafka.html
   controllers:
     description: A comma-separated list of hostnames that will act as Kafka controllers. These hosts will be responsible for managing the Kafka cluster. Note that only manager and receiver nodes are eligible to run Kafka. This configuration needs to be set before enabling Kafka. Failure to do so may result in Kafka topics becoming unavailable requiring manual intervention to restore functionality or reset Kafka, either of which can result in data loss.
-    forcedType: "string"
+    forcedType: string
     helpLink: kafka.html
   reset:
     description: Disable and reset the Kafka cluster. This will remove all Kafka data including logs that may have not yet been ingested into Elasticsearch and reverts the grid to using REDIS as the global pipeline. This is useful when testing different Kafka configurations such as rearranging Kafka brokers / controllers allowing you to reset the cluster rather than manually fixing any issues arising from attempting to reassign a Kafka broker into a controller. Enter 'YES_RESET_KAFKA' and submit to disable and reset Kafka. Make any configuration changes required and re-enable Kafka when ready. This action CANNOT be reversed.
     advanced: True
     helpLink: kafka.html
+  logstash:
+    description: By default logstash is disabled when Kafka is enabled. This option allows you to specify any hosts you would like to re-enable logstash on alongside Kafka.
+    forcedType: "[]string"
+    multiline: True
+    advanced: True
+    helpLink: kafka.html
   config:
+    password:
+      description: The password used for the Kafka certificates.
+      readonly: True
+      sensitive: True
+      helpLink: kafka.html
+    trustpass:
+      description: The password used for the Kafka truststore.
+      readonly: True
+      sensitive: True
+      helpLink: kafka.html
     broker:
       advertised_x_listeners:
         description: Specify the list of listeners (hostname and port) that Kafka brokers provide to clients for communication.
@@ -128,6 +140,10 @@ kafka:
         description: The trust store file location within the Docker container.
         title: ssl.truststore.location
         helpLink: kafka.html
+      ssl_x_truststore_x_type:
+        description: The trust store file format.
+        title: ssl.truststore.type
+        helpLink: kafka.html
       ssl_x_truststore_x_password:
         description: The trust store file password. If null, the trust store file is still use, but integrity checking is disabled. Invalid for PEM format.
         title: ssl.truststore.password
@@ -167,6 +183,10 @@ kafka:
         description: The trust store file location within the Docker container.
         title: ssl.truststore.location
         helpLink: kafka.html
+      ssl_x_truststore_x_type:
+        description: The trust store file format.
+        title: ssl.truststore.type
+        helpLink: kafka.html
       ssl_x_truststore_x_password:
         description: The trust store file password. If null, the trust store file is still use, but integrity checking is disabled. Invalid for PEM format.
         title: ssl.truststore.password

salt/kafka/ssl.sls

@@ -0,0 +1,201 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states or sls in allowed_states %}
+{%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   set kafka_password = salt['pillar.get']('kafka:config:password') %}
+
+include:
+  - ca.dirs
+    {% set global_ca_server = [] %}
+    {% set x509dict = salt['mine.get'](GLOBALS.manager | lower~'*', 'x509.get_pem_entries') %}
+    {% for host in x509dict %}
+      {% if 'manager' in host.split('_')|last or host.split('_')|last == 'standalone' %}
+        {% do global_ca_server.append(host) %}
+      {% endif %}
+    {% endfor %}
+    {% set ca_server = global_ca_server[0] %}
+
+{% if GLOBALS.pipeline == "KAFKA" %}
+
+{%   if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone'] %}
+kafka_client_key:
+  x509.private_key_managed:
+    - name: /etc/pki/kafka-client.key
+    - keysize: 4096
+    - backup: True
+    - new: True
+    {% if salt['file.file_exists']('/etc/pki/kafka-client.key') -%}
+    - prereq:
+      - x509: /etc/pki/kafka-client.crt
+    {%- endif %}
+    - retry:
+        attempts: 5
+        interval: 30
+
+kafka_client_crt:
+  x509.certificate_managed:
+    - name: /etc/pki/kafka-client.crt
+    - ca_server: {{ ca_server }}
+    - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
+    - signing_policy: kafka
+    - private_key: /etc/pki/kafka-client.key
+    - CN: {{ GLOBALS.hostname }}
+    - days_remaining: 0
+    - days_valid: 820
+    - backup: True
+    - timeout: 30
+    - retry:
+        attempts: 5
+        interval: 30
+
+kafka_client_key_perms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/kafka-client.key
+    - mode: 640
+    - user: 960
+    - group: 939
+
+kafka_client_crt_perms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/kafka-client.crt
+    - mode: 640
+    - user: 960
+    - group: 939
+{%   endif %}
+
+{%   if GLOBALS.role in ['so-manager', 'so-managersearch','so-receiver', 'so-standalone'] %}
+kafka_key:
+  x509.private_key_managed:
+    - name: /etc/pki/kafka.key
+    - keysize: 4096
+    - backup: True
+    - new: True
+    {% if salt['file.file_exists']('/etc/pki/kafka.key') -%}
+    - prereq:
+      - x509: /etc/pki/kafka.crt
+    {%- endif %}
+    - retry:
+        attempts: 5
+        interval: 30
+
+kafka_crt:
+  x509.certificate_managed:
+    - name: /etc/pki/kafka.crt
+    - ca_server: {{ ca_server }}
+    - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
+    - signing_policy: kafka
+    - private_key: /etc/pki/kafka.key
+    - CN: {{ GLOBALS.hostname }}
+    - days_remaining: 0
+    - days_valid: 820
+    - backup: True
+    - timeout: 30
+    - retry:
+        attempts: 5
+        interval: 30
+  cmd.run:
+    - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/kafka.key -in /etc/pki/kafka.crt -export -out /etc/pki/kafka.p12 -nodes -passout pass:{{ kafka_password }}"
+    - onchanges:
+      - x509: /etc/pki/kafka.key
+kafka_key_perms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/kafka.key
+    - mode: 640
+    - user: 960
+    - group: 939
+
+kafka_crt_perms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/kafka.crt
+    - mode: 640
+    - user: 960
+    - group: 939
+
+kafka_pkcs12_perms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/kafka.p12
+    - mode: 640
+    - user: 960
+    - group: 939
+{%   endif %}
+
+# Standalone needs kafka-logstash for automated testing. Searchnode/manager search need it for logstash to consume from Kafka.
+# Manager will have cert, but be unused until a pipeline is created and logstash enabled.
+{%   if GLOBALS.role in ['so-standalone', 'so-managersearch', 'so-searchnode', 'so-manager'] %}
+kafka_logstash_key:
+  x509.private_key_managed:
+    - name: /etc/pki/kafka-logstash.key
+    - keysize: 4096
+    - backup: True
+    - new: True
+    {% if salt['file.file_exists']('/etc/pki/kafka-logstash.key') -%}
+    - prereq:
+      - x509: /etc/pki/kafka-logstash.crt
+    {%- endif %}
+    - retry:
+        attempts: 5
+        interval: 30
+
+kafka_logstash_crt:
+  x509.certificate_managed:
+    - name: /etc/pki/kafka-logstash.crt
+    - ca_server: {{ ca_server }}
+    - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
+    - signing_policy: kafka
+    - private_key: /etc/pki/kafka-logstash.key
+    - CN: {{ GLOBALS.hostname }}
+    - days_remaining: 0
+    - days_valid: 820
+    - backup: True
+    - timeout: 30
+    - retry:
+        attempts: 5
+        interval: 30
+  cmd.run:
+    - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/kafka-logstash.key -in /etc/pki/kafka-logstash.crt -export -out /etc/pki/kafka-logstash.p12 -nodes -passout pass:{{ kafka_password }}"
+    - onchanges:
+      - x509: /etc/pki/kafka-logstash.key
+
+kafka_logstash_key_perms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/kafka-logstash.key
+    - mode: 640
+    - user: 931
+    - group: 939
+
+kafka_logstash_crt_perms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/kafka-logstash.crt
+    - mode: 640
+    - user: 931
+    - group: 939
+
+kafka_logstash_pkcs12_perms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/kafka-logstash.p12
+    - mode: 640
+    - user: 931
+    - group: 939
+
+{%   endif %}
+{% endif %}
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/kafka/tools/sbin_jinja/so-kafka-trust

@@ -0,0 +1,13 @@
+#!/bin/bash
+#
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+{% set TRUSTPASS = salt['pillar.get']('kafka:config:trustpass') %}
+
+if [ ! -f /opt/so/saltstack/local/salt/kafka/files/kafka-truststore ]; then
+    docker run -v /etc/pki/ca.crt:/etc/pki/ca.crt --name so-kafkatrust --user root --entrypoint /opt/java/openjdk/bin/keytool {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kafka:{{ GLOBALS.so_version }} -import -file /etc/pki/ca.crt -alias SOS -keystore /etc/pki/kafka-truststore -storepass {{ TRUSTPASS }} -storetype jks -noprompt
+    docker cp so-kafkatrust:/etc/pki/kafka-truststore /opt/so/saltstack/local/salt/kafka/files/kafka-truststore
+    docker rm so-kafkatrust
+fi

salt/kibana/files/config_saved_objects.ndjson

@@ -1 +1,2 @@
-{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n  \"from\": \"now-24h\",\n  \"to\": \"now\"\n}"},"coreMigrationVersion": "8.10.4","id": "8.10.4","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="}
+{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n  \"from\": \"now-24h\",\n  \"to\": \"now\"\n}"},"coreMigrationVersion": "8.14.3","id": "8.14.3","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="}
+

salt/kibana/tools/sbin_jinja/so-kibana-config-load

@@ -63,7 +63,7 @@ update() {
 
     IFS=$'\r\n' GLOBIGNORE='*' command eval  'LINES=($(cat $1))'
     for i in "${LINES[@]}"; do
-      RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.10.4" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ")
+      RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.14.3" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ")
       echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi
     done
  

salt/logstash/defaults.yaml

@@ -7,9 +7,7 @@ logstash:
         - search
       receiver:
         - receiver
-      heavynode:
+      heavynode: []
-        - manager
-        - search
       searchnode:
         - search
       manager:
@@ -27,7 +25,7 @@ logstash:
       - so/0011_input_endgame.conf
       - so/0012_input_elastic_agent.conf.jinja
       - so/0013_input_lumberjack_fleet.conf
-      - so/9999_output_redis.conf.jinja        
+      - so/9999_output_redis.conf.jinja
     receiver:
       - so/0011_input_endgame.conf
       - so/0012_input_elastic_agent.conf.jinja
@@ -37,7 +35,6 @@ logstash:
       - so/0900_input_redis.conf.jinja
       - so/9805_output_elastic_agent.conf.jinja
       - so/9900_output_endgame.conf.jinja
-      - so/0800_input_kafka.conf.jinja
     custom0: []
     custom1: []
     custom2: []

salt/logstash/enabled.sls

@@ -14,6 +14,11 @@
 include:
 {%   if GLOBALS.role not in ['so-receiver','so-fleet'] %}
   - elasticsearch.ca
+{%   endif %}
+{# Kafka ca runs on nodes that can run logstash for Kafka input / output. Only when Kafka is global pipeline #}
+{%   if GLOBALS.role in ['so-searchnode', 'so-manager', 'so-managersearch', 'so-receiver', 'so-standalone'] and GLOBALS.pipeline == 'KAFKA' %}
+  - kafka.ca
+  - kafka.ssl
 {%   endif %}
   - logstash.config
   - logstash.sostatus
@@ -79,8 +84,9 @@ so-logstash:
       - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro
       - /opt/so/conf/ca/tls-ca-bundle.pem:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem:ro
       {% endif %}
-      {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-searchnode'] %}
+      {% if GLOBALS.pipeline == "KAFKA" and GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-searchnode'] %}
       - /etc/pki/kafka-logstash.p12:/usr/share/logstash/kafka-logstash.p12:ro
+      - /opt/so/conf/kafka/kafka-truststore.jks:/etc/pki/kafka-truststore.jks:ro
       {% endif %}
       {% if GLOBALS.role == 'so-eval' %}
       - /nsm/zeek:/nsm/zeek:ro
@@ -105,6 +111,9 @@ so-logstash:
       - file: ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}
         {% endfor %}
       {% endfor %}
+      {% if GLOBALS.pipeline == 'KAFKA' and GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-searchnode'] %}
+      - file: kafkacertz
+      {% endif %}
     - require:
       {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-receiver'] %}
       - x509: etc_filebeat_crt
@@ -118,6 +127,9 @@ so-logstash:
       - file: cacertz
       - file: capemz
       {% endif %}
+      {% if GLOBALS.pipeline == 'KAFKA' and GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-searchnode'] %}
+      - file: kafkacertz
+      {% endif %}
 
 delete_so-logstash_so-status.disabled:
   file.uncomment:

salt/logstash/init.sls

@@ -4,13 +4,9 @@
 # Elastic License 2.0.
 
 {% from 'logstash/map.jinja' import LOGSTASH_MERGED %}
-{% from 'kafka/map.jinja' import KAFKAMERGED %}
 
 include:
-{# Disable logstash when Kafka is enabled except when the role is standalone #}
+{% if LOGSTASH_MERGED.enabled %}
-{% if LOGSTASH_MERGED.enabled and grains.role == 'so-standalone' %}
-  - logstash.enabled
-{% elif LOGSTASH_MERGED.enabled and not KAFKAMERGED.enabled %}
   - logstash.enabled
 {% else %}
   - logstash.disabled

salt/logstash/map.jinja

@@ -6,24 +6,40 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% import_yaml 'logstash/defaults.yaml' as LOGSTASH_DEFAULTS %}
 {% set LOGSTASH_MERGED = salt['pillar.get']('logstash', LOGSTASH_DEFAULTS.logstash, merge=True) %}
+{% set KAFKA_LOGSTASH = salt['pillar.get']('kafka:logstash', []) %}
 
-{% set REDIS_NODES = [] %}
+{# used to store the redis nodes that logstash needs to know about to pull from the queue #}
-{# LOGSTASH_NODES is the same as ES_LOGSTASH_NODES from elasticsearch/config.map.jinja but heavynodes are present #}
+{% set LOGSTASH_REDIS_NODES = [] %}
+{# stores all logstash nodes #}
 {% set LOGSTASH_NODES = [] %}
-{% set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
+{% set logstash_node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
+{% set redis_node_data = salt['pillar.get']('redis:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
 
-{% for node_type, node_details in node_data.items() | sort %}
+{% for node_type, node_details in redis_node_data.items() | sort %}
 {%   if GLOBALS.role in ['so-searchnode', 'so-standalone', 'so-managersearch', 'so-fleet'] %}
 {%     if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %}
-{%       for hostname in node_data[node_type].keys() %}
+{%       for hostname in redis_node_data[node_type].keys() %}
-{%         do REDIS_NODES.append({hostname:node_details[hostname].ip}) %}
+{%         do LOGSTASH_REDIS_NODES.append({hostname:node_details[hostname].ip}) %}
 {%       endfor %}
 {%     endif %}
-{%   else %}
-{%     do REDIS_NODES.append({GLOBALS.hostname:GLOBALS.node_ip}) %}
 {%   endif %}
+{% endfor %}
 
-{%   for hostname in node_data[node_type].keys() %}
+{% for node_type, node_details in logstash_node_data.items() | sort %}
+{%   for hostname in logstash_node_data[node_type].keys() %}
 {%     do LOGSTASH_NODES.append({hostname:node_details[hostname].ip}) %}
 {%   endfor %}
 {% endfor %}
+
+{# Append Kafka input pipeline when Kafka is enabled #}
+{% if GLOBALS.pipeline == 'KAFKA' %}
+{%   do LOGSTASH_MERGED.defined_pipelines.search.remove('so/0900_input_redis.conf.jinja') %}
+{%   do LOGSTASH_MERGED.defined_pipelines.search.append('so/0800_input_kafka.conf.jinja') %}
+{%   do LOGSTASH_MERGED.defined_pipelines.manager.append('so/0800_input_kafka.conf.jinja') %}
+{# Disable logstash on manager & receiver nodes unless it has an override configured #}
+{%   if not KAFKA_LOGSTASH %}
+{%     if GLOBALS.role in ['so-manager', 'so-receiver'] and GLOBALS.hostname not in KAFKA_LOGSTASH %}
+{%       do LOGSTASH_MERGED.update({'enabled': False}) %}
+{%     endif %}
+{%   endif %}
+{% endif %}

salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja

@@ -1,4 +1,5 @@
-{%- set kafka_password = salt['pillar.get']('kafka:password') %}
+{%- set kafka_password = salt['pillar.get']('kafka:config:password') %}
+{%- set kafka_trustpass = salt['pillar.get']('kafka:config:trustpass') %}
 {%- set kafka_brokers = salt['pillar.get']('kafka:nodes', {}) %}
 {%- set brokers = [] %}
 
@@ -22,8 +23,8 @@ input {
         ssl_keystore_location => '/usr/share/logstash/kafka-logstash.p12'
         ssl_keystore_password => '{{ kafka_password }}'
         ssl_keystore_type => 'PKCS12'
-        ssl_truststore_location => '/etc/pki/ca-trust/extracted/java/cacerts'
+        ssl_truststore_location => '/etc/pki/kafka-truststore.jks'
-        ssl_truststore_password => 'changeit'
+        ssl_truststore_password => '{{ kafka_trustpass }}'
         decorate_events => true
         tags => [ "elastic-agent", "input-{{ GLOBALS.hostname}}", "kafka" ]
     }

salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja

@@ -1,8 +1,8 @@
-{%- from 'logstash/map.jinja' import REDIS_NODES with context %}
+{%- from 'logstash/map.jinja' import LOGSTASH_REDIS_NODES with context %}
 {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %}
 
-{%- for index in range(REDIS_NODES|length) %}
+{%- for index in range(LOGSTASH_REDIS_NODES|length) %}
-{%-   for host in REDIS_NODES[index] %}
+{%-   for host in LOGSTASH_REDIS_NODES[index] %}
 input {
         redis {
                 host => '{{ host }}'

salt/manager/tools/sbin/so-minion

@@ -528,7 +528,6 @@ function createHEAVYNODE() {
 	pcapspace
 	add_elasticsearch_to_minion
 	add_elastic_agent_to_minion
-	add_logstash_to_minion
 	add_sensor_to_minion
 	add_strelka_to_minion
 	add_redis_to_minion

salt/manager/tools/sbin/so-user

@@ -234,10 +234,14 @@ function updatePassword() {
     passwordHash=$(hashPassword "$password")
     # Update DB with new hash
     echo "update identity_credentials set config=CAST('{\"hashed_password\":\"$passwordHash\"}' as BLOB), created_at=datetime('now'), updated_at=datetime('now') where identity_id='${identityId}' and identity_credential_type_id=(select id from identity_credential_types where name='password');" | sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath"
-    # Deactivate MFA
-    echo "delete from identity_credential_identifiers where identity_credential_id=(select id from identity_credentials where identity_id='${identityId}' and identity_credential_type_id=(select id from identity_credential_types where name in ('totp', 'webauthn', 'oidc')));" | sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath"
-    echo "delete from identity_credentials where identity_id='${identityId}' and identity_credential_type_id=(select id from identity_credential_types where name in ('totp', 'webauthn', 'oidc'));" | sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath"
     [[ $? != 0 ]] && fail "Unable to update password"
+    # Deactivate MFA
+    echo "delete from identity_credential_identifiers where identity_credential_id in (select id from identity_credentials where identity_id='${identityId}' and identity_credential_type_id in (select id from identity_credential_types where name in ('totp', 'webauthn', 'oidc')));" | sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath"
+    [[ $? != 0 ]] && fail "Unable to clear aal2 identity IDs"
+    echo "delete from identity_credentials where identity_id='${identityId}' and identity_credential_type_id in (select id from identity_credential_types where name in ('totp', 'webauthn', 'oidc'));" | sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath"
+    [[ $? != 0 ]] && fail "Unable to clear aal2 identity credentials"
+    echo "update identities set available_aal='aal1' where id='${identityId}';" | sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath"
+    [[ $? != 0 ]] && fail "Unable to reset aal"
   fi
 }
 

salt/manager/tools/sbin/so-user-add

@@ -1,2 +0,0 @@
-#!/bin/bash
-so-user add --email $1

salt/manager/tools/sbin/so-user-disable

@@ -1,2 +0,0 @@
-#!/bin/bash
-so-user disable --email $1

salt/manager/tools/sbin/so-user-enable

@@ -1,2 +0,0 @@
-#!/bin/bash
-so-user enable --email $1

salt/manager/tools/sbin/so-user-list

@@ -1,2 +0,0 @@
-#!/bin/bash
-so-user list

salt/manager/tools/sbin/soup

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
@@ -30,7 +30,7 @@ check_err() {
   [[ $ERR_HANDLED == true ]] && exit $exit_code
 
   if [[ $exit_code -ne 0 ]]; then
-  
+
     set +e
     systemctl_func "start" "$cron_service_name"
     systemctl_func "start" "salt-master"
@@ -108,7 +108,7 @@ add_common() {
 }
 
 airgap_mounted() {
-  # Let's see if the ISO is already mounted. 
+  # Let's see if the ISO is already mounted.
   if [[ -f /tmp/soagupdate/SecurityOnion/VERSION ]]; then
     echo "The ISO is already mounted"
   else
@@ -116,8 +116,8 @@ airgap_mounted() {
       echo "This is airgap. Ask for a location."
       echo ""
       cat << EOF
-In order for soup to proceed, the path to the downloaded Security Onion ISO file, or the path to the CD-ROM or equivalent device containing the ISO media must be provided. 
+In order for soup to proceed, the path to the downloaded Security Onion ISO file, or the path to the CD-ROM or equivalent device containing the ISO media must be provided.
-For example, if you have copied the new Security Onion ISO file to your home directory, then the path might look like /home/myuser/securityonion-2.x.y.iso. 
+For example, if you have copied the new Security Onion ISO file to your home directory, then the path might look like /home/myuser/securityonion-2.x.y.iso.
 Or, if you have burned the new ISO onto an optical disk then the path might look like /dev/cdrom.
 
 EOF
@@ -134,7 +134,7 @@ EOF
         exit 0
       else
         echo "ISO has been mounted!"
-      fi  
+      fi
     elif [[ -f $ISOLOC/SecurityOnion/VERSION ]]; then
       ln -s $ISOLOC /tmp/soagupdate
       echo "Found the update content"
@@ -149,7 +149,7 @@ EOF
         echo "Device has been mounted!"
       fi
     else
-      echo "Could not find Security Onion ISO content at ${ISOLOC}" 
+      echo "Could not find Security Onion ISO content at ${ISOLOC}"
       echo "Ensure the path you entered is correct, and that you verify the ISO that you downloaded."
       exit 0
     fi
@@ -195,7 +195,7 @@ check_airgap() {
       UPDATE_DIR=/tmp/soagupdate/SecurityOnion
       AGDOCKER=/tmp/soagupdate/docker
       AGREPO=/tmp/soagupdate/minimal/Packages
-  else 
+  else
       is_airgap=1
   fi
 }
@@ -308,6 +308,21 @@ clone_to_tmp() {
   fi
 }
 
+disable_logstash_heavynodes() {
+  c=0
+  printf "\nChecking for heavynodes and disabling Logstash if they exist\n"
+  for file in /opt/so/saltstack/local/pillar/minions/*.sls; do
+    if [[ "$file" =~ "_heavynode.sls" && ! "$file" =~ "/opt/so/saltstack/local/pillar/minions/adv_" ]]; then
+      if [ "$c" -eq 0 ]; then
+        c=$((c + 1))
+        FINAL_MESSAGE_QUEUE+=("Logstash has been disabled on all heavynodes. It can be re-enabled via Grid Configuration in SOC.")
+      fi
+      echo "Disabling Logstash for: $file"
+      so-yaml.py replace "$file" logstash.enabled False
+    fi
+  done
+}
+
 enable_highstate() {
     echo "Enabling highstate."
     salt-call state.enable highstate -l info --local
@@ -386,17 +401,18 @@ preupgrade_changes() {
     [[ "$INSTALLEDVERSION" == 2.4.60 ]] && up_to_2.4.70
     [[ "$INSTALLEDVERSION" == 2.4.70 ]] && up_to_2.4.80
     [[ "$INSTALLEDVERSION" == 2.4.80 ]] && up_to_2.4.90
+    [[ "$INSTALLEDVERSION" == 2.4.90 ]] && up_to_2.4.100
     true
 }
 
 postupgrade_changes() {
     # This function is to add any new pillar items if needed.
     echo "Running post upgrade processes."
-    
+
     [[ "$POSTVERSION" == 2.4.2 ]] && post_to_2.4.3
     [[ "$POSTVERSION" == 2.4.3 ]] && post_to_2.4.4
-    [[ "$POSTVERSION" == 2.4.4 ]] && post_to_2.4.5 
+    [[ "$POSTVERSION" == 2.4.4 ]] && post_to_2.4.5
-    [[ "$POSTVERSION" == 2.4.5 ]] && post_to_2.4.10 
+    [[ "$POSTVERSION" == 2.4.5 ]] && post_to_2.4.10
     [[ "$POSTVERSION" == 2.4.10 ]] && post_to_2.4.20
     [[ "$POSTVERSION" == 2.4.20 ]] && post_to_2.4.30
     [[ "$POSTVERSION" == 2.4.30 ]] && post_to_2.4.40
@@ -405,6 +421,7 @@ postupgrade_changes() {
     [[ "$POSTVERSION"  == 2.4.60 ]] && post_to_2.4.70
     [[ "$POSTVERSION"  == 2.4.70 ]] && post_to_2.4.80
     [[ "$POSTVERSION"  == 2.4.80 ]] && post_to_2.4.90
+    [[ "$POSTVERSION"  == 2.4.90 ]] && post_to_2.4.100
     true
 }
 
@@ -485,10 +502,15 @@ post_to_2.4.80() {
 }
 
 post_to_2.4.90() {
-  echo "Nothing to apply"
+  disable_logstash_heavynodes
   POSTVERSION=2.4.90
 }
 
+post_to_2.4.100() {
+  echo "Nothing to apply"
+  POSTVERSION=2.4.100
+}
+
 repo_sync() {
   echo "Sync the local repo."
   su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync."
@@ -554,13 +576,13 @@ up_to_2.4.5() {
 
 up_to_2.4.10() {
   echo "Nothing to do for 2.4.10"
-  
+
   INSTALLEDVERSION=2.4.10
 }
 
 up_to_2.4.20() {
   echo "Nothing to do for 2.4.20"
-  
+
   INSTALLEDVERSION=2.4.20
 }
 
@@ -613,7 +635,7 @@ up_to_2.4.50() {
   mkdir /opt/so/rules/nids/suri
   chown socore:socore /opt/so/rules/nids/suri
   mv -v /opt/so/rules/nids/*.rules /opt/so/rules/nids/suri/.
-  
+
   echo "Adding /nsm/elastic-fleet/artifacts to file_roots in /etc/salt/master using so-yaml"
   so-yaml.py append /etc/salt/master file_roots.base  /nsm/elastic-fleet/artifacts
 
@@ -658,9 +680,23 @@ up_to_2.4.80() {
 }
 
 up_to_2.4.90() {
-  echo "Nothing to apply"
+  kafkatrust=$(get_random_value)
+  # rearranging the kafka pillar to reduce clutter in SOC UI
+  kafkasavedpass=$(so-yaml.py get /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.password)
+  kafkatrimpass=$(echo "$kafkasavedpass" | sed -n '1 p' )
+  echo "Making changes to the Kafka pillar layout"
+  so-yaml.py remove /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.password
+  so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.password "$kafkatrimpass"
+  so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.trustpass "$kafkatrust"
+  so-elasticsearch-query so-detection*/_settings -X PUT -d '{"index":{"refresh_interval":"1s"}}'
+
   INSTALLEDVERSION=2.4.90
 }
+up_to_2.4.100() {
+  # Elastic Update for this release, so download Elastic Agent files
+  determine_elastic_agent_upgrade
+  INSTALLEDVERSION=2.4.100
+}
 
 add_detection_test_pillars() {
   if [[ -n "$SOUP_INTERNAL_TESTING" ]]; then
@@ -691,7 +727,7 @@ Documentation: https://docs.securityonion.net/en/2.4/telemetry.html
 ASSIST_EOF
 
     echo -n "Continue the upgrade with SOC Telemetry enabled [Y/n]? "
-  
+
     read -r input
     input=$(echo "${input,,}" | xargs echo -n)
     echo ""
@@ -732,7 +768,7 @@ suricata_idstools_migration() {
   rsync -av /opt/so/rules/nids/suri/local.rules /nsm/backup/detections-migration/suricata/local-rules
   if [[ -f /opt/so/saltstack/local/salt/idstools/rules/local.rules ]]; then
       rsync -av /opt/so/saltstack/local/salt/idstools/rules/local.rules /nsm/backup/detections-migration/suricata/local-rules/local.rules.bak
-  fi 
+  fi
 
   #Tell SOC to migrate
   mkdir -p /opt/so/conf/soc/migrations
@@ -749,7 +785,7 @@ playbook_migration() {
   crontab -l | grep -v 'so-playbook-ruleupdate_cron' | crontab -
 
   if grep -A 1 'playbook:'  /opt/so/saltstack/local/pillar/minions/* | grep -q 'enabled: True'; then
-  
+
     # Check for active Elastalert rules
     active_rules_count=$(find /opt/so/rules/elastalert/playbook/ -type f \( -name "*.yaml" -o -name "*.yml" \) | wc -l)
 
@@ -841,7 +877,7 @@ upgrade_space() {
     fi
   else
       echo "You have enough space for upgrade. Proceeding with soup."
-  fi  
+  fi
 }
 
 unmount_update() {
@@ -899,7 +935,7 @@ upgrade_check() {
     fi
   else
     is_hotfix=false
-  fi 
+  fi
 
 }
 
@@ -911,7 +947,7 @@ upgrade_check_salt() {
     echo "Salt needs to be upgraded to $NEWSALTVERSION."
     UPGRADESALT=1
   fi
-}   
+}
 
 upgrade_salt() {
   SALTUPGRADED=True
@@ -921,7 +957,9 @@ upgrade_salt() {
   if [[ $is_rpm ]]; then
     echo "Removing yum versionlock for Salt."
     echo ""
-    yum versionlock delete "salt-*"
+    yum versionlock delete "salt"
+    yum versionlock delete "salt-minion"
+    yum versionlock delete "salt-master"
     echo "Updating Salt packages."
     echo ""
     set +e
@@ -939,7 +977,9 @@ upgrade_salt() {
     set -e
     echo "Applying yum versionlock for Salt."
     echo ""
-    yum versionlock add "salt-*"
+    yum versionlock add "salt-0:$NEWSALTVERSION-0.*"
+    yum versionlock add "salt-minion-0:$NEWSALTVERSION-0.*"
+    yum versionlock add "salt-master-0:$NEWSALTVERSION-0.*"
   # Else do Ubuntu things
   elif [[ $is_deb ]]; then
     echo "Removing apt hold for Salt."
@@ -1029,7 +1069,7 @@ apply_hotfix() {
       mv /etc/pki/managerssl.key /etc/pki/managerssl.key.old
       systemctl_func "start" "salt-minion"
       (wait_for_salt_minion "$MINIONID" "5" '/dev/stdout' || fail "Salt minion was not running or ready.") 2>&1 | tee -a "$SOUP_LOG"
-    fi  
+    fi
   else
    echo "No actions required. ($INSTALLEDVERSION/$HOTFIXVERSION)"
   fi
@@ -1058,7 +1098,7 @@ apply_hotfix() {
 
 main() {
   trap 'check_err $?' EXIT
-  
+
   if [ -n "$BRANCH" ]; then
     echo "SOUP will use the $BRANCH branch."
     echo ""
@@ -1242,7 +1282,7 @@ main() {
     echo "Waiting on the Salt Master service to be ready."
     check_salt_master_status || fail  "Can't access salt master or it is not ready. Check $SOUP_LOG for details."
     set -e
-    
+
     echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes."
     (wait_for_salt_minion "$MINIONID" "5" '/dev/stdout' || fail "Salt minion was not running or ready.") 2>&1 | tee -a "$SOUP_LOG"
     highstate
@@ -1293,9 +1333,9 @@ main() {
     if [[ $NUM_MINIONS -gt 1 ]]; then
 
       cat << EOF
-    
+
-  
+
-  
+
 This appears to be a distributed deployment. Other nodes should update themselves at the next Salt highstate (typically within 15 minutes). Do not manually restart anything until you know that all the search/heavy nodes in your deployment are updated. This is especially important if you are using true clustering for Elasticsearch.
 
 Each minion is on a random 15 minute check-in period and things like network bandwidth can be a factor in how long the actual upgrade takes. If you have a heavy node on a slow link, it is going to take a while to get the containers to it. Depending on what changes happened between the versions, Elasticsearch might not be able to talk to said heavy node until the update is complete.
@@ -1348,13 +1388,13 @@ while getopts ":b:f:y" opt; do
         echo "Cannot run soup in unattended mode. You must run soup manually to accept the Elastic License."
         exit 1
       else
-        UNATTENDED=true 
+        UNATTENDED=true
       fi
     ;;
     f )
       ISOLOC="$OPTARG"
     ;;
-    \? ) 
+    \? )
       echo "Usage: soup [-b] [-y] [-f <iso location>]"
       exit 1
     ;;
@@ -1380,6 +1420,8 @@ Please review the following for more information about the update process and re
 $DOC_BASE_URL/soup.html
 https://blog.securityonion.net
 
+WARNING: If you run soup via an SSH session and that SSH session terminates, then any processes running in that session would terminate. You should avoid leaving soup unattended especially if the machine you are SSHing from is configured to sleep after a period of time. You might also consider using something like screen or tmux so that if your SSH session terminates, the processes will continue running on the server.
+
 EOF
 
   cat << EOF

salt/motd/files/so_motd.jinja

@@ -15,12 +15,11 @@ Access the Security Onion web interface at https://{{ GLOBALS.url_base }}
   {%- endfor -%}
 
   {%- if minions_need_restarted | length > 0 %}
-****************************************************************************************************
+####################################################################################################
-* The following nodes in your Security Onion grid may need to be restarted due to package updates. * 
+# The following nodes in your Security Onion grid may need to be restarted due to package updates. #
-* If the node has already been patched, restarted and been up for less than 15 minutes, then it    *
+# If a node has already been patched and restarted but has been up for less than 15 minutes,       #
-* may not have updated it's restart_needed status yet. This will cause it to be listed below, even *
+# then it may not have updated its status yet.                                                     #
-* if it has already been restarted. This feature will be improved in the future.                   *
+####################################################################################################
-****************************************************************************************************
 
     {% for minion in minions_need_restarted -%}
   {{ minion }}

salt/registry/enabled.sls

@@ -14,7 +14,7 @@ include:
 # Install the registry container
 so-dockerregistry:
   docker_container.running:
-    - image: ghcr.io/security-onion-solutions/registry:2.8.2
+    - image: ghcr.io/security-onion-solutions/registry:2.8.3
     - hostname: so-registry
     - networks:
       - sobridge:

salt/salt/files/engines.conf

@@ -43,20 +43,20 @@ engines:
                   - cmd.run:
                       cmd: /usr/sbin/so-yaml.py replace /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.enabled True
                   - cmd.run:
-                      cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver' saltutil.kill_all_jobs
+                      cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver or G@role:so-searchnode' saltutil.kill_all_jobs
                   - cmd.run:
                       cmd: salt-call state.apply kafka.nodes
                   - cmd.run:
-                      cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver' state.highstate
+                      cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver or G@role:so-searchnode' state.highstate
               'KAFKA':
                 to:
                   'REDIS':
                   - cmd.run:
                       cmd: /usr/sbin/so-yaml.py replace /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.enabled False
                   - cmd.run:
-                      cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver' saltutil.kill_all_jobs
+                      cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver or G@role:so-searchnode' saltutil.kill_all_jobs
                   - cmd.run:
-                      cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver' state.highstate
+                      cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver or G@role:so-searchnode' state.highstate
         - files:
             - /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls
             - /opt/so/saltstack/local/pillar/kafka/adv_kafka.sls

salt/salt/map.jinja

@@ -3,12 +3,10 @@
 
 {% if grains.os_family == 'Debian' %}
   {% set SPLITCHAR = '+' %}
-  {% set SALTNOTHELD = salt['cmd.run']('apt-mark showhold | grep -q salt ; echo $?', python_shell=True) %}
   {% set SALTPACKAGES = ['salt-common', 'salt-master', 'salt-minion'] %}
   {% set SYSTEMD_UNIT_FILE = '/lib/systemd/system/salt-minion.service' %}
 {% else %}
   {% set SPLITCHAR = '-' %}
-  {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep -q salt ; echo $?', python_shell=True) %}
   {% set SALTPACKAGES = ['salt', 'salt-master', 'salt-minion'] %}
   {% set SYSTEMD_UNIT_FILE = '/usr/lib/systemd/system/salt-minion.service' %}
 {% endif %}

salt/salt/master.defaults.yaml

@@ -1,4 +1,4 @@
 # version cannot be used elsewhere in this pillar as soup is grepping for it to determine if Salt needs to be patched
 salt:
   master:
-    version: 3006.6
+    version: 3006.9

salt/salt/master.sls

@@ -1,16 +1,13 @@
-{% from 'salt/map.jinja' import SALTNOTHELD %}
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls in allowed_states %}
 
 include:
   - salt.minion
 
-{% if SALTNOTHELD == 1 %}
 hold_salt_master_package:
   module.run:
     - pkg.hold:
       - name: salt-master
-{% endif %}
 
 # prior to 2.4.30 this engine ran on the manager with salt-minion
 # this has changed to running with the salt-master in 2.4.30

salt/salt/minion.defaults.yaml

@@ -1,6 +1,6 @@
 # version cannot be used elsewhere in this pillar as soup is grepping for it to determine if Salt needs to be patched
 salt:
   minion:
-    version: 3006.6
+    version: 3006.9
     check_threshold: 3600 # in seconds, threshold used for so-salt-minion-check. any value less than 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default
     service_start_delay: 30 # in seconds.

salt/salt/minion.sls

@@ -2,13 +2,13 @@
 {% from 'salt/map.jinja' import UPGRADECOMMAND with context %}
 {% from 'salt/map.jinja' import SALTVERSION %}
 {% from 'salt/map.jinja' import INSTALLEDSALTVERSION %}
-{% from 'salt/map.jinja' import SALTNOTHELD %}
 {% from 'salt/map.jinja' import SALTPACKAGES %}
 {% from 'salt/map.jinja' import SYSTEMD_UNIT_FILE %}
 {% import_yaml 'salt/minion.defaults.yaml' as SALTMINION %}
 {% set service_start_delay = SALTMINION.salt.minion.service_start_delay %}
 
 include:
+  - salt.python_modules
   - salt
   - systemd.reload
   - repo.client
@@ -19,15 +19,12 @@ include:
 
 {% if INSTALLEDSALTVERSION|string != SALTVERSION|string %}
 
-{% if SALTNOTHELD | int == 0 %}
 unhold_salt_packages:
-  module.run:
+  pkg.unheld:
-    - pkg.unhold:
+    - pkgs:
-      - pkgs:
 {% for package in SALTPACKAGES %}
-        - {{ package }}
+      - {{ package }}
 {% endfor %}
-{% endif %}
 
 install_salt_minion:
   cmd.run:
@@ -41,15 +38,12 @@ install_salt_minion:
 
 {% if INSTALLEDSALTVERSION|string == SALTVERSION|string %}
 
-{% if SALTNOTHELD | int == 1 %}
 hold_salt_packages:
-  module.run:
+  pkg.held:
-    - pkg.hold:
+    - pkgs:
-      - pkgs:
 {% for package in SALTPACKAGES %}
-        - {{ package }}
+      - {{ package }}: {{SALTVERSION}}-0.*
 {% endfor %}
-{% endif %}
 
 remove_error_log_level_logfile:
   file.line:

salt/salt/python_modules.sls

@@ -0,0 +1,21 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+docker_module_package:
+  file.recurse:
+    - name: /opt/so/conf/salt/module_packages/docker
+    - source: salt://salt/module_packages/docker
+    - clean: True
+    - makedirs: True
+
+# fail hard on this state so that soup would be cancelled on a manager (eventhough salt would have already updated)
+# on a non manager, failing hard here will prevent the minion from upgrading
+# we want to fail hard here to prevent the minion from upgrading and potetially being able to manager docker containers from a dep mismatch
+docker_python_module_install:
+  cmd.run:
+    - name: /opt/saltstack/salt/bin/python3.10 -m pip install docker --no-index --find-links=/opt/so/conf/salt/module_packages/docker/ --upgrade
+    - onchanges:
+      - file: docker_module_package
+    - failhard: True

salt/soc/config.sls

@@ -90,7 +90,7 @@ filedetectionsbackup:
 
 crondetectionsruntime:
   cron.present:
-    - name: /usr/sbin/so-detections-runtime-status cron 
+    - name: /usr/sbin/so-detections-runtime-status cron
     - identifier: detections-runtime-status
     - user: root
     - minute: '*/10'
@@ -190,6 +190,14 @@ socsigmarepo:
     - group: 939
     - mode: 775
 
+socsensoronirepos:
+  file.directory:
+    - name: /opt/so/conf/soc/ai_summary_repos
+    - user: 939
+    - group: 939
+    - mode: 775
+    - makedirs: True
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/soc/defaults.map.jinja

@@ -14,7 +14,7 @@
 {% endfor %}
 
 {# add all grid heavy nodes to soc.server.modules.elastic.remoteHostUrls #}
-{% for node_type, minions in salt['pillar.get']('logstash:nodes', {}).items() %}
+{% for node_type, minions in salt['pillar.get']('elasticsearch:nodes', {}).items() %}
 {%   if node_type in ['heavynode'] %}
 {%     for m in minions.keys() %}
 {%       do SOCDEFAULTS.soc.config.server.modules.elastic.remoteHostUrls.append('https://' ~ m ~ ':9200') %}

salt/soc/defaults.yaml

@@ -96,6 +96,12 @@ soc:
         links:
           - '/#/alerts?q=rule.uuid: {:so_detection.publicId|escape} | groupby rule.name event.module* event.severity_label'
         target: ''
+      - name: actionAdd
+        description: actionAddHelp
+        icon: fa-plus
+        links:
+          - '/#/config?s=soc.config.actions'
+        target: ''
     eventFields:
       default:
         - soc_timestamp
@@ -1298,6 +1304,7 @@ soc:
       maxPacketCount: 5000
       htmlDir: html
       importUploadDir: /nsm/soc/uploads
+      forceUserOtp: false
       modules:
         cases: soc
         filedatastore:
@@ -1305,7 +1312,10 @@ soc:
         kratos:
           hostUrl:
         elastalertengine:
-          allowRegex: ''
+          aiRepoUrl: https://github.com/Security-Onion-Solutions/securityonion-resources
+          aiRepoBranch: generated-summaries-stable
+          aiRepoPath: /opt/sensoroni/ai_summary_repos
+          showAiSummaries: true
           autoUpdateEnabled: true
           autoEnabledSigmaRules:
             default:
@@ -1321,7 +1331,6 @@ soc:
           communityRulesImportFrequencySeconds: 86400
           communityRulesImportErrorSeconds: 300
           failAfterConsecutiveErrorCount: 10
-          denyRegex: ''
           elastAlertRulesFolder: /opt/sensoroni/elastalert
           reposFolder: /opt/sensoroni/sigma/repos
           rulesFingerprintFile: /opt/sensoroni/fingerprints/sigma.fingerprint
@@ -1360,6 +1369,8 @@ soc:
           maxLogLength: 1024
           asyncThreshold: 10
           lookupTunnelParent: true
+          maxScrollSize: 10000
+          bulkIndexerWorkerCount: -1
         influxdb:
           hostUrl:
           token:
@@ -1384,7 +1395,10 @@ soc:
           userFiles:
             - rbac/users_roles
         strelkaengine:
-          allowRegex: ''
+          aiRepoUrl: https://github.com/Security-Onion-Solutions/securityonion-resources
+          aiRepoBranch: generated-summaries-stable
+          aiRepoPath: /opt/sensoroni/ai_summary_repos
+          showAiSummaries: true
           autoEnabledYaraRules:
             - securityonion-yara
           autoUpdateEnabled: true
@@ -1392,7 +1406,6 @@ soc:
           communityRulesImportErrorSeconds: 300
           failAfterConsecutiveErrorCount: 10
           compileYaraPythonScriptPath: /opt/sensoroni/yara/compile_yara.py
-          denyRegex: ''
           reposFolder: /opt/sensoroni/yara/repos
           rulesRepos:
             default:
@@ -1407,14 +1420,18 @@ soc:
           stateFilePath: /opt/sensoroni/fingerprints/strelkaengine.state
           integrityCheckFrequencySeconds: 1200
         suricataengine:
-          allowRegex: ''
+          aiRepoUrl: https://github.com/Security-Onion-Solutions/securityonion-resources
+          aiRepoBranch: generated-summaries-stable
+          aiRepoPath: /opt/sensoroni/ai_summary_repos
+          showAiSummaries: true
           autoUpdateEnabled: true
           communityRulesImportFrequencySeconds: 86400
           communityRulesImportErrorSeconds: 300
           customRulesets:
+          disableRegex: []
+          enableRegex: []
           failAfterConsecutiveErrorCount: 10
           communityRulesFile: /nsm/rules/suricata/emerging-all.rules
-          denyRegex: ''
           rulesFingerprintFile: /opt/sensoroni/fingerprints/emerging-all.fingerprint
           stateFilePath: /opt/sensoroni/fingerprints/suricataengine.state
           integrityCheckFrequencySeconds: 1200
@@ -2286,15 +2303,15 @@ soc:
 
               alert http $EXTERNAL_NET any -> $HOME_NET any (msg:"Example Rule Title - 'example' String Detected"; content:"example"; sid:[publicId]; rev:1;)
             strelka: |
-              /* 
+              /*
               This is a YARA rule template. Replace all template values with your own values.
               The YARA rule name is the unique identifier for the rule.
               Docs: https://yara.readthedocs.io/en/stable/writingrules.html#writing-yara-rules
-              */ 
+              */
 
               rule Example // This identifier _must_ be unique
               {
-                  meta: 
+                  meta:
                       description = "Generic YARA Rule"
                       author = "@SecurityOnion"
                       date = "YYYY-MM-DD"
@@ -2317,7 +2334,7 @@ soc:
               id: [publicId]
               status: 'experimental'
               description: |
-                  This should be a detailed description of what this Detection focuses on: what we are trying to find and why we are trying to find it. 
+                  This should be a detailed description of what this Detection focuses on: what we are trying to find and why we are trying to find it.
                   For example, from rule 97a80ec7-0e2f-4d05-9ef4-65760e634f6b: "Detects a whoami.exe executed with the /priv command line flag instructing the tool to show all current user privileges. This is often used after a privilege escalation attempt."
               references:
                 - 'https://local.invalid'
@@ -2326,7 +2343,7 @@ soc:
               tags:
                 - detection.threat_hunting
                 - attack.technique_id
-              logsource: 
+              logsource:
                 category: process_creation
                 product: windows
               detection:

salt/soc/enabled.sls

@@ -33,6 +33,7 @@ so-soc:
       - /nsm/soc/uploads:/nsm/soc/uploads:rw
       - /opt/so/log/soc/:/opt/sensoroni/logs/:rw
       - /opt/so/conf/soc/soc.json:/opt/sensoroni/sensoroni.json:ro
+      - /opt/so/conf/soc/ai_summary_repos:/opt/sensoroni/ai_summary_repos:rw
 {% if SOCMERGED.telemetryEnabled and not GLOBALS.airgap %}
       - /opt/so/conf/soc/analytics.js:/opt/sensoroni/html/js/analytics.js:ro
 {% endif %}

salt/soc/files/soc/motd.md

@@ -1,6 +1,6 @@
 ## Getting Started
 
-New to Security Onion? Click the menu in the upper-right corner and you'll find links for [Help](/docs/) and a [Cheat Sheet](/docs/cheatsheet.pdf) that will help you best utilize Security Onion to hunt for evil! In addition, check out our free Security Onion Essentials online course, available on our [Training](https://securityonionsolutions.com/training) website.
+New to Security Onion? Click the menu in the upper-right corner and you'll find links for [Help](/docs/) and a [Cheat Sheet](/docs/cheatsheet.pdf) that will help you best utilize Security Onion to hunt for evil! In addition, check out our free Security Onion Essentials online course, available on our [Training](https://securityonion.com/training) website.
 
 If you're ready to dive in, take a look at the [Alerts](/#/alerts) interface to see what Security Onion has detected so far. If you find any false positives, then you can tune those in [Detections](/#/detections).
 
@@ -20,13 +20,17 @@ For more coverage of your enterprise, you can deploy the Elastic Agent to endpoi
 
 To see all the latest features and fixes in this version of Security Onion, click the upper-right menu and then click the [What's New](/docs/release-notes.html) link.
 
+## Security Onion Pro
+
+Need enterprise features and premium support? Check out [Security Onion Pro](https://securityonion.com/pro/)!
+
 ## Enterprise Appliances
 
-Want the best hardware for your enterprise deployment? Check out our [enterprise appliances](https://securityonionsolutions.com/hardware/)!
+Want the best hardware for your enterprise deployment? Check out our [enterprise appliances](https://securityonion.com/hardware/)!
 
 ## Premium Support
 
-Experiencing difficulties and need priority support or remote assistance? We offer a [premium support plan](https://securityonionsolutions.com/support/) to assist corporate, educational, and government organizations.
+Experiencing difficulties and need priority support or remote assistance? We offer a [premium support plan](https://securityonion.com/support/) to assist corporate, educational, and government organizations.
 
 ## Customize This Space
 

salt/soc/merged.map.jinja

@@ -5,9 +5,9 @@
 
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% from 'soc/defaults.map.jinja' import SOCDEFAULTS with context %}
-{% from 'logstash/map.jinja' import LOGSTASH_NODES %}
+{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
 {% from 'manager/map.jinja' import MANAGERMERGED %}
-{% set DOCKER_EXTRA_HOSTS = LOGSTASH_NODES %}
+{% set DOCKER_EXTRA_HOSTS = ELASTICSEARCH_NODES %}
 {% do DOCKER_EXTRA_HOSTS.append({GLOBALS.influxdb_host:pillar.node_data[GLOBALS.influxdb_host].ip}) %}
 
 {% set SOCMERGED = salt['pillar.get']('soc', SOCDEFAULTS, merge=true) %}

salt/soc/soc_soc.yaml

@@ -81,8 +81,27 @@ soc:
         description: Maximum number of packets to show in the PCAP viewer. Larger values can cause more resource utilization on both the SOC server and the browser.
         global: True
         advanced: True
+      forceUserOtp:
+        title: Require TOTP
+        description: Require all users to enable Time-based One Time Passwords (MFA) upon login to SOC.
+        global: True
       modules:
         elastalertengine:
+          aiRepoUrl:
+            description: URL to the AI repository. This is used to pull in AI models for use in ElastAlert rules.
+            global: True
+            advanced: True
+          aiRepoBranch:
+            description: The branch to pull from the AI repository. Leaving this blank will pull the default branch.
+            global: True
+            advanced: True
+          aiRepoPath:
+            description: Path to the AI repository. This is used to pull in AI models for use in ElastAlert rules.
+            global: True
+            advanced: True
+          showAiSummaries:
+            description: Show AI summaries for ElastAlert rules.
+            global: True
           additionalAlerters:
             title: Additional Alerters
             description: Specify additional alerters to enable for all Sigma rules, one alerter name per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. Note that the configuration parameters for these alerters must be provided in the ElastAlert configuration section. Filter for 'Alerter' to find this related setting. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key.
@@ -90,11 +109,6 @@ soc:
             helpLink: sigma.html
             forcedType: "[]string"
             multiline: True
-          allowRegex:
-            description: 'Regex used to filter imported Sigma rules. Deny regex takes precedence over the Allow regex setting.'
-            global: True
-            advanced: True
-            helpLink: sigma.html
           autoEnabledSigmaRules:
             default: &autoEnabledSigmaRules
               description: 'Sigma rules to automatically enable on initial import. Format is $Ruleset+$Level - for example, for the core community ruleset and critical level rules: core+critical. These will be applied based on role if defined and default if not.'
@@ -103,11 +117,6 @@ soc:
               helpLink: sigma.html
             so-eval: *autoEnabledSigmaRules
             so-import: *autoEnabledSigmaRules
-          denyRegex:
-            description: 'Regex used to filter imported Sigma rules. Deny regex takes precedence over the Allow regex setting.'
-            global: True
-            advanced: True
-            helpLink: sigma.html
           communityRulesImportFrequencySeconds:
             description: 'How often to check for new Sigma rules (in seconds). This applies to both Community Rule Packages and any configured Git repos.'
             global: True
@@ -174,6 +183,10 @@ soc:
           lookupTunnelParent:
             description: When true, if a pivoted event appears to be encapsulated, such as in a VXLAN packet, then SOC will pivot to the VXLAN packet stream. When false, SOC will attempt to pivot to the encapsulated packet stream itself, but at the risk that it may be unable to locate it in the stored PCAP data.
             global: True
+          maxScrollSize:
+            description: The maximum number of documents to request in a single Elasticsearch scroll request.
+          bulkIndexWorkerCount:
+            description: The number of worker threads to use when bulk indexing data into Elasticsearch. A value below 1 will default to the number of CPUs available.
         sostatus:
           refreshIntervalMs:
             description: Duration (in milliseconds) between refreshes of the grid status. Shortening this duration may not have expected results, as the backend systems feeding this sostatus data will continue their updates as scheduled.
@@ -195,21 +208,26 @@ soc:
             advanced: True
             forcedType: int
         strelkaengine:
-          allowRegex:
+          aiRepoUrl:
-            description: 'Regex used to filter imported YARA rules. Deny regex takes precedence over the Allow regex setting.'
+            description: URL to the AI repository. This is used to pull in AI models for use in Strelka rules.
             global: True
             advanced: True
-            helpLink: yara.html
+          aiRepoBranch:
+            description: The branch to pull from the AI repository. Leaving this blank will pull the default branch.
+            global: True
+            advanced: True
+          aiRepoPath:
+            description: Path to the AI repository. This is used to pull in AI models for use in Strelka rules.
+            global: True
+            advanced: True
+          showAiSummaries:
+            description: Show AI summaries for Strelka rules.
+            global: True
           autoEnabledYaraRules:
             description: 'YARA rules to automatically enable on initial import. Format is $Ruleset - for example, for the default shipped ruleset: securityonion-yara'
             global: True
             advanced: True
             helpLink: sigma.html
-          denyRegex:
-            description: 'Regex used to filter imported YARA rules. Deny regex takes precedence over the Allow regex setting.'
-            global: True
-            advanced: True
-            helpLink: yara.html
           communityRulesImportFrequencySeconds:
             description: 'How often to check for new YARA rules (in seconds). This applies to both Community Rules and any configured Git repos.'
             global: True
@@ -228,21 +246,34 @@ soc:
               helpLink: yara.html
             airgap: *serulesRepos
         suricataengine:
-          allowRegex:
+          aiRepoUrl:
-            description: 'Regex used to filter imported Suricata rules. Deny regex takes precedence over the Allow regex setting.'
+            description: URL to the AI repository. This is used to pull in AI models for use in Suricata rules.
             global: True
             advanced: True
-            helpLink: suricata.html
+          aiRepoBranch:
-          denyRegex:
+            description: The branch to pull from the AI repository. Leaving this blank will pull the default branch.
-            description: 'Regex used to filter imported Suricata rules. Deny regex takes precedence over the Allow regex setting.'
             global: True
             advanced: True
-            helpLink: suricata.html
+          aiRepoPath:
+            description: Path to the AI repository. This is used to pull in AI models for use in Suricata rules.
+            global: True
+            advanced: True
+          showAiSummaries:
+            description: Show AI summaries for Suricata rules.
+            global: True
           communityRulesImportFrequencySeconds:
             description: 'How often to check for new Suricata rules (in seconds).'
             global: True
             advanced: True
             helpLink: suricata.html
+          disableRegex:
+            description: A list of regular expressions used to automatically disable rules that match any of them. Each regular expression is tested against the rule's content.
+            global: True
+            forcedType: "[]string"
+          enableRegex:
+            description: A list of regular expressions used to automatically enable rules that match any of them. Each regular expression is tested against the rule's content. Takes priority over disableRegex matches.
+            global: True
+            forcedType: "[]string"
           integrityCheckFrequencySeconds:
             description: 'How often the Suricata integrity checker runs (in seconds). This verifies the integrity of deployed rules.'
             global: True

salt/ssl/init.sls

@@ -17,8 +17,6 @@
   {% set COMMONNAME = GLOBALS.manager %}
 {% endif %}
 
-{% set kafka_password = salt['pillar.get']('kafka:password') %}
-
 {% if grains.id.split('_')|last in ['manager', 'managersearch', 'eval', 'standalone', 'import'] %}
 include:
   - ca
@@ -666,7 +664,6 @@ elastickeyperms:
 {%- endif %}
 
 {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone'] %}
-
 elasticfleet_kafka_key:
   x509.private_key_managed:
     - name: /etc/pki/elasticfleet-kafka.key
@@ -696,17 +693,13 @@ elasticfleet_kafka_crt:
     - retry:
         attempts: 5
         interval: 30
-  cmd.run:
-  - name: "/usr/bin/openssl pkcs8 -in /etc/pki/elasticfleet-kafka.key -topk8 -out /etc/pki/elasticfleet-kafka.p8 -nocrypt"
-  - onchanges:
-    - x509: elasticfleet_kafka_key
 
 elasticfleet_kafka_cert_perms:
   file.managed:
     - replace: False
     - name: /etc/pki/elasticfleet-kafka.crt
     - mode: 640
-    - user: 960
+    - user: 947
     - group: 939
 
 elasticfleet_kafka_key_perms:
@@ -714,187 +707,8 @@ elasticfleet_kafka_key_perms:
     - replace: False
     - name: /etc/pki/elasticfleet-kafka.key
     - mode: 640
-    - user: 960
+    - user: 947
     - group: 939
-
-elasticfleet_kafka_pkcs8_perms:
-  file.managed:
-    - replace: False
-    - name: /etc/pki/elasticfleet-kafka.p8
-    - mode: 640
-    - user: 960
-    - group: 939
-
-kafka_client_key:
-  x509.private_key_managed:
-    - name: /etc/pki/kafka-client.key
-    - keysize: 4096
-    - backup: True
-    - new: True
-    {% if salt['file.file_exists']('/etc/pki/kafka-client.key') -%}
-    - prereq:
-      - x509: /etc/pki/kafka-client.crt
-    {%- endif %}
-    - retry:
-        attempts: 5
-        interval: 30
-
-kafka_client_crt:
-  x509.certificate_managed:
-    - name: /etc/pki/kafka-client.crt
-    - ca_server: {{ ca_server }}
-    - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
-    - signing_policy: kafka
-    - private_key: /etc/pki/kafka-client.key
-    - CN: {{ GLOBALS.hostname }}
-    - days_remaining: 0
-    - days_valid: 820
-    - backup: True
-    - timeout: 30
-    - retry:
-        attempts: 5
-        interval: 30
-
-kafka_client_key_perms:
-  file.managed:
-    - replace: False
-    - name: /etc/pki/kafka-client.key
-    - mode: 640
-    - user: 960
-    - group: 939
-
-kafka_client_crt_perms:
-  file.managed:
-    - replace: False
-    - name: /etc/pki/kafka-client.crt
-    - mode: 640
-    - user: 960
-    - group: 939
-
-{% endif %}
-
-{% if grains['role'] in ['so-manager', 'so-managersearch','so-receiver', 'so-standalone'] %}
-
-kafka_key:
-  x509.private_key_managed:
-    - name: /etc/pki/kafka.key
-    - keysize: 4096
-    - backup: True
-    - new: True
-    {% if salt['file.file_exists']('/etc/pki/kafka.key') -%}
-    - prereq:
-      - x509: /etc/pki/kafka.crt
-    {%- endif %}
-    - retry:
-        attempts: 5
-        interval: 30
-
-kafka_crt:
-  x509.certificate_managed:
-    - name: /etc/pki/kafka.crt
-    - ca_server: {{ ca_server }}
-    - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
-    - signing_policy: kafka
-    - private_key: /etc/pki/kafka.key
-    - CN: {{ GLOBALS.hostname }}
-    - days_remaining: 0
-    - days_valid: 820
-    - backup: True
-    - timeout: 30
-    - retry:
-        attempts: 5
-        interval: 30
-  cmd.run:
-    - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/kafka.key -in /etc/pki/kafka.crt -export -out /etc/pki/kafka.p12 -nodes -passout pass:{{ kafka_password }}"
-    - onchanges:
-      - x509: /etc/pki/kafka.key
-kafka_key_perms:
-  file.managed:
-    - replace: False
-    - name: /etc/pki/kafka.key
-    - mode: 640
-    - user: 960
-    - group: 939
-
-kafka_crt_perms:
-  file.managed:
-    - replace: False
-    - name: /etc/pki/kafka.crt
-    - mode: 640
-    - user: 960
-    - group: 939
-
-kafka_pkcs12_perms:
-  file.managed:
-    - replace: False
-    - name: /etc/pki/kafka.p12
-    - mode: 640
-    - user: 960
-    - group: 939
-
-{% endif %}
-
-# Standalone needs kafka-logstash for automated testing. Searchnode/manager search need it for logstash to consume from Kafka.
-# Manager will have cert, but be unused until a pipeline is created and logstash enabled.
-{% if grains['role'] in ['so-standalone', 'so-managersearch', 'so-searchnode', 'so-manager'] %}
-kafka_logstash_key:
-  x509.private_key_managed:
-    - name: /etc/pki/kafka-logstash.key
-    - keysize: 4096
-    - backup: True
-    - new: True
-    {% if salt['file.file_exists']('/etc/pki/kafka-logstash.key') -%}
-    - prereq:
-      - x509: /etc/pki/kafka-logstash.crt
-    {%- endif %}
-    - retry:
-        attempts: 5
-        interval: 30
-
-kafka_logstash_crt:
-  x509.certificate_managed:
-    - name: /etc/pki/kafka-logstash.crt
-    - ca_server: {{ ca_server }}
-    - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
-    - signing_policy: kafka
-    - private_key: /etc/pki/kafka-logstash.key
-    - CN: {{ GLOBALS.hostname }}
-    - days_remaining: 0
-    - days_valid: 820
-    - backup: True
-    - timeout: 30
-    - retry:
-        attempts: 5
-        interval: 30
-  cmd.run:
-    - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/kafka-logstash.key -in /etc/pki/kafka-logstash.crt -export -out /etc/pki/kafka-logstash.p12 -nodes -passout pass:{{ kafka_password }}"
-    - onchanges:
-      - x509: /etc/pki/kafka-logstash.key
-
-kafka_logstash_key_perms:
-  file.managed:
-    - replace: False
-    - name: /etc/pki/kafka-logstash.key
-    - mode: 640
-    - user: 960
-    - group: 939
-
-kafka_logstash_crt_perms:
-  file.managed:
-    - replace: False
-    - name: /etc/pki/kafka-logstash.crt
-    - mode: 640
-    - user: 960
-    - group: 939
-
-kafka_logstash_pkcs12_perms:
-  file.managed:
-    - replace: False
-    - name: /etc/pki/kafka-logstash.p12
-    - mode: 640
-    - user: 960
-    - group: 931
-
 {% endif %}
 
 {% else %}

salt/suricata/map.jinja

@@ -101,20 +101,20 @@
 
 {# change address-groups vars from list to comma seperated string #}
 {% for k, v in SURICATAMERGED.config.vars['address-groups'].items() %}
+{%   if v is string %}
+{%     do SURICATAMERGED.config.vars['address-groups'].update({k: '[' ~ v ~ ']'}) %}
 {#   if address-group value is a list #}
-{%   if v is iterable and (v is not string and v is not mapping and v | length > 1) %}
+{%   elif v is iterable and v is not mapping %}
 {%     do SURICATAMERGED.config.vars['address-groups'].update({k: '[' ~ v | join(',') ~ ']'}) %}
-{%   else %}
-{%     do SURICATAMERGED.config.vars['address-groups'].update({k: v[0]}) %}
 {%   endif %}
 {% endfor %}
 
 {# change port-groups vars from list to comma seperated string #}
 {% for k, v in SURICATAMERGED.config.vars['port-groups'].items() %}
+{%   if v is string %}
+{%     do SURICATAMERGED.config.vars['port-groups'].update({k: '[' ~ v ~ ']'}) %}
 {#   if address-group value is a list #}
-{%   if v is iterable and (v is not string and v is not mapping and v | length > 1) %}
+{%   elif v is iterable and v is not mapping %}
 {%     do SURICATAMERGED.config.vars['port-groups'].update({k: '[' ~ v | join(',') ~ ']'}) %}
-{%   else %}
-{%     do SURICATAMERGED.config.vars['port-groups'].update({k: v[0]}) %}
 {%   endif %}
 {% endfor %}

salt/suricata/soc_suricata.yaml

@@ -154,12 +154,14 @@ suricata:
           description: Assign a list of hosts, or networks, using CIDR notation, to this Suricata variable. The variable can then be re-used within Suricata rules. This allows for a single adjustment to the variable that will then affect all rules referencing the variable.
           regex: ^(((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\/([0-9]|[1-2][0-9]|3[0-2]))?$|^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?))|:))|(([0-9A-Fa-f]{1,4}:){5}((:[0-9A-Fa-f]{1,4}){1,2}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){4}((:[0-9A-Fa-f]{1,4}){1,3}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){3}((:[0-9A-Fa-f]{1,4}){1,4}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){2}((:[0-9A-Fa-f]{1,4}){1,5}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){1}((:[0-9A-Fa-f]{1,4}){1,6}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(:((:[0-9A-Fa-f]{1,4}){1,7}|:)))(\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8]))?$
           regexFailureMessage: You must enter a valid IP address or CIDR.
-          helpLink: suricata.html
+          forcedType: "[]string"
           duplicates: True
+          helpLink: suricata.html
         EXTERNAL_NET: &suriaddressgroup
           description: Assign a list of hosts, or networks, or other customization, to this Suricata variable. The variable can then be re-used within Suricata rules. This allows for a single adjustment to the variable that will then affect all rules referencing the variable.
-          helpLink: suricata.html
+          forcedType: "[]string"
           duplicates: True
+          helpLink: suricata.html
         HTTP_SERVERS: *suriaddressgroup
         SMTP_SERVERS: *suriaddressgroup
         SQL_SERVERS: *suriaddressgroup
@@ -176,8 +178,9 @@ suricata:
       port-groups:
         HTTP_PORTS: &suriportgroup
           description: Assign a list of network port numbers to this Suricata variable. The variable can then be re-used within Suricata rules. This allows for a single adjustment to the variable that will then affect all rules referencing the variable.
-          helpLink: suricata.html
+          forcedType: "[]string"
           duplicates: True
+          helpLink: suricata.html
         SHELLCODE_PORTS: *suriportgroup
         ORACLE_PORTS: *suriportgroup
         SSH_PORTS: *suriportgroup

salt/top.sls

@@ -155,6 +155,7 @@ base:
     - nginx
     - elasticfleet.install_agent_grid
     - stig
+    - kafka
 
   '*_managersearch and G@saltversion:{{saltversion}}':
     - match: compound
@@ -184,6 +185,7 @@ base:
     - utility
     - elasticfleet
     - stig
+    - kafka
 
   '*_heavynode and G@saltversion:{{saltversion}}':
     - match: compound

setup/so-functions

@@ -1180,13 +1180,16 @@ kibana_pillar() {
 kafka_pillar() {
 	KAFKACLUSTERID=$(get_random_value 22)
 	KAFKAPASS=$(get_random_value)
+	KAFKATRUST=$(get_random_value)
 	logCmd "mkdir -p $local_salt_dir/pillar/kafka"
 	logCmd "touch $adv_kafka_pillar_file"
 	logCmd "touch $kafka_pillar_file"
 	printf '%s\n'\
 		"kafka:"\
 		"  cluster_id: $KAFKACLUSTERID"\
-		"  password: $KAFKAPASS" > $kafka_pillar_file
+		"  config:"\
+		"    password: $KAFKAPASS"\
+		"    trustpass: $KAFKATRUST" > $kafka_pillar_file
 }
 
 logrotate_pillar() {
@@ -1254,30 +1257,7 @@ soc_pillar() {
 		"soc:"\
 		"  config:"\
 		"    server:"\
-		"      srvKey: '$SOCSRVKEY'"\
+		"      srvKey: '$SOCSRVKEY'" > "$soc_pillar_file"
-		"      modules:"\
-		"        elastalertengine:"\
-		"          allowRegex: '$ELASTALERT_ALLOW_REGEX'" > "$soc_pillar_file"
-	if [[ -n "$ELASTALERT_FAIL_ERROR_COUNT" ]]; then
-		printf '%s\n'\
-	    "          failAfterConsecutiveErrorCount: $ELASTALERT_FAIL_ERROR_COUNT" >> "$soc_pillar_file"
-	fi
-	
-	printf '%s\n'\
-		"        strelkaengine:"\
-		"          allowRegex: '$STRELKA_ALLOW_REGEX'" >> "$soc_pillar_file"
-	if [[ -n "$STRELKA_FAIL_ERROR_COUNT" ]]; then
-	 	printf '%s\n'\
-		"          failAfterConsecutiveErrorCount: $STRELKA_FAIL_ERROR_COUNT" >> "$soc_pillar_file"
-	fi
-	
-	printf '%s\n'\
-		"        suricataengine:"\
-		"          allowRegex: '$SURICATA_ALLOW_REGEX'" >> "$soc_pillar_file"
-	if [[ -n "$SURICATA_FAIL_ERROR_COUNT" ]]; then
-		printf '%s\n'\
-		"          failAfterConsecutiveErrorCount: $SURICATA_FAIL_ERROR_COUNT" >> "$soc_pillar_file"
-	fi
 
 	if [[ $telemetry -ne 0 ]]; then
 		echo "  telemetryEnabled: false" >> $soc_pillar_file
@@ -1834,7 +1814,7 @@ repo_sync_local() {
 		
 		if [[ ! $is_airgap ]]; then
         	curl --retry 5 --retry-delay 60 -A "netinstall/$SOVERSION/$OS/$(uname -r)/1" https://sigs.securityonion.net/checkup --output /tmp/install
-			logCmd "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/"
+			retry 5 60 "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/" >> "$setup_log" 2>&1 || fail_setup
 			# After the download is complete run createrepo
 			create_repo
 		fi
@@ -1951,7 +1931,7 @@ saltify() {
 }
 
 salt_install_module_deps() {
-    logCmd "salt-pip install docker --no-index --only-binary=:all: --find-links files/salt_module_deps/docker/"
+    logCmd "salt-call state.apply salt.python_modules --local --file-root=../salt/"
 }
 
 salt_patch_x509_v2() {

setup/so-setup

@@ -245,12 +245,6 @@ if [ -n "$test_profile" ]; then
 	WEBUSER=onionuser@somewhere.invalid
 	WEBPASSWD1=0n10nus3r
 	WEBPASSWD2=0n10nus3r
-	STRELKA_ALLOW_REGEX="EquationGroup_Toolset_Apr17__ELV_.*"
-	STRELKA_FAIL_ERROR_COUNT=1
-	ELASTALERT_ALLOW_REGEX="Security Onion"
-	ELASTALERT_FAIL_ERROR_COUNT=1
-	SURICATA_ALLOW_REGEX="(200033\\d|2100538|2102466)"
-	SURICATA_FAIL_ERROR_COUNT=1
 
 	update_sudoers_for_testing
 fi