Merge pull request #14773 from Security-Onion-Solutions/2.4/dev

2.4.160

.github/DISCUSSION_TEMPLATE/2-4.yml

@@ -25,6 +25,10 @@ body:
         - 2.4.111
         - 2.4.120
         - 2.4.130
+        - 2.4.140
+        - 2.4.141
+        - 2.4.150
+        - 2.4.160
         - Other (please provide detail below)
     validations:
       required: true

.github/workflows/pythontest.yml

@@ -1,10 +1,6 @@
 name: python-test
 
 on:
-  push:
-    paths: 
-      - "salt/sensoroni/files/analyzers/**"
-      - "salt/manager/tools/sbin"
   pull_request:
     paths:
       - "salt/sensoroni/files/analyzers/**"
@@ -17,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10"]
+        python-version: ["3.13"]
         python-code-path: ["salt/sensoroni/files/analyzers", "salt/manager/tools/sbin"]
 
     steps:
@@ -36,4 +32,4 @@ jobs:
         flake8 ${{ matrix.python-code-path }} --show-source --max-complexity=12  --doctests --max-line-length=200 --statistics
     - name: Test with pytest
       run: |
-        pytest ${{ matrix.python-code-path }} --cov=${{ matrix.python-code-path }} --doctest-modules --cov-report=term --cov-fail-under=100 --cov-config=pytest.ini
+        PYTHONPATH=${{ matrix.python-code-path }} pytest ${{ matrix.python-code-path }} --cov=${{ matrix.python-code-path }} --doctest-modules --cov-report=term --cov-fail-under=100 --cov-config=pytest.ini

DOWNLOAD_AND_VERIFY_ISO.md

@@ -1,17 +1,17 @@
-### 2.4.130-20250311 ISO image released on 2025/03/11
+### 2.4.160-20250625 ISO image released on 2025/06/25
 
 
 ### Download and Verify
 
-2.4.130-20250311 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.4.130-20250311.iso
+2.4.160-20250625 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-2.4.160-20250625.iso
  
-MD5: 4641CA710570CCE18CD7D50653373DC0  
-SHA1: 786EF73F7945FDD80126C9AE00BDD29E58743715  
-SHA256: 48C7A042F20C46B8087BAE0F971696DADE9F9364D52F416718245C16E7CCB977  
+MD5: 78CF5602EFFAB84174C56AD2826E6E4E  
+SHA1: FC7EEC3EC95D97D3337501BAA7CA8CAE7C0E15EA  
+SHA256: 0ED965E8BEC80EE16AE90A0F0F96A3046CEF2D92720A587278DDDE3B656C01C2  
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.130-20250311.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.160-20250625.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS  
@@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.130-20250311.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.160-20250625.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.4.130-20250311.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.4.160-20250625.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.4.130-20250311.iso.sig securityonion-2.4.130-20250311.iso
+gpg --verify securityonion-2.4.160-20250625.iso.sig securityonion-2.4.160-20250625.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Mon 10 Mar 2025 06:30:49 PM EDT using RSA key ID FE507013
+gpg: Signature made Wed 25 Jun 2025 10:13:33 AM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

VERSION

@@ -1 +1 @@
-2.4.130
+2.4.160

pillar/node_data/ips.sls

@@ -24,6 +24,7 @@
 {%   endif %}
 {% endfor %}
 
+{% if node_types %}
 node_data:
 {% for node_type, host_values in node_types.items() %}
 {%   for hostname, details in host_values.items() %}
@@ -33,3 +34,6 @@ node_data:
     role: {{node_type}}
 {%   endfor %}
 {% endfor %}
+{% else %}
+node_data: False
+{% endif %}

pillar/top.sls

@@ -24,10 +24,10 @@ base:
     - firewall.adv_firewall
     - nginx.soc_nginx
     - nginx.adv_nginx
-    - node_data.ips
 
   '*_manager or *_managersearch':
     - match: compound
+    - node_data.ips
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
     {% endif %}
@@ -90,6 +90,7 @@ base:
     - soc.license
 
   '*_eval':
+    - node_data.ips
     - secrets
     - healthcheck.eval
     - elasticsearch.index_templates
@@ -138,6 +139,7 @@ base:
     - minions.adv_{{ grains.id }}
 
   '*_standalone':
+    - node_data.ips
     - logstash.nodes
     - logstash.soc_logstash
     - logstash.adv_logstash
@@ -260,6 +262,7 @@ base:
     - soc.license
 
   '*_import':
+    - node_data.ips
     - secrets
     - elasticsearch.index_templates
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
@@ -305,6 +308,7 @@ base:
     - minions.adv_{{ grains.id }}
 
   '*_fleet':
+    - node_data.ips
     - backup.soc_backup
     - backup.adv_backup
     - logstash.nodes

salt/common/init.sls

@@ -129,6 +129,10 @@ common_sbin:
     - group: 939
     - file_mode: 755
     - show_changes: False
+{% if GLOBALS.role == 'so-heavynode' %}
+    - exclude_pat:
+      - so-pcap-import
+{% endif %}
 
 common_sbin_jinja:
   file.recurse:
@@ -139,6 +143,20 @@ common_sbin_jinja:
     - file_mode: 755
     - template: jinja
     - show_changes: False
+{% if GLOBALS.role == 'so-heavynode' %}
+    - exclude_pat:
+      - so-import-pcap
+{% endif %}
+
+{% if GLOBALS.role == 'so-heavynode' %}
+remove_so-pcap-import_heavynode:
+  file.absent:
+    - name: /usr/sbin/so-pcap-import
+
+remove_so-import-pcap_heavynode:
+  file.absent:
+    - name: /usr/sbin/so-import-pcap
+{% endif %}
 
 {% if not GLOBALS.is_manager%}
 # prior to 2.4.50 these scripts were in common/tools/sbin on the manager because of soup and distributed to non managers

salt/common/soup_scripts.sls

@@ -64,6 +64,12 @@ copy_so-repo-sync_manager_tools_sbin:
     - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-repo-sync
     - preserve: True
 
+copy_bootstrap-salt_manager_tools_sbin:
+  file.copy:
+    - name: /opt/so/saltstack/default/salt/salt/scripts/bootstrap-salt.sh
+    - source: {{UPDATE_DIR}}/salt/salt/scripts/bootstrap-salt.sh
+    - preserve: True
+
 # This section is used to put the new script in place so that it can be called during soup.
 # It is faster than calling the states that normally manage them to put them in place.
 copy_so-common_sbin:
@@ -108,6 +114,13 @@ copy_so-repo-sync_sbin:
     - force: True
     - preserve: True
 
+copy_bootstrap-salt_sbin:
+  file.copy:
+    - name: /usr/sbin/bootstrap-salt.sh
+    - source: {{UPDATE_DIR}}/salt/salt/scripts/bootstrap-salt.sh
+    - force: True
+    - preserve: True
+
 {# this is added in 2.4.120 to remove salt repo files pointing to saltproject.io to accomodate the move to broadcom and new bootstrap-salt script #}
 {%   if salt['pkg.version_cmp'](SOVERSION, '2.4.120') == -1 %}
 {%     set saltrepofile = '/etc/yum.repos.d/salt.repo' %}

salt/common/tools/sbin/so-common

@@ -99,6 +99,17 @@ add_interface_bond0() {
 	fi
 }
 
+airgap_playbooks() {
+	SRC_DIR=$1
+	# Copy playbooks if using airgap
+	mkdir -p /nsm/airgap-resources
+	# Purge old airgap playbooks to ensure SO only uses the latest released playbooks
+	rm -fr /nsm/airgap-resources/playbooks
+	tar xf $SRC_DIR/airgap-resources/playbooks.tgz -C /nsm/airgap-resources/
+	chown -R socore:socore /nsm/airgap-resources/playbooks
+	git config --global --add safe.directory /nsm/airgap-resources/playbooks
+}
+
 check_container() {
 	docker ps | grep "$1:" > /dev/null 2>&1
 	return $?
@@ -299,7 +310,8 @@ fail() {
 
 get_agent_count() {
 	if [ -f /opt/so/log/agents/agentstatus.log ]; then
-		AGENTCOUNT=$(cat /opt/so/log/agents/agentstatus.log | grep -wF active | awk '{print $2}')
+		AGENTCOUNT=$(cat /opt/so/log/agents/agentstatus.log | grep -wF active | awk '{print $2}' | sed 's/,//')
+		[[ -z "$AGENTCOUNT" ]] && AGENTCOUNT="0"
 	else
 		AGENTCOUNT=0
 	fi

salt/common/tools/sbin/so-common-status-check

@@ -45,7 +45,7 @@ def check_for_fps():
         result = subprocess.run([feat_full + '-mode-setup', '--is-enabled'], stdout=subprocess.PIPE)
         if result.returncode == 0:
             fps = 1
-    except FileNotFoundError:
+    except:
         fn = '/proc/sys/crypto/' + feat_full + '_enabled'
         try:
           with open(fn, 'r') as f:

salt/common/tools/sbin/so-docker-prune

@@ -4,22 +4,16 @@
 # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
-
-
-
-import sys, argparse, re, docker
+import sys, argparse, re, subprocess, json
 from packaging.version import Version, InvalidVersion
 from itertools import groupby, chain
 
-
 def get_image_name(string) -> str:
   return ':'.join(string.split(':')[:-1])
 
-
 def get_so_image_basename(string) -> str:
   return get_image_name(string).split('/so-')[-1]
 
-
 def get_image_version(string) -> str:
   ver = string.split(':')[-1]
   if ver == 'latest':
@@ -35,56 +29,75 @@ def get_image_version(string) -> str:
         return '999999.9.9'
     return ver
 
+def run_command(command):
+    process = subprocess.run(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+    if process.returncode != 0:
+        print(f"Error executing command: {command}", file=sys.stderr)
+        print(f"Error message: {process.stderr}", file=sys.stderr)
+        exit(1)
+    return process.stdout
 
 def main(quiet):
-  client = docker.from_env()
-
-  # Prune old/stopped containers
-  if not quiet: print('Pruning old containers')
-  client.containers.prune()
-
-  image_list = client.images.list(filters={ 'dangling': False })
-
-  # Map list of image objects to flattened list of tags (format: "name:version")
-  tag_list = list(chain.from_iterable(list(map(lambda x: x.attrs.get('RepoTags'), image_list))))
-
-  # Filter to only SO images (base name begins with "so-")
-  tag_list = list(filter(lambda x: re.match(r'^.*\/so-[^\/]*$', get_image_name(x)), tag_list))
-
-  # Group tags into lists by base name (sort by same projection first)
-  tag_list.sort(key=lambda x: get_so_image_basename(x))
-  grouped_tag_lists = [ list(it) for _, it in groupby(tag_list, lambda x: get_so_image_basename(x)) ]
-
-  no_prunable = True
-  for t_list in grouped_tag_lists:
     try:
-      # Group tags by version, in case multiple images exist with the same version string
-      t_list.sort(key=lambda x: Version(get_image_version(x)), reverse=True)
-      grouped_t_list = [ list(it) for _,it in groupby(t_list, lambda x: get_image_version(x)) ]
-
-      # Keep the 2 most current version groups
-      if len(grouped_t_list) <= 2:
-        continue
-      else:
-        no_prunable = False
-        for group in grouped_t_list[2:]:
-          for tag in group:
-            if not quiet: print(f'Removing image {tag}')
+        # Prune old/stopped containers using docker CLI
+        if not quiet: print('Pruning old containers')
+        run_command('docker container prune -f')
+        
+        # Get list of images using docker CLI
+        images_json = run_command('docker images --format "{{json .}}"')
+        
+        # Parse the JSON output
+        image_list = []
+        for line in images_json.strip().split('\n'):
+            if line:  # Skip empty lines
+                image_list.append(json.loads(line))
+        
+        # Extract tags in the format "name:version"
+        tag_list = []
+        for img in image_list:
+            # Skip dangling images
+            if img.get('Repository') != "<none>" and img.get('Tag') != "<none>":
+                tag = f"{img.get('Repository')}:{img.get('Tag')}"
+                # Filter to only SO images (base name begins with "so-")
+                if re.match(r'^.*\/so-[^\/]*$', get_image_name(tag)):
+                    tag_list.append(tag)
+        
+        # Group tags into lists by base name (sort by same projection first)
+        tag_list.sort(key=lambda x: get_so_image_basename(x))
+        grouped_tag_lists = [list(it) for k, it in groupby(tag_list, lambda x: get_so_image_basename(x))]
+        
+        no_prunable = True
+        for t_list in grouped_tag_lists:
             try:
-              client.images.remove(tag, force=True)
-            except docker.errors.ClientError as e:
-              print(f'Could not remove image {tag}, continuing...')
-    except (docker.errors.APIError, InvalidVersion) as e:
-      print(f'so-{get_so_image_basename(t_list[0])}: {e}', file=sys.stderr)
-      exit(1)
+                # Group tags by version, in case multiple images exist with the same version string
+                t_list.sort(key=lambda x: Version(get_image_version(x)), reverse=True)
+                grouped_t_list = [list(it) for k, it in groupby(t_list, lambda x: get_image_version(x))]
+                # Keep the 2 most current version groups
+                if len(grouped_t_list) <= 2:
+                    continue
+                else:
+                    no_prunable = False
+                    for group in grouped_t_list[2:]:
+                        for tag in group:
+                            if not quiet: print(f'Removing image {tag}')
+                            try:
+                                run_command(f'docker rmi -f {tag}')
+                            except Exception as e:
+                                print(f'Could not remove image {tag}, continuing...')
+            except (InvalidVersion) as e:
+                print(f'so-{get_so_image_basename(t_list[0])}: {e}', file=sys.stderr)
+                exit(1)
+            except Exception as e:
+                print('Unhandled exception occurred:')
+                print(f'so-{get_so_image_basename(t_list[0])}: {e}', file=sys.stderr)
+                exit(1)
+        
+        if no_prunable and not quiet:
+            print('No Security Onion images to prune')
+    
     except Exception as e:
-      print('Unhandled exception occurred:')
-      print(f'so-{get_so_image_basename(t_list[0])}: {e}', file=sys.stderr)
-      exit(1)
-
-  if no_prunable and not quiet:
-    print('No Security Onion images to prune')
-
+        print(f"Error: {e}", file=sys.stderr)
+        exit(1)
 
 if __name__ == "__main__":
   main_parser = argparse.ArgumentParser(add_help=False)

salt/common/tools/sbin/so-log-check

@@ -127,6 +127,8 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|process already finished"     # Telegraf script finished just as the auto kill timeout kicked in
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|No shard available"           # Typical error when making a query before ES has finished loading all indices
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|responded with status-code 503" # telegraf getting 503 from ES during startup
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|process_cluster_event_timeout_exception" # logstash waiting for elasticsearch to start
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|not configured for GeoIP"     # SO does not bundle the maxminddb with Zeek
 fi
 
 if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
@@ -155,6 +157,7 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|request_unauthorized"         # false positive (login failures to Hydra result in an 'error' log) 
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|adding index lifecycle policy" # false positive (elasticsearch policy names contain 'error') 
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|adding ingest pipeline"       # false positive (elasticsearch ingest pipeline names contain 'error') 
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|updating index template"      # false positive (elasticsearch index or template names contain 'error') 
 fi
 
 if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then

salt/docker/defaults.yaml

@@ -200,6 +200,7 @@ docker:
       final_octet: 88
       port_bindings:
         - 0.0.0.0:9092:9092
+        - 0.0.0.0:29092:29092
         - 0.0.0.0:9093:9093
         - 0.0.0.0:8778:8778
       custom_bind_mounts: []

salt/elasticfleet/defaults.yaml

@@ -11,6 +11,7 @@ elasticfleet:
     defend_filters:
       enable_auto_configuration: False
     subscription_integrations: False
+    auto_upgrade_integrations: False
   logging:
     zeek:
       excluded:

salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json

@@ -0,0 +1,46 @@
+{%- set identities = salt['sqlite3.fetch']('/nsm/kratos/db/db.sqlite', 'SELECT id, json_extract(traits, "$.email") as email FROM identities;') -%}
+{%- set valid_identities = false -%}
+{%- if identities -%}
+  {%- set valid_identities = true -%}
+  {%- for id, email in identities -%}
+    {%- if not id or not email -%}
+      {%- set valid_identities = false -%}
+      {%- break -%}
+    {%- endif -%}
+  {%- endfor -%}
+{%- endif -%}
+{
+  "package": {
+    "name": "log",
+    "version": ""
+  },
+  "name": "kratos-logs",
+  "namespace": "so",
+  "description": "Kratos logs",
+  "policy_id": "so-grid-nodes_general",
+  "inputs": {
+    "logs-logfile": {
+      "enabled": true,
+      "streams": {
+        "log.logs": {
+          "enabled": true,
+          "vars": {
+            "paths": [
+              "/opt/so/log/kratos/kratos.log"
+            ],
+            "data_stream.dataset": "kratos",
+            "tags": ["so-kratos"],
+            {%- if valid_identities -%}
+            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true\n- add_fields:\n    target: event\n    fields:\n      category: iam\n      module: kratos\n- if:\n    has_fields:\n      - identity_id\n  then:{% for id, email in identities %}\n    - if:\n        equals:\n          identity_id: \"{{ id }}\"\n      then:\n        - add_fields:\n            target: ''\n            fields:\n              user.name: \"{{ email }}\"{% endfor %}",
+            {%- else -%}
+            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true\n- add_fields:\n    target: event\n    fields:\n      category: iam\n      module: kratos",
+            {%- endif -%}
+            "custom": "pipeline: kratos"
+          }
+        }
+      }
+    }
+  },
+  "force": true
+}
+

salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json

@@ -1,30 +0,0 @@
-{
-  "package": {
-    "name": "log",
-    "version": ""
-  },
-  "name": "kratos-logs",
-  "namespace": "so",
-  "description": "Kratos logs",
-  "policy_id": "so-grid-nodes_general",
-  "inputs": {
-    "logs-logfile": {
-      "enabled": true,
-      "streams": {
-        "log.logs": {
-          "enabled": true,
-          "vars": {
-            "paths": [
-              "/opt/so/log/kratos/kratos.log"
-            ],
-            "data_stream.dataset": "kratos",
-            "tags": ["so-kratos"],
-            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: iam\n      module: kratos",
-            "custom": "pipeline: kratos"
-          }
-        }
-      }
-    }
-  },
-  "force": true
-}

salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json

@@ -31,7 +31,8 @@
             ],
             "tags": [
               "so-grid-node"
-            ]
+            ],
+            "processors": "- if:\n    contains:\n      message: \"salt-minion\"\n  then: \n    - dissect:\n        tokenizer: \"%{} %{} %{} %{} %{} %{}: [%{log.level}] %{*}\"\n        field: \"message\"\n        trim_values: \"all\"\n        target_prefix: \"\"\n    - drop_event:\n        when:\n          equals:\n            log.level: \"INFO\""
           }
         }
       }

salt/elasticfleet/soc_elasticfleet.yaml

@@ -45,6 +45,11 @@ elasticfleet:
       global: True
       forcedType: bool
       helpLink: elastic-fleet.html
+    auto_upgrade_integrations:
+      description: Enables or disables automatically upgrading Elastic Agent integrations.
+      global: True
+      forcedType: bool
+      helpLink: elastic-fleet.html
     server:
       custom_fqdn:
         description: Custom FQDN for Agents to connect to. One per line.

salt/elasticfleet/tools/sbin/so-elastic-fleet-common

@@ -108,7 +108,7 @@ elastic_fleet_package_is_installed() {
 }
 
 elastic_fleet_installed_packages() {
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET -H 'kbn-xsrf: true' -H 'Content-Type: application/json' "localhost:5601/api/fleet/epm/packages/installed?perPage=300"
+    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET -H 'kbn-xsrf: true' -H 'Content-Type: application/json' "localhost:5601/api/fleet/epm/packages/installed?perPage=500"
 }
 
 elastic_fleet_agent_policy_ids() {

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-upgrade

@@ -1,62 +0,0 @@
-#!/bin/bash
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-. /usr/sbin/so-elastic-fleet-common
-
-curl_output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/)
-if [ $? -ne 0 ]; then
-    echo "Error: Failed to connect to Kibana."
-    exit 1
-fi
-
-IFS=$'\n'
-agent_policies=$(elastic_fleet_agent_policy_ids)
-if [ $? -ne 0 ]; then
-    echo "Error: Failed to retrieve agent policies."
-    exit 1
-fi
-
-for AGENT_POLICY in $agent_policies; do
-    integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY")
-    for INTEGRATION in $integrations; do
-        if ! [[ "$INTEGRATION" == "elastic-defend-endpoints" ]] && ! [[ "$INTEGRATION" == "fleet_server-"* ]]; then
-            # Get package name so we know what package to look for when checking the current and latest available version
-            PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION")
-
-            # Get currently installed version of package
-            PACKAGE_VERSION=$(elastic_fleet_integration_policy_package_version "$AGENT_POLICY" "$INTEGRATION")
-
-            # Get latest available version of package
-            AVAILABLE_VERSION=$(elastic_fleet_package_latest_version_check "$PACKAGE_NAME")
-
-            # Get integration ID
-            INTEGRATION_ID=$(elastic_fleet_integration_id "$AGENT_POLICY" "$INTEGRATION")
-
-            if [[ "$PACKAGE_VERSION" != "$AVAILABLE_VERSION" ]]; then
-                # Dry run of the upgrade
-                echo "Current $PACKAGE_NAME package version ($PACKAGE_VERSION) is not the same as the latest available package ($AVAILABLE_VERSION)..."
-                echo "Upgrading $INTEGRATION..."
-                echo "Starting dry run..."
-                DRYRUN_OUTPUT=$(elastic_fleet_integration_policy_dryrun_upgrade "$INTEGRATION_ID")
-                DRYRUN_ERRORS=$(echo "$DRYRUN_OUTPUT" | jq .[].hasErrors)
-                
-                # If no errors with dry run, proceed with actual upgrade
-                if [[ "$DRYRUN_ERRORS" == "false" ]]; then
-                    echo "No errors detected. Proceeding with upgrade..."
-                    elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"
-                    if [ $? -ne 0 ]; then
-                        echo "Error: Upgrade failed for integration ID '$INTEGRATION_ID'."
-                        exit 1
-                    fi
-                else
-                    echo "Errors detected during dry run. Stopping upgrade..."
-                    exit 1
-                fi
-            fi
-        fi
-    done
-done
-echo

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers

@@ -83,5 +83,10 @@ docker run \
 {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o so-elastic-agent_windows_amd64_msi --arch x64 /workspace/so-elastic-agent.wxs
 printf "\n### MSI Generated...\n"
 
-printf "\n### Cleaning up temp files in /nsm/elastic-agent-workspace\n"
+printf "\n### Cleaning up temp files \n"
 rm -rf /nsm/elastic-agent-workspace
+rm -rf /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64.exe
+
+printf "\n### Copying so_agent-installers to /nsm/elastic-fleet/ for nginx.\n"
+\cp -vr /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/ /nsm/elastic-fleet/
+chmod 644 /nsm/elastic-fleet/so_agent-installers/*

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade

@@ -14,7 +14,7 @@ if ! is_manager_node; then
 fi
 
 # Get current list of Grid Node Agents that need to be upgraded
-RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/agents?perPage=20&page=1&kuery=policy_id%20%3A%20so-grid-nodes_%2A&showInactive=false&showUpgradeable=true&getStatusSummary=true")
+RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/agents?perPage=20&page=1&kuery=NOT%20agent.version%20:%20%22{{ELASTICSEARCHDEFAULTS.elasticsearch.version}}%22%20and%20policy_id%20:%20%22so-grid-nodes_general%22&showInactive=false&getStatusSummary=true")
 
 # Check to make sure that the server responded with good data - else, bail from script
 CHECKSUM=$(jq -r '.page' <<< "$RAW_JSON")

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade

@@ -0,0 +1,73 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
+{%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %}
+{%- set AUTO_UPGRADE_INTEGRATIONS = salt['pillar.get']('elasticfleet:config:auto_upgrade_integrations', default=false) %}
+
+. /usr/sbin/so-elastic-fleet-common
+
+curl_output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/)
+if [ $? -ne 0 ]; then
+    echo "Error: Failed to connect to Kibana."
+    exit 1
+fi
+
+IFS=$'\n'
+agent_policies=$(elastic_fleet_agent_policy_ids)
+if [ $? -ne 0 ]; then
+    echo "Error: Failed to retrieve agent policies."
+    exit 1
+fi
+
+default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %})
+
+for AGENT_POLICY in $agent_policies; do
+    integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY")
+    for INTEGRATION in $integrations; do
+        if ! [[ "$INTEGRATION" == "elastic-defend-endpoints" ]] && ! [[ "$INTEGRATION" == "fleet_server-"* ]]; then
+            # Get package name so we know what package to look for when checking the current and latest available version
+            PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION")
+            {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+            if [[ " ${default_packages[@]} " =~ " $PACKAGE_NAME " ]]; then
+            {%- endif %}
+                # Get currently installed version of package
+                PACKAGE_VERSION=$(elastic_fleet_integration_policy_package_version "$AGENT_POLICY" "$INTEGRATION")
+
+                # Get latest available version of package
+                AVAILABLE_VERSION=$(elastic_fleet_package_latest_version_check "$PACKAGE_NAME")
+
+                # Get integration ID
+                INTEGRATION_ID=$(elastic_fleet_integration_id "$AGENT_POLICY" "$INTEGRATION")
+
+                if [[ "$PACKAGE_VERSION" != "$AVAILABLE_VERSION" ]]; then
+                    # Dry run of the upgrade
+                    echo ""
+                    echo "Current $PACKAGE_NAME package version ($PACKAGE_VERSION) is not the same as the latest available package ($AVAILABLE_VERSION)..."
+                    echo "Upgrading $INTEGRATION..."
+                    echo "Starting dry run..."
+                    DRYRUN_OUTPUT=$(elastic_fleet_integration_policy_dryrun_upgrade "$INTEGRATION_ID")
+                    DRYRUN_ERRORS=$(echo "$DRYRUN_OUTPUT" | jq .[].hasErrors)
+
+                    # If no errors with dry run, proceed with actual upgrade
+                    if [[ "$DRYRUN_ERRORS" == "false" ]]; then
+                        echo "No errors detected. Proceeding with upgrade..."
+                        elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"
+                        if [ $? -ne 0 ]; then
+                            echo "Error: Upgrade failed for $PACKAGE_NAME with integration ID '$INTEGRATION_ID'."
+                            exit 1
+                        fi
+                    else
+                        echo "Errors detected during dry run for $PACKAGE_NAME policy upgrade..."
+                        exit 1
+                    fi
+                fi
+            {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+            fi
+            {%- endif %}
+        fi
+    done
+done
+echo

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load

@@ -3,7 +3,10 @@
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
 # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
 # this file except in compliance with the Elastic License 2.0.
+{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
 {% set SUB = salt['pillar.get']('elasticfleet:config:subscription_integrations', default=false) %}
+{% set AUTO_UPGRADE_INTEGRATIONS = salt['pillar.get']('elasticfleet:config:auto_upgrade_integrations', default=false) %}
+{%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %}
 
 . /usr/sbin/so-common
 . /usr/sbin/so-elastic-fleet-common
@@ -46,6 +49,28 @@ compare_versions() {
     fi
 }
 
+IFS=$'\n'
+agent_policies=$(elastic_fleet_agent_policy_ids)
+if [ $? -ne 0 ]; then
+    echo "Error: Failed to retrieve agent policies."
+    exit 1
+fi
+
+default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %})
+
+in_use_integrations=()
+
+for AGENT_POLICY in $agent_policies; do
+    integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY")
+    for INTEGRATION in $integrations; do
+        PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION")
+        # non-default integrations that are in-use in any policy
+        if ! [[ " ${default_packages[@]} " =~ " $PACKAGE_NAME " ]]; then
+            in_use_integrations+=("$PACKAGE_NAME")
+        fi
+    done
+done
+
 if [[ -f $STATE_FILE_SUCCESS  ]]; then
     if retry 3 1 "curl -s -K /opt/so/conf/elasticsearch/curl.config --output /dev/null --silent --head --fail localhost:5601/api/fleet/epm/packages"; then
         # Package_list contains all integrations beta / non-beta.
@@ -77,10 +102,19 @@ if [[ -f $STATE_FILE_SUCCESS  ]]; then
                     else
                         results=$(compare_versions "$latest_version" "$installed_version")
                         if [ $results == "greater" ]; then
-                            echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update."
-                            jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
+                            {#- When auto_upgrade_integrations is false, skip upgrading in_use_integrations  #}
+                            {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+                            if ! [[ " ${in_use_integrations[@]} " =~ " $package_name " ]]; then
+                            {%- endif %}
+                                echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update."
+                                jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
 
-                            PENDING_UPDATE=true
+                                PENDING_UPDATE=true
+                            {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+                            else
+                                echo "skipping available upgrade for in use integration - $package_name."
+                            fi
+                            {%- endif %}
                         fi
                     fi
                 fi
@@ -92,9 +126,18 @@ if [[ -f $STATE_FILE_SUCCESS  ]]; then
                 else
                     results=$(compare_versions "$latest_version" "$installed_version")
                     if [ $results == "greater" ]; then
-                        echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update."
-                        jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
-                        PENDING_UPDATE=true
+                        {#- When auto_upgrade_integrations is false, skip upgrading in_use_integrations  #}
+                        {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+                        if ! [[ " ${in_use_integrations[@]} " =~ " $package_name " ]]; then
+                        {%- endif %}
+                            echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update."
+                            jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
+                            PENDING_UPDATE=true
+                        {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+                        else
+                            echo "skipping available upgrade for in use integration - $package_name."
+                        fi
+                        {%- endif %}
                     fi
                 fi
             {% endif %}

salt/elasticfleet/tools/sbin_jinja/so-kafka-fleet-output-policy

@@ -32,7 +32,7 @@ if ! echo "$output" | grep -q "so-manager_kafka"; then
                 --arg KAFKACA "$KAFKACA" \
                 --arg MANAGER_IP "{{ GLOBALS.manager_ip }}:9092" \
                 --arg KAFKA_OUTPUT_VERSION "$KAFKA_OUTPUT_VERSION" \
-                    '{ "name": "grid-kafka", "id": "so-manager_kafka", "type": "kafka", "hosts": [ $MANAGER_IP ], "is_default": false, "is_default_monitoring": false, "config_yaml": "", "ssl": { "certificate_authorities": [ $KAFKACA ], "certificate": $KAFKACRT, "key": $KAFKAKEY, "verification_mode": "full" }, "proxy_id": null, "client_id": "Elastic", "version": $KAFKA_OUTPUT_VERSION, "compression": "none", "auth_type": "ssl", "partition": "round_robin", "round_robin": { "group_events": 1 }, "topics":[{"topic":"%{[event.module]}-securityonion","when":{"type":"regexp","condition":"event.module:.+"}},{"topic":"default-securityonion"}], "headers": [ { "key": "", "value": "" } ], "timeout": 30, "broker_timeout": 30, "required_acks": 1 }'
+                    '{ "name": "grid-kafka", "id": "so-manager_kafka", "type": "kafka", "hosts": [ $MANAGER_IP ], "is_default": false, "is_default_monitoring": false, "config_yaml": "", "ssl": { "certificate_authorities": [ $KAFKACA ], "certificate": $KAFKACRT, "key": $KAFKAKEY, "verification_mode": "full" }, "proxy_id": null, "client_id": "Elastic", "version": $KAFKA_OUTPUT_VERSION, "compression": "none", "auth_type": "ssl", "partition": "round_robin", "round_robin": { "group_events": 10 }, "topics":[{"topic":"default-securityonion"}], "headers": [ { "key": "", "value": "" } ], "timeout": 30, "broker_timeout": 30, "required_acks": 1 }'
                 )
   curl -sK /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" -o /dev/null
   refresh_output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs" | jq -r .items[].id)

salt/elasticsearch/auth.sls

@@ -15,7 +15,7 @@
 elastic_auth_pillar:
   file.managed:
     - name: /opt/so/saltstack/local/pillar/elasticsearch/auth.sls
-    - mode: 600
+    - mode: 640
     - reload_pillar: True
     - contents: |
         elasticsearch:

salt/elasticsearch/defaults.yaml

@@ -162,6 +162,7 @@ elasticsearch:
         - http-mappings
         - dtc-http-mappings
         - log-mappings
+        - metadata-mappings
         - network-mappings
         - dtc-network-mappings
         - observer-mappings
@@ -274,7 +275,7 @@ elasticsearch:
               number_of_replicas: 0
               auto_expand_replicas: 0-2
               number_of_shards: 1
-              refresh_interval: 30s
+              refresh_interval: 1s
               sort:
                 field: '@timestamp'
                 order: desc
@@ -316,6 +317,7 @@ elasticsearch:
         - http-mappings
         - dtc-http-mappings
         - log-mappings
+        - metadata-mappings
         - network-mappings
         - dtc-network-mappings
         - observer-mappings
@@ -427,6 +429,7 @@ elasticsearch:
         - http-mappings
         - dtc-http-mappings
         - log-mappings
+        - metadata-mappings
         - network-mappings
         - dtc-network-mappings
         - observer-mappings
@@ -534,6 +537,7 @@ elasticsearch:
         - http-mappings
         - dtc-http-mappings
         - log-mappings
+        - metadata-mappings
         - network-mappings
         - dtc-network-mappings
         - observer-mappings
@@ -697,6 +701,7 @@ elasticsearch:
         - client-mappings
         - device-mappings
         - network-mappings
+        - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
         - so-fleet_agent_id_verification-1
         data_stream:
@@ -768,6 +773,7 @@ elasticsearch:
         - http-mappings
         - dtc-http-mappings
         - log-mappings
+        - metadata-mappings
         - network-mappings
         - dtc-network-mappings
         - observer-mappings
@@ -878,6 +884,7 @@ elasticsearch:
         - http-mappings
         - dtc-http-mappings
         - log-mappings
+        - metadata-mappings
         - network-mappings
         - dtc-network-mappings
         - observer-mappings
@@ -998,6 +1005,7 @@ elasticsearch:
       index_template:
         composed_of:
         - so-data-streams-mappings
+        - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
         - so-fleet_agent_id_verification-1
         - so-logs-mappings
@@ -2832,6 +2840,7 @@ elasticsearch:
         - http-mappings
         - dtc-http-mappings
         - log-mappings
+        - metadata-mappings
         - network-mappings
         - dtc-network-mappings
         - observer-mappings
@@ -3062,6 +3071,7 @@ elasticsearch:
         - event-mappings
         - logs-system.syslog@package
         - logs-system.syslog@custom
+        - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
         - so-fleet_agent_id_verification-1
         - so-system-mappings
@@ -3421,6 +3431,7 @@ elasticsearch:
         - dtc-http-mappings
         - log-mappings
         - logstash-mappings
+        - metadata-mappings
         - network-mappings
         - dtc-network-mappings
         - observer-mappings
@@ -3505,6 +3516,7 @@ elasticsearch:
         composed_of:
         - metrics-endpoint.metadata@package
         - metrics-endpoint.metadata@custom
+        - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
         - so-fleet_agent_id_verification-1
         data_stream:
@@ -3551,6 +3563,7 @@ elasticsearch:
         composed_of:
         - metrics-endpoint.metrics@package
         - metrics-endpoint.metrics@custom
+        - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
         - so-fleet_agent_id_verification-1
         data_stream:
@@ -3597,6 +3610,7 @@ elasticsearch:
         composed_of:
         - metrics-endpoint.policy@package
         - metrics-endpoint.policy@custom
+        - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
         - so-fleet_agent_id_verification-1
         data_stream:
@@ -3645,6 +3659,7 @@ elasticsearch:
         - metrics-fleet_server.agent_status@package
         - metrics-fleet_server.agent_status@custom
         - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
         - so-fleet_agent_id_verification-1
         data_stream:
@@ -3668,6 +3683,7 @@ elasticsearch:
         - metrics-fleet_server.agent_versions@package
         - metrics-fleet_server.agent_versions@custom
         - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
         - so-fleet_agent_id_verification-1
         data_stream:
@@ -3715,6 +3731,7 @@ elasticsearch:
         - http-mappings
         - dtc-http-mappings
         - log-mappings
+        - metadata-mappings
         - network-mappings
         - dtc-network-mappings
         - observer-mappings
@@ -3827,6 +3844,7 @@ elasticsearch:
         - http-mappings
         - dtc-http-mappings
         - log-mappings
+        - metadata-mappings
         - network-mappings
         - dtc-network-mappings
         - observer-mappings
@@ -3939,6 +3957,7 @@ elasticsearch:
         - http-mappings
         - dtc-http-mappings
         - log-mappings
+        - metadata-mappings
         - network-mappings
         - dtc-network-mappings
         - observer-mappings
@@ -4051,6 +4070,7 @@ elasticsearch:
         - http-mappings
         - dtc-http-mappings
         - log-mappings
+        - metadata-mappings
         - network-mappings
         - dtc-network-mappings
         - observer-mappings
@@ -4163,6 +4183,7 @@ elasticsearch:
         - http-mappings
         - dtc-http-mappings
         - log-mappings
+        - metadata-mappings
         - network-mappings
         - dtc-network-mappings
         - observer-mappings
@@ -4276,6 +4297,7 @@ elasticsearch:
         - http-mappings
         - dtc-http-mappings
         - log-mappings
+        - metadata-mappings
         - network-mappings
         - dtc-network-mappings
         - observer-mappings

salt/elasticsearch/enabled.sls

@@ -204,12 +204,17 @@ so-elasticsearch-roles-load:
       - docker_container: so-elasticsearch
       - file: elasticsearch_sbin_jinja
 
-{%     if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
+{%     if grains.role in ['so-managersearch', 'so-manager'] %}
+{%       set ap = "absent" %}
+{%     endif %}
+{%     if grains.role in ['so-eval', 'so-standalone', 'so-heavynode'] %}
 {%       if ELASTICSEARCHMERGED.index_clean %}
 {%         set ap = "present" %}
 {%       else %}
 {%         set ap = "absent" %}
 {%       endif %}
+{%     endif %}  
+{%     if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
 so-elasticsearch-indices-delete:
   cron.{{ap}}:
     - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1

salt/elasticsearch/files/ingest/global@custom

@@ -8,7 +8,7 @@
   "processors": [
     { "set":        { "ignore_failure": true,  "field": "event.module", "value": "elastic_agent" } },
     { "split":      { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "module_temp"  } },
-    { "split":      { "if": "ctx.data_stream?.dataset.contains('.')", "field":"data_stream.dataset", "separator":"\\.", "target_field":"datastream_dataset_temp", "ignore_missing":true } },
+    { "split":      { "if": "ctx.data_stream?.dataset != null && ctx.data_stream?.dataset.contains('.')", "field":"data_stream.dataset", "separator":"\\.", "target_field":"datastream_dataset_temp", "ignore_missing":true } },
     { "set":        { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}" } },
     { "set":        { "if": "ctx.datastream_dataset_temp != null && ctx.datastream_dataset_temp[0] == 'network_traffic'", "field":"event.module", "value":"{{ datastream_dataset_temp.0 }}", "ignore_failure":true, "ignore_empty_value":true, "description":"Fix EA network packet capture" } },
     { "gsub":       { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "pattern": "^[^.]*.", "replacement": "", "target_field": "dataset_tag_temp" } },
@@ -19,11 +19,12 @@
     { "set":        { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } },
     { "set":        { "if": "ctx.tags != null && ctx.tags.contains('import')", "override": true, "field": "data_stream.dataset", "value": "import" } },
     { "set":        { "if": "ctx.tags != null && ctx.tags.contains('import')", "override": true, "field": "data_stream.namespace", "value": "so" } },
-    { "date":       { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp","ignore_failure": true, "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSX","yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } },
     { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true }  },
     { "set":        { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } },
-    { "rename":        { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } },
+    { "rename":       { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } },
     { "set":        { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } },
+    {"append":      {"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"if":"ctx?.event?.dataset == 'endpoint.events.network' && ctx?.source?.ip != null","ignore_failure":true}},
+    {"foreach":     {"field":"host.ip","processor":{"append":{"field":"related.ip","value":"{{_ingest._value}}","allow_duplicates":false}},"if":"ctx?.event?.module == 'endpoint'","description":"Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip"}},
     { "remove":     { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp", "datastream_dataset_temp" ], "ignore_missing": true, "ignore_failure": true } }
   ]
 }

salt/elasticsearch/files/ingest/import.wel

@@ -1,11 +0,0 @@
-{
-  "description" : "import.wel",
-  "processors" : [
-    { "set":           { "field": "event.ingested", "value": "{{ @timestamp }}"  } },
-    { "set" :          { "field" : "@timestamp", "value" : "{{ event.created }}" } },
-    { "remove":        { "field": [ "event_record_id", "event.created" , "timestamp" , "winlog.event_data.UtcTime" ], "ignore_failure": true } },
-    { "pipeline":      { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'",   "name": "sysmon" } },
-    { "pipeline":      { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational'",  "name":"win.eventlogs" } },
-    { "pipeline":      { "name": "common" } }
-  ]
-}

salt/elasticsearch/files/ingest/suricata.alert

@@ -9,6 +9,7 @@
     { "rename":{ "field": "rule.signature_id",     "target_field": "rule.uuid", "ignore_failure": true  } },
     { "rename":{ "field": "rule.signature_id",     "target_field": "rule.signature", "ignore_failure": true  } },
     { "rename":{ "field": "message2.payload_printable",     "target_field": "network.data.decoded", "ignore_failure": true  } },
+    { "dissect": { "field": "rule.rule", "pattern": "%{?prefix}content:\"%{dns.query_name}\"%{?remainder}", "ignore_missing": true, "ignore_failure": true } },
     { "pipeline": { "name": "common.nids" } }
   ]
-}
+}

salt/elasticsearch/files/ingest/suricata.common

@@ -18,6 +18,13 @@
     { "set":    { "field": "event.ingested",            "value": "{{@timestamp}}"                                       } },
     { "date": { "field": "message2.timestamp", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "timezone": "UTC", "ignore_failure": true } },
     { "remove":{ "field": "agent",                                                              "ignore_failure": true  } },
+    {"append":{"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"ignore_failure":true}},
+    {
+      "script": {
+        "source": "boolean isPrivate(def ip) { if (ip == null) return false; int dot1 = ip.indexOf('.'); if (dot1 == -1) return false; int dot2 = ip.indexOf('.', dot1 + 1); if (dot2 == -1) return false; int first = Integer.parseInt(ip.substring(0, dot1)); if (first == 10) return true; if (first == 192 && ip.startsWith('168.', dot1 + 1)) return true; if (first == 172) { int second = Integer.parseInt(ip.substring(dot1 + 1, dot2)); return second >= 16 && second <= 31; } return false; } String[] fields = new String[] {\"source\", \"destination\"}; for (int i = 0; i < fields.length; i++) { def field = fields[i]; def ip = ctx[field]?.ip; if (ip != null) { if (ctx.network == null) ctx.network = new HashMap(); if (isPrivate(ip)) { if (ctx.network.private_ip == null) ctx.network.private_ip = new ArrayList(); if (!ctx.network.private_ip.contains(ip)) ctx.network.private_ip.add(ip); } else { if (ctx.network.public_ip == null) ctx.network.public_ip = new ArrayList(); if (!ctx.network.public_ip.contains(ip)) ctx.network.public_ip.add(ip); } } }",
+        "ignore_failure": false
+      }
+    },
     { "pipeline": { "if": "ctx?.event?.dataset != null", "name": "suricata.{{event.dataset}}" } }
   ]
 }

salt/elasticsearch/files/ingest/zeek.common

@@ -12,7 +12,8 @@
     { "rename":         { "field": "message2.id.orig_p",        "target_field": "source.port",          "ignore_missing": true  } },
     { "rename":         { "field": "message2.id.resp_h",        "target_field": "destination.ip",       "ignore_missing": true  } },
     { "rename":         { "field": "message2.id.resp_p",        "target_field": "destination.port",     "ignore_missing": true  } },
-    { "community_id": {} },
+    { "rename":         { "field": "message2.community_id",     "target_field": "network.community_id", "ignore_missing": true  } },
+    { "community_id":   { "if":    "ctx.network?.community_id == null"     } },
     { "set":         { "if": "ctx.source?.ip != null", "field": "client.ip",        "value": "{{source.ip}}"           } },
     { "set":         { "if": "ctx.source?.port != null", "field": "client.port",        "value": "{{source.port}}"       } },
     { "set":         { "if": "ctx.destination?.ip != null", "field": "server.ip",        "value": "{{destination.ip}}"  } },

salt/elasticsearch/files/ingest/zeek.dns

@@ -20,7 +20,8 @@
     { "rename": 	{ "field": "message2.RD", 		"target_field": "dns.recursion.desired",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.RA", 		"target_field": "dns.recursion.available",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.Z",		"target_field": "dns.reserved",			"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.answers", 		"target_field": "dns.answers.name",		"ignore_missing": true 	} },
+    { "rename":  	{ "field": "message2.answers", 		"target_field": "dns.answers.name", 		"ignore_missing": true  	} },
+    { "script": { "lang": "painless", "if": "ctx.dns != null && ctx.dns.answers != null && ctx.dns.answers.name != null", "source": "def ips = []; for (item in ctx.dns.answers.name) { if (item =~ /^(?:[0-9]{1,3}\\.){3}[0-9]{1,3}$/ || item =~ /^([a-fA-F0-9:]+:+)+[a-fA-F0-9]+$/) { ips.add(item); } } ctx.dns.resolved_ip = ips;" } },
     { "rename": 	{ "field": "message2.TTLs", 		"target_field": "dns.ttls",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.rejected", 	"target_field": "dns.query.rejected",		"ignore_missing": true 	} },
     { "script": 	{ "lang": "painless", "source": "ctx.dns.query.length = ctx.dns.query.name.length()",	"ignore_failure": true  } },
@@ -28,4 +29,4 @@
     { "pipeline": { "if": "ctx.dns?.query?.name != null && ctx.dns.query.name.contains('.')", "name": "dns.tld" } },
     { "pipeline":       { "name": "zeek.common"											} }
   ]
-}
+}

salt/elasticsearch/files/ingest/zeek.ldap_search

@@ -1,9 +1,25 @@
 {
     "description":"zeek.ldap_search",
     "processors":[
-        {"pipeline":    {"name": "zeek.ldap",       "ignore_missing_pipeline":true,"ignore_failure":true}},
-        {"set":         {"field": "event.dataset",  "value":"ldap_search"}},
-        {"remove":      {"field": "tags",           "ignore_missing":true}},
+        {"set":         {"field": "event.dataset",                 "value":"ldap_search"}},
+        {"json":        {"field": "message",                       "target_field": "message2",                     "ignore_failure": true}},
+        {"rename":      {"field": "message2.message_id",           "target_field": "ldap.message_id",              "ignore_missing": true}},
+        {"rename":      {"field": "message2.opcode",               "target_field": "ldap.opcode",                  "ignore_missing": true}},
+        {"rename":      {"field": "message2.result",               "target_field": "ldap.result",                  "ignore_missing": true}},
+        {"rename":      {"field": "message2.diagnostic_message",   "target_field": "ldap.diagnostic_message",      "ignore_missing": true}},
+        {"rename":      {"field": "message2.version",              "target_field": "ldap.version",                 "ignore_missing": true}},
+        {"rename":      {"field": "message2.object",               "target_field": "ldap.object",                  "ignore_missing": true}},
+        {"rename":      {"field": "message2.argument",             "target_field": "ldap.argument",                "ignore_missing": true}},
+        {"rename":      {"field": "message2.scope",                "target_field": "ldap_search.scope",            "ignore_missing":true}},
+        {"rename":      {"field": "message2.deref_aliases",        "target_field": "ldap_search.deref_aliases",    "ignore_missing":true}},
+        {"rename":      {"field": "message2.base_object",          "target_field": "ldap.object",                  "ignore_missing":true}},
+        {"rename":      {"field": "message2.result_count",         "target_field": "ldap_search.result_count",     "ignore_missing":true}},
+        {"rename":      {"field": "message2.filter",               "target_field": "ldap_search.filter",           "ignore_missing":true}},
+        {"rename":      {"field": "message2.attributes",           "target_field": "ldap_search.attributes",       "ignore_missing":true}},
+        {"script":      {"source": "if (ctx.containsKey('ldap') && ctx.ldap.containsKey('diagnostic_message') && ctx.ldap.diagnostic_message != null) {\n          String message = ctx.ldap.diagnostic_message;\n\n          // get user and property from SASL success\n          if (message.toLowerCase().contains(\"sasl(0): successful result\")) {\n            Pattern pattern = /user:\\s*([^ ]+)\\s*property:\\s*([^ ]+)/i;\n            Matcher matcher = pattern.matcher(message);\n            if (matcher.find()) {\n              ctx.ldap.user_email = matcher.group(1);  // Extract user email\n              ctx.ldap.property = matcher.group(2);    // Extract property\n            }\n          }\n          if (message.toLowerCase().contains(\"ldaperr:\")) {\n            Pattern pattern = /comment:\\s*([^,]+)/i;\n            Matcher matcher = pattern.matcher(message);\n\n            if (matcher.find()) {\n              ctx.ldap.comment = matcher.group(1);\n            }\n          }\n        }","ignore_failure": true}},
+        {"script":      {"source": "if (ctx.containsKey('ldap') && ctx.ldap.containsKey('object') && ctx.ldap.object != null) {\n    String message = ctx.ldap.object;\n\n    // parse common name from ldap object\n    if (message.toLowerCase().contains(\"cn=\")) {\n        Pattern pattern = /cn=([^,]+)/i;\n        Matcher matcher = pattern.matcher(message);\n        if (matcher.find()) {\n            ctx.ldap.common_name = matcher.group(1);  // Extract CN\n        }\n    }\n    // build domain from ldap object\n    if (message.toLowerCase().contains(\"dc=\")) {\n        Pattern dcPattern = /dc=([^,]+)/i;\n        Matcher dcMatcher = dcPattern.matcher(message);\n\n        StringBuilder domainBuilder = new StringBuilder();\n        while (dcMatcher.find()) {\n            if (domainBuilder.length() > 0 ){\n                domainBuilder.append(\".\");\n            }\n            domainBuilder.append(dcMatcher.group(1));\n        }\n        if (domainBuilder.length() > 0) {\n            ctx.ldap.domain = domainBuilder.toString();\n        }\n    }\n    // create list of any organizational units from ldap object\n    if (message.toLowerCase().contains(\"ou=\")) {\n        Pattern ouPattern = /ou=([^,]+)/i;\n        Matcher ouMatcher = ouPattern.matcher(message);\n        ctx.ldap.organizational_unit = [];\n\n        while (ouMatcher.find()) {\n            ctx.ldap.organizational_unit.add(ouMatcher.group(1));\n        }\n        if(ctx.ldap.organizational_unit.isEmpty()) {\n          ctx.remove(\"ldap.organizational_unit\");\n        }\n    }\n}\n","ignore_failure": true}},
+        {"remove":      {"field": "message2.tags",                 "ignore_failure": true}},
+        {"remove":      {"field": ["host"],                        "ignore_failure": true}},
         {"pipeline":    {"name": "zeek.common"}}
     ]
 }

salt/elasticsearch/soc_elasticsearch.yaml

@@ -12,7 +12,7 @@ elasticsearch:
     description: Specify the memory heap size in (m)egabytes for Elasticsearch.
     helpLink: elasticsearch.html
   index_clean:
-    description: Determines if indices should be considered for deletion by available disk space in the cluster. Otherwise, indices will only be deleted by the age defined in the ILM settings.
+    description: Determines if indices should be considered for deletion by available disk space in the cluster. Otherwise, indices will only be deleted by the age defined in the ILM settings. This setting only applies to EVAL, STANDALONE, and HEAVY NODE installations. Other installations can only use ILM settings.
     forcedType: bool
     helpLink: elasticsearch.html
   retention: 

salt/elasticsearch/templates/component/ecs/log.json

@@ -29,7 +29,7 @@
                 "file": {
                   "properties": {
                     "line": {
-                      "type": "integer"
+                      "type": "long"
                     },
                     "name": {
                       "ignore_above": 1024,

salt/elasticsearch/templates/component/ecs/metadata.json

@@ -0,0 +1,26 @@
+{
+  "template": {
+    "mappings": {
+      "dynamic_templates": [],
+      "properties": {
+        "metadata": {
+          "properties": {
+            "kafka": {
+              "properties": {
+                "timestamp": {
+                  "type": "date"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "_meta": {
+    "_meta": {
+      "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-log.html",
+      "ecs_version": "1.12.2"
+    }
+  }
+}

salt/elasticsearch/templates/component/elastic-agent/so-fleet_globals-1.json

@@ -5,6 +5,7 @@
         "managed_by": "security_onion",
         "managed": true
       },
+      "date_detection": false,
       "dynamic_templates": [
         {
           "strings_as_keyword": {
@@ -16,7 +17,19 @@
           }
         }
       ],
-      "date_detection": false
+      "properties": {
+        "metadata": {
+          "properties": {
+            "kafka": {
+              "properties": {
+                "timestamp": {
+                  "type": "date"
+                }
+              }
+            }
+          }
+        }
+      }
     }
   },
   "_meta": {

salt/elasticsearch/templates/component/elastic-agent/so-fleet_integrations.ip_mappings-1.json

@@ -1,37 +1,59 @@
 {
-    "template": {
-      "mappings": {
-        "properties": {
-          "host": {
-        "properties":{
-              "ip": {
-                "type": "ip"
-              }
-        }
-      },
-      "related": {
-            "properties":{
-              "ip": {
-                "type": "ip"
-              }
+  "template": {
+    "mappings": {
+      "properties": {
+        "host": {
+          "properties": {
+            "ip": {
+              "type": "ip"
             }
-          },
-      "destination": {
-            "properties":{
-              "ip": {
-                "type": "ip"
-              }
+          }
+        },
+        "related": {
+          "properties": {
+            "ip": {
+              "type": "ip"
             }
-          },
-      "source": {
-            "properties":{
-              "ip": {
-                "type": "ip"
+          }
+        },
+        "destination": {
+          "properties": {
+            "ip": {
+              "type": "ip"
+            }
+          }
+        },
+        "source": {
+          "properties": {
+            "ip": {
+              "type": "ip"
+            }
+          }
+        },
+        "metadata": {
+          "properties": {
+            "input": {
+              "properties": {
+                "beats": {
+                  "properties": {
+                    "host": {
+                      "properties": {
+                        "ip": {
+                          "type": "ip"
+                        }
+                      }
+                    }
+                  }
+                }
               }
             }
           }
         }
       }
     }
+  },
+  "_meta": {
+    "managed_by": "security_onion",
+    "managed": true
   }
-  
+}

salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-start

@@ -1,4 +1,4 @@
-/bin/bash
+#!/bin/bash
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
 # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
 # https://securityonion.net/license; you may not use this file except in compliance with the
@@ -6,6 +6,6 @@
 
 . /usr/sbin/so-common
 
-
 echo "Starting ILM..."
 curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -X POST https://localhost:9200/_ilm/start
+echo

salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-stop

@@ -8,3 +8,4 @@
 
 echo "Stopping ILM..."
 curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -X POST https://localhost:9200/_ilm/stop
+echo

salt/elasticsearch/tools/sbin/so-elasticsearch-indices-growth

@@ -0,0 +1,113 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+INFLUX_URL="https://localhost:8086/api/v2"
+
+. /usr/sbin/so-common
+
+request() {
+    curl -skK /opt/so/conf/influxdb/curl.config "$INFLUX_URL/$@"
+}
+
+lookup_org_id() {
+  response=$(request orgs?org=Security+Onion)
+  echo "$response" | jq -r ".orgs[] | select(.name == \"Security Onion\").id"
+}
+
+ORG_ID=$(lookup_org_id)
+
+run_flux_query() {
+    local query=$1
+    request "query?org=$ORG_ID" -H 'Accept:application/csv' -H 'Content-type:application/vnd.flux' -d "$query" -XPOST 2>/dev/null
+}
+
+read_csv_result() {
+    local result="$1"
+    echo "$result" | grep '^,_result,' | head -1 | awk -F',' '{print $NF}' | tr -d '\r\n\t '
+}
+
+bytes_to_gb() {
+    local bytes="${1:-0}"
+    if [[ "$bytes" =~ ^-?[0-9]+$ ]]; then
+        echo "$bytes" | awk '{printf "%.2f", $1 / 1024 / 1024 / 1024}'
+    else
+        echo "0.00"
+    fi
+}
+
+indexes_query='from(bucket: "telegraf/so_long_term")
+|> range(start: -7d)
+|> filter(fn: (r) => r._measurement == "elasticsearch_index_size")
+|> distinct(column: "_field")
+|> keep(columns: ["_field"])'
+
+indexes_result=$(run_flux_query "$indexes_query")
+indexes=$(echo "$indexes_result" | tail -n +2 | cut -d',' -f4 | grep -v '^$' | grep -v '^_field$' | sed 's/\r$//' | sort -u)
+
+printf "%-50s %15s %15s %15s\n" "Index Name" "Last 24hr (GB)" "Last 7d (GB)" "Last 30d (GB)"
+printf "%-50s %15s %15s %15s\n" "$(printf '%.0s-' {1..50})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})"
+
+for index in $indexes; do
+  [[ -z "$index" ]] && continue
+  current_query="from(bucket: \"telegraf/so_long_term\")
+    |> range(start: -4h)
+    |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
+    |> last()
+    |> keep(columns: [\"_value\"])"
+  current_result=$(run_flux_query "$current_query")
+  current_size=$(read_csv_result "$current_result")
+  current_size=${current_size:-0}
+
+  size_24h_query="from(bucket: \"telegraf/so_long_term\")
+    |> range(start: -25h, stop: -23h)
+    |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
+    |> last()
+    |> keep(columns: [\"_value\"])"
+  size_24h_result=$(run_flux_query "$size_24h_query")
+  size_24h_ago=$(read_csv_result "$size_24h_result")
+  size_24h_ago=${size_24h_ago:-$current_size}
+
+  size_7d_query="from(bucket: \"telegraf/so_long_term\")
+    |> range(start: -7d8h, stop: -7d)
+    |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
+    |> last()
+    |> keep(columns: [\"_value\"])"
+  size_7d_result=$(run_flux_query "$size_7d_query")
+  size_7d_ago=$(read_csv_result "$size_7d_result")
+  size_7d_ago=${size_7d_ago:-$current_size}
+
+  size_30d_query="from(bucket: \"telegraf/so_long_term\")
+    |> range(start: -30d8h, stop: -30d)
+    |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
+    |> last()
+    |> keep(columns: [\"_value\"])"
+  size_30d_result=$(run_flux_query "$size_30d_query")
+  size_30d_ago=$(read_csv_result "$size_30d_result")
+  size_30d_ago=${size_30d_ago:-$current_size}
+
+  # if an index was recently cleaned up by ilm it will result in a negative number for 'index growth'.
+  growth_24h=$(( current_size > size_24h_ago ? current_size - size_24h_ago : 0 ))
+
+  growth_7d=$(( current_size > size_7d_ago ? current_size - size_7d_ago : 0 ))
+
+  growth_30d=$(( current_size > size_30d_ago ? current_size - size_30d_ago : 0 ))
+
+  growth_24h_gb=$(bytes_to_gb "$growth_24h")
+  growth_7d_gb=$(bytes_to_gb "$growth_7d")
+  growth_30d_gb=$(bytes_to_gb "$growth_30d")
+
+  # Only results for indices with atleast 1 metric above 0.00
+  if [[ "$growth_24h_gb" != "0.00" ]] || [[ "$growth_7d_gb" != "0.00" ]] || [[ "$growth_30d_gb" != "0.00" ]]; then
+    printf "%020.2f|%-50s %15s %15s %15s\n" \
+      "$growth_24h" \
+      "$index" \
+      "$growth_24h_gb" \
+      "$growth_7d_gb" \
+      "$growth_30d_gb"
+  fi
+done | sort -t'|' -k1,1nr | cut -d'|' -f2-
+

salt/firewall/defaults.yaml

@@ -11,6 +11,7 @@ firewall:
     endgame: []
     eval: []
     external_suricata: []
+    external_kafka: []
     fleet: []
     heavynode: []
     idh: []
@@ -103,6 +104,10 @@ firewall:
       tcp:
         - 9092
       udp: []
+    kafka_external_access:
+      tcp:
+        - 29092
+      udp: []
     kibana:
       tcp:
         - 5601
@@ -473,6 +478,8 @@ firewall:
             external_suricata:
               portgroups:
                 - external_suricata
+            external_kafka:
+              portgroups: []
             desktop:
               portgroups:
                 - docker_registry
@@ -668,6 +675,8 @@ firewall:
             external_suricata:
               portgroups:
                 - external_suricata
+            external_kafka:
+              portgroups: []
             desktop:
               portgroups:
                 - docker_registry
@@ -867,6 +876,8 @@ firewall:
             external_suricata:
               portgroups:
                 - external_suricata
+            external_kafka:
+              portgroups: []
             strelka_frontend:
               portgroups:
                 - strelka_frontend
@@ -1337,6 +1348,8 @@ firewall:
             endgame:
               portgroups:
                 - endgame
+            external_kafka:
+              portgroups: []
             receiver:
               portgroups: []
             customhostgroup0:

salt/firewall/map.jinja

@@ -21,25 +21,38 @@
 {# Only add Kafka firewall items when Kafka enabled #}
 {% set role = GLOBALS.role.split('-')[1] %}
 
-{% if GLOBALS.pipeline == 'KAFKA' and role in ['manager', 'managersearch', 'standalone'] %}
-{%   do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[role].portgroups.append('kafka_controller') %}
-{%   do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.receiver.portgroups.append('kafka_controller') %}
-{% endif %}
+{% if GLOBALS.pipeline == 'KAFKA' %}
+{%   set KAFKA_EXTERNAL_ACCESS = salt['pillar.get']('kafka:config:external_access:enabled', default=False) %}
+{%   set kafka_node_type = salt['pillar.get']('kafka:nodes:'+ GLOBALS.hostname + ':role') %}
 
-{% if GLOBALS.pipeline == 'KAFKA' and role == 'receiver' %}
-{%   do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.self.portgroups.append('kafka_controller') %}
-{%   do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.standalone.portgroups.append('kafka_controller') %}
-{%   do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.manager.portgroups.append('kafka_controller') %}
-{%   do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.managersearch.portgroups.append('kafka_controller') %}
-{%   do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.receiver.portgroups.append('kafka_controller') %}
-{% endif %}
+{%   if role in ['manager', 'managersearch', 'standalone'] %}
+{%     do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[role].portgroups.append('kafka_controller') %}
+{%     do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.receiver.portgroups.append('kafka_controller') %}
+{%   endif %}
 
-{% if GLOBALS.pipeline == 'KAFKA' and role in ['manager', 'managersearch', 'standalone', 'receiver'] %}
-{%   for r in ['manager', 'managersearch', 'standalone', 'receiver', 'fleet', 'idh', 'sensor', 'searchnode','heavynode', 'elastic_agent_endpoint', 'desktop'] %}
-{%     if FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[r] is defined %}
-{%       do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[r].portgroups.append('kafka_data') %}
+{%   if role == 'receiver' %}
+{%     do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.self.portgroups.append('kafka_controller') %}
+{%     do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.standalone.portgroups.append('kafka_controller') %}
+{%     do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.manager.portgroups.append('kafka_controller') %}
+{%     do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.managersearch.portgroups.append('kafka_controller') %}
+{%     do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.receiver.portgroups.append('kafka_controller') %}
+{%   endif %}
+
+{%   if role in ['manager', 'managersearch', 'standalone', 'receiver'] %}
+{%     for r in ['manager', 'managersearch', 'standalone', 'receiver', 'fleet', 'idh', 'sensor', 'searchnode','heavynode', 'elastic_agent_endpoint', 'desktop'] %}
+{%       if FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[r] is defined %}
+{%         do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[r].portgroups.append('kafka_data') %}
+{%       endif %}
+{%     endfor %}
+{%   endif %}
+
+{%   if KAFKA_EXTERNAL_ACCESS %}
+{#     Kafka external access only applies for Kafka nodes with the broker role. #}
+{%     if role in ['manager', 'managersearch', 'standalone', 'receiver'] and 'broker' in kafka_node_type %}
+{%       do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.external_kafka.portgroups.append('kafka_external_access') %}
 {%     endif %}
-{%   endfor %}
+{%   endif %}
+
 {% endif %}
 
-{% set FIREWALL_MERGED = salt['pillar.get']('firewall', FIREWALL_DEFAULT.firewall, merge=True) %}
+{% set FIREWALL_MERGED = salt['pillar.get']('firewall', FIREWALL_DEFAULT.firewall, merge=True) %}

salt/firewall/soc_firewall.yaml

@@ -33,6 +33,7 @@ firewall:
     endgame: *hostgroupsettingsadv
     eval: *hostgroupsettings
     external_suricata: *hostgroupsettings
+    external_kafka: *hostgroupsettings
     fleet: *hostgroupsettings
     heavynode: *hostgroupsettings
     idh: *hostgroupsettings
@@ -130,6 +131,9 @@ firewall:
     kafka_data:
       tcp: *tcpsettings
       udp: *udpsettings
+    kafka_external_access:
+      tcp: *tcpsettings
+      udp: *udpsettings
     kibana:
       tcp: *tcpsettings
       udp: *udpsettings

salt/global/soc_global.yaml

@@ -43,7 +43,7 @@ global:
     global: True
     advanced: True
   pipeline:
-    description: Sets which pipeline technology for events to use. Currently only Redis is fully supported. Kafka is experimental and requires a Security Onion Pro license.
+    description: Sets which pipeline technology for events to use. The use of Kafka requires a Security Onion Pro license.
     regex: ^(REDIS|KAFKA)$
     options:
       - REDIS

salt/idstools/config.sls

@@ -24,13 +24,23 @@ idstools_sbin:
     - group: 939
     - file_mode: 755
 
-idstools_sbin_jinja:
-  file.recurse:
-    - name: /usr/sbin
-    - source: salt://idstools/tools/sbin_jinja
+# If this is used, exclude so-rule-update
+#idstools_sbin_jinja:
+#  file.recurse:
+#    - name: /usr/sbin
+#    - source: salt://idstools/tools/sbin_jinja
+#    - user: 934
+#    - group: 939 
+#    - file_mode: 755
+#    - template: jinja
+
+idstools_so-rule-update:
+  file.managed:
+    - name: /usr/sbin/so-rule-update
+    - source: salt://idstools/tools/sbin_jinja/so-rule-update
     - user: 934
     - group: 939 
-    - file_mode: 755
+    - mode: 755
     - template: jinja
 
 suricatacustomdirsfile:

salt/idstools/enabled.sls

@@ -55,6 +55,7 @@ so-idstools:
     {% endif %}
     - watch:
       - file: idstoolsetcsync
+      - file: idstools_so-rule-update
 
 delete_so-idstools_so-status.disabled:
   file.uncomment:
@@ -76,6 +77,7 @@ run_so-rule-update:
     - require:
       - docker_container: so-idstools
     - onchanges:
+      - file: idstools_so-rule-update
       - file: idstoolsetcsync
       - file: synclocalnidsrules
     - order: last

salt/idstools/rules/extraction.rules

@@ -1,26 +1,26 @@
 # Extract all PDF mime type
-alert http any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; sid:1100000; rev:1;)
-alert smtp any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; sid:1100001; rev:1;)
-alert nfs any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; sid:1100002; rev:1;)
-alert smb any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; sid:1100003; rev:1;)
+alert http any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100000; rev:1;)
+alert smtp any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100001; rev:1;)
+alert nfs any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100002; rev:1;)
+alert smb any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100003; rev:1;)
 # Extract EXE/DLL file types
-alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:1100004; rev:1;)
-alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:1100005; rev:1;)
-alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:1100006; rev:1;)
-alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:1100007; rev:1;)
-alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:1100008; rev:1;)
-alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:1100009; rev:1;)
-alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:1100010; rev:1;)
-alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:1100011; rev:1;)
+alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100004; rev:1;)
+alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100005; rev:1;)
+alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100006; rev:1;)
+alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100007; rev:1;)
+alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100008; rev:1;)
+alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100009; rev:1;)
+alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100010; rev:1;)
+alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100011; rev:1;)
 
 # Extract all Zip files
-alert http any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; sid:1100012; rev:1;)
-alert smtp any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; sid:1100013; rev:1;)
-alert nfs any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; sid:1100014; rev:1;)
-alert smb any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; sid:1100015; rev:1;)
+alert http any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100012; rev:1;)
+alert smtp any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100013; rev:1;)
+alert nfs any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100014; rev:1;)
+alert smb any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100015; rev:1;)
 
 # Extract Word Docs
-alert http any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100016; rev:1;)
-alert smtp any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100017; rev:1;)
-alert nfs any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100018; rev:1;)
-alert smb any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100019; rev:1;)
+alert http any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100016; rev:1;)
+alert smtp any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100017; rev:1;)
+alert nfs any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100018; rev:1;)
+alert smb any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100019; rev:1;)

salt/influxdb/templates/alarm_nsm_disk.json

@@ -5,10 +5,10 @@
     "name": "alarm-nsm-disk"
   },
   "spec": {
-    "description": "Percent used space on the root partition of at least one node has exceeded the alarm threshold.",
+    "description": "Percent used space on the nsm partition of at least one node has exceeded the alarm threshold.",
     "every": "1m0s",
     "name": "NSM Disk High Usage",
-    "query": "from(bucket: \"telegraf/so_short_term\")\n  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n  |> filter(fn: (r) => r[\"_measurement\"] == \"disk\")\n  |> filter(fn: (r) => r[\"path\"] == \"/\")\n  |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n  |> aggregateWindow(every: 1m, fn: max, createEmpty: false)\n  |> yield(name: \"max\")",
+    "query": "from(bucket: \"telegraf/so_short_term\")\n  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n  |> filter(fn: (r) => r[\"_measurement\"] == \"disk\")\n  |> filter(fn: (r) => r[\"path\"] == \"/nsm\")\n  |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n  |> aggregateWindow(every: 1m, fn: max, createEmpty: false)\n  |> yield(name: \"max\")",
     "status": "active",
     "statusMessageTemplate": "Check: ${ r._check_name } is: ${ r._level }",
     "thresholds": [

salt/kafka/config.map.jinja

@@ -8,6 +8,7 @@
 {% set KAFKA_NODES_PILLAR = salt['pillar.get']('kafka:nodes') %}
 {% set KAFKA_PASSWORD = salt['pillar.get']('kafka:config:password') %}
 {% set KAFKA_TRUSTPASS = salt['pillar.get']('kafka:config:trustpass') %}
+{% set KAFKA_EXTERNAL_ACCESS = salt['pillar.get']('kafka:config:external_access:enabled', default=False) %}
 
 {# Create list of KRaft controllers #}
 {% set controllers = [] %}
@@ -15,7 +16,7 @@
 {# Check for Kafka nodes with controller in process_x_roles #}
 {% for node in KAFKA_NODES_PILLAR %}
 {%   if 'controller' in KAFKA_NODES_PILLAR[node].role %}
-{%     do controllers.append(KAFKA_NODES_PILLAR[node].nodeid ~ "@" ~ node ~ ":9093") %}
+{%     do controllers.append(KAFKA_NODES_PILLAR[node].nodeid ~ "@" ~ KAFKA_NODES_PILLAR[node].ip ~ ":9093") %}
 {%   endif %}
 {% endfor %}
 
@@ -28,7 +29,15 @@
 {# Generate server.properties for 'broker' , 'controller', 'broker,controller' node types
     anything above this line is a configuration needed for ALL Kafka nodes #}
 {% if node_type == 'broker' %}
-{%   do KAFKAMERGED.config.broker.update({'advertised_x_listeners': 'BROKER://'+ GLOBALS.node_ip +':9092' }) %}
+{%   if KAFKA_EXTERNAL_ACCESS %}
+{%     do KAFKAMERGED.config.broker.update({'advertised_x_listeners': 'BROKER://'+ GLOBALS.node_ip +':9092' + ',' + 'EXTERNAL_ACCESS://' + GLOBALS.node_ip + ':29092' }) %}
+{%     do KAFKAMERGED.config.broker.update({'listeners': KAFKAMERGED.config.broker.listeners + ',' + KAFKAMERGED.config.external_access.listeners }) %}
+{%     do KAFKAMERGED.config.broker.update({'listener_x_security_x_protocol_x_map': KAFKAMERGED.config.broker.listener_x_security_x_protocol_x_map + ',' + KAFKAMERGED.config.external_access.listener_x_security_x_protocol_x_map }) %}
+{%     do KAFKAMERGED.config.broker.update({'sasl_x_enabled_x_mechanisms': KAFKAMERGED.config.external_access.sasl_x_enabled_x_mechanisms }) %}
+{%     do KAFKAMERGED.config.broker.update({'sasl_x_mechanism_x_inter_x_broker_x_protocol': KAFKAMERGED.config.external_access.sasl_x_mechanism_x_inter_x_broker_x_protocol }) %}
+{%   else %}
+{%     do KAFKAMERGED.config.broker.update({'advertised_x_listeners': 'BROKER://'+ GLOBALS.node_ip +':9092' }) %}
+{%   endif %}
 {%   do KAFKAMERGED.config.broker.update({'controller_x_quorum_x_voters': kafka_controller_quorum_voters }) %}
 {%   do KAFKAMERGED.config.broker.update({'node_x_id': salt['pillar.get']('kafka:nodes:'+ GLOBALS.hostname +':nodeid') }) %}
 {%   do KAFKAMERGED.config.broker.update({'ssl_x_keystore_x_password': KAFKA_PASSWORD }) %}
@@ -42,6 +51,7 @@
 {% endif %}
 
 {% if node_type == 'controller' %}
+{%   do KAFKAMERGED.config.controller.update({'advertised_x_listeners': 'CONTROLLER://' + GLOBALS.node_ip + ':9093'}) %}
 {%   do KAFKAMERGED.config.controller.update({'controller_x_quorum_x_voters': kafka_controller_quorum_voters }) %}
 {%   do KAFKAMERGED.config.controller.update({'node_x_id': salt['pillar.get']('kafka:nodes:'+ GLOBALS.hostname +':nodeid') }) %}
 {%   do KAFKAMERGED.config.controller.update({'ssl_x_keystore_x_password': KAFKA_PASSWORD }) %}
@@ -50,7 +60,15 @@
 
 {# Kafka nodes of this type are not recommended for use outside of development / testing. #}
 {% if node_type == 'broker,controller' %}
-{%   do KAFKAMERGED.config.broker.update({'advertised_x_listeners': 'BROKER://'+ GLOBALS.node_ip +':9092' }) %}
+{%   if KAFKA_EXTERNAL_ACCESS %}
+{%     do KAFKAMERGED.config.broker.update({'advertised_x_listeners': 'BROKER://'+ GLOBALS.node_ip +':9092' + ',' + 'CONTROLLER://'+ GLOBALS.node_ip +':9093' + ',' + 'EXTERNAL_ACCESS://' + GLOBALS.node_ip + ':29092' }) %}
+{%     do KAFKAMERGED.config.broker.update({'listeners': KAFKAMERGED.config.broker.listeners + ',' + KAFKAMERGED.config.external_access.listeners }) %}
+{%     do KAFKAMERGED.config.broker.update({'listener_x_security_x_protocol_x_map': KAFKAMERGED.config.broker.listener_x_security_x_protocol_x_map + ',' + KAFKAMERGED.config.external_access.listener_x_security_x_protocol_x_map }) %}
+{%     do KAFKAMERGED.config.broker.update({'sasl_x_enabled_x_mechanisms': KAFKAMERGED.config.external_access.sasl_x_enabled_x_mechanisms }) %}
+{%     do KAFKAMERGED.config.broker.update({'sasl_x_mechanism_x_inter_x_broker_x_protocol': KAFKAMERGED.config.external_access.sasl_x_mechanism_x_inter_x_broker_x_protocol }) %}
+{%   else %}
+{%     do KAFKAMERGED.config.broker.update({'advertised_x_listeners': 'BROKER://'+ GLOBALS.node_ip +':9092' + ',' + 'CONTROLLER://'+ GLOBALS.node_ip +':9093' }) %}
+{%   endif %}
 {%   do KAFKAMERGED.config.broker.update({'controller_x_listener_x_names': KAFKAMERGED.config.controller.controller_x_listener_x_names }) %}
 {%   do KAFKAMERGED.config.broker.update({'controller_x_quorum_x_voters': kafka_controller_quorum_voters }) %}
 {%   do KAFKAMERGED.config.broker.update({'node_x_id': salt['pillar.get']('kafka:nodes:'+ GLOBALS.hostname +':nodeid') }) %}

salt/kafka/config.sls

@@ -6,6 +6,8 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   set KAFKA_EXTERNAL_ACCESS = salt['pillar.get']('kafka:config:external_access:enabled', default=False) %}
+{%   set KAFKA_EXTERNAL_USERS = salt['pillar.get']('kafka:config:external_access:remote_users', default=None) %}
 
 kafka_group:
   group.present:
@@ -69,6 +71,29 @@ kafka_kraft_{{sc}}_properties:
     - show_changes: False
 {%   endfor %}
 
+{%   if KAFKA_EXTERNAL_ACCESS and KAFKA_EXTERNAL_USERS != None %}
+kafka_server_jaas_properties:
+  file.managed:
+    - source: salt://kafka/etc/jaas.conf.jinja
+    - name: /opt/so/conf/kafka/kafka_server_jaas.conf
+    - template: jinja
+    - user: 960
+    - group: 960
+    - show_changes: False
+{%   else %}
+remove_kafka_server_jaas_properties:
+  file.absent:
+    - name: /opt/so/conf/kafka/kafka_server_jaas.conf
+{%   endif %}
+
+kafka_log4j_properties:
+  file.managed:
+    - source: salt://kafka/etc/log4j.properties
+    - name: /opt/so/conf/kafka/log4j.properties
+    - user: 960
+    - group: 960
+    - show_changes: False
+
 reset_quorum_on_changes:
   cmd.run:
     - name: rm -f /nsm/kafka/data/__cluster_metadata-0/quorum-state
@@ -81,4 +106,4 @@ reset_quorum_on_changes:
   test.fail_without_changes:
     - name: {{sls}}_state_not_allowed
 
-{% endif %}
+{% endif %}

salt/kafka/defaults.yaml

@@ -21,7 +21,7 @@ kafka:
       log_x_segment_x_bytes: 1073741824
       node_x_id:
       num_x_io_x_threads: 8
-      num_x_network_x_threads: 3
+      num_x_network_x_threads: 5
       num_x_partitions: 3
       num_x_recovery_x_threads_x_per_x_data_x_dir: 1
       offsets_x_topic_x_replication_x_factor: 1
@@ -46,6 +46,7 @@ kafka:
       ssl_x_keystore_x_type: PKCS12
       ssl_x_keystore_x_password:
     controller:
+      advertsied_x_listeners:
       controller_x_listener_x_names: CONTROLLER
       controller_x_quorum_x_voters:
       listeners: CONTROLLER://0.0.0.0:9093
@@ -61,4 +62,10 @@ kafka:
       ssl_x_keystore_x_password:
       ssl_x_truststore_x_location: /etc/pki/kafka-truststore.jks
       ssl_x_truststore_x_type: JKS
-      ssl_x_truststore_x_password:
+      ssl_x_truststore_x_password:
+    external_access:
+      enabled: False
+      listeners: EXTERNAL_ACCESS://0.0.0.0:29092
+      listener_x_security_x_protocol_x_map: EXTERNAL_ACCESS:SASL_SSL
+      sasl_x_enabled_x_mechanisms: PLAIN
+      sasl_x_mechanism_x_inter_x_broker_x_protocol: SSL

salt/kafka/enabled.sls

@@ -14,6 +14,7 @@
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKER %}
 {%   set KAFKANODES = salt['pillar.get']('kafka:nodes') %}
+{%   set KAFKA_EXTERNAL_ACCESS = salt['pillar.get']('kafka:config:external_access:enabled', default=False) %}
 {%   if 'gmd' in salt['pillar.get']('features', []) %}
 
 include:
@@ -34,7 +35,7 @@ so-kafka:
     - user: kafka
     - environment:
         KAFKA_HEAP_OPTS: -Xmx2G -Xms1G
-        KAFKA_OPTS: -javaagent:/opt/jolokia/agents/jolokia-agent-jvm-javaagent.jar=port=8778,host={{ DOCKER.containers['so-kafka'].ip }},policyLocation=file:/opt/jolokia/jolokia.xml
+        KAFKA_OPTS: "-javaagent:/opt/jolokia/agents/jolokia-agent-jvm-javaagent.jar=port=8778,host={{ DOCKER.containers['so-kafka'].ip }},policyLocation=file:/opt/jolokia/jolokia.xml {%- if KAFKA_EXTERNAL_ACCESS %} -Djava.security.auth.login.config=/opt/kafka/config/kafka_server_jaas.conf {% endif -%}"
     - extra_hosts:
       {% for node in KAFKANODES %}
       - {{ node }}:{{ KAFKANODES[node].ip }}
@@ -54,11 +55,18 @@ so-kafka:
       - /nsm/kafka/data/:/nsm/kafka/data/:rw
       - /opt/so/log/kafka:/opt/kafka/logs/:rw
       - /opt/so/conf/kafka/server.properties:/opt/kafka/config/kraft/server.properties:ro
-      - /opt/so/conf/kafka/client.properties:/opt/kafka/config/kraft/client.properties
+      - /opt/so/conf/kafka/client.properties:/opt/kafka/config/kraft/client.properties:ro
+      - /opt/so/conf/kafka/log4j.properties:/opt/kafka/config/log4j.properties:ro
+      {% if KAFKA_EXTERNAL_ACCESS %}
+      - /opt/so/conf/kafka/kafka_server_jaas.conf:/opt/kafka/config/kafka_server_jaas.conf:ro
+      {% endif %}
     - watch:
       {% for sc in ['server', 'client'] %}
       - file: kafka_kraft_{{sc}}_properties
       {% endfor %}
+      {% if KAFKA_EXTERNAL_ACCESS %}
+      - file: kafka_server_jaas_properties
+      {% endif %}
       - file: kafkacertz
     - require:
       - file: kafkacertz
@@ -87,4 +95,4 @@ include:
   test.fail_without_changes:
     - name: {{sls}}_state_not_allowed
 
-{% endif %}
+{% endif %}

salt/kafka/etc/jaas.conf.jinja

@@ -0,0 +1,16 @@
+{% set KAFKA_EXTERNAL_USERS = salt['pillar.get']('kafka:config:external_access:remote_users') -%}
+
+{%- set valid_users = [] -%}
+
+{%- for item, user in KAFKA_EXTERNAL_USERS.items() -%}
+{%   if 'password' in user and user.password is not none and user.password.strip() != "" -%}
+{%     do valid_users.append('user_' ~ user.username ~ '="' ~ user.password ~ '"') -%}
+{%   endif -%}
+{%- endfor -%}
+
+KafkaServer {
+    org.apache.kafka.common.security.plain.PlainLoginModule required
+    {% for user_entry in valid_users -%}
+    {{ user_entry }}{{ ";" if loop.last }}
+    {% endfor %}
+};

salt/kafka/etc/log4j.properties

@@ -0,0 +1,101 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more                                                                                                                                                                          
+# contributor license agreements.  See the NOTICE file distributed with                                                                                                                                                                       
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with                                                                                                                                                                        
+# the License.  You may obtain a copy of the License at                                                                                                                                                                                       
+#                                                                                                                                                                                             
+#    http://www.apache.org/licenses/LICENSE-2.0                                                
+#                                                                                                                                                                                             
+# Unless required by applicable law or agreed to in writing, software                                                                                                                         
+# distributed under the License is distributed on an "AS IS" BASIS,                                                                                                                           
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.                     
+# See the License for the specific language governing permissions and                          
+# limitations under the License.                                                                                                                                                              
+                                                                                               
+# Unspecified loggers and loggers with additivity=true output to server.log and stdout         
+# Note that INFO only applies to unspecified loggers, the log level of the child logger is used otherwise                                                                                                                                                                       
+log4j.rootLogger=INFO, stdout, kafkaAppender                                                   
+                                                                                               
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender                                         
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout                                    
+log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n                               
+                                                                                               
+log4j.appender.kafkaAppender=org.apache.log4j.RollingFileAppender
+log4j.appender.kafkaAppender.File=${kafka.logs.dir}/server.log                                 
+log4j.appender.kafkaAppender.MaxFileSize=10MB
+log4j.appender.kafkaAppender.MaxBackupIndex=10
+log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout                                                                                                                                                                                                                                                           
+log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n                        
+                                                                                               
+log4j.appender.stateChangeAppender=org.apache.log4j.RollingFileAppender
+log4j.appender.stateChangeAppender.File=${kafka.logs.dir}/state-change.log                     
+log4j.appender.stateChangeAppender.MaxFileSize=10MB
+log4j.appender.stateChangeAppender.MaxBackupIndex=10
+log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout                                                                                                                      
+log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n                                                                                                                 
+                                                                                                                                                                                                                                                                                                                             
+log4j.appender.requestAppender=org.apache.log4j.RollingFileAppender
+log4j.appender.requestAppender.File=${kafka.logs.dir}/kafka-request.log                                                                 
+log4j.appender.requestAppender.MaxFileSize=10MB
+log4j.appender.requestAppender.MaxBackupIndex=10
+log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout   
+log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n                                                                                     
+                                                                                                                                        
+log4j.appender.cleanerAppender=org.apache.log4j.RollingFileAppender
+log4j.appender.cleanerAppender.File=${kafka.logs.dir}/log-cleaner.log                                                                   
+log4j.appender.cleanerAppender.MaxFileSize=10MB
+log4j.appender.cleanerAppender.MaxBackupIndex=10
+log4j.appender.cleanerAppender.layout=org.apache.log4j.PatternLayout                                                                                          
+log4j.appender.cleanerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n                      
+                                                                                                                                        
+log4j.appender.controllerAppender=org.apache.log4j.RollingFileAppender
+log4j.appender.controllerAppender.File=${kafka.logs.dir}/controller.log                        
+log4j.appender.controllerAppender.MaxFileSize=10MB
+log4j.appender.controllerAppender.MaxBackupIndex=10
+log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout                                                                                                                                                                       
+log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n                   
+                                                                                                                       
+log4j.appender.authorizerAppender=org.apache.log4j.RollingFileAppender
+log4j.appender.authorizerAppender.File=${kafka.logs.dir}/kafka-authorizer.log                  
+log4j.appender.authorizerAppender.MaxFileSize=10MB
+log4j.appender.authorizerAppender.MaxBackupIndex=10
+log4j.appender.authorizerAppender.layout=org.apache.log4j.PatternLayout                                                                                       
+log4j.appender.authorizerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n                                                                                  
+                                                                                                                                                                                                                                              
+# Change the line below to adjust ZK client logging                                                                    
+log4j.logger.org.apache.zookeeper=INFO                                                         
+                                                                                               
+# Change the two lines below to adjust the general broker logging level (output to server.log and stdout)                                                                                                                                                                                                                    
+log4j.logger.kafka=INFO                                                                        
+log4j.logger.org.apache.kafka=INFO                                                             
+                                                                                               
+# Change to DEBUG or TRACE to enable request logging                                                                   
+log4j.logger.kafka.request.logger=WARN, requestAppender                                        
+log4j.additivity.kafka.request.logger=false                                                                            
+
+# Uncomment the lines below and change log4j.logger.kafka.network.RequestChannel$ to TRACE for additional output                                                                                                                                                                                                             
+# related to the handling of requests                                                                                  
+#log4j.logger.kafka.network.Processor=TRACE, requestAppender                                                           
+#log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender                                                                                                   
+#log4j.additivity.kafka.server.KafkaApis=false                                                                         
+log4j.logger.kafka.network.RequestChannel$=WARN, requestAppender                                                       
+log4j.additivity.kafka.network.RequestChannel$=false                                                                                                          
+                                                                                                                       
+# Change the line below to adjust KRaft mode controller logging                                                        
+log4j.logger.org.apache.kafka.controller=INFO, controllerAppender                                                                                             
+log4j.additivity.org.apache.kafka.controller=false                                                                     
+                                                                                                                       
+# Change the line below to adjust ZK mode controller logging                                                           
+log4j.logger.kafka.controller=TRACE, controllerAppender                                                                                                       
+log4j.additivity.kafka.controller=false                                                                                
+
+log4j.logger.kafka.log.LogCleaner=INFO, cleanerAppender                                                                                                       
+log4j.additivity.kafka.log.LogCleaner=false                                                                                                                   
+
+log4j.logger.state.change.logger=INFO, stateChangeAppender                                                                                                    
+log4j.additivity.state.change.logger=false                                                                                                                    
+
+# Access denials are logged at INFO level, change to DEBUG to also log allowed accesses                                                                       
+log4j.logger.kafka.authorizer.logger=INFO, authorizerAppender                                                                                                 
+log4j.additivity.kafka.authorizer.logger=false  

salt/kafka/nodes.sls

@@ -10,9 +10,9 @@
 write_kafka_pillar_yaml:
   file.managed:
     - name: /opt/so/saltstack/local/pillar/kafka/nodes.sls
-    - mode: 644
+    - mode: 640
     - user: socore
     - source: salt://kafka/files/managed_node_pillar.jinja
     - template: jinja
     - context:
-        COMBINED_KAFKANODES: {{ COMBINED_KAFKANODES }}
+        COMBINED_KAFKANODES: {{ COMBINED_KAFKANODES }}

salt/kafka/soc_kafka.yaml

@@ -34,10 +34,6 @@ kafka:
       sensitive: True
       helpLink: kafka.html
     broker:
-      advertised_x_listeners:
-        description: Specify the list of listeners (hostname and port) that Kafka brokers provide to clients for communication.
-        title: advertised.listeners
-        helpLink: kafka.html
       auto_x_create_x_topics_x_enable:
         description: Enable the auto creation of topics.
         title: auto.create.topics.enable
@@ -226,4 +222,52 @@ kafka:
         description: The role performed by controller node.
         title: process.roles
         readonly: True
-        helpLink: kafka.html
+        helpLink: kafka.html
+    external_access:
+      enabled:
+        description: Enables or disables access to Kafka topics using user/password authentication. Used for producing / consuming messages via an external client.
+        forcedType: bool
+        helpLink: kafka.html
+      listeners:
+        description: Set of URIs that is listened on and the listener names in a comma-seperated list.
+        title: listeners
+        readonly: True
+        advanced: True
+        helpLink: kafka.html
+      listener_x_security_x_protocol_x_map:
+        description: External listener name and mapped security protocol.
+        title: listener.security.protocol.map
+        readonly: True
+        advanced: True
+        helpLink: kafka.html
+      sasl_x_enabled_x_mechanisms:
+        description: SASL/PLAIN is a simple username/password authentication mechanism, used with TLS to implement secure authentication.
+        title: sasl.enabled.mechanisms
+        readonly: True
+        advanced: True
+        helpLink: kafka.html
+      sasl_x_mechanism_x_inter_x_broker_x_protocol:
+        description: SASL mechanism used for inter-broker communication
+        title: sasl.mechanism.inter.broker.protocol
+        readonly: True
+        advanced: True
+        helpLink: kafka.html
+      remote_users:
+        user01: &remote_user
+          username:
+            description: Username to be used for custom account
+            forcedType: string
+            global: True
+          password:
+            description: Password to be used for custom account
+            forcedType: string
+            global: True
+            sensitive: True
+        user02: *remote_user
+        user03: *remote_user
+        user04: *remote_user
+        user05: *remote_user
+        user06: *remote_user
+        user07: *remote_user
+        user08: *remote_user
+        user09: *remote_user

salt/kibana/secrets.sls

@@ -22,7 +22,7 @@ kibana_pillar_directory:
 kibana_secrets_pillar:
   file.managed:
     - name: /opt/so/saltstack/local/pillar/kibana/secrets.sls
-    - mode: 600
+    - mode: 640
     - reload_pillar: True
     - contents: |
         kibana:

salt/kratos/config.sls

@@ -13,13 +13,21 @@ kratosgroup:
     - name: kratos
     - gid: 928
 
+kratoshome:
+  file.directory:
+    - name: /opt/so/conf/kratos
+    - user: 928
+    - group: 928
+    - mode: 700
+    - makedirs: True
+
 # Add Kratos user
 kratos:
   user.present:
     - uid: 928
     - gid: 928
     - home: /opt/so/conf/kratos
-    
+
 kratosdir:
   file.directory:
     - name: /nsm/kratos

salt/logstash/config.sls

@@ -22,7 +22,15 @@ logstashgroup:
     - name: logstash
     - gid: 931
 
-# Add the logstash user for the jog4j settings
+logstashhome:
+  file.directory:
+    - name: /opt/so/conf/logstash
+    - user: 931
+    - group: 931
+    - mode: 700
+    - makedirs: True
+
+# Add the logstash user for the log4j settings
 logstash:
   user.present:
     - uid: 931

salt/logstash/etc/log4j2.properties

@@ -23,6 +23,8 @@ appender.rolling.policies.type = Policies
 appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
 appender.rolling.policies.time.interval = 1
 appender.rolling.policies.time.modulate = true
+appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
+appender.rolling.policies.size.size = 1GB
 appender.rolling.strategy.type = DefaultRolloverStrategy
 appender.rolling.strategy.action.type = Delete
 appender.rolling.strategy.action.basepath = /var/log/logstash

salt/manager/elasticsearch.sls

@@ -3,5 +3,5 @@ elastic_curl_config_distributed:
     - name: /opt/so/saltstack/local/salt/elasticsearch/curl.config
     - source: salt://elasticsearch/files/curl.config.template
     - template: jinja
-    - mode: 600
+    - mode: 640
     - show_changes: False

salt/manager/init.sls

@@ -127,15 +127,28 @@ so_fleetagent_status:
     - month: '*'
     - dayweek: '*'
 
-socore_own_saltstack:
+socore_own_saltstack_default:
   file.directory:
-    - name: /opt/so/saltstack
+    - name: /opt/so/saltstack/default
     - user: socore
     - group: socore
     - recurse:
       - user
       - group
 
+socore_own_saltstack_local:
+  file.directory:
+    - name: /opt/so/saltstack/local
+    - user: socore
+    - group: socore
+    - dir_mode: 750
+    - file_mode: 640
+    - replace: False
+    - recurse:
+      - user
+      - group
+      - mode
+
 rules_dir:
   file.directory:
     - name: /nsm/rules/yara
@@ -143,6 +156,13 @@ rules_dir:
     - group: socore
     - makedirs: True
 
+nsm_playbooks_dir:
+  file.directory:
+    - name: /nsm/airgap-resources/playbooks
+    - user: socore
+    - group: socore
+    - makedirs: True
+
 git_config_set_safe_dirs:
   git.config_set:
     - name: safe.directory
@@ -153,6 +173,8 @@ git_config_set_safe_dirs:
       - /nsm/rules/custom-local-repos/local-yara
       - /nsm/securityonion-resources
       - /opt/so/conf/soc/ai_summary_repos/securityonion-resources
+      - /nsm/airgap-resources/playbooks
+      - /opt/so/conf/soc/playbooks
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/manager/tools/sbin/so-minion

@@ -126,7 +126,7 @@ function testMinion() {
 }
 
 function restartMinion() {
-	salt "$MINION_ID" system.reboot
+	salt "$MINION_ID" system.reboot --async
 	result=$?
 
 	exit $result

salt/manager/tools/sbin/so-user

@@ -356,7 +356,7 @@ function syncElastic() {
     [[ $? != 0 ]] && fail "Unable to read credential hashes from database"
 
     user_data_formatted=$(echo "${userData}" | jq -r '.user + ":" + .data.hashed_password')
-    if lookup_salt_value "features" "" "pillar" | grep -x odc; then
+    if lookup_salt_value "features" "" "pillar" | grep -qx odc; then
       # generate random placeholder salt/hash for users without passwords
       random_crypt=$(get_random_value 53)
       user_data_formatted=$(echo "${user_data_formatted}" | sed -r "s/^(.+:)\$/\\1\$2a\$12${random_crypt}/")

salt/manager/tools/sbin/soup

@@ -243,6 +243,13 @@ check_pillar_items() {
   fi
 }
 
+check_saltmaster_status() {
+  set +e
+  echo "Waiting on the Salt Master service to be ready."
+  check_salt_master_status || fail  "Can't access salt master or it is not ready. Check $SOUP_LOG for details."
+  set -e
+}
+
 check_sudoers() {
   if grep -q "so-setup" /etc/sudoers; then
     echo "There is an entry for so-setup in the sudoers file, this can be safely deleted using \"visudo\"."
@@ -407,6 +414,10 @@ preupgrade_changes() {
     [[ "$INSTALLEDVERSION" == 2.4.110 ]] && up_to_2.4.111
     [[ "$INSTALLEDVERSION" == 2.4.111 ]] && up_to_2.4.120
     [[ "$INSTALLEDVERSION" == 2.4.120 ]] && up_to_2.4.130
+    [[ "$INSTALLEDVERSION" == 2.4.130 ]] && up_to_2.4.140
+    [[ "$INSTALLEDVERSION" == 2.4.140 ]] && up_to_2.4.141
+    [[ "$INSTALLEDVERSION" == 2.4.141 ]] && up_to_2.4.150
+    [[ "$INSTALLEDVERSION" == 2.4.150 ]] && up_to_2.4.160
     true
 }
 
@@ -431,6 +442,10 @@ postupgrade_changes() {
     [[ "$POSTVERSION" == 2.4.110 ]] && post_to_2.4.111
     [[ "$POSTVERSION"  == 2.4.111 ]] && post_to_2.4.120
     [[ "$POSTVERSION"  == 2.4.120 ]] && post_to_2.4.130
+    [[ "$POSTVERSION"  == 2.4.130 ]] && post_to_2.4.140
+    [[ "$POSTVERSION"  == 2.4.140 ]] && post_to_2.4.141
+    [[ "$POSTVERSION"  == 2.4.141 ]] && post_to_2.4.150
+    [[ "$POSTVERSION"  == 2.4.150 ]] && post_to_2.4.160
     true
 }
 
@@ -547,10 +562,28 @@ post_to_2.4.130() {
   echo "Updating Kibana default space"
   /usr/sbin/so-kibana-space-defaults
 
+  POSTVERSION=2.4.130
+}
+
+post_to_2.4.140() {
+  echo "Nothing to apply"
+  POSTVERSION=2.4.140
+}
+
+post_to_2.4.141() {
+  echo "Nothing to apply"
+  POSTVERSION=2.4.141
+}
+
+post_to_2.4.150() {
   echo "Regenerating Elastic Agent Installers"
   /sbin/so-elastic-agent-gen-installers
+  POSTVERSION=2.4.150
+}
 
-  POSTVERSION=2.4.130
+post_to_2.4.160() {
+  echo "Nothing to apply"
+  POSTVERSION=2.4.160
 }
 
 repo_sync() {
@@ -725,8 +758,6 @@ up_to_2.4.90() {
   so-yaml.py remove /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.password
   so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.password "$kafkatrimpass"
   so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.trustpass "$kafkatrust"
-  echo "If the Detection index exists, update the refresh_interval"
-  so-elasticsearch-query so-detection*/_settings -X PUT -d '{"index":{"refresh_interval":"1s"}}'
 
   INSTALLEDVERSION=2.4.90
 }
@@ -773,6 +804,31 @@ up_to_2.4.130() {
   INSTALLEDVERSION=2.4.130
 }
 
+up_to_2.4.140() {
+  echo "Nothing to do for 2.4.140"
+
+  INSTALLEDVERSION=2.4.140
+}
+
+up_to_2.4.141() {
+  echo "Nothing to do for 2.4.141"
+
+  INSTALLEDVERSION=2.4.141
+}
+
+up_to_2.4.150() {
+  echo "If the Detection indices exists, update the refresh_interval"
+  so-elasticsearch-query so-detection*/_settings -X PUT -d '{"index":{"refresh_interval":"1s"}}'
+
+  INSTALLEDVERSION=2.4.150
+}
+
+up_to_2.4.160() {
+  echo "Nothing to do for 2.4.160"
+
+  INSTALLEDVERSION=2.4.160
+}
+
 add_hydra_pillars() {
   mkdir -p /opt/so/saltstack/local/pillar/hydra
   touch /opt/so/saltstack/local/pillar/hydra/soc_hydra.sls
@@ -1085,7 +1141,7 @@ upgrade_check() {
 }
 
 upgrade_check_salt() {
-  NEWSALTVERSION=$(grep version: $UPDATE_DIR/salt/salt/master.defaults.yaml | awk '{print $2}')
+  NEWSALTVERSION=$(grep "version:" $UPDATE_DIR/salt/salt/master.defaults.yaml | grep -o "[0-9]\+\.[0-9]\+")
   if [ "$INSTALLEDSALTVERSION" == "$NEWSALTVERSION" ]; then
     echo "You are already running the correct version of Salt for Security Onion."
   else
@@ -1231,26 +1287,6 @@ failed_soup_restore_items() {
   masterunlock
 }
 
-#upgrade salt to 3004.1
-#2_3_10_hotfix_1() {
-#    systemctl_func "stop" "$cron_service_name"
-#    # update mine items prior to stopping salt-minion and salt-master
-#    update_salt_mine
-#    stop_salt_minion
-#    stop_salt_master
-#    update_repo
-#    # Does salt need upgraded. If so update it.
-#    if [[ $UPGRADESALT -eq 1 ]]; then
-#      echo "Upgrading Salt"
-#      # Update the repo files so it can actually upgrade
-#      upgrade_salt
-#    fi
-#    systemctl_func "start" "salt-master"
-#    systemctl_func "start" "salt-minion"
-#    systemctl_func "start" "$cron_service_name"
-
-#}
-
 main() {
   trap 'check_err $?' EXIT
 
@@ -1382,6 +1418,8 @@ main() {
     if [[ $is_airgap -eq 0 ]]; then
       echo "Updating Rule Files to the Latest."
       update_airgap_rules
+      echo "Updating Playbooks to the Latest."
+      airgap_playbooks "$UPDATE_DIR"
     fi
 
     # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars
@@ -1405,10 +1443,7 @@ main() {
     systemctl_func "start" "salt-master"
 
     # Testing that salt-master is up by checking that is it connected to itself
-    set +e
-    echo "Waiting on the Salt Master service to be ready."
-    check_salt_master_status || fail  "Can't access salt master or it is not ready. Check $SOUP_LOG for details."
-    set -e
+    check_saltmaster_status
 
     # update the salt-minion configs here and start the minion
     # since highstate are disabled above, minion start should not trigger a highstate
@@ -1435,10 +1470,7 @@ main() {
 
     systemctl_func "start" "salt-master"
 
-    set +e
-    echo "Waiting on the Salt Master service to be ready."
-    check_salt_master_status || fail  "Can't access salt master or it is not ready. Check $SOUP_LOG for details."
-    set -e
+    check_saltmaster_status
 
     echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes."
     (wait_for_salt_minion "$MINIONID" "5" '/dev/stdout' || fail "Salt minion was not running or ready.") 2>&1 | tee -a "$SOUP_LOG"
@@ -1446,7 +1478,11 @@ main() {
     # Stop long-running scripts to allow potentially updated scripts to load on the next execution.
     killall salt-relay.sh
 
+    # ensure the mine is updated and populated before highstates run, following the salt-master restart
+    update_salt_mine
+
     highstate
+    check_saltmaster_status
     postupgrade_changes
     [[ $is_airgap -eq 0 ]] && unmount_update
 

salt/nginx/enabled.sls

@@ -63,7 +63,7 @@ managerssl_crt:
     - signing_policy: managerssl
     - private_key: /etc/pki/managerssl.key
     - CN: {{ GLOBALS.hostname }}
-    - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
+    - subjectAltName: "DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}, DNS:{{ GLOBALS.url_base }}" 
     - days_remaining: 0
     - days_valid: 820
     - backup: True
@@ -121,7 +121,7 @@ so-nginx:
       - /opt/so/log/nginx/:/var/log/nginx:rw
       - /opt/so/tmp/nginx/:/var/lib/nginx:rw
       - /opt/so/tmp/nginx/:/run:rw
-      - /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/:/opt/socore/html/packages
+      - /nsm/elastic-fleet/so_agent-installers/:/opt/socore/html/packages
       - /nsm/elastic-fleet/artifacts/:/opt/socore/html/artifacts 
       {% if grains.role in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone', 'so-import'] %}
       - /etc/pki/managerssl.crt:/etc/pki/nginx/server.crt:ro

salt/nginx/etc/nginx.conf

@@ -383,6 +383,7 @@ http {
 			}
 			if ($request_uri ~* ^/(?!(^/api/.*))) {
 				add_header    Set-Cookie "AUTH_REDIRECT=$request_uri;Path=/;Max-Age=14400";
+				add_header    Strict-Transport-Security   "max-age=31536000; includeSubDomains";
 			}
 			return        302 /auth/self-service/login/browser;
 		}
@@ -392,6 +393,7 @@ http {
 				return	  403;
 			}
 			add_header    Set-Cookie "ory_kratos_session=;Path=/;Max-Age=0;expires=Thu, 01 Jan 1970 00:00:00 GMT;";
+			add_header    Strict-Transport-Security   "max-age=31536000; includeSubDomains";
 			return        302 /auth/self-service/login/browser;
 		}
 

salt/pcap/config.sls

@@ -79,13 +79,6 @@ pcaptmpdir:
     - group: 941
     - makedirs: True
 
-pcapoutdir:
-  file.directory:
-    - name: /nsm/pcapout
-    - user: 939
-    - group: 939
-    - makedirs: True
-
 pcapindexdir:
   file.directory:
     - name: /nsm/pcapindex

salt/pcap/init.sls

@@ -23,4 +23,11 @@ pcapdir:
     - name: /nsm/pcap
     - user: 941
     - group: 941
-    - makedirs: True
+    - makedirs: True
+
+pcapoutdir:
+  file.directory:
+    - name: /nsm/pcapout
+    - user: 939
+    - group: 939
+    - makedirs: True

salt/registry/enabled.sls

@@ -14,7 +14,7 @@ include:
 # Install the registry container
 so-dockerregistry:
   docker_container.running:
-    - image: ghcr.io/security-onion-solutions/registry:2.8.3
+    - image: ghcr.io/security-onion-solutions/registry:3.0.0
     - hostname: so-registry
     - networks:
       - sobridge:
@@ -25,7 +25,7 @@ so-dockerregistry:
       - {{ BINDING }}
       {% endfor %}
     - binds:
-      - /opt/so/conf/docker-registry/etc/config.yml:/etc/docker/registry/config.yml:ro
+      - /opt/so/conf/docker-registry/etc/config.yml:/etc/distribution/config.yml:ro
       - /opt/so/conf/docker-registry:/var/lib/registry:rw
       - /nsm/docker-registry/docker:/var/lib/registry/docker:rw
       - /etc/pki/registry.crt:/etc/pki/registry.crt:ro

salt/salt/map.jinja

@@ -1,5 +1,11 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+
 {% import_yaml 'salt/minion.defaults.yaml' as saltminion %}
-{% set SALTVERSION = saltminion.salt.minion.version %}
+{% set SALTVERSION = saltminion.salt.minion.version | string %}
+{% set INSTALLEDSALTVERSION = grains.saltversion | string %}
 
 {% if grains.os_family == 'Debian' %}
   {% set SPLITCHAR = '+' %}
@@ -11,9 +17,7 @@
   {% set SYSTEMD_UNIT_FILE = '/usr/lib/systemd/system/salt-minion.service' %}
 {% endif %}
 
-{% set INSTALLEDSALTVERSION = grains.saltversion %}
-
-{% if grains.saltversion|string != SALTVERSION|string %}
+{% if INSTALLEDSALTVERSION != SALTVERSION %}
   {% if grains.os_family|lower == 'redhat' %}
       {% set UPGRADECOMMAND = 'yum clean all ; /usr/sbin/bootstrap-salt.sh -s 120 -r -F stable ' ~ SALTVERSION %}
   {% elif grains.os_family|lower == 'debian' %}

salt/salt/master.defaults.yaml

@@ -1,4 +1,4 @@
 # version cannot be used elsewhere in this pillar as soup is grepping for it to determine if Salt needs to be patched
 salt:
   master:
-    version: 3006.9
+    version: '3006.9'

salt/salt/master/mine_update_highstate.sls

@@ -0,0 +1,29 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+# This state should only be run on managers and should never be run manually
+
+{% set MINION_ID = grains.id %}
+
+# Run mine.update on all minions
+salt.master.mine_update_highstate.update_mine_all_minions:
+  salt.function:
+    - name: mine.update
+    - tgt: '*'
+    - batch: 50
+    - retry:
+        attempts: 3
+        interval: 1
+
+# Run highstate on the original minion
+# we can use concurrent on this highstate because no other highstate would be running when this is called
+# this state will run onlyif there is not an instance of it already running
+salt.master.mine_update_highstate.run_highstate_on_{{ MINION_ID }}:
+  salt.state:
+    - tgt: {{ MINION_ID }}
+    - highstate: True
+    - concurrent: True
+    - onlyif:
+      - 'ps -ef | grep -v grep | grep "/usr/bin/salt-minion.*ProcessPayload.*jid=.*Minion._thread_return" | wc -l | grep -q "^0$"'

salt/salt/minion.defaults.yaml

@@ -1,6 +1,5 @@
 # version cannot be used elsewhere in this pillar as soup is grepping for it to determine if Salt needs to be patched
 salt:
   minion:
-    version: 3006.9
+    version: '3006.9'
     check_threshold: 3600 # in seconds, threshold used for so-salt-minion-check. any value less than 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default
-    service_start_delay: 30 # in seconds.

salt/salt/minion.sls

@@ -5,10 +5,10 @@
 {% from 'salt/map.jinja' import SALTPACKAGES %}
 {% from 'salt/map.jinja' import SYSTEMD_UNIT_FILE %}
 {% import_yaml 'salt/minion.defaults.yaml' as SALTMINION %}
-{% set service_start_delay = SALTMINION.salt.minion.service_start_delay %}
 
 include:
   - salt.python_modules
+  - salt.patch.x509_v2
   - salt
   - systemd.reload
   - repo.client
@@ -89,8 +89,6 @@ salt_minion_service_unit_file:
     - name: {{ SYSTEMD_UNIT_FILE }}
     - source: salt://salt/service/salt-minion.service.jinja
     - template: jinja
-    - defaults:
-        service_start_delay: {{ service_start_delay }}
     - onchanges_in:
       - module: systemd_reload
 

salt/salt/patch/x509_v2/init.sls

@@ -0,0 +1,6 @@
+patch_x509_v2_state_module:
+  file.replace:
+    - name: /opt/saltstack/salt/lib/python3.10/site-packages/salt/states/x509_v2.py
+    - pattern: 'res = __salt__\["state.single"\]\("file.managed", name, test=test, \*\*kwargs\)'
+    - repl: 'res = __salt__["state.single"]("file.managed", name, test=test, concurrent=True, **kwargs)'
+    - backup: .bak

salt/salt/service/salt-minion.service.jinja

@@ -8,8 +8,9 @@ KillMode=process
 Type=notify
 NotifyAccess=all
 LimitNOFILE=8192
+ExecStartPre=/bin/bash -c 'until /sbin/ip -4 addr show dev {{ salt["pillar.get"]("host:mainint") }} | grep -q "inet "; do sleep 1; done'
 ExecStart=/usr/bin/salt-minion
-ExecStartPre=/bin/sleep {{ salt['pillar.get']('salt:minion:service_start_delay', service_start_delay) }}
+TimeoutStartSec=120
 
 [Install]
 WantedBy=multi-user.target