Remove header

Refactor for simplicity
Add error handling
2025-12-06 09:12:45 +01:00 · 2025-11-29 17:45:22 -05:00 · 2025-11-29 17:00:51 -05:00 · 2025-11-29 16:27:22 -05:00 · 2025-11-29 15:40:10 -05:00 · 2025-11-29 15:17:28 -05:00
57 changed files with 1668 additions and 502 deletions
--- a/.github/workflows/pythontest.yml
+++ b/.github/workflows/pythontest.yml
@@ -4,7 +4,7 @@ on:
  pull_request:
    paths:
      - "salt/sensoroni/files/analyzers/**"
-      - "salt/manager/tools/sbin"
+      - "salt/manager/tools/sbin/**"

 jobs:
  build:
--- a/salt/_runners/setup_hypervisor.py
+++ b/salt/_runners/setup_hypervisor.py
@@ -172,7 +172,15 @@ MANAGER_HOSTNAME = socket.gethostname()

 def _download_image():
    """
-    Download and validate the Oracle Linux KVM image.
+    Download and validate the Oracle Linux KVM image with retry logic and progress monitoring.
+    
+    Features:
+    - Detects stalled downloads (no progress for 30 seconds)
+    - Retries up to 3 times on failure
+    - Connection timeout of 30 seconds
+    - Read timeout of 60 seconds
+    - Cleans up partial downloads on failure
+    
    Returns:
        bool: True if successful or file exists with valid checksum, False on error
    """
@@ -186,44 +194,106 @@ def _download_image():
    
    log.info("Starting image download process")
    
-    try:
-        # Download file
-        log.info("Downloading Oracle Linux KVM image from %s to %s", IMAGE_URL, IMAGE_PATH)
-        response = requests.get(IMAGE_URL, stream=True)
-        response.raise_for_status()
+    # Retry configuration
+    max_attempts = 3
+    retry_delay = 5  # seconds to wait between retry attempts
+    stall_timeout = 30  # seconds without progress before considering download stalled
+    connection_timeout = 30  # seconds to establish connection
+    read_timeout = 60  # seconds to wait for data chunks
    
-        # Get total file size for progress tracking
-        total_size = int(response.headers.get('content-length', 0))
-        downloaded_size = 0
-        last_log_time = 0
+    for attempt in range(1, max_attempts + 1):
+        log.info("Download attempt %d of %d", attempt, max_attempts)
        
-        # Save file with progress logging
-        with salt.utils.files.fopen(IMAGE_PATH, 'wb') as f:
-            for chunk in response.iter_content(chunk_size=8192):
-                f.write(chunk)
-                downloaded_size += len(chunk)
+        try:
+            # Download file with timeouts
+            log.info("Downloading Oracle Linux KVM image from %s to %s", IMAGE_URL, IMAGE_PATH)
+            response = requests.get(
+                IMAGE_URL,
+                stream=True,
+                timeout=(connection_timeout, read_timeout)
+            )
+            response.raise_for_status()

-                # Log progress every second
-                current_time = time.time()
-                if current_time - last_log_time >= 1:
-                    progress = (downloaded_size / total_size) * 100 if total_size > 0 else 0
-                    log.info("Progress - %.1f%% (%d/%d bytes)", 
-                            progress, downloaded_size, total_size)
-                    last_log_time = current_time
+            # Get total file size for progress tracking
+            total_size = int(response.headers.get('content-length', 0))
+            downloaded_size = 0
+            last_log_time = 0
+            last_progress_time = time.time()
+            last_downloaded_size = 0

-        # Validate downloaded file
-        if not _validate_image_checksum(IMAGE_PATH, IMAGE_SHA256):
-            os.unlink(IMAGE_PATH)
-            return False
+            # Save file with progress logging and stall detection
+            with salt.utils.files.fopen(IMAGE_PATH, 'wb') as f:
+                for chunk in response.iter_content(chunk_size=8192):
+                    if chunk:  # filter out keep-alive new chunks
+                        f.write(chunk)
+                        downloaded_size += len(chunk)
+                        current_time = time.time()
                        
-        log.info("Successfully downloaded and validated Oracle Linux KVM image")
-        return True
+                        # Check for stalled download
+                        if downloaded_size > last_downloaded_size:
+                            # Progress made, reset stall timer
+                            last_progress_time = current_time
+                            last_downloaded_size = downloaded_size
+                        elif current_time - last_progress_time > stall_timeout:
+                            # No progress for stall_timeout seconds
+                            raise Exception(
+                                f"Download stalled: no progress for {stall_timeout} seconds "
+                                f"at {downloaded_size}/{total_size} bytes"
+                            )
                        
-    except Exception as e:
-        log.error("Error downloading hypervisor image: %s", str(e))
-        if os.path.exists(IMAGE_PATH):
-            os.unlink(IMAGE_PATH)
-        return False
+                        # Log progress every second
+                        if current_time - last_log_time >= 1:
+                            progress = (downloaded_size / total_size) * 100 if total_size > 0 else 0
+                            log.info("Progress - %.1f%% (%d/%d bytes)",
+                                    progress, downloaded_size, total_size)
+                            last_log_time = current_time
+
+            # Validate downloaded file
+            log.info("Download complete, validating checksum...")
+            if not _validate_image_checksum(IMAGE_PATH, IMAGE_SHA256):
+                log.error("Checksum validation failed on attempt %d", attempt)
+                os.unlink(IMAGE_PATH)
+                if attempt < max_attempts:
+                    log.info("Will retry download...")
+                    continue
+                else:
+                    log.error("All download attempts failed due to checksum mismatch")
+                    return False
+
+            log.info("Successfully downloaded and validated Oracle Linux KVM image")
+            return True
+
+        except requests.exceptions.Timeout as e:
+            log.error("Download attempt %d failed: Timeout - %s", attempt, str(e))
+            if os.path.exists(IMAGE_PATH):
+                os.unlink(IMAGE_PATH)
+            if attempt < max_attempts:
+                log.info("Will retry download in %d seconds...", retry_delay)
+                time.sleep(retry_delay)
+            else:
+                log.error("All download attempts failed due to timeout")
+                
+        except requests.exceptions.RequestException as e:
+            log.error("Download attempt %d failed: Network error - %s", attempt, str(e))
+            if os.path.exists(IMAGE_PATH):
+                os.unlink(IMAGE_PATH)
+            if attempt < max_attempts:
+                log.info("Will retry download in %d seconds...", retry_delay)
+                time.sleep(retry_delay)
+            else:
+                log.error("All download attempts failed due to network errors")
+                
+        except Exception as e:
+            log.error("Download attempt %d failed: %s", attempt, str(e))
+            if os.path.exists(IMAGE_PATH):
+                os.unlink(IMAGE_PATH)
+            if attempt < max_attempts:
+                log.info("Will retry download in %d seconds...", retry_delay)
+                time.sleep(retry_delay)
+            else:
+                log.error("All download attempts failed")
+    
+    return False

 def _check_ssh_keys_exist():
    """
@@ -419,25 +489,28 @@ def _ensure_hypervisor_host_dir(minion_id: str = None):
        log.error(f"Error creating hypervisor host directory: {str(e)}")
        return False

-def _apply_dyanno_hypervisor_state():
+def _apply_dyanno_hypervisor_state(status):
    """
    Apply the soc.dyanno.hypervisor state on the salt master.
    
    This function applies the soc.dyanno.hypervisor state on the salt master
    to update the hypervisor annotation and ensure all hypervisor host directories exist.
    
+    Args:
+        status: Status passed to the hypervisor annotation state
+    
    Returns:
        bool: True if state was applied successfully, False otherwise
    """
    try:
-        log.info("Applying soc.dyanno.hypervisor state on salt master")
+        log.info(f"Applying soc.dyanno.hypervisor state on salt master with status: {status}")
        
        # Initialize the LocalClient
        local = salt.client.LocalClient()
        
        # Target the salt master to apply the soc.dyanno.hypervisor state
        target = MANAGER_HOSTNAME + '_*'
-        state_result = local.cmd(target, 'state.apply', ['soc.dyanno.hypervisor', "pillar={'baseDomain': {'status': 'PreInit'}}", 'concurrent=True'], tgt_type='glob')
+        state_result = local.cmd(target, 'state.apply', ['soc.dyanno.hypervisor', f"pillar={{'baseDomain': {{'status': '{status}'}}}}", 'concurrent=True'], tgt_type='glob')
        log.debug(f"state_result: {state_result}")
        # Check if state was applied successfully
        if state_result:
@@ -454,17 +527,17 @@ def _apply_dyanno_hypervisor_state():
                        success = False
            
            if success:
-                log.info("Successfully applied soc.dyanno.hypervisor state")
+                log.info(f"Successfully applied soc.dyanno.hypervisor state with status: {status}")
                return True
            else:
-                log.error("Failed to apply soc.dyanno.hypervisor state")
+                log.error(f"Failed to apply soc.dyanno.hypervisor state with status: {status}")
                return False
        else:
-            log.error("No response from salt master when applying soc.dyanno.hypervisor state")
+            log.error(f"No response from salt master when applying soc.dyanno.hypervisor state with status: {status}")
            return False
            
    except Exception as e:
-        log.error(f"Error applying soc.dyanno.hypervisor state: {str(e)}")
+        log.error(f"Error applying soc.dyanno.hypervisor state with status: {status}: {str(e)}")
        return False

 def _apply_cloud_config_state():
@@ -598,11 +671,6 @@ def setup_environment(vm_name: str = 'sool9', disk_size: str = '220G', minion_id
        log.warning("Failed to apply salt.cloud.config state, continuing with setup")
        # We don't return an error here as we want to continue with the setup process

-    # Apply the soc.dyanno.hypervisor state on the salt master
-    if not _apply_dyanno_hypervisor_state():
-        log.warning("Failed to apply soc.dyanno.hypervisor state, continuing with setup")
-        # We don't return an error here as we want to continue with the setup process
-
    log.info("Starting setup_environment in setup_hypervisor runner")
    
    # Check if environment is already set up
@@ -616,9 +684,12 @@ def setup_environment(vm_name: str = 'sool9', disk_size: str = '220G', minion_id
    
    # Handle image setup if needed
    if not image_valid:
+        _apply_dyanno_hypervisor_state('ImageDownloadStart')
        log.info("Starting image download/validation process")
        if not _download_image():
            log.error("Image download failed")
+            # Update hypervisor annotation with failure status
+            _apply_dyanno_hypervisor_state('ImageDownloadFailed')
            return {
                'success': False,
                'error': 'Image download failed',
@@ -631,6 +702,8 @@ def setup_environment(vm_name: str = 'sool9', disk_size: str = '220G', minion_id
        log.info("Setting up SSH keys")
        if not _setup_ssh_keys():
            log.error("SSH key setup failed")
+            # Update hypervisor annotation with failure status
+            _apply_dyanno_hypervisor_state('SSHKeySetupFailed')
            return {
                'success': False,
                'error': 'SSH key setup failed',
@@ -655,6 +728,12 @@ def setup_environment(vm_name: str = 'sool9', disk_size: str = '220G', minion_id
    success = vm_result.get('success', False)
    log.info("Setup environment completed with status: %s", "SUCCESS" if success else "FAILED")
    
+    # Update hypervisor annotation with success status
+    if success:
+        _apply_dyanno_hypervisor_state('PreInit')
+    else:
+        _apply_dyanno_hypervisor_state('SetupFailed')
+    
    # If setup was successful and we have a minion_id, run highstate
    if success and minion_id:
        log.info("Running highstate on hypervisor %s", minion_id)
--- a/salt/allowed_states.map.jinja
+++ b/salt/allowed_states.map.jinja
@@ -38,7 +38,6 @@
    'hydra',
    'elasticfleet',
    'elastic-fleet-package-registry',
-    'suricata.manager',
    'utility'
 ] %}

--- a/salt/bpf/pcap.map.jinja
+++ b/salt/bpf/pcap.map.jinja
@@ -1,4 +1,7 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
+{% set PCAP_BPF_STATUS = 0  %}
+{% set STENO_BPF_COMPILED = "" %}
+
 {% if GLOBALS.pcap_engine == "TRANSITION" %}
 {%   set PCAPBPF = ["ip and host 255.255.255.1 and port 1"] %}
 {% else %}
@@ -8,3 +11,11 @@
 {{   MACROS.remove_comments(BPFMERGED, 'pcap') }}
 {%   set PCAPBPF = BPFMERGED.pcap %}
 {% endif %}
+
+{% if PCAPBPF %}
+   {% set PCAP_BPF_CALC = salt['cmd.run_all']('/usr/sbin/so-bpf-compile ' ~ GLOBALS.sensor.interface ~ ' ' ~ PCAPBPF|join(" "), cwd='/root') %}
+   {% if PCAP_BPF_CALC['retcode'] == 0 %}
+      {% set PCAP_BPF_STATUS = 1 %}
+      {% set STENO_BPF_COMPILED =  ",\\\"--filter=" + PCAP_BPF_CALC['stdout'] + "\\\""  %}
+   {% endif %}
+{% endif %}
--- a/salt/bpf/soc_bpf.yaml
+++ b/salt/bpf/soc_bpf.yaml
@@ -1,11 +1,11 @@
 bpf:
  pcap:
-    description: List of BPF filters to apply to Stenographer.
+    description: List of BPF filters to apply to the PCAP engine.
    multiline: True
    forcedType: "[]string"
    helpLink: bpf.html
  suricata:
-    description: List of BPF filters to apply to Suricata.
+    description: List of BPF filters to apply to Suricata. This will apply to alerts and, if enabled, to metadata and PCAP logs generated by Suricata.
    multiline: True
    forcedType: "[]string"
    helpLink: bpf.html
--- a/salt/bpf/suricata.map.jinja
+++ b/salt/bpf/suricata.map.jinja
@@ -1,7 +1,16 @@
+{% from 'vars/globals.map.jinja' import GLOBALS %}
 {% import_yaml 'bpf/defaults.yaml' as BPFDEFAULTS %}
 {% set BPFMERGED = salt['pillar.get']('bpf', BPFDEFAULTS.bpf, merge=True) %}
+{% set SURICATA_BPF_STATUS = 0 %}
 {% import 'bpf/macros.jinja' as MACROS %}

 {{ MACROS.remove_comments(BPFMERGED, 'suricata') }}

 {% set SURICATABPF = BPFMERGED.suricata %}
+
+{% if SURICATABPF %}
+   {% set SURICATA_BPF_CALC = salt['cmd.run_all']('/usr/sbin/so-bpf-compile ' ~ GLOBALS.sensor.interface ~ ' ' ~ SURICATABPF|join(" "), cwd='/root') %}
+   {% if SURICATA_BPF_CALC['retcode'] == 0 %}
+      {% set SURICATA_BPF_STATUS = 1 %}
+   {% endif %}
+{% endif %}
--- a/salt/bpf/zeek.map.jinja
+++ b/salt/bpf/zeek.map.jinja
@@ -1,7 +1,16 @@
+{% from 'vars/globals.map.jinja' import GLOBALS %}
 {% import_yaml 'bpf/defaults.yaml' as BPFDEFAULTS %}
 {% set BPFMERGED = salt['pillar.get']('bpf', BPFDEFAULTS.bpf, merge=True) %}
+{% set ZEEK_BPF_STATUS = 0 %}
 {% import 'bpf/macros.jinja' as MACROS %}

 {{ MACROS.remove_comments(BPFMERGED, 'zeek') }}

 {% set ZEEKBPF = BPFMERGED.zeek %}
+
+{% if ZEEKBPF %}
+   {% set ZEEK_BPF_CALC = salt['cmd.run_all']('/usr/sbin/so-bpf-compile ' ~ GLOBALS.sensor.interface ~ ' ' ~ ZEEKBPF|join(" "), cwd='/root') %}
+   {% if ZEEK_BPF_CALC['retcode'] == 0 %}
+      {% set ZEEK_BPF_STATUS = 1 %}
+   {% endif %}
+{% endif %}
--- a/salt/common/tools/sbin/so-bpf-compile
+++ b/salt/common/tools/sbin/so-bpf-compile
@@ -29,9 +29,26 @@ fi

 interface="$1"
 shift
-tcpdump -i $interface -ddd $@ | tail -n+2 |
-while read line; do
+
+# Capture tcpdump output and exit code
+tcpdump_output=$(tcpdump -i "$interface" -ddd "$@" 2>&1)
+tcpdump_exit=$?
+
+if [ $tcpdump_exit -ne 0 ]; then
+  echo "$tcpdump_output" >&2
+  exit $tcpdump_exit
+fi
+
+# Process the output, skipping the first line
+echo "$tcpdump_output" | tail -n+2 | while read -r line; do
  cols=( $line )
-  printf "%04x%02x%02x%08x" ${cols[0]} ${cols[1]} ${cols[2]} ${cols[3]}
+  printf "%04x%02x%02x%08x" "${cols[0]}" "${cols[1]}" "${cols[2]}" "${cols[3]}"
 done
+
+# Check if the pipeline succeeded
+if [ "${PIPESTATUS[0]}" -ne 0 ]; then
+  exit 1
+fi
+
 echo ""
+exit 0
--- a/salt/common/tools/sbin/so-common
+++ b/salt/common/tools/sbin/so-common
@@ -395,7 +395,7 @@ is_manager_node() {
 }

 is_sensor_node() {
-	# Check to see if this is a sensor (forward) node
+	# Check to see if this is a sensor node
 	is_single_node_grid && return 0
 	grep "role: so-" /etc/salt/grains | grep -E "sensor|heavynode" &> /dev/null
 }
--- a/salt/elasticfleet/defaults.yaml
+++ b/salt/elasticfleet/defaults.yaml
@@ -15,7 +15,6 @@ elasticfleet:
  logging:
    zeek:
      excluded:
-        - analyzer
        - broker
        - capture_loss
        - cluster
--- a/salt/elasticsearch/files/ingest/suricata.alert
+++ b/salt/elasticsearch/files/ingest/suricata.alert
@@ -1,15 +1,79 @@
 {
-  "description" : "suricata.alert",
-  "processors" : [
-    { "set":   { "if": "ctx.event?.imported != true", "field": "_index", "value": "logs-suricata.alerts-so" } },
-    { "set":   { "field": "tags","value": "alert" }},
-    { "rename":{ "field": "message2.alert",	"target_field": "rule",	"ignore_failure": true 	} },
-    { "rename":{ "field": "rule.signature",     "target_field": "rule.name", "ignore_failure": true  } },
-    { "rename":{ "field": "rule.ref",     "target_field": "rule.version", "ignore_failure": true  } },
-    { "rename":{ "field": "rule.signature_id",     "target_field": "rule.uuid", "ignore_failure": true  } },
-    { "rename":{ "field": "rule.signature_id",     "target_field": "rule.signature", "ignore_failure": true  } },
-    { "rename":{ "field": "message2.payload_printable",     "target_field": "network.data.decoded", "ignore_failure": true  } },
-    { "dissect": { "field": "rule.rule", "pattern": "%{?prefix}content:\"%{dns.query_name}\"%{?remainder}", "ignore_missing": true, "ignore_failure": true } },
-    { "pipeline": { "name": "common.nids" } }
-  ]
+    "description": "suricata.alert",
+    "processors": [
+        {
+            "set": {
+                "if": "ctx.event?.imported != true",
+                "field": "_index",
+                "value": "logs-suricata.alerts-so"
+            }
+        },
+        {
+            "set": {
+                "field": "tags",
+                "value": "alert"
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.alert",
+                "target_field": "rule",
+                "ignore_missing": true,
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "rule.signature",
+                "target_field": "rule.name",
+                "ignore_missing": true,
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "rule.ref",
+                "target_field": "rule.version",
+                "ignore_missing": true,
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "rule.signature_id",
+                "target_field": "rule.uuid",
+                "ignore_missing": true,
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "rule.signature_id",
+                "target_field": "rule.signature",
+                "ignore_missing": true,
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.payload_printable",
+                "target_field": "network.data.decoded",
+                "ignore_missing": true,
+                "ignore_failure": true
+            }
+        },
+        {
+            "dissect": {
+                "field": "rule.rule",
+                "pattern": "%{?prefix}content:\"%{dns.query_name}\"%{?remainder}",
+                "ignore_missing": true,
+                "ignore_failure": true
+            }
+        },
+        {
+            "pipeline": {
+                "name": "common.nids"
+            }
+        }
+    ]
 }
--- a/salt/elasticsearch/files/ingest/suricata.common
+++ b/salt/elasticsearch/files/ingest/suricata.common
@@ -1,30 +1,155 @@
 {
-  "description" : "suricata.common",
-  "processors" : [
-    { "json":   { "field": "message",                   "target_field": "message2",             "ignore_failure": true  } },
-    { "rename": { "field": "message2.pkt_src",          "target_field": "network.packet_source","ignore_failure": true  } },
-    { "rename": { "field": "message2.proto",            "target_field": "network.transport",    "ignore_failure": true  } },
-    { "rename": { "field": "message2.in_iface",         "target_field": "observer.ingress.interface.name",    "ignore_failure": true  } },
-    { "rename": { "field": "message2.flow_id",          "target_field": "log.id.uid",           "ignore_failure": true  } },
-    { "rename": { "field": "message2.src_ip",           "target_field": "source.ip",            "ignore_failure": true  } },
-    { "rename": { "field": "message2.src_port",         "target_field": "source.port",          "ignore_failure": true  } },
-    { "rename": { "field": "message2.dest_ip",          "target_field": "destination.ip",       "ignore_failure": true  } },
-    { "rename": { "field": "message2.dest_port",        "target_field": "destination.port",     "ignore_failure": true  } },
-    { "rename": { "field": "message2.vlan",             "target_field": "network.vlan.id",      "ignore_failure": true  } },
-    { "rename": { "field": "message2.community_id",     "target_field": "network.community_id", "ignore_missing": true  } },
-    { "rename": { "field": "message2.xff",              "target_field": "xff.ip",               "ignore_missing": true  } },
-    { "set":    { "field": "event.dataset",             "value": "{{ message2.event_type }}"                            } },
-    { "set":    { "field": "observer.name",             "value": "{{agent.name}}"                                       } },
-    { "set":    { "field": "event.ingested",            "value": "{{@timestamp}}"                                       } },
-    { "date": { "field": "message2.timestamp", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "timezone": "UTC", "ignore_failure": true } },
-    { "remove":{ "field": "agent",                                                              "ignore_failure": true  } },
-    {"append":{"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"ignore_failure":true}},
-    {
-      "script": {
-        "source": "boolean isPrivate(def ip) { if (ip == null) return false; int dot1 = ip.indexOf('.'); if (dot1 == -1) return false; int dot2 = ip.indexOf('.', dot1 + 1); if (dot2 == -1) return false; int first = Integer.parseInt(ip.substring(0, dot1)); if (first == 10) return true; if (first == 192 && ip.startsWith('168.', dot1 + 1)) return true; if (first == 172) { int second = Integer.parseInt(ip.substring(dot1 + 1, dot2)); return second >= 16 && second <= 31; } return false; } String[] fields = new String[] {\"source\", \"destination\"}; for (int i = 0; i < fields.length; i++) { def field = fields[i]; def ip = ctx[field]?.ip; if (ip != null) { if (ctx.network == null) ctx.network = new HashMap(); if (isPrivate(ip)) { if (ctx.network.private_ip == null) ctx.network.private_ip = new ArrayList(); if (!ctx.network.private_ip.contains(ip)) ctx.network.private_ip.add(ip); } else { if (ctx.network.public_ip == null) ctx.network.public_ip = new ArrayList(); if (!ctx.network.public_ip.contains(ip)) ctx.network.public_ip.add(ip); } } }",
-        "ignore_failure": false
-      }
-    },
-    { "pipeline": { "if": "ctx?.event?.dataset != null", "name": "suricata.{{event.dataset}}" } }
-  ]
+    "description": "suricata.common",
+    "processors": [
+        {
+            "json": {
+                "field": "message",
+                "target_field": "message2",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.pkt_src",
+                "target_field": "network.packet_source",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.proto",
+                "target_field": "network.transport",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.in_iface",
+                "target_field": "observer.ingress.interface.name",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.flow_id",
+                "target_field": "log.id.uid",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.src_ip",
+                "target_field": "source.ip",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.src_port",
+                "target_field": "source.port",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dest_ip",
+                "target_field": "destination.ip",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dest_port",
+                "target_field": "destination.port",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.vlan",
+                "target_field": "network.vlan.id",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.community_id",
+                "target_field": "network.community_id",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.xff",
+                "target_field": "xff.ip",
+                "ignore_missing": true
+            }
+        },
+        {
+            "set": {
+                "field": "event.dataset",
+                "value": "{{ message2.event_type }}"
+            }
+        },
+        {
+            "set": {
+                "field": "observer.name",
+                "value": "{{agent.name}}"
+            }
+        },
+        {
+            "set": {
+                "field": "event.ingested",
+                "value": "{{@timestamp}}"
+            }
+        },
+        {
+            "date": {
+                "field": "message2.timestamp",
+                "target_field": "@timestamp",
+                "formats": [
+                    "ISO8601",
+                    "UNIX"
+                ],
+                "timezone": "UTC",
+                "ignore_failure": true
+            }
+        },
+        {
+            "remove": {
+                "field": "agent",
+                "ignore_failure": true
+            }
+        },
+        {
+            "append": {
+                "field": "related.ip",
+                "value": [
+                    "{{source.ip}}",
+                    "{{destination.ip}}"
+                ],
+                "allow_duplicates": false,
+                "ignore_failure": true
+            }
+        },
+        {
+            "script": {
+                "source": "boolean isPrivate(def ip) { if (ip == null) return false; int dot1 = ip.indexOf('.'); if (dot1 == -1) return false; int dot2 = ip.indexOf('.', dot1 + 1); if (dot2 == -1) return false; int first = Integer.parseInt(ip.substring(0, dot1)); if (first == 10) return true; if (first == 192 && ip.startsWith('168.', dot1 + 1)) return true; if (first == 172) { int second = Integer.parseInt(ip.substring(dot1 + 1, dot2)); return second >= 16 && second <= 31; } return false; } String[] fields = new String[] {\"source\", \"destination\"}; for (int i = 0; i < fields.length; i++) { def field = fields[i]; def ip = ctx[field]?.ip; if (ip != null) { if (ctx.network == null) ctx.network = new HashMap(); if (isPrivate(ip)) { if (ctx.network.private_ip == null) ctx.network.private_ip = new ArrayList(); if (!ctx.network.private_ip.contains(ip)) ctx.network.private_ip.add(ip); } else { if (ctx.network.public_ip == null) ctx.network.public_ip = new ArrayList(); if (!ctx.network.public_ip.contains(ip)) ctx.network.public_ip.add(ip); } } }",
+                "ignore_failure": false
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.capture_file",
+                "target_field": "suricata.capture_file",
+                "ignore_missing": true
+            }
+        },
+        {
+            "pipeline": {
+                "if": "ctx?.event?.dataset != null",
+                "name": "suricata.{{event.dataset}}"
+            }
+        }
+    ]
 }
--- a/salt/elasticsearch/files/ingest/suricata.dns
+++ b/salt/elasticsearch/files/ingest/suricata.dns
@@ -1,21 +1,136 @@
 {
-  "description" : "suricata.dns",
-  "processors" : [
-    { "rename": 	{ "field": "message2.proto", 			"target_field": "network.transport",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.app_proto", 		"target_field": "network.protocol",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dns.type", 		"target_field": "dns.query.type",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dns.tx_id",		"target_field": "dns.id",			"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dns.version", 		"target_field": "dns.version",			"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dns.rrname", 		"target_field": "dns.query.name",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dns.rrtype", 		"target_field": "dns.query.type_name",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dns.flags", 		"target_field": "dns.flags",			"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dns.qr", 			"target_field": "dns.qr",			"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dns.rd", 			"target_field": "dns.recursion.desired",	"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dns.ra", 			"target_field": "dns.recursion.available",	"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dns.rcode", 		"target_field": "dns.response.code_name",	"ignore_missing": true 	} },
-    { "rename":		{ "field": "message2.dns.grouped.A", 		"target_field": "dns.answers.data",		"ignore_missing": true 	} },
-    { "rename":		{ "field": "message2.dns.grouped.CNAME",	"target_field": "dns.answers.name",		"ignore_missing": true 	} },
-    { "pipeline":	{ "if": "ctx.dns.query?.name != null && ctx.dns.query.name.contains('.')", "name": "dns.tld"				} },
-    { "pipeline": 	{ "name": "common"													} }
-  ]
+    "description": "suricata.dns",
+    "processors": [
+        {
+            "rename": {
+                "field": "message2.proto",
+                "target_field": "network.transport",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.app_proto",
+                "target_field": "network.protocol",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dns.type",
+                "target_field": "dns.query.type",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dns.tx_id",
+                "target_field": "dns.tx_id",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dns.id",
+                "target_field": "dns.id",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dns.version",
+                "target_field": "dns.version",
+                "ignore_missing": true
+            }
+        },
+        {
+            "pipeline": {
+                "name": "suricata.dnsv3",
+                "ignore_missing_pipeline": true,
+                "if": "ctx?.dns?.version != null && ctx?.dns?.version == 3",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dns.rrname",
+                "target_field": "dns.query.name",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dns.rrtype",
+                "target_field": "dns.query.type_name",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dns.flags",
+                "target_field": "dns.flags",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dns.qr",
+                "target_field": "dns.qr",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dns.rd",
+                "target_field": "dns.recursion.desired",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dns.ra",
+                "target_field": "dns.recursion.available",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dns.opcode",
+                "target_field": "dns.opcode",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dns.rcode",
+                "target_field": "dns.response.code_name",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dns.grouped.A",
+                "target_field": "dns.answers.data",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dns.grouped.CNAME",
+                "target_field": "dns.answers.name",
+                "ignore_missing": true
+            }
+        },
+        {
+            "pipeline": {
+                "if": "ctx.dns.query?.name != null && ctx.dns.query.name.contains('.')",
+                "name": "dns.tld"
+            }
+        },
+        {
+            "pipeline": {
+                "name": "common"
+            }
+        }
+    ]
 }
--- a/salt/elasticsearch/files/ingest/suricata.dnsv3
+++ b/salt/elasticsearch/files/ingest/suricata.dnsv3
@@ -0,0 +1,56 @@
+{
+  "processors": [
+    {
+      "rename": {
+        "field": "message2.dns.queries",
+        "target_field": "dns.queries",
+        "ignore_missing": true,
+        "ignore_failure": true
+      }
+    },
+    {
+      "script": {
+        "source": "if (ctx?.dns?.queries != null && ctx?.dns?.queries.length > 0) {\n    if (ctx.dns == null) {\n      ctx.dns = new HashMap();\n    }\n    if (ctx.dns.query == null) {\n      ctx.dns.query = new HashMap();\n    }\n    ctx.dns.query.name = ctx?.dns?.queries[0].rrname;\n}"
+      }
+    },
+    {
+      "script": {
+        "source": "if (ctx?.dns?.queries != null && ctx?.dns?.queries.length > 0) {\n    if (ctx.dns == null) {\n      ctx.dns = new HashMap();\n    }\n    if (ctx.dns.query == null) {\n      ctx.dns.query = new HashMap();\n    }\n    ctx.dns.query.type_name = ctx?.dns?.queries[0].rrtype;\n}"
+      }
+    },
+    {
+      "foreach": {
+        "field": "dns.queries",
+        "processor": {
+          "rename": {
+            "field": "_ingest._value.rrname",
+            "target_field": "_ingest._value.name",
+            "ignore_missing": true
+          }
+        },
+        "ignore_failure": true
+      }
+    },
+    {
+      "foreach": {
+        "field": "dns.queries",
+        "processor": {
+          "rename": {
+            "field": "_ingest._value.rrtype",
+            "target_field": "_ingest._value.type_name",
+            "ignore_missing": true
+          }
+        },
+        "ignore_failure": true
+      }
+    },
+    {
+      "pipeline": {
+        "name": "suricata.tld",
+        "ignore_missing_pipeline": true,
+        "if": "ctx?.dns?.queries != null && ctx?.dns?.queries.length > 0",
+        "ignore_failure": true
+      }
+    }
+  ]
+}
--- a/salt/elasticsearch/files/ingest/suricata.tld
+++ b/salt/elasticsearch/files/ingest/suricata.tld
@@ -0,0 +1,52 @@
+{
+    "processors": [
+        {
+            "script": {
+                "source": "if (ctx.dns != null && ctx.dns.queries != null) {\n  for (def q : ctx.dns.queries) {\n    if (q.name != null && q.name.contains('.')) {\n      q.top_level_domain = q.name.substring(q.name.lastIndexOf('.') + 1);\n    }\n  }\n}",
+                "ignore_failure": true
+            }
+        },
+        {
+            "script": {
+                "source": "if (ctx.dns != null && ctx.dns.queries != null) {\n  for (def q : ctx.dns.queries) {\n    if (q.name != null && q.name.contains('.')) {\n      q.query_without_tld = q.name.substring(0, q.name.lastIndexOf('.'));\n    }\n  }\n}",
+                "ignore_failure": true
+            }
+        },
+        {
+            "script": {
+                "source": "if (ctx.dns != null && ctx.dns.queries != null) {\n  for (def q : ctx.dns.queries) {\n    if (q.query_without_tld != null && q.query_without_tld.contains('.')) {\n      q.parent_domain = q.query_without_tld.substring(q.query_without_tld.lastIndexOf('.') + 1);\n    }\n  }\n}",
+                "ignore_failure": true
+            }
+        },
+        {
+            "script": {
+                "source": "if (ctx.dns != null && ctx.dns.queries != null) {\n  for (def q : ctx.dns.queries) {\n    if (q.query_without_tld != null && q.query_without_tld.contains('.')) {\n      q.subdomain = q.query_without_tld.substring(0, q.query_without_tld.lastIndexOf('.'));\n    }\n  }\n}",
+                "ignore_failure": true
+            }
+        },
+        {
+            "script": {
+                "source": "if (ctx.dns != null && ctx.dns.queries != null) {\n  for (def q : ctx.dns.queries) {\n    if (q.parent_domain != null && q.top_level_domain != null) {\n      q.highest_registered_domain = q.parent_domain + \".\" + q.top_level_domain;\n    }\n  }\n}",
+                "ignore_failure": true
+            }
+        },
+        {
+            "script": {
+                "source": "if (ctx.dns != null && ctx.dns.queries != null) {\n  for (def q : ctx.dns.queries) {\n    if (q.subdomain != null) {\n      q.subdomain_length = q.subdomain.length();\n    }\n  }\n}",
+                "ignore_failure": true
+            }
+        },
+        {
+            "script": {
+                "source": "if (ctx.dns != null && ctx.dns.queries != null) {\n  for (def q : ctx.dns.queries) {\n    if (q.parent_domain != null) {\n      q.parent_domain_length = q.parent_domain.length();\n    }\n  }\n}",
+                "ignore_failure": true
+            }
+        },
+        {
+            "script": {
+                "source": "if (ctx.dns != null && ctx.dns.queries != null) {\n  for (def q : ctx.dns.queries) {\n    q.remove('query_without_tld');\n  }\n}",
+                "ignore_failure": true
+            }
+        }
+    ]
+}
--- a/salt/elasticsearch/files/ingest/zeek.analyzer
+++ b/salt/elasticsearch/files/ingest/zeek.analyzer
@@ -0,0 +1,61 @@
+{
+    "description": "zeek.analyzer",
+    "processors": [
+        {
+            "set": {
+                "field": "event.dataset",
+                "value": "analyzer"
+            }
+        },
+        {
+            "remove": {
+                "field": [
+                    "host"
+                ],
+                "ignore_failure": true
+            }
+        },
+        {
+            "json": {
+                "field": "message",
+                "target_field": "message2",
+                "ignore_failure": true
+            }
+        },
+        {
+            "set": {
+                "field": "network.protocol",
+                "copy_from": "message2.analyzer_name",
+                "ignore_empty_value": true,
+                "if": "ctx?.message2?.analyzer_kind == 'protocol'"
+            }
+        },
+        {
+            "set": {
+                "field": "network.protocol",
+                "ignore_empty_value": true,
+                "if": "ctx?.message2?.analyzer_kind != 'protocol'",
+                "copy_from": "message2.proto"
+            }
+        },
+        {
+            "lowercase": {
+                "field": "network.protocol",
+                "ignore_missing": true,
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.failure_reason",
+                "target_field": "error.reason",
+                "ignore_missing": true
+            }
+        },
+        {
+            "pipeline": {
+                "name": "zeek.common"
+            }
+        }
+    ]
+}
--- a/salt/elasticsearch/files/ingest/zeek.dns
+++ b/salt/elasticsearch/files/ingest/zeek.dns
@@ -1,35 +1,227 @@
 {
-  "description" : "zeek.dns",
-  "processors" : [
-    { "set": { "field": "event.dataset", "value": "dns" } },
-    { "remove":         { "field": ["host"],     "ignore_failure": true                                                                  } },
-    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
-    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.proto", 		"target_field": "network.transport",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.trans_id",		"target_field": "dns.id",	"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.rtt",	 	"target_field": "event.duration",			"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.query", 		"target_field": "dns.query.name",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.qclass", 		"target_field": "dns.query.class",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.qclass_name", 	"target_field": "dns.query.class_name",	"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.qtype", 		"target_field": "dns.query.type",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.qtype_name", 	"target_field": "dns.query.type_name",	"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.rcode",	 	"target_field": "dns.response.code",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.rcode_name", 	"target_field": "dns.response.code_name",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.AA", 		"target_field": "dns.authoritative",			"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.TC", 		"target_field": "dns.truncated",			"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.RD", 		"target_field": "dns.recursion.desired",			"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.RA", 		"target_field": "dns.recursion.available",			"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.Z",		"target_field": "dns.reserved",			"ignore_missing": true 	} },
-    { "rename":  	{ "field": "message2.answers", 		"target_field": "dns.answers.name", 		"ignore_missing": true  	} },
-    { "foreach":  {"field": "dns.answers.name","processor": {"pipeline": {"name": "common.ip_validation"}},"if": "ctx.dns != null && ctx.dns.answers != null && ctx.dns.answers.name != null","ignore_failure": true}},
-    { "foreach":  {"field": "temp._valid_ips","processor": {"append": {"field": "dns.resolved_ip","allow_duplicates": false,"value": "{{{_ingest._value}}}","ignore_failure": true}},"ignore_failure": true}},
-    { "script":   { "source": "if (ctx.dns.resolved_ip != null && ctx.dns.resolved_ip instanceof List) {\n        ctx.dns.resolved_ip.removeIf(item -> item == null || item.toString().trim().isEmpty());\n      }","ignore_failure": true }},
-    { "remove": {"field": ["temp"], "ignore_missing": true ,"ignore_failure": true } },
-    { "rename": 	{ "field": "message2.TTLs", 		"target_field": "dns.ttls",			"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.rejected", 	"target_field": "dns.query.rejected",		"ignore_missing": true 	} },
-    { "script": 	{ "lang": "painless", "source": "ctx.dns.query.length = ctx.dns.query.name.length()",	"ignore_failure": true  } },
-    { "set":      { "if": "ctx._index == 'so-zeek'", "field": "_index",      "value": "so-zeek_dns", "override": true   } },
-    { "pipeline": { "if": "ctx.dns?.query?.name != null && ctx.dns.query.name.contains('.')", "name": "dns.tld" } },
-    { "pipeline":       { "name": "zeek.common"											} }
-  ]
+    "description": "zeek.dns",
+    "processors": [
+        {
+            "set": {
+                "field": "event.dataset",
+                "value": "dns"
+            }
+        },
+        {
+            "remove": {
+                "field": [
+                    "host"
+                ],
+                "ignore_failure": true
+            }
+        },
+        {
+            "json": {
+                "field": "message",
+                "target_field": "message2",
+                "ignore_failure": true
+            }
+        },
+        {
+            "dot_expander": {
+                "field": "id.orig_h",
+                "path": "message2",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.proto",
+                "target_field": "network.transport",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.trans_id",
+                "target_field": "dns.id",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.rtt",
+                "target_field": "event.duration",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.query",
+                "target_field": "dns.query.name",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.qclass",
+                "target_field": "dns.query.class",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.qclass_name",
+                "target_field": "dns.query.class_name",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.qtype",
+                "target_field": "dns.query.type",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.qtype_name",
+                "target_field": "dns.query.type_name",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.rcode",
+                "target_field": "dns.response.code",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.rcode_name",
+                "target_field": "dns.response.code_name",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.AA",
+                "target_field": "dns.authoritative",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.TC",
+                "target_field": "dns.truncated",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.RD",
+                "target_field": "dns.recursion.desired",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.RA",
+                "target_field": "dns.recursion.available",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.Z",
+                "target_field": "dns.reserved",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.answers",
+                "target_field": "dns.answers.name",
+                "ignore_missing": true
+            }
+        },
+        {
+            "foreach": {
+                "field": "dns.answers.name",
+                "processor": {
+                    "pipeline": {
+                        "name": "common.ip_validation"
+                    }
+                },
+                "if": "ctx.dns != null && ctx.dns.answers != null && ctx.dns.answers.name != null",
+                "ignore_failure": true
+            }
+        },
+        {
+            "foreach": {
+                "field": "temp._valid_ips",
+                "processor": {
+                    "append": {
+                        "field": "dns.resolved_ip",
+                        "allow_duplicates": false,
+                        "value": "{{{_ingest._value}}}",
+                        "ignore_failure": true
+                    }
+                },
+                "if": "ctx.dns != null && ctx.dns.answers != null && ctx.dns.answers.name != null",
+                "ignore_failure": true
+            }
+        },
+        {
+            "script": {
+                "source": "if (ctx.dns.resolved_ip != null && ctx.dns.resolved_ip instanceof List) {\n        ctx.dns.resolved_ip.removeIf(item -> item == null || item.toString().trim().isEmpty());\n      }",
+                "ignore_failure": true
+            }
+        },
+        {
+            "remove": {
+                "field": [
+                    "temp"
+                ],
+                "ignore_missing": true,
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.TTLs",
+                "target_field": "dns.ttls",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.rejected",
+                "target_field": "dns.query.rejected",
+                "ignore_missing": true
+            }
+        },
+        {
+            "script": {
+                "lang": "painless",
+                "source": "ctx.dns.query.length = ctx.dns.query.name.length()",
+                "ignore_failure": true
+            }
+        },
+        {
+            "set": {
+                "if": "ctx._index == 'so-zeek'",
+                "field": "_index",
+                "value": "so-zeek_dns",
+                "override": true
+            }
+        },
+        {
+            "pipeline": {
+                "if": "ctx.dns?.query?.name != null && ctx.dns.query.name.contains('.')",
+                "name": "dns.tld"
+            }
+        },
+        {
+            "pipeline": {
+                "name": "zeek.common"
+            }
+        }
+    ]
 }
--- a/salt/elasticsearch/files/ingest/zeek.dpd
+++ b/salt/elasticsearch/files/ingest/zeek.dpd
@@ -1,20 +0,0 @@
-{
-  "description" : "zeek.dpd",
-  "processors" : [
-    { "set": { "field": "event.dataset", "value": "dpd" } },
-    { "remove":         { "field": ["host"],     "ignore_failure": true                                                                  } },
-    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
-    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source.ip",		"ignore_missing": true 	} },
-    { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source.port",		"ignore_missing": true 	} },
-    { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
-    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.proto", 		"target_field": "network.protocol",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.analyzer", 	"target_field": "observer.analyzer",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.failure_reason", 	"target_field": "error.reason",	"ignore_missing": true 	} },
-    { "pipeline":       { "name": "zeek.common"                                                                                   } }
-  ]
-}
--- a/salt/elasticsearch/templates/component/ecs/suricata.json
+++ b/salt/elasticsearch/templates/component/ecs/suricata.json
@@ -841,6 +841,10 @@
                  "type": "long"
                }
              }
+            },
+            "capture_file": {
+                "type": "keyword",
+                "ignore_above": 1024
            }
          }
        }
--- a/salt/elasticsearch/tools/sbin/so-elasticsearch-retention-estimate
+++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-retention-estimate
@@ -206,7 +206,7 @@ fail() {
    exit 1
 }

-echo -e "\nDISCLAIMER: Script output is based on current data patterns, but are approximations soley intended to assist with getting a general ILM policy configured."
+echo -e "\nDISCLAIMER: Script output is based on current data patterns, but are approximations solely intended to assist with getting a general ILM policy configured."

 ORG_ID=$(lookup_org_id)
 [ -n "$ORG_ID" ] || fail "Unable to resolve InfluxDB org id"
@@ -756,7 +756,7 @@ if [ "$should_trigger_recommendations" = true ]; then

            ilm_output=$(so-elasticsearch-query "${index}/_ilm/explain" --fail 2>/dev/null) || true
            if [ -n "$ilm_output" ]; then
-                policy=$(echo "$ilm_output" | jq --arg idx "$index" -r ".indices[$idx].policy // empty" 2>/dev/null)
+                policy=$(echo "$ilm_output" | jq -r '.indices | to_entries | .[0].value.policy // empty' 2>/dev/null)
            fi
            if [ -n "$policy" ] && [ -n "${policy_ages[$policy]:-}" ]; then
                delete_min_age=${policy_ages[$policy]}
@@ -1024,8 +1024,12 @@ else
    if [ "$ilm_indices_immediate" -gt 0 ]; then
        echo -e "${BOLD}Deleting now:${NC}            $ilm_indices_immediate indices (~${ilm_delete_immediate_gb} GB, $ilm_shards_immediate shards)"
    fi
-    if [ "$ilm_indices_7d" -gt 0 ]; then
-        echo -e "${BOLD}Storage to be freed (7d):${NC}  $ilm_indices_7d indices (~${ilm_delete_7d_gb} GB, $ilm_shards_7d shards)"
+    if [ "$ilm_indices_30d" -gt 0 ]; then
+        if [ "$ilm_delete_scheduled_30d" -gt 0 ] && [ "$ilm_indices_scheduled_30d" -gt 0 ]; then
+            echo -e "${BOLD}Storage to be freed (30d):${NC} $ilm_indices_30d indices (~${ilm_delete_30d_gb} GB, $ilm_shards_30d shards)"
+        elif [ "$ilm_indices_7d" -gt 0 ]; then
+            echo -e "${BOLD}Storage to be freed (7d):${NC}  $ilm_indices_7d indices (~${ilm_delete_7d_gb} GB, $ilm_shards_7d shards)"
+        fi
    fi

    log_title "LOG" "Retention Projection"
--- a/salt/hypervisor/tools/sbin_jinja/so-kvm-create-volume
+++ b/salt/hypervisor/tools/sbin_jinja/so-kvm-create-volume
@@ -45,7 +45,7 @@ used during VM provisioning to add dedicated NSM storage volumes.
    This command creates and attaches a volume with the following settings:
    - VM Name: `vm1_sensor`
    - Volume Size: `500` GB
-    - Volume Path: `/nsm/libvirt/volumes/vm1_sensor-nsm.img`
+    - Volume Path: `/nsm/libvirt/volumes/vm1_sensor-nsm-<epoch_timestamp>.img`
    - Device: `/dev/vdb` (virtio-blk)
    - VM remains stopped after attachment

@@ -75,7 +75,8 @@ used during VM provisioning to add dedicated NSM storage volumes.

 - The script automatically stops the VM if it's running before creating and attaching the volume.
 - Volumes are created with full pre-allocation for optimal performance.
- Volume files are stored in `/nsm/libvirt/volumes/` with naming pattern `<vm_name>-nsm.img`.
+- Volume files are stored in `/nsm/libvirt/volumes/` with naming pattern `<vm_name>-nsm-<epoch_timestamp>.img`.
+- The epoch timestamp ensures unique volume names and prevents conflicts.
 - Volumes are attached as `/dev/vdb` using virtio-blk for high performance.
 - The script checks available disk space before creating the volume.
 - Ownership is set to `qemu:qemu` with permissions `640`.
@@ -142,6 +143,7 @@ import socket
 import subprocess
 import pwd
 import grp
+import time
 import xml.etree.ElementTree as ET
 from io import StringIO
 from so_vm_utils import start_vm, stop_vm
@@ -242,10 +244,13 @@ def create_volume_file(vm_name, size_gb, logger):
    Raises:
        VolumeCreationError: If volume creation fails
    """
-    # Define volume path (directory already created in main())
-    volume_path = os.path.join(VOLUME_DIR, f"{vm_name}-nsm.img")
+    # Generate epoch timestamp for unique volume naming
+    epoch_timestamp = int(time.time())
    
-    # Check if volume already exists
+    # Define volume path with epoch timestamp for uniqueness
+    volume_path = os.path.join(VOLUME_DIR, f"{vm_name}-nsm-{epoch_timestamp}.img")
+    
+    # Check if volume already exists (shouldn't be possible with timestamp)
    if os.path.exists(volume_path):
        logger.error(f"VOLUME: Volume already exists: {volume_path}")
        raise VolumeCreationError(f"Volume already exists: {volume_path}")
--- a/salt/manager/init.sls
+++ b/salt/manager/init.sls
@@ -206,10 +206,33 @@ git_config_set_safe_dirs:
    - multivar:
      - /nsm/rules/custom-local-repos/local-sigma
      - /nsm/rules/custom-local-repos/local-yara
+      - /nsm/rules/custom-local-repos/local-suricata
      - /nsm/securityonion-resources
      - /opt/so/conf/soc/ai_summary_repos/securityonion-resources
      - /nsm/airgap-resources/playbooks
      - /opt/so/conf/soc/playbooks
+
+surinsmrulesdir:
+  file.directory:
+    - name: /nsm/rules/suricata
+    - user: 939
+    - group: 939
+    - makedirs: True
+
+suriextractionrules:
+  file.managed:
+    - name: /nsm/rules/suricata/so_extraction.rules
+    - source: salt://suricata/files/so_extraction.rules
+    - user: 939
+    - group: 939
+
+surifiltersrules:
+  file.managed:
+    - name: /nsm/rules/suricata/so_filters.rules
+    - source: salt://suricata/files/so_filters.rules
+    - user: 939
+    - group: 939
+
 {% else %}

 {{sls}}_state_not_allowed:
--- a/salt/manager/tools/sbin/so-yaml.py
+++ b/salt/manager/tools/sbin/so-yaml.py
@@ -26,8 +26,8 @@ def showUsage(args):
    print('  Where:', file=sys.stderr)
    print('   YAML_FILE       - Path to the file that will be modified. Ex: /opt/so/conf/service/conf.yaml', file=sys.stderr)
    print('   KEY             - YAML key, does not support \' or " characters at this time. Ex: level1.level2', file=sys.stderr)
-    print('   VALUE           - Value to set for a given key', file=sys.stderr)
-    print('   LISTITEM        - Item to append to a given key\'s list value', file=sys.stderr)
+    print('   VALUE           - Value to set for a given key. Can be a literal value or file:<path> to load from a YAML file.', file=sys.stderr)
+    print('   LISTITEM        - Item to append to a given key\'s list value. Can be a literal value or file:<path> to load from a YAML file.', file=sys.stderr)
    sys.exit(1)


@@ -58,7 +58,13 @@ def appendItem(content, key, listItem):


 def convertType(value):
-    if isinstance(value, str) and len(value) > 0 and (not value.startswith("0") or len(value) == 1):
+    if isinstance(value, str) and value.startswith("file:"):
+        path = value[5:]  # Remove "file:" prefix
+        if not os.path.exists(path):
+            print(f"File '{path}' does not exist.", file=sys.stderr)
+            sys.exit(1)
+        return loadYaml(path)
+    elif isinstance(value, str) and len(value) > 0 and (not value.startswith("0") or len(value) == 1):
        if "." in value:
            try:
                value = float(value)
--- a/salt/manager/tools/sbin/so-yaml_test.py
+++ b/salt/manager/tools/sbin/so-yaml_test.py
@@ -361,6 +361,29 @@ class TestRemove(unittest.TestCase):
        self.assertEqual(soyaml.convertType("FALSE"), False)
        self.assertEqual(soyaml.convertType(""), "")

+    def test_convert_file(self):
+        import tempfile
+        import os
+
+        # Create a temporary YAML file
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f:
+            f.write("test:\n  - name: hi\n    color: blue\n")
+            temp_file = f.name
+
+        try:
+            result = soyaml.convertType(f"file:{temp_file}")
+            expected = {"test": [{"name": "hi", "color": "blue"}]}
+            self.assertEqual(result, expected)
+        finally:
+            os.unlink(temp_file)
+
+    def test_convert_file_nonexistent(self):
+        with self.assertRaises(SystemExit) as cm:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
+                soyaml.convertType("file:/nonexistent/file.yaml")
+                self.assertEqual(cm.exception.code, 1)
+                self.assertIn("File '/nonexistent/file.yaml' does not exist.", mock_stderr.getvalue())
+
    def test_get_int(self):
        with patch('sys.stdout', new=StringIO()) as mock_stdout:
            filename = "/tmp/so-yaml_test-get.yaml"
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -274,7 +274,7 @@ check_os_updates() {
        if [[ "$confirm" == [cC] ]]; then
          echo "Continuing without updating packages"
        elif [[ "$confirm" == [uU] ]]; then
-          echo "Applying Grid Updates"
+          echo "Applying Grid Updates. The following patch.os salt state may take a while depending on how many packages need to be updated."
          update_flag=true
        else
          echo "Exiting soup"
@@ -426,6 +426,7 @@ preupgrade_changes() {
    [[ "$INSTALLEDVERSION" == 2.4.160 ]] && up_to_2.4.170
    [[ "$INSTALLEDVERSION" == 2.4.170 ]] && up_to_2.4.180
    [[ "$INSTALLEDVERSION" == 2.4.180 ]] && up_to_2.4.190
+    [[ "$INSTALLEDVERSION" == 2.4.190 ]] && up_to_2.4.200
    true
 }

@@ -457,6 +458,7 @@ postupgrade_changes() {
    [[ "$POSTVERSION"  == 2.4.160 ]] && post_to_2.4.170
    [[ "$POSTVERSION"  == 2.4.170 ]] && post_to_2.4.180
    [[ "$POSTVERSION"  == 2.4.180 ]] && post_to_2.4.190
+    [[ "$POSTVERSION"  == 2.4.190 ]] && post_to_2.4.200
    true
 }

@@ -636,6 +638,13 @@ post_to_2.4.190() {
  POSTVERSION=2.4.190
 }

+post_to_2.4.200() {
+  echo "Initiating Suricata idstools migration..."
+  suricata_idstools_removal_post
+
+  POSTVERSION=2.4.200
+}
+
 repo_sync() {
  echo "Sync the local repo."
  su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync."
@@ -903,6 +912,13 @@ up_to_2.4.190() {
  INSTALLEDVERSION=2.4.190
 }

+up_to_2.4.200() {
+  echo "Backing up idstools config..."
+  suricata_idstools_removal_pre
+
+  INSTALLEDVERSION=2.4.200
+}
+
 add_hydra_pillars() {
  mkdir -p /opt/so/saltstack/local/pillar/hydra
  touch /opt/so/saltstack/local/pillar/hydra/soc_hydra.sls
@@ -986,6 +1002,8 @@ rollover_index() {
 }

 suricata_idstools_migration() {
+  # For 2.4.70
+
  #Backup the pillars for idstools
  mkdir -p /nsm/backup/detections-migration/idstools
  rsync -av /opt/so/saltstack/local/pillar/idstools/* /nsm/backup/detections-migration/idstools
@@ -1086,6 +1104,159 @@ playbook_migration() {
  echo "Playbook Migration is complete...."
 }

+suricata_idstools_removal_pre() {
+# For SOUPs beginning with 2.4.200 - pre SOUP checks
+
+# Create syncBlock file
+install -d -o 939 -g 939 -m 755 /opt/so/conf/soc/fingerprints
+install -o 939 -g 939 -m 644 /dev/null /opt/so/conf/soc/fingerprints/suricataengine.syncBlock
+cat > /opt/so/conf/soc/fingerprints/suricataengine.syncBlock << EOF
+Suricata ruleset sync is blocked until this file is removed. Make sure that you have manually added any custom Suricata rulesets via SOC config - review the documentation for more details: securityonion.net/docs
+EOF
+
+# Backup custom rules & overrides
+mkdir -p /nsm/backup/detections-migration/2-4-200
+cp /usr/sbin/so-rule-update /nsm/backup/detections-migration/2-4-200
+cp /opt/so/conf/idstools/etc/rulecat.conf /nsm/backup/detections-migration/2-4-200
+
+if [[ -f /opt/so/conf/soc/so-detections-backup.py ]]; then
+    python3 /opt/so/conf/soc/so-detections-backup.py
+
+    # Verify backup by comparing counts
+    echo "Verifying detection overrides backup..."
+    es_override_count=$(/sbin/so-elasticsearch-query 'so-detection/_count' \
+       -d '{"query": {"bool": {"must": [{"exists": {"field": "so_detection.overrides"}}]}}}' | jq -r '.count') || {
+        echo "  Error: Failed to query Elasticsearch for override count"
+        exit 1
+    }
+
+    if [[ ! "$es_override_count" =~ ^[0-9]+$ ]]; then
+        echo "  Error: Invalid override count from Elasticsearch: '$es_override_count'"
+        exit 1
+    fi
+
+    backup_override_count=$(find /nsm/backup/detections/repo/*/overrides -type f 2>/dev/null | wc -l)
+
+    echo "  Elasticsearch overrides: $es_override_count"
+    echo "  Backed up overrides: $backup_override_count"
+
+    if [[ "$es_override_count" -gt 0 ]]; then
+        if [[ "$backup_override_count" -gt 0 ]]; then
+            echo "  Override backup verified successfully"
+        else
+            echo "  Error: Elasticsearch has $es_override_count overrides but backup has 0 files"
+            exit 1
+        fi
+    else
+        echo "  No overrides to backup"
+    fi
+else
+    echo "SOC Detections backup script not found, skipping detection backup"
+fi
+
+}
+
+suricata_idstools_removal_post() {
+# For SOUPs beginning with 2.4.200 - post SOUP checks
+
+echo "Checking idstools configuration for custom modifications..."
+
+# Normalize and hash file content for consistent comparison
+# Args: $1 - file path
+# Outputs: SHA256 hash to stdout
+# Returns: 0 on success, 1 on failure
+hash_normalized_file() {
+    local file="$1"
+
+    if [[ ! -r "$file" ]]; then
+        return 1
+    fi
+
+    sed -E \
+        -e 's/^[[:space:]]+//; s/[[:space:]]+$//' \
+        -e '/^$/d' \
+        -e 's|--url=http://[^:]+:7788|--url=http://MANAGER:7788|' \
+        "$file" | sha256sum | awk '{print $1}'
+}
+
+# Known-default hashes
+KNOWN_SO_RULE_UPDATE_HASHES=(
+    "8f1fe1cb65c08aab78830315b952785c7ccdcc108c5c0474f427e29d4e39ee5f"  # non-Airgap
+    "d23ac5a962c709dcb888103effb71444df72b46009b6c426e280dbfbc7d74d40"  # Airgap
+)
+
+KNOWN_RULECAT_CONF_HASHES=(
+    "17fc663a83b30d4ba43ac6643666b0c96343c5ea6ea833fe6a8362fe415b666b"  # default
+)
+
+# Check a config file against known hashes
+# Args: $1 - file path, $2 - array name of known hashes
+check_config_file() {
+    local file="$1"
+    local known_hashes_array="$2"
+    local file_display_name=$(basename "$file")
+
+    if [[ ! -f "$file" ]]; then
+        echo "Warning: $file not found"
+        echo "$file_display_name not found - manual verification required" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock
+        return 1
+    fi
+
+    echo "Hashing $file..."
+    local file_hash
+    if ! file_hash=$(hash_normalized_file "$file"); then
+        echo "Warning: Could not read $file"
+        echo "$file_display_name not readable - manual verification required" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock
+        return 1
+    fi
+
+    echo "  Hash: $file_hash"
+
+    # Check if hash matches any known default
+    local -n known_hashes=$known_hashes_array
+    for known_hash in "${known_hashes[@]}"; do
+        if [[ "$file_hash" == "$known_hash" ]]; then
+            echo "  Matches known default configuration"
+            return 0
+        fi
+    done
+
+    # No match - custom configuration detected
+    echo "Does not match known default - custom configuration detected"
+    echo "Custom $file_display_name detected (hash: $file_hash)" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock
+
+    # If this is so-rule-update, check for ETPRO license code and write out to the syncBlock file
+    # If ETPRO is enabled, the license code already exists in the so-rule-update script, this is just making it easier to migrate
+    if [[ "$file_display_name" == "so-rule-update" ]]; then
+        local etpro_code
+        etpro_code=$(grep -oP '\-\-etpro=\K[0-9a-fA-F]+' "$file" 2>/dev/null) || true
+        if [[ -n "$etpro_code" ]]; then
+            echo "ETPRO code found: $etpro_code" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock
+        fi
+    fi
+
+    return 1
+}
+
+# Check so-rule-update and rulecat.conf
+SO_RULE_UPDATE="/usr/sbin/so-rule-update"
+RULECAT_CONF="/opt/so/conf/idstools/etc/rulecat.conf"
+
+custom_found=0
+
+check_config_file "$SO_RULE_UPDATE" "KNOWN_SO_RULE_UPDATE_HASHES" || custom_found=1
+check_config_file "$RULECAT_CONF" "KNOWN_RULECAT_CONF_HASHES" || custom_found=1
+
+# If no custom configs found, remove syncBlock
+if [[ $custom_found -eq 0 ]]; then
+    echo "idstools migration completed successfully - removing Suricata engine syncBlock"
+    rm -f /opt/so/conf/soc/fingerprints/suricataengine.syncBlock
+else
+    echo "Custom idstools configuration detected - syncBlock remains in place"
+    echo "Review /opt/so/conf/soc/fingerprints/suricataengine.syncBlock for details"
+fi
+}
+
 determine_elastic_agent_upgrade() {
  if [[ $is_airgap -eq 0 ]]; then
    update_elastic_agent_airgap
@@ -1318,6 +1489,8 @@ upgrade_salt() {
    fi
  # Else do Ubuntu things
  elif [[ $is_deb ]]; then
+    # ensure these files don't exist when upgrading from 3006.9 to 3006.16
+    rm -f /etc/apt/keyrings/salt-archive-keyring-2023.pgp /etc/apt/sources.list.d/salt.list
    echo "Removing apt hold for Salt."
    echo ""
    apt-mark unhold "salt-common"
@@ -1679,7 +1852,7 @@ This appears to be a distributed deployment. Other nodes should update themselve

 Each minion is on a random 15 minute check-in period and things like network bandwidth can be a factor in how long the actual upgrade takes. If you have a heavy node on a slow link, it is going to take a while to get the containers to it. Depending on what changes happened between the versions, Elasticsearch might not be able to talk to said heavy node until the update is complete.

-If it looks like you’re missing data after the upgrade, please avoid restarting services and instead make sure at least one search node has completed its upgrade. The best way to do this is to run 'sudo salt-call state.highstate' from a search node and make sure there are no errors. Typically if it works on one node it will work on the rest. Forward nodes are less complex and will update as they check in so you can monitor those from the Grid section of SOC.
+If it looks like you’re missing data after the upgrade, please avoid restarting services and instead make sure at least one search node has completed its upgrade. The best way to do this is to run 'sudo salt-call state.highstate' from a search node and make sure there are no errors. Typically if it works on one node it will work on the rest. Sensor nodes are less complex and will update as they check in so you can monitor those from the Grid section of SOC.

 For more information, please see $DOC_BASE_URL/soup.html#distributed-deployments.

--- a/salt/pcap/config.sls
+++ b/salt/pcap/config.sls
@@ -8,12 +8,9 @@

 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% from "pcap/config.map.jinja" import PCAPMERGED %}
-{% from 'bpf/pcap.map.jinja' import PCAPBPF %}
-
-{% set BPF_COMPILED = "" %}
+{% from 'bpf/pcap.map.jinja' import PCAPBPF, PCAP_BPF_STATUS, PCAP_BPF_CALC, STENO_BPF_COMPILED %}

 # PCAP Section
-
 stenographergroup:
  group.present:
    - name: stenographer
@@ -40,18 +37,12 @@ pcap_sbin:
    - group: 939
    - file_mode: 755

-{% if PCAPBPF %}
-   {% set BPF_CALC = salt['cmd.script']('salt://common/tools/sbin/so-bpf-compile', GLOBALS.sensor.interface + ' ' + PCAPBPF|join(" "),cwd='/root') %}
-   {% if BPF_CALC['stderr'] == "" %}
-      {% set BPF_COMPILED =  ",\\\"--filter=" + BPF_CALC['stdout'] + "\\\""  %}
-   {% else  %}
-
-bpfcompilationfailure:
+{% if PCAPBPF and not PCAP_BPF_STATUS %}
+stenoPCAPbpfcompilationfailure:
  test.configurable_test_state:
   - changes: False
   - result: False
-   - comment: "BPF Compilation Failed - Discarding Specified BPF"
-   {% endif %}
+  - comment: "BPF Syntax Error - Discarding Specified BPF. Error: {{ PCAP_BPF_CALC['stderr'] }}"
 {% endif %}

 stenoconf:
@@ -64,7 +55,7 @@ stenoconf:
    - template: jinja
    - defaults:
        PCAPMERGED: {{ PCAPMERGED }}
-        BPF_COMPILED: "{{ BPF_COMPILED }}"
+        STENO_BPF_COMPILED: "{{ STENO_BPF_COMPILED }}"

 stenoca:
  file.directory:
--- a/salt/pcap/files/config.jinja
+++ b/salt/pcap/files/config.jinja
@@ -6,6 +6,6 @@
  , "Interface": "{{ pillar.sensor.interface }}"
  , "Port": 1234
  , "Host": "127.0.0.1"
-  , "Flags": ["-v", "--blocks={{ PCAPMERGED.config.blocks }}", "--preallocate_file_mb={{ PCAPMERGED.config.preallocate_file_mb }}", "--aiops={{ PCAPMERGED.config.aiops }}", "--uid=stenographer", "--gid=stenographer"{{ BPF_COMPILED }}]
+  , "Flags": ["-v", "--blocks={{ PCAPMERGED.config.blocks }}", "--preallocate_file_mb={{ PCAPMERGED.config.preallocate_file_mb }}", "--aiops={{ PCAPMERGED.config.aiops }}", "--uid=stenographer", "--gid=stenographer"{{ STENO_BPF_COMPILED }}]
  , "CertPath": "/etc/stenographer/certs"
 }
--- a/salt/pcap/soc_pcap.yaml
+++ b/salt/pcap/soc_pcap.yaml
@@ -7,7 +7,7 @@ pcap:
      description: By default, Stenographer limits the number of files in the pcap directory to 30000 to avoid limitations with the ext3 filesystem. However, if you're using the ext4 or xfs filesystems, then it is safe to increase this value. So if you have a large amount of storage and find that you only have 3 weeks worth of PCAP on disk while still having plenty of free space, then you may want to increase this default setting.
      helpLink: stenographer.html
    diskfreepercentage:
-      description: Stenographer will purge old PCAP on a regular basis to keep the disk free percentage at this level. If you have a distributed deployment with dedicated forward nodes, then the default value of 10 should be reasonable since Stenographer should be the main consumer of disk space in the /nsm partition. However, if you have systems that run both Stenographer and Elasticsearch at the same time (like eval and standalone installations), then you’ll want to make sure that this value is no lower than 21 so that you avoid Elasticsearch hitting its watermark setting at 80% disk usage. If you have an older standalone installation, then you may need to manually change this value to 21.
+      description: Stenographer will purge old PCAP on a regular basis to keep the disk free percentage at this level. If you have a distributed deployment with dedicated Sensor nodes, then the default value of 10 should be reasonable since Stenographer should be the main consumer of disk space in the /nsm partition. However, if you have systems that run both Stenographer and Elasticsearch at the same time (like eval and standalone installations), then you’ll want to make sure that this value is no lower than 21 so that you avoid Elasticsearch hitting its watermark setting at 80% disk usage. If you have an older standalone installation, then you may need to manually change this value to 21.
      helpLink: stenographer.html
    blocks:
      description: The number of 1MB packet blocks used by Stenographer and AF_PACKET to store packets in memory, per thread. You shouldn't need to change this. 
--- a/salt/registry/enabled.sls
+++ b/salt/registry/enabled.sls
@@ -5,6 +5,7 @@

 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
+{%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKER %}

 include:
@@ -57,6 +58,17 @@ so-dockerregistry:
      - x509: registry_crt
      - x509: registry_key

+wait_for_so-dockerregistry:
+  http.wait_for_successful_query:
+    - name: 'https://{{ GLOBALS.registry_host }}:5000/v2/'
+    - ssl: True
+    - verify_ssl: False
+    - status: 200
+    - wait_for: 120
+    - request_interval: 5
+    - require:
+      -  docker_container: so-dockerregistry
+
 delete_so-dockerregistry_so-status.disabled:
  file.uncomment:
    - name: /opt/so/conf/so-status/so-status.conf
--- a/salt/salt/engines/master/virtual_node_manager.py
+++ b/salt/salt/engines/master/virtual_node_manager.py
@@ -727,7 +727,8 @@ def check_hypervisor_disk_space(hypervisor: str, size_gb: int) -> Tuple[bool, Op
        result = local.cmd(
            hypervisor_minion,
            'cmd.run',
-            ["df -BG /nsm/libvirt/volumes | tail -1 | awk '{print $4}' | sed 's/G//'"]
+            ["df -BG /nsm/libvirt/volumes | tail -1 | awk '{print $4}' | sed 's/G//'"],
+            kwarg={'python_shell': True}
        )
        
        if not result or hypervisor_minion not in result:
--- a/salt/salt/scripts/bootstrap-salt.sh
+++ b/salt/salt/scripts/bootstrap-salt.sh
@@ -26,7 +26,7 @@
 #======================================================================================================================
 set -o nounset                              # Treat unset variables as an error

-__ScriptVersion="2025.02.24"
+__ScriptVersion="2025.09.03"
 __ScriptName="bootstrap-salt.sh"

 __ScriptFullName="$0"
@@ -48,6 +48,7 @@ __ScriptArgs="$*"
 #   * BS_GENTOO_USE_BINHOST:    If 1 add `--getbinpkg` to gentoo's emerge
 #   * BS_SALT_MASTER_ADDRESS:   The IP or DNS name of the salt-master the minion should connect to
 #   * BS_SALT_GIT_CHECKOUT_DIR: The directory where to clone Salt on git installations
+#   * BS_TMP_DIR:               The directory to use for executing the installation (defaults to /tmp)
 #======================================================================================================================


@@ -171,12 +172,12 @@ __check_config_dir() {

    case "$CC_DIR_NAME" in
        http://*|https://*)
-            __fetch_url "/tmp/${CC_DIR_BASE}" "${CC_DIR_NAME}"
-            CC_DIR_NAME="/tmp/${CC_DIR_BASE}"
+            __fetch_url "${_TMP_DIR}/${CC_DIR_BASE}" "${CC_DIR_NAME}"
+            CC_DIR_NAME="${_TMP_DIR}/${CC_DIR_BASE}"
            ;;
        ftp://*)
-            __fetch_url "/tmp/${CC_DIR_BASE}" "${CC_DIR_NAME}"
-            CC_DIR_NAME="/tmp/${CC_DIR_BASE}"
+            __fetch_url "${_TMP_DIR}/${CC_DIR_BASE}" "${CC_DIR_NAME}"
+            CC_DIR_NAME="${_TMP_DIR}/${CC_DIR_BASE}"
            ;;
        *://*)
            echoerror "Unsupported URI scheme for $CC_DIR_NAME"
@@ -194,22 +195,22 @@ __check_config_dir() {

    case "$CC_DIR_NAME" in
        *.tgz|*.tar.gz)
-            tar -zxf "${CC_DIR_NAME}" -C /tmp
+            tar -zxf "${CC_DIR_NAME}" -C ${_TMP_DIR}
            CC_DIR_BASE=$(basename "${CC_DIR_BASE}" ".tgz")
            CC_DIR_BASE=$(basename "${CC_DIR_BASE}" ".tar.gz")
-            CC_DIR_NAME="/tmp/${CC_DIR_BASE}"
+            CC_DIR_NAME="${_TMP_DIR}/${CC_DIR_BASE}"
            ;;
        *.tbz|*.tar.bz2)
-            tar -xjf "${CC_DIR_NAME}" -C /tmp
+            tar -xjf "${CC_DIR_NAME}" -C ${_TMP_DIR}
            CC_DIR_BASE=$(basename "${CC_DIR_BASE}" ".tbz")
            CC_DIR_BASE=$(basename "${CC_DIR_BASE}" ".tar.bz2")
-            CC_DIR_NAME="/tmp/${CC_DIR_BASE}"
+            CC_DIR_NAME="${_TMP_DIR}/${CC_DIR_BASE}"
            ;;
        *.txz|*.tar.xz)
-            tar -xJf "${CC_DIR_NAME}" -C /tmp
+            tar -xJf "${CC_DIR_NAME}" -C ${_TMP_DIR}
            CC_DIR_BASE=$(basename "${CC_DIR_BASE}" ".txz")
            CC_DIR_BASE=$(basename "${CC_DIR_BASE}" ".tar.xz")
-            CC_DIR_NAME="/tmp/${CC_DIR_BASE}"
+            CC_DIR_NAME="${_TMP_DIR}/${CC_DIR_BASE}"
            ;;
    esac

@@ -245,6 +246,7 @@ __check_unparsed_options() {
 #----------------------------------------------------------------------------------------------------------------------
 _KEEP_TEMP_FILES=${BS_KEEP_TEMP_FILES:-$BS_FALSE}
 _TEMP_CONFIG_DIR="null"
+_TMP_DIR=${BS_TMP_DIR:-"/tmp"}
 _SALTSTACK_REPO_URL="https://github.com/saltstack/salt.git"
 _SALT_REPO_URL=${_SALTSTACK_REPO_URL}
 _TEMP_KEYS_DIR="null"
@@ -281,7 +283,7 @@ _SIMPLIFY_VERSION=$BS_TRUE
 _LIBCLOUD_MIN_VERSION="0.14.0"
 _EXTRA_PACKAGES=""
 _HTTP_PROXY=""
-_SALT_GIT_CHECKOUT_DIR=${BS_SALT_GIT_CHECKOUT_DIR:-/tmp/git/salt}
+_SALT_GIT_CHECKOUT_DIR=${BS_SALT_GIT_CHECKOUT_DIR:-${_TMP_DIR}/git/salt}
 _NO_DEPS=$BS_FALSE
 _FORCE_SHALLOW_CLONE=$BS_FALSE
 _DISABLE_SSL=$BS_FALSE
@@ -367,7 +369,7 @@ __usage() {
        also be specified. Salt installation will be ommitted, but some of the
        dependencies could be installed to write configuration with -j or -J.
    -d  Disables checking if Salt services are enabled to start on system boot.
-        You can also do this by touching /tmp/disable_salt_checks on the target
+        You can also do this by touching ${BS_TMP_DIR}/disable_salt_checks on the target
        host. Default: \${BS_FALSE}
    -D  Show debug output
    -f  Force shallow cloning for git installations.
@@ -424,6 +426,9 @@ __usage() {
    -r  Disable all repository configuration performed by this script. This
        option assumes all necessary repository configuration is already present
        on the system.
+    -T  If set this overrides the use of /tmp for script execution. This is
+        to allow for systems in which noexec is applied to temp filesystem mounts
+        for security reasons
    -U  If set, fully upgrade the system prior to bootstrapping Salt
    -v  Display script version
    -V  Install Salt into virtualenv
@@ -436,7 +441,7 @@ __usage() {
 EOT
 }   # ----------  end of function __usage  ----------

-while getopts ':hvnDc:g:Gx:k:s:MSWNXCPFUKIA:i:Lp:dH:bflV:J:j:rR:aqQ' opt
+while getopts ':hvnDc:g:Gx:k:s:MSWNXCPFUKIA:i:Lp:dH:bflV:J:j:rR:T:aqQ' opt
 do
  case "${opt}" in

@@ -478,6 +483,7 @@ do
    a )  _PIP_ALL=$BS_TRUE                              ;;
    r )  _DISABLE_REPOS=$BS_TRUE                        ;;
    R )  _CUSTOM_REPO_URL=$OPTARG                       ;;
+    T )  _TMP_DIR="$OPTARG"                             ;;
    J )  _CUSTOM_MASTER_CONFIG=$OPTARG                  ;;
    j )  _CUSTOM_MINION_CONFIG=$OPTARG                  ;;
    q )  _QUIET_GIT_INSTALLATION=$BS_TRUE               ;;
@@ -495,10 +501,10 @@ done
 shift $((OPTIND-1))

 # Define our logging file and pipe paths
-LOGFILE="/tmp/$( echo "$__ScriptName" | sed s/.sh/.log/g )"
-LOGPIPE="/tmp/$( echo "$__ScriptName" | sed s/.sh/.logpipe/g )"
+LOGFILE="${_TMP_DIR}/$( echo "$__ScriptName" | sed s/.sh/.log/g )"
+LOGPIPE="${_TMP_DIR}/$( echo "$__ScriptName" | sed s/.sh/.logpipe/g )"
 # Ensure no residual pipe exists
-rm "$LOGPIPE" 2>/dev/null
+rm -f "$LOGPIPE" 2>/dev/null

 # Create our logging pipe
 # On FreeBSD we have to use mkfifo instead of mknod
@@ -534,7 +540,7 @@ exec 2>"$LOGPIPE"
 #              14               SIGALRM
 #              15               SIGTERM
 #----------------------------------------------------------------------------------------------------------------------
-APT_ERR=$(mktemp /tmp/apt_error.XXXXXX)
+APT_ERR=$(mktemp ${_TMP_DIR}/apt_error.XXXXXX)
 __exit_cleanup() {
    EXIT_CODE=$?

@@ -927,6 +933,11 @@ if [ -d "${_VIRTUALENV_DIR}" ]; then
    exit 1
 fi

+# Make sure the designated temp directory exists
+if [ ! -d "${_TMP_DIR}" ]; then
+    mkdir -p "${_TMP_DIR}"
+fi
+
 #---  FUNCTION  -------------------------------------------------------------------------------------------------------
 #         NAME:  __fetch_url
 #  DESCRIPTION:  Retrieves a URL and writes it to a given path
@@ -1941,11 +1952,6 @@ __wait_for_apt(){
    # Timeout set at 15 minutes
    WAIT_TIMEOUT=900

-    ## see if sync'ing the clocks helps
-    if [ -f /usr/sbin/hwclock ]; then
-        /usr/sbin/hwclock -s
-    fi
-
    # Run our passed in apt command
    "${@}" 2>"$APT_ERR"
    APT_RETURN=$?
@@ -1996,14 +2002,14 @@ __apt_get_upgrade_noinput() {
 #----------------------------------------------------------------------------------------------------------------------
 __temp_gpg_pub() {
    if __check_command_exists mktemp; then
-        tempfile="$(mktemp /tmp/salt-gpg-XXXXXXXX.pub 2>/dev/null)"
+        tempfile="$(mktemp ${_TMP_DIR}/salt-gpg-XXXXXXXX.pub 2>/dev/null)"

        if [ -z "$tempfile" ]; then
-            echoerror "Failed to create temporary file in /tmp"
+            echoerror "Failed to create temporary file in ${_TMP_DIR}"
            return 1
        fi
    else
-        tempfile="/tmp/salt-gpg-$$.pub"
+        tempfile="${_TMP_DIR}/salt-gpg-$$.pub"
    fi

    echo $tempfile
@@ -2043,7 +2049,7 @@ __rpm_import_gpg() {
    __fetch_url "$tempfile" "$url" || return 1

    # At least on CentOS 8, a missing newline at the end causes:
-    #   error: /tmp/salt-gpg-n1gKUb1u.pub: key 1 not an armored public key.
+    #   error: ${_TMP_DIR}/salt-gpg-n1gKUb1u.pub: key 1 not an armored public key.
    # shellcheck disable=SC1003,SC2086
    sed -i -e '$a\' $tempfile

@@ -2109,7 +2115,7 @@ __git_clone_and_checkout() {
    fi

    __SALT_GIT_CHECKOUT_PARENT_DIR=$(dirname "${_SALT_GIT_CHECKOUT_DIR}" 2>/dev/null)
-    __SALT_GIT_CHECKOUT_PARENT_DIR="${__SALT_GIT_CHECKOUT_PARENT_DIR:-/tmp/git}"
+    __SALT_GIT_CHECKOUT_PARENT_DIR="${__SALT_GIT_CHECKOUT_PARENT_DIR:-${_TMP_DIR}/git}"
    __SALT_CHECKOUT_REPONAME="$(basename "${_SALT_GIT_CHECKOUT_DIR}" 2>/dev/null)"
    __SALT_CHECKOUT_REPONAME="${__SALT_CHECKOUT_REPONAME:-salt}"
    [ -d "${__SALT_GIT_CHECKOUT_PARENT_DIR}" ] || mkdir "${__SALT_GIT_CHECKOUT_PARENT_DIR}"
@@ -2162,7 +2168,7 @@ __git_clone_and_checkout() {

        if [ "$__SHALLOW_CLONE" -eq $BS_TRUE ]; then
            # Let's try 'treeless' cloning to speed up. Treeless cloning omits trees and blobs ('files')
-	    # but includes metadata (commit history, tags, branches etc.
+            # but includes metadata (commit history, tags, branches etc.
            # Test for "--filter" option introduced in git 2.19, the minimal version of git where the treeless
            # cloning we need actually works
            if [ "$(git clone 2>&1 | grep 'filter')" != "" ]; then
@@ -2390,14 +2396,14 @@ __overwriteconfig() {

    # Make a tempfile to dump any python errors into.
    if __check_command_exists mktemp; then
-        tempfile="$(mktemp /tmp/salt-config-XXXXXXXX 2>/dev/null)"
+        tempfile="$(mktemp ${_TMP_DIR}/salt-config-XXXXXXXX 2>/dev/null)"

        if [ -z "$tempfile" ]; then
-            echoerror "Failed to create temporary file in /tmp"
+            echoerror "Failed to create temporary file in ${_TMP_DIR}"
            return 1
        fi
    else
-        tempfile="/tmp/salt-config-$$"
+        tempfile="${_TMP_DIR}/salt-config-$$"
    fi

    if [ -n "$_PY_EXE" ]; then
@@ -2760,8 +2766,8 @@ __install_salt_from_repo() {
    echoinfo "Installing salt using ${_py_exe}, $(${_py_exe} --version)"
    cd "${_SALT_GIT_CHECKOUT_DIR}" || return 1

-    mkdir -p /tmp/git/deps
-    echodebug "Created directory /tmp/git/deps"
+    mkdir -p ${_TMP_DIR}/git/deps
+    echodebug "Created directory ${_TMP_DIR}/git/deps"

    if [ ${DISTRO_NAME_L} = "ubuntu" ] && [ "$DISTRO_MAJOR_VERSION" -eq 22 ]; then
        echodebug "Ubuntu 22.04 has problem with base.txt requirements file, not parsing sys_platform == 'win32', upgrading from default pip works"
@@ -2774,7 +2780,7 @@ __install_salt_from_repo() {
        fi
    fi

-    rm -f /tmp/git/deps/*
+    rm -f ${_TMP_DIR}/git/deps/*

    echodebug "Installing Salt requirements from PyPi, ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed ${_PIP_INSTALL_ARGS} -r requirements/static/ci/py${_py_version}/linux.txt"
    ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed ${_PIP_INSTALL_ARGS} -r "requirements/static/ci/py${_py_version}/linux.txt"
@@ -2799,7 +2805,7 @@ __install_salt_from_repo() {

    echodebug "Running '${_py_exe} setup.py --salt-config-dir=$_SALT_ETC_DIR --salt-cache-dir=${_SALT_CACHE_DIR} ${SETUP_PY_INSTALL_ARGS} bdist_wheel'"
    ${_py_exe} setup.py --salt-config-dir="$_SALT_ETC_DIR" --salt-cache-dir="${_SALT_CACHE_DIR} ${SETUP_PY_INSTALL_ARGS}" bdist_wheel || return 1
-    mv dist/salt*.whl /tmp/git/deps/ || return 1
+    mv dist/salt*.whl ${_TMP_DIR}/git/deps/ || return 1

    cd "${__SALT_GIT_CHECKOUT_PARENT_DIR}" || return 1

@@ -2813,14 +2819,14 @@ __install_salt_from_repo() {
        ${_pip_cmd} install --force-reinstall --break-system-packages "${_arch_dep}"
    fi

-    echodebug "Running '${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --no-deps --force-reinstall ${_PIP_INSTALL_ARGS} /tmp/git/deps/salt*.whl'"
+    echodebug "Running '${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --no-deps --force-reinstall ${_PIP_INSTALL_ARGS} ${_TMP_DIR}/git/deps/salt*.whl'"

-    echodebug "Running ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --no-deps --force-reinstall ${_PIP_INSTALL_ARGS} --global-option=--salt-config-dir=$_SALT_ETC_DIR --salt-cache-dir=${_SALT_CACHE_DIR} ${SETUP_PY_INSTALL_ARGS} /tmp/git/deps/salt*.whl"
+    echodebug "Running ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --no-deps --force-reinstall ${_PIP_INSTALL_ARGS} --global-option=--salt-config-dir=$_SALT_ETC_DIR --salt-cache-dir=${_SALT_CACHE_DIR} ${SETUP_PY_INSTALL_ARGS} ${_TMP_DIR}/git/deps/salt*.whl"

    ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --no-deps --force-reinstall \
        ${_PIP_INSTALL_ARGS} \
        --global-option="--salt-config-dir=$_SALT_ETC_DIR --salt-cache-dir=${_SALT_CACHE_DIR} ${SETUP_PY_INSTALL_ARGS}" \
-        /tmp/git/deps/salt*.whl || return 1
+        ${_TMP_DIR}/git/deps/salt*.whl || return 1

    echoinfo "Checking if Salt can be imported using ${_py_exe}"
    CHECK_SALT_SCRIPT=$(cat << EOM
@@ -6295,8 +6301,8 @@ __get_packagesite_onedir_latest() {
 }


-__install_saltstack_photon_onedir_repository() {
-    echodebug "__install_saltstack_photon_onedir_repository() entry"
+__install_saltstack_vmware_photon_os_onedir_repository() {
+    echodebug "__install_saltstack_vmware_photon_os_onedir_repository() entry"

    if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -ne 3 ]; then
        echoerror "Python version is no longer supported, only Python 3"
@@ -6376,8 +6382,8 @@ __install_saltstack_photon_onedir_repository() {
    return 0
 }

-install_photon_deps() {
-    echodebug "install_photon_deps() entry"
+install_vmware_photon_os_deps() {
+    echodebug "install_vmware_photon_os_deps() entry"

    if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -ne 3 ]; then
        echoerror "Python version is no longer supported, only Python 3"
@@ -6406,8 +6412,8 @@ install_photon_deps() {
    return 0
 }

-install_photon_stable_post() {
-    echodebug "install_photon_stable_post() entry"
+install_vmware_photon_os_stable_post() {
+    echodebug "install_vmware_photon_os_stable_post() entry"

    for fname in api master minion syndic; do
        # Skip salt-api since the service should be opt-in and not necessarily started on boot
@@ -6424,8 +6430,8 @@ install_photon_stable_post() {
    done
 }

-install_photon_git_deps() {
-    echodebug "install_photon_git_deps() entry"
+install_vmware_photon_os_git_deps() {
+    echodebug "install_vmware_photon_os_git_deps() entry"

    if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -ne 3 ]; then
        echoerror "Python version is no longer supported, only Python 3"
@@ -6463,7 +6469,7 @@ install_photon_git_deps() {

    __PACKAGES="python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc glibc-devel linux-devel.x86_64 cython${PY_PKG_VER}"

-    echodebug "install_photon_git_deps() distro major version, ${DISTRO_MAJOR_VERSION}"
+    echodebug "install_vmware_photon_os_git_deps() distro major version, ${DISTRO_MAJOR_VERSION}"

    ## Photon 5 container is missing systemd on default installation
    if [ "${DISTRO_MAJOR_VERSION}" -lt 5  ]; then
@@ -6489,8 +6495,8 @@ install_photon_git_deps() {
    return 0
 }

-install_photon_git() {
-    echodebug "install_photon_git() entry"
+install_vmware_photon_os_git() {
+    echodebug "install_vmware_photon_os_git() entry"

    if [ "${_PY_EXE}" != "" ]; then
        _PYEXE=${_PY_EXE}
@@ -6500,7 +6506,7 @@ install_photon_git() {
        return 1
    fi

-    install_photon_git_deps
+    install_vmware_photon_os_git_deps

    if [ -f "${_SALT_GIT_CHECKOUT_DIR}/salt/syspaths.py" ]; then
        ${_PYEXE} setup.py --salt-config-dir="$_SALT_ETC_DIR" --salt-cache-dir="${_SALT_CACHE_DIR}" ${SETUP_PY_INSTALL_ARGS} install --prefix=/usr || return 1
@@ -6510,8 +6516,8 @@ install_photon_git() {
    return 0
 }

-install_photon_git_post() {
-    echodebug "install_photon_git_post() entry"
+install_vmware_photon_os_git_post() {
+    echodebug "install_vmware_photon_os_git_post() entry"

    for fname in api master minion syndic; do
        # Skip if not meant to be installed
@@ -6543,9 +6549,9 @@ install_photon_git_post() {
    done
 }

-install_photon_restart_daemons() {
+install_vmware_photon_os_restart_daemons() {
    [ "$_START_DAEMONS" -eq $BS_FALSE ] && return
-    echodebug "install_photon_restart_daemons() entry"
+    echodebug "install_vmware_photon_os_restart_daemons() entry"


    for fname in api master minion syndic; do
@@ -6567,8 +6573,8 @@ install_photon_restart_daemons() {
    done
 }

-install_photon_check_services() {
-    echodebug "install_photon_check_services() entry"
+install_vmware_photon_os_check_services() {
+    echodebug "install_vmware_photon_os_check_services() entry"

    for fname in api master minion syndic; do
        # Skip salt-api since the service should be opt-in and not necessarily started on boot
@@ -6585,8 +6591,8 @@ install_photon_check_services() {
    return 0
 }

-install_photon_onedir_deps() {
-    echodebug "install_photon_onedir_deps() entry"
+install_vmware_photon_os_onedir_deps() {
+    echodebug "install_vmware_photon_os_onedir_deps() entry"


    if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then
@@ -6600,17 +6606,17 @@ install_photon_onedir_deps() {
    fi

    if [ "$_DISABLE_REPOS" -eq "$BS_FALSE" ]; then
-        __install_saltstack_photon_onedir_repository || return 1
+        __install_saltstack_vmware_photon_os_onedir_repository || return 1
    fi

    # If -R was passed, we need to configure custom repo url with rsync-ed packages
    # Which was handled in __install_saltstack_rhel_repository buu that hanlded old-stable which is for
    # releases which are End-Of-Life. This call has its own check in case -r was passed without -R.
    if [ "$_CUSTOM_REPO_URL" != "null" ]; then
-        __install_saltstack_photon_onedir_repository || return 1
+        __install_saltstack_vmware_photon_os_onedir_repository || return 1
    fi

-    __PACKAGES="procps-ng sudo shadow"
+    __PACKAGES="procps-ng sudo shadow wget"

    # shellcheck disable=SC2086
    __tdnf_install_noinput ${__PACKAGES} || return 1
@@ -6626,9 +6632,9 @@ install_photon_onedir_deps() {
 }


-install_photon_onedir() {
+install_vmware_photon_os_onedir() {

-    echodebug "install_photon_onedir() entry"
+    echodebug "install_vmware_photon_os_onedir() entry"

    STABLE_REV=$ONEDIR_REV
    _GENERIC_PKG_VERSION=""
@@ -6672,9 +6678,9 @@ install_photon_onedir() {
    return 0
 }

-install_photon_onedir_post() {
+install_vmware_photon_os_onedir_post() {
    STABLE_REV=$ONEDIR_REV
-    install_photon_stable_post || return 1
+    install_vmware_photon_os_stable_post || return 1

    return 0
 }
@@ -7797,7 +7803,7 @@ install_macosx_git_deps() {
        export PATH=/usr/local/bin:$PATH
    fi

-    __fetch_url "/tmp/get-pip.py" "https://bootstrap.pypa.io/get-pip.py" || return 1
+    __fetch_url "${_TMP_DIR}/get-pip.py" "https://bootstrap.pypa.io/get-pip.py" || return 1

    if [ -n "$_PY_EXE" ]; then
        _PYEXE="${_PY_EXE}"
@@ -7807,7 +7813,7 @@ install_macosx_git_deps() {
    fi

    # Install PIP
-    $_PYEXE /tmp/get-pip.py || return 1
+    $_PYEXE ${_TMP_DIR}/get-pip.py || return 1

    # shellcheck disable=SC2119
    __git_clone_and_checkout || return 1
@@ -7819,9 +7825,9 @@ install_macosx_stable() {

    install_macosx_stable_deps || return 1

-    __fetch_url "/tmp/${PKG}" "${SALTPKGCONFURL}" || return 1
+    __fetch_url "${_TMP_DIR}/${PKG}" "${SALTPKGCONFURL}" || return 1

-    /usr/sbin/installer -pkg "/tmp/${PKG}" -target / || return 1
+    /usr/sbin/installer -pkg "${_TMP_DIR}/${PKG}" -target / || return 1

    return 0
 }
@@ -7830,9 +7836,9 @@ install_macosx_onedir() {

    install_macosx_onedir_deps || return 1

-    __fetch_url "/tmp/${PKG}" "${SALTPKGCONFURL}" || return 1
+    __fetch_url "${_TMP_DIR}/${PKG}" "${SALTPKGCONFURL}" || return 1

-    /usr/sbin/installer -pkg "/tmp/${PKG}" -target / || return 1
+    /usr/sbin/installer -pkg "${_TMP_DIR}/${PKG}" -target / || return 1

    return 0
 }
--- a/salt/sensoroni/defaults.yaml
+++ b/salt/sensoroni/defaults.yaml
@@ -15,7 +15,7 @@ sensoroni:
    sensoronikey:
    soc_host:
    suripcap:
-      pcapMaxCount: 999999
+      pcapMaxCount: 100000
  analyzers:
      echotrail:
        base_url: https://api.echotrail.io/insights/
--- a/salt/soc/config.sls
+++ b/salt/soc/config.sls
@@ -215,7 +215,6 @@ socsensoronirepos:
    - mode: 775
    - makedirs: True

-
 create_custom_local_yara_repo_template:
  git.present:
    - name: /nsm/rules/custom-local-repos/local-yara
@@ -249,6 +248,39 @@ add_readme_custom_local_sigma_repo_template:
    - context:
        repo_type: "sigma"

+create_custom_local_suricata_repo_template:
+  git.present:
+    - name: /nsm/rules/custom-local-repos/local-suricata
+    - bare: False
+    - force: True
+
+add_readme_custom_local_suricata_repo_template:
+  file.managed:
+    - name: /nsm/rules/custom-local-repos/local-suricata/README
+    - source: salt://soc/files/soc/detections_custom_repo_template_readme.jinja
+    - user: 939
+    - group: 939
+    - template: jinja
+    - context:
+        repo_type: "suricata"
+
+etpro_airgap_folder:
+  file.directory:
+    - name: /nsm/rules/custom-local-repos/local-etpro-suricata
+    - user: 939
+    - group: 939
+    - makedirs: True
+
+add_readme_etpro_airgap_template:
+  file.managed:
+    - name: /nsm/rules/custom-local-repos/local-etpro-suricata/README
+    - source: salt://soc/files/soc/detections_custom_repo_template_readme.jinja
+    - user: 939
+    - group: 939
+    - template: jinja
+    - context:
+        repo_type: "suricata-etpro"
+
 socore_own_custom_repos:
  file.directory:
    - name: /nsm/rules/custom-local-repos/
--- a/salt/soc/defaults.yaml
+++ b/salt/soc/defaults.yaml
@@ -1364,6 +1364,8 @@ soc:
        cases: soc
        filedatastore:
          jobDir: jobs
+          retryFailureIntervalMs: 600000
+          retryFailureMaxAttempts: 5
        kratos:
          hostUrl:
        hydra:
@@ -1586,17 +1588,38 @@ soc:
                insecureSkipVerify: false
                readOnly: true
                deleteUnreferenced: true
+              - name: ABUSECH-SSLBL
+                deleteUnreferenced: true
+                description: 'Abuse.ch SSL Blacklist'
+                enabled: false
+                license: CC0-1.0
+                readOnly: true
+                sourcePath: https://sslbl.abuse.ch/blacklist/sslblacklist_tls_cert.tar.gz
+                sourceType: url
              - name: local-rules
-                id: local-rules
-                description: "Local custom rules from files (*.rules) in a directory on the filesystem"
+                description: "Local rules from files (*.rules) in a directory on the filesystem"
                license: "custom"
                sourceType: directory
-                sourcePath: /nsm/rules/local/
+                sourcePath: /nsm/rules/custom-local-repos/local-suricata
                readOnly: false
                deleteUnreferenced: false
+                enabled: true
+              - name: SO_FILTERS
+                deleteUnreferenced: true
+                description: Filter rules for when Suricata is set as the metadata engine
                enabled: false
-                excludeFiles:
-                  - "*backup*"
+                license: Elastic-2.0
+                readOnly: true
+                sourcePath: /nsm/rules/suricata/so_filters.rules
+                sourceType: directory
+              - name: SO_EXTRACTIONS
+                description: Extraction rules for when Suricata is set as the metadata engine
+                deleteUnreferenced: true
+                enabled: false
+                license: Elastic-2.0
+                readOnly: true
+                sourcePath: /nsm/rules/suricata/so_extraction.rules
+                sourceType: directory
            airgap:
              - name: Emerging-Threats
                description: "Emerging Threats ruleset - To enable ET Pro, enter your license key below. Leave empty for ET Open (free) rules."
@@ -1617,16 +1640,29 @@ soc:
                readOnly: true
                deleteUnreferenced: true
              - name: local-rules
-                id: local-rules
-                description: "Local custom rules from files (*.rules) in a directory on the filesystem"
+                description: "Local rules from files (*.rules) in a directory on the filesystem"
                license: "custom"
                sourceType: directory
-                sourcePath: /nsm/rules/local/
+                sourcePath: /nsm/rules/custom-local-repos/local-suricata
                readOnly: false
                deleteUnreferenced: false
+                enabled: true
+              - name: SO_FILTERS
+                deleteUnreferenced: true
+                description: Filter rules for when Suricata is set as the metadata engine
                enabled: false
-                excludeFiles:
-                  - "*backup*"
+                license: Elastic-2.0
+                readOnly: true
+                sourcePath: /nsm/rules/suricata/so_filters.rules
+                sourceType: directory
+              - name: SO_EXTRACTIONS
+                description: Extraction rules for when Suricata is set as the metadata engine
+                deleteUnreferenced: true
+                enabled: false
+                license: Elastic-2.0
+                readOnly: true
+                sourcePath: /nsm/rules/suricata/so_extraction.rules
+                sourceType: directory
        navigator:
          intervalMinutes: 30
          outputPath: /opt/sensoroni/navigator
@@ -1804,7 +1840,7 @@ soc:
              showSubtitle: true
            - name: DPD
              description: Dynamic Protocol Detection errors
-              query: 'tags:dpd | groupby error.reason'
+              query: '(tags:dpd OR tags:analyzer) | groupby error.reason'
              showSubtitle: true
            - name: Files
              description: Files grouped by mimetype
@@ -2070,7 +2106,7 @@ soc:
              query: 'tags:dns | groupby dns.query.name | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby dns.highest_registered_domain | groupby dns.parent_domain | groupby dns.query.type_name | groupby dns.response.code_name | groupby dns.answers.name | groupby destination.as.organization.name'
            - name: DPD
              description: DPD (Dynamic Protocol Detection) errors
-              query: 'tags:dpd | groupby error.reason | groupby -sankey error.reason source.ip | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby network.protocol | groupby destination.as.organization.name'
+              query: '(tags:dpd OR tags:analyzer) | groupby error.reason | groupby -sankey error.reason source.ip | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby network.protocol | groupby destination.as.organization.name'
            - name: Files
              description: Files seen in network traffic
              query: 'tags:file | groupby file.mime_type | groupby -sankey file.mime_type file.source | groupby file.source | groupby file.bytes.total | groupby source.ip | groupby destination.ip | groupby destination.as.organization.name'
@@ -2612,6 +2648,7 @@ soc:
        assistant:
          enabled: false
          investigationPrompt: Investigate Alert ID {socId}
+          compressContextPrompt: Summarize the conversation for context compaction
          thresholdColorRatioLow: 0.5
          thresholdColorRatioMed: 0.75
          thresholdColorRatioMax: 1
@@ -2621,13 +2658,22 @@ soc:
              contextLimitSmall: 200000
              contextLimitLarge: 1000000
              lowBalanceColorAlert: 500000
+              enabled: true
            - id: sonnet-4.5
              displayName: Claude Sonnet 4.5
              contextLimitSmall: 200000
              contextLimitLarge: 1000000
              lowBalanceColorAlert: 500000
+              enabled: true
            - id: gptoss-120b
              displayName: GPT-OSS 120B
              contextLimitSmall: 128000
              contextLimitLarge: 128000
              lowBalanceColorAlert: 500000
+              enabled: true
+            - id: qwen-235b
+              displayName: QWEN 235B
+              contextLimitSmall: 256000
+              contextLimitLarge: 256000
+              lowBalanceColorAlert: 500000
+              enabled: true
--- a/salt/soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja
+++ b/salt/soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja
@@ -43,10 +43,26 @@

 No Virtual Machines Found
 {%-   endif %}
-{%- else %}
+{%- elif baseDomainStatus == 'ImageDownloadStart' %}
+#### INFO
+
+Base domain image download started.
+{%- elif baseDomainStatus == 'ImageDownloadFailed' %}
+#### ERROR
+
+Base domain image download failed. Please check the salt-master log for details and verify network connectivity.
+{%- elif baseDomainStatus == 'SSHKeySetupFailed' %}
+#### ERROR
+
+SSH key setup failed. Please check the salt-master log for details.
+{%- elif baseDomainStatus == 'SetupFailed' %}
 #### WARNING

-Base domain has not been initialized.
+Setup failed. Please check the salt-master log for details.
+{%- elif baseDomainStatus == 'PreInit' %}
+#### WARNING
+
+Base domain has not been initialized. Waiting for hypervisor to highstate.
 {%- endif %}
 {%- endmacro -%}

--- a/salt/soc/enabled.sls
+++ b/salt/soc/enabled.sls
@@ -27,7 +27,8 @@ so-soc:
      - /opt/so/conf/strelka:/opt/sensoroni/yara:rw
      - /opt/so/conf/sigma:/opt/sensoroni/sigma:rw
      - /opt/so/rules/elastalert/rules:/opt/sensoroni/elastalert:rw
-      - /opt/so/rules/nids/suri:/opt/sensoroni/nids:rw
+      - /opt/so/saltstack/local/salt/suricata/rules:/opt/sensoroni/suricata/rules:rw
+      - /opt/so/saltstack/local/salt/suricata/files:/opt/sensoroni/suricata/threshold:rw
      - /opt/so/conf/soc/fingerprints:/opt/sensoroni/fingerprints:rw
      - /nsm/soc/jobs:/opt/sensoroni/jobs:rw
      - /nsm/soc/uploads:/nsm/soc/uploads:rw
--- a/salt/soc/files/soc/detections_custom_repo_template_readme.jinja
+++ b/salt/soc/files/soc/detections_custom_repo_template_readme.jinja
@@ -45,6 +45,61 @@ Finally, commit it:
 The next time the Strelka / YARA engine syncs, the new rule should be imported
 If there are errors, review the sync log to troubleshoot further.

+{% elif repo_type == 'suricata' %}
+# Suricata Local Custom Rules Repository
+
+This folder has already been initialized as a git repo 
+and your Security Onion grid is configured to import any Suricata rule files found here. 
+
+Just add your rule file and commit it.
+
+For example:
+
+** Note: If this is your first time making changes to this repo, you may run into the following error:
+
+fatal: detected dubious ownership in repository at '/nsm/rules/custom-local-repos/local-suricata'
+To add an exception for this directory, call:
+	git config --global --add safe.directory /nsm/rules/custom-local-repos/local-suricata
+
+This means that the user you are running commands as does not match the user that is used for this git repo (socore).
+You will need to make sure your rule files are accessible to the socore user, so either su to socore 
+or add the exception and then chown the rule files later.
+
+Also, you will be asked to set some configuration:
+```
+Author identity unknown
+*** Please tell me who you are.
+Run
+  git config --global user.email "you@example.com"
+  git config --global user.name "Your Name"
+to set your account's default identity.
+Omit --global to set the identity only in this repository.
+```
+
+Run these commands, ommitting the `--global`.
+
+With that out of the way:
+
+First, create the rule file with a .rules extension:
+`vi my_custom_rules.rules`
+
+Next, use git to stage the new rule to be committed:
+`git add my_custom_rules.rules`
+
+Finally, commit it:
+`git commit -m "Initial commit of my_custom_rule.rules"`
+
+The next time the Suricata engine syncs, the new rule/s should be imported
+If there are errors, review the sync log to troubleshoot further.
+
+{% elif repo_type == 'suricata-etpro' %}
+# Suricata ETPRO - Airgap
+
+This folder has been initialized for use with ETPRO during Airgap deployment.
+
+Just add your ETPRO rule/s file to this folder and the Suricata engine will import them.
+
+If there are errors, review the sync log to troubleshoot further.
 {% elif repo_type == 'sigma' %}
 # Sigma Local Custom Rules Repository

--- a/salt/soc/merged.map.jinja
+++ b/salt/soc/merged.map.jinja
@@ -90,6 +90,17 @@
 {%   endif %}
 {% endif %}

+{# Enable SO_FILTERS and SO_EXTRACTIONS when Suricata is the metadata engine #}
+{% if SOCMERGED.config.server.modules.suricataengine is defined and SOCMERGED.config.server.modules.suricataengine.rulesetSources is defined %}
+{%   if SOCMERGED.config.server.modules.suricataengine.rulesetSources is not mapping %}
+{%     for ruleset in SOCMERGED.config.server.modules.suricataengine.rulesetSources %}
+{%       if ruleset.name in ['SO_FILTERS', 'SO_EXTRACTIONS'] and GLOBALS.md_engine == 'SURICATA' %}
+{%         do ruleset.update({'enabled': true}) %}
+{%       endif %}
+{%     endfor %}
+{%   endif %}
+{% endif %}
+
 {# Transform Emerging-Threats ruleset based on license key #}
 {% if SOCMERGED.config.server.modules.suricataengine is defined and SOCMERGED.config.server.modules.suricataengine.rulesetSources is defined %}
 {%   if SOCMERGED.config.server.modules.suricataengine.rulesetSources is not mapping %}
@@ -97,6 +108,7 @@
 {%     if ruleset.name == 'Emerging-Threats' %}
 {%       if ruleset.licenseKey and ruleset.licenseKey != '' %}
 {#         License key is defined - transform to ETPRO #}
+{#         Engine Version is hardcoded in the URL - this does not change often: https://community.emergingthreats.net/t/supported-engines/71 #}
 {%         do ruleset.update({
             'name': 'ETPRO',
             'sourcePath': 'https://rules.emergingthreatspro.com/' ~ ruleset.licenseKey ~ '/suricata-7.0.3/etpro.rules.tar.gz',
--- a/salt/soc/soc_soc.yaml
+++ b/salt/soc/soc_soc.yaml
@@ -424,6 +424,17 @@ soc:
            description: The maximum number of documents to request in a single Elasticsearch scroll request.
          bulkIndexWorkerCount:
            description: The number of worker threads to use when bulk indexing data into Elasticsearch. A value below 1 will default to the number of CPUs available.
+        filedatastore:
+          jobDir:
+            description: The location where local job files are stored on the manager.
+            global: True
+            advanced: True
+          retryFailureIntervalMs:
+            description: The interval, in milliseconds, to wait before attempting to reprocess a failed job.
+            global: True
+          retryFailureMaxAttempts:
+            description: The max number of attempts to process a job, in the event the job fails to complete.
+            global: True
        sostatus:
          refreshIntervalMs:
            description: Duration (in milliseconds) between refreshes of the grid status. Shortening this duration may not have expected results, as the backend systems feeding this sostatus data will continue their updates as scheduled.
@@ -590,11 +601,11 @@ soc:
                label: Ruleset License
                required: True
              - field: readOnly
-                label: Read Only
+                label: Read Only (Prevents changes to the rule itself - can still be enabled/disabled/tuned)
                forcedType: bool
                required: False
              - field: deleteUnreferenced
-                label: Delete Unreferenced
+                label: Delete Unreferenced (Deletes rules that are no longer referenced by ruleset source)
                forcedType: bool
                required: False
            airgap: *serulesetSources
@@ -652,6 +663,9 @@ soc:
          investigationPrompt: 
            description: Prompt given to Onion AI when beginning an investigation.
            global: True
+          compressContextPrompt:
+            description: Prompt given to Onion AI when summarizing a conversation in order to compress context.
+            global: True
          thresholdColorRatioLow:
            description: Lower visual context color change threshold.
            global: True
@@ -694,6 +708,9 @@ soc:
              label: Low Balance Color Alert
              forcedType: int
              required: True
+            - field: enabled
+              label: Enabled
+              forcedType: bool
        apiTimeoutMs:
          description: Duration (in milliseconds) to wait for a response from the SOC server API before giving up and showing an error on the SOC UI.
          global: True
--- a/salt/suricata/config.sls
+++ b/salt/suricata/config.sls
@@ -7,9 +7,41 @@
 {% if sls.split('.')[0] in allowed_states %}

 {%   from 'vars/globals.map.jinja' import GLOBALS %}
-{%   from 'bpf/suricata.map.jinja' import SURICATABPF %}
 {%   from 'suricata/map.jinja' import SURICATAMERGED %}
-{%   set BPF_STATUS = 0  %}
+{%   from 'bpf/suricata.map.jinja' import SURICATABPF, SURICATA_BPF_STATUS, SURICATA_BPF_CALC %}
+
+{%   if GLOBALS.pcap_engine in ["SURICATA", "TRANSITION"] %}
+{%     from 'bpf/pcap.map.jinja' import PCAPBPF, PCAP_BPF_STATUS, PCAP_BPF_CALC %}
+# BPF compilation and configuration
+{%     if PCAPBPF and not PCAP_BPF_STATUS %}
+suriPCAPbpfcompilationfailure:
+  test.configurable_test_state:
+    - changes: False
+    - result: False
+    - comment: "BPF Syntax Error - Discarding Specified BPF. Error: {{ PCAP_BPF_CALC['stderr'] }}"
+{%     endif %}
+{%   endif %}
+
+# BPF applied to all of Suricata - alerts/metadata/pcap
+suribpf:
+  file.managed:
+    - name: /opt/so/conf/suricata/bpf
+    - user: 940
+    - group: 940
+   {% if SURICATA_BPF_STATUS %}
+    - contents: {{ SURICATABPF }}
+   {% else %}
+    - contents:
+      - ""
+   {% endif %}
+
+{%   if SURICATABPF and not SURICATA_BPF_STATUS %}
+suribpfcompilationfailure:
+  test.configurable_test_state:
+    - changes: False
+    - result: False
+    - comment: "BPF Syntax Error - Discarding Specified BPF. Error: {{ SURICATA_BPF_CALC['stderr'] }}"
+{%   endif %}

 # Add Suricata Group
 suricatagroup:
@@ -53,13 +85,15 @@ suridir:
  file.directory:
    - name: /opt/so/conf/suricata
    - user: 940
-    - group: 940
+    - group: 939
+    - mode: 775

 suriruledir:
  file.directory:
-    - name: /opt/so/conf/suricata/rules
+    - name: /opt/so/rules/suricata
    - user: 940
-    - group: 940
+    - group: 939
+    - mode: 775
    - makedirs: True

 surilogdir:
@@ -84,12 +118,10 @@ suridatadir:
    - mode: 770
    - makedirs: True

-# salt:// would resolve to /opt/so/rules/nids because of the defined file_roots and
-#  not existing under /opt/so/saltstack/local/salt or /opt/so/saltstack/default/salt
 surirulesync:
  file.recurse:
-    - name: /opt/so/conf/suricata/rules/
-    - source: salt://suri/
+    - name: /opt/so/rules/suricata/
+    - source: salt://suricata/rules/
    - user: 940
    - group: 940
    - show_changes: False
@@ -124,10 +156,10 @@ suriconfig:
 surithresholding:
  file.managed:
    - name: /opt/so/conf/suricata/threshold.conf
-    - source: salt://suricata/files/threshold.conf.jinja
+    - source: salt://suricata/files/threshold.conf
    - user: 940
    - group: 940
-    - template: jinja
+    - onlyif: salt://suricata/files/threshold.conf

 suriclassifications:
  file.managed:
@@ -136,32 +168,6 @@ suriclassifications:
    - user: 940
    - group: 940

-# BPF compilation and configuration
-{% if SURICATABPF %}
-   {% set BPF_CALC = salt['cmd.script']('salt://common/tools/sbin/so-bpf-compile', GLOBALS.sensor.interface + ' ' + SURICATABPF|join(" "),cwd='/root') %}
-   {% if BPF_CALC['stderr'] == "" %}
-      {% set BPF_STATUS = 1  %}
-   {% else  %}
-suribpfcompilationfailure:
-  test.configurable_test_state:
-   - changes: False
-   - result: False
-   - comment: "BPF Syntax Error - Discarding Specified BPF"
-   {% endif %}
-{% endif %}
-
-suribpf:
-  file.managed:
-    - name: /opt/so/conf/suricata/bpf
-    - user: 940
-    - group: 940
-   {% if BPF_STATUS %}
-    - contents: {{ SURICATABPF }}
-   {% else %}
-    - contents:
-      - ""
-   {% endif %}
-
 so-suricata-eve-clean:
  file.managed:
    - name: /usr/sbin/so-suricata-eve-clean
--- a/salt/suricata/defaults.yaml
+++ b/salt/suricata/defaults.yaml
@@ -34,7 +34,7 @@ suricata:
      threads: 1
      tpacket-v3: "yes"
      ring-size: 5000
-      block-size: 32768
+      block-size: 69632
      block-timeout: 10
      use-emergency-flush: "yes"
      buffer-size: 32768
@@ -97,6 +97,11 @@ suricata:
          - 4789
        TEREDO_PORTS:
          - 3544
+        SIP_PORTS:
+          - 5060
+          - 5061
+        GENEVE_PORTS:
+          - 6081
    default-log-dir: /var/log/suricata/
    stats:
      enabled: "yes"
@@ -134,14 +139,6 @@ suricata:
              header: X-Forwarded-For
      unified2-alert:
        enabled: "no"
-      http-log:
-        enabled: "no"
-        filename: http.log
-        append: "yes"
-      tls-log:
-        enabled: "no"
-        filename: tls.log
-        append: "yes"
      tls-store:
        enabled: "no"
      pcap-log:
@@ -157,9 +154,6 @@ suricata:
        totals: "yes"
        threads: "no"
        null-values: "yes"
-      syslog:
-        enabled: "no"
-        facility: local5
      drop:
        enabled: "no"
      file-store:
@@ -206,6 +200,9 @@ suricata:
          enabled: "yes"
          detection-ports:
            dp: 443
+          ja3-fingerprints: auto
+          ja4-fingerprints: auto
+          encryption-handling: track-only
        dcerpc:
          enabled: "yes"
        ftp:
@@ -255,19 +252,21 @@ suricata:
          libhtp:
             default-config:
               personality: IDS
-               request-body-limit: 100kb
-               response-body-limit: 100kb
-               request-body-minimal-inspect-size: 32kb
-               request-body-inspect-window: 4kb
-               response-body-minimal-inspect-size: 40kb
-               response-body-inspect-window: 16kb
+               request-body-limit: 100 KiB
+               response-body-limit: 100 KiB
+               request-body-minimal-inspect-size: 32 KiB
+               request-body-inspect-window: 4 KiB
+               response-body-minimal-inspect-size: 40 KiB
+               response-body-inspect-window: 16 KiB
               response-body-decompress-layer-limit: 2
               http-body-inline: auto
               swf-decompression:
-                 enabled: "yes"
+                 enabled: "no"
                 type: both
-                 compress-depth: 0
-                 decompress-depth: 0
+                 compress-depth: 100 KiB
+                 decompress-depth: 100 KiB
+               randomize-inspection-sizes: "yes"
+               randomize-inspection-range: 10
               double-decode-path: "no"
               double-decode-query: "no"
             server-config:
@@ -401,8 +400,12 @@ suricata:
      vxlan:
        enabled: true
        ports: $VXLAN_PORTS
-      erspan:
+      geneve:
        enabled: true
+        ports: $GENEVE_PORTS
+        max-layers: 16
+      recursion-level:
+        use-for-tracking: true
    detect:
      profile: medium
      custom-values:
@@ -422,7 +425,12 @@ suricata:
    spm-algo: auto
    luajit:
      states: 128
-
+    security:
+      lua:
+        allow-rules: false
+        max-bytes: 500000 
+        max-instructions: 500000
+        allow-restricted-functions: false
    profiling:
      rules:
        enabled: "yes"
--- a/salt/suricata/enabled.sls
+++ b/salt/suricata/enabled.sls
@@ -36,7 +36,7 @@ so-suricata:
      - /opt/so/conf/suricata/suricata.yaml:/etc/suricata/suricata.yaml:ro
      - /opt/so/conf/suricata/threshold.conf:/etc/suricata/threshold.conf:ro
      - /opt/so/conf/suricata/classification.config:/etc/suricata/classification.config:ro
-      - /opt/so/conf/suricata/rules:/etc/suricata/rules:ro
+      - /opt/so/rules/suricata:/etc/suricata/rules:ro
      - /opt/so/log/suricata/:/var/log/suricata/:rw
      - /nsm/suricata/:/nsm/:rw
      - /nsm/suricata/extracted:/var/log/suricata//filestore:rw
--- a/salt/suricata/files/so_extraction.rules
+++ b/salt/suricata/files/so_extraction.rules
@@ -0,0 +1,26 @@
+# Extract all PDF mime type
+alert http any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100000; rev:1;)
+alert smtp any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100001; rev:1;)
+alert nfs any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100002; rev:1;)
+alert smb any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100003; rev:1;)
+# Extract EXE/DLL file types
+alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100004; rev:1;)
+alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100005; rev:1;)
+alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100006; rev:1;)
+alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100007; rev:1;)
+alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100008; rev:1;)
+alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100009; rev:1;)
+alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100010; rev:1;)
+alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100011; rev:1;)
+
+# Extract all Zip files
+alert http any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100012; rev:1;)
+alert smtp any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100013; rev:1;)
+alert nfs any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100014; rev:1;)
+alert smb any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100015; rev:1;)
+
+# Extract Word Docs
+alert http any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100016; rev:1;)
+alert smtp any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100017; rev:1;)
+alert nfs any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100018; rev:1;)
+alert smb any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100019; rev:1;)
--- a/salt/suricata/files/so_filters.rules
+++ b/salt/suricata/files/so_filters.rules
@@ -0,0 +1,12 @@
+# Start the filters at sid 1200000
+# Example of filtering out *google.com from being in the dns log.
+#config dns any any -> any any (dns.query; content:"google.com"; config: logging disable, type tx, scope tx; sid:1200000;)
+# Example of filtering out  *google.com from being in the http log.
+#config http any any -> any any (http.host; content:"google.com"; config: logging disable, type tx, scope tx; sid:1200001;)
+# Example of filtering out  someuseragent from being in the http log.
+#config http any any -> any any (http.user_agent; content:"someuseragent"; config: logging disable, type tx, scope tx; sid:1200002;)
+# Example of filtering out  Google's certificate from being in the ssl log.
+#config tls any any -> any any (tls.fingerprint; content:"4f:a4:5e:58:7e:d9:db:20:09:d7:b6:c7:ff:58:c4:7b:dc:3f:55:b4"; config: logging disable, type tx, scope tx; sid:1200003;)
+# Example of filtering out a md5 of a file from being in the files log.
+#config fileinfo any any -> any any (fileinfo.filemd5; content:"7a125dc69c82d5caf94d3913eecde4b5"; config: logging disable, type tx, scope tx; sid:1200004;)
+
--- a/salt/suricata/files/threshold.conf.jinja
+++ b/salt/suricata/files/threshold.conf.jinja
@@ -1,35 +0,0 @@
-{% import_yaml 'suricata/thresholding/sids.yaml' as THRESHOLDING %}
-{% if THRESHOLDING -%}
-  
-  {% for EACH_SID in THRESHOLDING -%}
-    {% for ACTIONS_LIST in THRESHOLDING[EACH_SID] -%}
-      {% for EACH_ACTION in ACTIONS_LIST -%}
-        
-        {%- if EACH_ACTION == 'threshold' %}
-{{ EACH_ACTION }} gen_id {{ ACTIONS_LIST[EACH_ACTION].gen_id }}, sig_id {{ EACH_SID }}, type {{ ACTIONS_LIST[EACH_ACTION].type }}, track {{ ACTIONS_LIST[EACH_ACTION].track }}, count {{ ACTIONS_LIST[EACH_ACTION].count }}, seconds {{ ACTIONS_LIST[EACH_ACTION].seconds }}
-        
-        {%- elif EACH_ACTION == 'rate_filter' %}
-          {%- if ACTIONS_LIST[EACH_ACTION].new_action not in ['drop','reject'] %}
-{{ EACH_ACTION }} gen_id {{ ACTIONS_LIST[EACH_ACTION].gen_id }}, sig_id {{ EACH_SID }}, track {{ ACTIONS_LIST[EACH_ACTION].track }}, count {{ ACTIONS_LIST[EACH_ACTION].count }}, seconds {{ ACTIONS_LIST[EACH_ACTION].seconds }}, new_action {{ ACTIONS_LIST[EACH_ACTION].new_action }}, timeout {{ ACTIONS_LIST[EACH_ACTION].timeout }}
-          {%- else %}
-##### Security Onion does not support drop or reject actions for rate_filter
-##### {{ EACH_ACTION }} gen_id {{ ACTIONS_LIST[EACH_ACTION].gen_id }}, sig_id {{ EACH_SID }}, track {{ ACTIONS_LIST[EACH_ACTION].track }}, count {{ ACTIONS_LIST[EACH_ACTION].count }}, seconds {{ ACTIONS_LIST[EACH_ACTION].seconds }}, new_action {{ ACTIONS_LIST[EACH_ACTION].new_action }}, timeout {{ ACTIONS_LIST[EACH_ACTION].timeout }}
-          {%- endif %}
-        
-        {%- elif EACH_ACTION == 'suppress' %}
-          {%- if ACTIONS_LIST[EACH_ACTION].track is defined %}
-{{ EACH_ACTION }} gen_id {{ ACTIONS_LIST[EACH_ACTION].gen_id }}, sig_id {{ EACH_SID }}, track {{ ACTIONS_LIST[EACH_ACTION].track }}, ip {{ ACTIONS_LIST[EACH_ACTION].ip }}
-          {%- else %}
-{{ EACH_ACTION }} gen_id {{ ACTIONS_LIST[EACH_ACTION].gen_id }}, sig_id {{ EACH_SID }}
-          {%- endif %}
-        
-        {%- endif %}
-      
-      {%- endfor %}
-    {%- endfor %}
-  {%- endfor %}
-
-{%- else %}
-##### Navigate to suricata > thresholding > SIDS in SOC to define thresholding
-
-{%- endif %}
--- a/salt/suricata/manager.sls
+++ b/salt/suricata/manager.sls
@@ -1,30 +0,0 @@
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls in allowed_states %}
-
-surilocaldir:
-  file.directory:
-    - name: /opt/so/saltstack/local/salt/suricata
-    - user: socore
-    - group: socore
-    - makedirs: True
-
-ruleslink:
-  file.symlink:
-    - name: /opt/so/saltstack/local/salt/suricata/rules
-    - user: socore
-    - group: socore
-    - target: /opt/so/rules/nids/suri
-
-refresh_salt_master_fileserver_suricata_ruleslink:
-  salt.runner:
-    - name: fileserver.update
-    - onchanges:
-      - file: ruleslink
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}
--- a/salt/suricata/map.jinja
+++ b/salt/suricata/map.jinja
@@ -10,6 +10,12 @@

 {# before we change outputs back to list, enable pcap-log if suricata is the pcapengine #}
 {% if GLOBALS.pcap_engine in ["SURICATA", "TRANSITION"] %}
+
+{%   from 'bpf/pcap.map.jinja' import PCAPBPF, PCAP_BPF_STATUS %}
+{%   if PCAPBPF and PCAP_BPF_STATUS %}
+{%     do SURICATAMERGED.config.outputs['pcap-log'].update({'bpf-filter': PCAPBPF|join(" ")}) %}
+{%   endif %}
+
 {%   do SURICATAMERGED.config.outputs['pcap-log'].update({'enabled': 'yes'}) %}
 {#   move the items in suricata.pcap into suricata.config.outputs.pcap-log. these items were placed under suricata.config for ease of access in SOC #}
 {%   do SURICATAMERGED.config.outputs['pcap-log'].update({'compression': SURICATAMERGED.pcap.compression}) %}
--- a/salt/suricata/rules/PLACEHOLDER
+++ b/salt/suricata/rules/PLACEHOLDER
--- a/salt/suricata/soc_suricata.yaml
+++ b/salt/suricata/soc_suricata.yaml
@@ -190,6 +190,8 @@ suricata:
        FTP_PORTS: *suriportgroup
        VXLAN_PORTS: *suriportgroup
        TEREDO_PORTS: *suriportgroup
+        SIP_PORTS: *suriportgroup
+        GENEVE_PORTS: *suriportgroup
    outputs:
      eve-log:
        types:
@@ -209,7 +211,7 @@ suricata:
                helpLink: suricata.html
      pcap-log:
        enabled:
-          description: This value is ignored by SO. pcapengine in globals takes precidence.
+          description: This value is ignored by SO. pcapengine in globals takes precedence.
          readonly: True
          helpLink: suricata.html
          advanced: True        
@@ -297,3 +299,10 @@ suricata:
        ports: 
          description: Ports to listen for. This should be a variable.    
          helpLink: suricata.html
+      geneve:
+        enabled:
+          description: Enable VXLAN capabilities.
+          helpLink: suricata.html
+        ports: 
+          description: Ports to listen for. This should be a variable.    
+          helpLink: suricata.html
--- a/salt/suricata/suricata_mdengine.yaml
+++ b/salt/suricata/suricata_mdengine.yaml
@@ -29,7 +29,7 @@ suricata:
                #custom: [Accept-Encoding, Accept-Language, Authorization]
                # dump-all-headers: none
            - dns:
-                version: 2
+                version: 3
                enabled: "yes"
                #requests: "no"
                #responses: "no"
--- a/salt/suricata/tools/sbin/so-suricata-reload-rules
+++ b/salt/suricata/tools/sbin/so-suricata-reload-rules
@@ -7,5 +7,5 @@

 . /usr/sbin/so-common

-retry 60 3 'docker exec so-suricata /opt/suricata/bin/suricatasc -c reload-rules /var/run/suricata/suricata-command.socket' '{"message": "done", "return": "OK"}' || fail "The Suricata container was not ready in time."
-retry 60 3 'docker exec so-suricata /opt/suricata/bin/suricatasc -c ruleset-reload-nonblocking /var/run/suricata/suricata-command.socket' '{"message": "done", "return": "OK"}' || fail "The Suricata container was not ready in time."
+retry 60 3 'docker exec so-suricata /opt/suricata/bin/suricatasc -c reload-rules /var/run/suricata/suricata-command.socket' '{"message":"done","return":"OK"}' || fail "The Suricata container was not ready in time."
+retry 60 3 'docker exec so-suricata /opt/suricata/bin/suricatasc -c ruleset-reload-nonblocking /var/run/suricata/suricata-command.socket' '{"message":"done","return":"OK"}' || fail "The Suricata container was not ready in time."
--- a/salt/top.sls
+++ b/salt/top.sls
@@ -74,7 +74,6 @@ base:
    - sensoroni
    - telegraf
    - firewall
-    - suricata.manager
    - healthcheck
    - elasticsearch
    - elastic-fleet-package-registry
@@ -105,7 +104,6 @@ base:
    - firewall
    - sensoroni
    - telegraf
-    - suricata.manager
    - healthcheck
    - elasticsearch
    - logstash
@@ -140,7 +138,6 @@ base:
    - sensoroni
    - telegraf
    - backup.config_backup
-    - suricata.manager
    - elasticsearch
    - logstash
    - redis
@@ -174,7 +171,6 @@ base:
    - sensoroni
    - telegraf
    - backup.config_backup
-    - suricata.manager
    - elasticsearch
    - logstash
    - redis
@@ -204,7 +200,6 @@ base:
    - sensoroni
    - telegraf
    - firewall
-    - suricata.manager
    - pcap
    - elasticsearch
    - elastic-fleet-package-registry
--- a/salt/zeek/config.sls
+++ b/salt/zeek/config.sls
@@ -8,8 +8,7 @@

 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% from "zeek/config.map.jinja" import ZEEKMERGED %}
-{% from 'bpf/zeek.map.jinja' import ZEEKBPF %}
-{% set BPF_STATUS = 0  %}
+{% from 'bpf/zeek.map.jinja' import ZEEKBPF, ZEEK_BPF_STATUS, ZEEK_BPF_CALC %}

 # Add Zeek group
 zeekgroup:
@@ -158,18 +157,13 @@ zeekja4cfg:
    - user: 937
    - group: 939

-# BPF compilation and configuration
-{% if ZEEKBPF %}
-   {% set BPF_CALC = salt['cmd.script']('salt://common/tools/sbin/so-bpf-compile', GLOBALS.sensor.interface + ' ' + ZEEKBPF|join(" "),cwd='/root') %}
-   {% if BPF_CALC['stderr'] == "" %}
-       {% set BPF_STATUS = 1  %}
-  {% else  %}
+# BPF compilation failed
+{% if ZEEKBPF and not ZEEK_BPF_STATUS %}
 zeekbpfcompilationfailure:
  test.configurable_test_state:
    - changes: False
    - result: False
-    - comment: "BPF Syntax Error - Discarding Specified BPF"
-   {% endif %}
+    - comment: "BPF Syntax Error - Discarding Specified BPF. Error: {{ ZEEK_BPF_CALC['stderr'] }}"
 {% endif %}

 zeekbpf:
@@ -177,7 +171,7 @@ zeekbpf:
    - name: /opt/so/conf/zeek/bpf
    - user: 940
    - group: 940
-{% if BPF_STATUS %}
+{% if ZEEK_BPF_STATUS %}
    - contents: {{ ZEEKBPF }}
 {% else %}
    - contents:
--- a/salt/zeek/defaults.yaml
+++ b/salt/zeek/defaults.yaml
@@ -45,7 +45,7 @@ zeek:
        - protocols/ssh/geo-data
        - protocols/ssh/detect-bruteforcing
        - protocols/ssh/interesting-hostnames
-        - protocols/http/detect-sqli
+        - protocols/http/detect-sql-injection
        - frameworks/files/hash-all-files
        - frameworks/files/detect-MHR
        - policy/frameworks/notice/extend-email/hostnames
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -502,6 +502,7 @@ configure_minion() {
 		minion_type=desktop
 	fi
 	info "Configuring minion type as $minion_type" 
+	logCmd "mkdir -p /etc/salt/minion.d"
 	echo "role: so-$minion_type" > /etc/salt/grains
 	
 	local minion_config=/etc/salt/minion
@@ -541,20 +542,6 @@ configure_minion() {
 		"log_file: /opt/so/log/salt/minion"\
 		"#startup_states: highstate" >> "$minion_config"

-	# At the time the so-managerhype node does not yet have the bridge configured.
-	# The so-hypervisor node doesn't either, but it doesn't cause issues here.
-	local usebr0=false
-	if [ "$minion_type" == 'hypervisor' ]; then
-    	usebr0=true
-	fi
-	local pillar_json="{\"host\": {\"mainint\": \"$MNIC\"}, \"usebr0\": $usebr0}"
-	info "Running: salt-call state.apply salt.mine_functions --local --file-root=../salt/ -l info pillar='$pillar_json'"
-	salt-call state.apply salt.mine_functions --local --file-root=../salt/ -l info pillar="$pillar_json"
-
-	{
-		logCmd "systemctl enable salt-minion";
-		logCmd "systemctl restart salt-minion";
-	} >> "$setup_log" 2>&1
 }

 checkin_at_boot() {
@@ -729,7 +716,7 @@ configure_network_sensor() {
 	fi

 	# Create the bond interface only if it doesn't already exist
-	nmcli -f name,uuid -p con | grep -q '$INTERFACE'
+	nmcli -f name,uuid -p con | grep -q "$INTERFACE"
 	local found_int=$?

 	if [[ $found_int != 0 ]]; then
@@ -798,25 +785,18 @@ configure_hyper_bridge() {
 }

 copy_salt_master_config() {
-
+	logCmd "mkdir /etc/salt"
 	title "Copy the Salt master config template to the proper directory"
 	if [ "$setup_type" = 'iso' ]; then
 		logCmd "cp /root/SecurityOnion/files/salt/master/master /etc/salt/master"
-		#logCmd "cp /root/SecurityOnion/files/salt/master/salt-master.service /usr/lib/systemd/system/salt-master.service"
 	else
 		logCmd "cp ../files/salt/master/master /etc/salt/master"
-		#logCmd "cp ../files/salt/master/salt-master.service /usr/lib/systemd/system/salt-master.service"
 	fi
 	info "Copying pillar and salt files in $temp_install_dir to $local_salt_dir"
 	logCmd "cp -R $temp_install_dir/pillar/ $local_salt_dir/"
 	if [ -d "$temp_install_dir"/salt ] ; then
 		logCmd "cp -R $temp_install_dir/salt/ $local_salt_dir/"
 	fi
-
-	# Restart the service so it picks up the changes
-	logCmd "systemctl daemon-reload"
-	logCmd "systemctl enable salt-master"
-	logCmd "systemctl restart salt-master"
 }

 create_local_nids_rules() {
@@ -1935,11 +1915,12 @@ repo_sync_local() {
 }

 saltify() {
-	info "Installing Salt"
 	SALTVERSION=$(grep "version:" ../salt/salt/master.defaults.yaml | grep -o "[0-9]\+\.[0-9]\+")
+	info "Installing Salt $SALTVERSION"
+	chmod u+x ../salt/salt/scripts/bootstrap-salt.sh
 	if [[ $is_deb ]]; then

-		DEBIAN_FRONTEND=noninteractive retry 150 20 "apt-get -y -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" upgrade" >> "$setup_log" 2>&1 || fail_setup
+		DEBIAN_FRONTEND=noninteractive retry 30 10 "apt-get -y -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" upgrade" >> "$setup_log" 2>&1 || fail_setup
 		if [ $OSVER == "focal" ]; then update-alternatives --install /usr/bin/python python /usr/bin/python3.10 10; fi
 		local pkg_arr=(
 			'apache2-utils'
@@ -1952,16 +1933,11 @@ saltify() {
 			'jq'
 			'gnupg'
 		)
-		retry 150 20 "apt-get -y install ${pkg_arr[*]}" || fail_setup
+		retry 30 10 "apt-get -y install ${pkg_arr[*]}" || fail_setup

 		logCmd "mkdir -vp /etc/apt/keyrings"
 		logCmd "wget -q --inet4-only -O /etc/apt/keyrings/docker.pub https://download.docker.com/linux/ubuntu/gpg"

-		# Download public key
-		logCmd "curl -fsSL -o /etc/apt/keyrings/salt-archive-keyring-2023.pgp https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public"
-		# Create apt repo target configuration
-		echo "deb [signed-by=/etc/apt/keyrings/salt-archive-keyring-2023.pgp arch=amd64] https://packages.broadcom.com/artifactory/saltproject-deb/ stable main" | sudo tee /etc/apt/sources.list.d/salt.list
-
 		if [[ $is_ubuntu ]]; then
 			# Add Docker Repo
 			add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" 
@@ -1972,45 +1948,50 @@ saltify() {
 			echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian $OSVER stable" > /etc/apt/sources.list.d/docker.list 
 		fi

-		logCmd "apt-key add /etc/apt/keyrings/salt-archive-keyring-2023.pgp"
-
-		#logCmd "apt-key add /opt/so/gpg/SALTSTACK-GPG-KEY.pub"
 		logCmd "apt-key add /etc/apt/keyrings/docker.pub"

-		# Add SO Saltstack Repo
-		#echo "deb https://repo.securityonion.net/file/securityonion-repo/ubuntu/20.04/amd64/salt3004.2/ focal main" > /etc/apt/sources.list.d/saltstack.list
-
-		# Ain't nothing but a GPG 
-
-		retry 150 20 "apt-get update" "" "Err:" || fail_setup
+		retry 30 10 "apt-get update" "" "Err:" || fail_setup
 		if [[ $waitforstate ]]; then
-			retry 150 20 "apt-get -y install salt-common=$SALTVERSION salt-minion=$SALTVERSION salt-master=$SALTVERSION" || fail_setup
-			retry 150 20 "apt-mark hold salt-minion salt-common salt-master" || fail_setup
-			retry 150 20 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-packaging python3-influxdb python3-lxml" || exit 1
+			retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -M -X stable $SALTVERSION" || fail_setup
+			retry 30 10 "apt-mark hold salt-minion salt-common salt-master" || fail_setup
+			retry 30 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-packaging python3-influxdb python3-lxml" || exit 1
 		else
-			retry 150 20 "apt-get -y install salt-common=$SALTVERSION salt-minion=$SALTVERSION" || fail_setup
-			retry 150 20 "apt-mark hold salt-minion salt-common" || fail_setup
+			retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -X stable $SALTVERSION" || fail_setup
+			retry 30 10 "apt-mark hold salt-minion salt-common" || fail_setup
 		fi
 	fi
 		
 	if [[ $is_rpm ]]; then
 		if [[ $waitforstate ]]; then 
 			# install all for a manager
-			logCmd "dnf -y install salt-$SALTVERSION salt-master-$SALTVERSION salt-minion-$SALTVERSION"
+			retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -M -X stable $SALTVERSION" || fail_setup
 		else
-			# We just need the minion
-			if [[ $is_airgap ]]; then
-			    logCmd "dnf -y install salt salt-minion"
-			else
-			    logCmd "dnf -y install salt-$SALTVERSION salt-minion-$SALTVERSION"
-			fi
+			# just a minion
+			retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -X stable $SALTVERSION" || fail_setup
 		fi
 	fi

-	logCmd "mkdir -p /etc/salt/minion.d"
 	salt_install_module_deps
 	salt_patch_x509_v2

+	# At the time the so-managerhype node does not yet have the bridge configured.
+	# The so-hypervisor node doesn't either, but it doesn't cause issues here.
+	local usebr0=false
+	if [ "$minion_type" == 'hypervisor' ]; then
+    	usebr0=true
+	fi
+	local pillar_json="{\"host\": {\"mainint\": \"$MNIC\"}, \"usebr0\": $usebr0}"
+	info "Running: salt-call state.apply salt.mine_functions --local --file-root=../salt/ -l info pillar='$pillar_json'"
+	salt-call state.apply salt.mine_functions --local --file-root=../salt/ -l info pillar="$pillar_json"
+
+	if [[ $waitforstate ]]; then 
+		logCmd "systemctl enable salt-master";
+		logCmd "systemctl start salt-master";
+	fi
+
+	logCmd "systemctl enable salt-minion";
+	logCmd "systemctl restart salt-minion";
+
 }

 salt_install_module_deps() {
--- a/setup/so-setup
+++ b/setup/so-setup
@@ -745,13 +745,12 @@ if ! [[ -f $install_opt_file ]]; then
 		securityonion_repo
 		# Update existing packages
 		update_packages
-		# Install salt
-		saltify
-		# Start the master service
+		# Put salt-master config in place
 		copy_salt_master_config
 		configure_minion "$minion_type"
+		# Install salt
+		saltify
 		check_sos_appliance
-
 		logCmd "salt-key -yd $MINION_ID"
 		sleep 2 # Debug RSA Key format errors
 		logCmd "salt-call state.show_top"
@@ -852,8 +851,8 @@ if ! [[ -f $install_opt_file ]]; then
 		gpg_rpm_import
 		securityonion_repo
 		update_packages
-		saltify
 		configure_minion "$minion_type"
+		saltify
 		check_sos_appliance
 		drop_install_options
 		hypervisor_local_states
--- a/setup/so-whiptail
+++ b/setup/so-whiptail
@@ -676,8 +676,8 @@ whiptail_install_type_dist_existing() {
 	EOM
 	
 	install_type=$(whiptail --title "$whiptail_title" --menu "$node_msg" 19 75 7 \
-		"SENSOR" "Create a forward only sensor " \
-		"SEARCHNODE" "Add a search node with parsing " \
+		"SENSOR" "Add a Sensor Node for monitoring network traffic " \
+		"SEARCHNODE" "Add a Search Node with parsing " \
 		"FLEET" "Dedicated Elastic Fleet Node " \
 		"HEAVYNODE" "Sensor + Search Node " \
 		"IDH" "Intrusion Detection Honeypot Node " \
Author	SHA1	Message	Date
DefensiveDepth	bda83a47a2	Remove header	2025-11-29 17:45:22 -05:00
DefensiveDepth	e96cfd35f7	Refactor for simplicity	2025-11-29 17:00:51 -05:00
DefensiveDepth	65c96b2edf	Add error handling	2025-11-29 16:27:22 -05:00
DefensiveDepth	87477ae4f6	Removed uneeded bind	2025-11-29 15:40:10 -05:00
DefensiveDepth	89a9106d79	Add context	2025-11-29 15:17:28 -05:00
DefensiveDepth	1284150382	Move to manager init	2025-11-27 08:39:19 -05:00
DefensiveDepth	4bb0a7c9d9	Merge remote-tracking branch 'origin/2.4/dev' into idstools-refactor	2025-11-25 13:52:21 -05:00
DefensiveDepth	ced3af818c	Refactor for Airgap	2025-11-25 13:51:50 -05:00
Josh Patterson	9c06713f32	Merge pull request #15251 from Security-Onion-Solutions/bravo use timestamp in volume path to prevent duplicates	2025-11-21 14:54:30 -05:00
Josh Patterson	23da0d4ba0	use timestamp in filename to prevent duplicates	2025-11-21 14:49:03 -05:00
Josh Patterson	d5f2cfb354	Merge pull request #15248 from Security-Onion-Solutions/bravo clarify hypervisor annotation	2025-11-20 17:28:32 -05:00
Josh Patterson	fb5ad4193d	indicate base image download start	2025-11-20 17:13:36 -05:00
Josh Patterson	1f5f283c06	update hypervisor annotaion. preinit instead of initialized	2025-11-20 16:53:55 -05:00
Josh Patterson	cf048030c4	Merge pull request #15247 from Security-Onion-Solutions/bravo Notify user of hypervisor environment setup failures	2025-11-20 16:04:49 -05:00
Josh Patterson	2d716b44a8	update comment	2025-11-20 15:52:21 -05:00
Jorge Reyes	d70d652310	Merge pull request #15244 from Security-Onion-Solutions/reyesj2/suricapfile suricata capture file	2025-11-20 14:31:43 -06:00
reyesj2	c5db7c8752	suricata.capture_file keyword	2025-11-20 14:26:12 -06:00
reyesj2	6f42ff3442	suricata capture_file	2025-11-20 14:16:49 -06:00
reyesj2	433dab7376	format json	2025-11-20 14:16:10 -06:00
Josh Patterson	97c1a46013	update annotation for general failure	2025-11-20 15:08:04 -05:00
Josh Patterson	fbe97221bb	set initialized status	2025-11-20 14:43:09 -05:00
Josh Patterson	841ce6b6ec	update hypervisor annotation for image download or ssh key creation failure	2025-11-20 13:55:22 -05:00
Josh Patterson	dd0b4c3820	fix failed or hung qcow2 image download	2025-11-19 15:48:53 -05:00
Josh Patterson	b407c68d88	Merge remote-tracking branch 'origin/2.4/dev' into bravo	2025-11-19 10:23:11 -05:00
Josh Patterson	5b6a7035af	need python_shell for pipes	2025-11-19 10:22:58 -05:00
Jason Ertel	12d490ad4a	Merge pull request #15240 from Security-Onion-Solutions/jertel/wip communicate to the viewer that OS patches may take some time	2025-11-19 10:01:03 -05:00
Jason Ertel	76cbd18d2c	communicate to the viewer that OS patches may take some time	2025-11-19 09:56:42 -05:00
DefensiveDepth	148ef7ef21	add default ruleset	2025-11-18 11:57:30 -05:00
DefensiveDepth	1b55642c86	Refactor rules location	2025-11-18 09:58:14 -05:00
DefensiveDepth	af7f7d0728	Fix file paths	2025-11-17 12:00:08 -05:00
Jorge Reyes	a7337c95e1	Merge pull request #15234 from Security-Onion-Solutions/reyesj2/pipeline-upd update zeek pipelines	2025-11-17 10:36:10 -06:00
Josh Patterson	3f7c3326ea	Merge pull request #15237 from Security-Onion-Solutions/bravo rm salt keyring and repo file for deb	2025-11-17 09:27:53 -05:00
Josh Patterson	bf41de8c14	rm salt keyring and repo file for deb	2025-11-17 08:56:02 -05:00
reyesj2	136a829509	detect-sqli deprecated in favor of detect-sql-injection	2025-11-14 16:51:00 -06:00
reyesj2	bcec999be4	zeek.dns reduce errors	2025-11-14 15:47:29 -06:00
reyesj2	7c73b4713f	update analyzer pipeline	2025-11-14 15:47:29 -06:00
reyesj2	45b4b1d963	ingest zeek analyzer.log + update dpd dashboard with analyzer tag	2025-11-14 15:47:29 -06:00
reyesj2	fcfd74ec1e	zeek.analyzer format json	2025-11-14 15:47:29 -06:00
reyesj2	68b0cd7549	rename zeek.dpd zeek.analyzer	2025-11-14 15:47:29 -06:00
reyesj2	715d801ce8	format json zeek.dns	2025-11-14 15:47:19 -06:00
Jorge Reyes	4a810696e7	Merge pull request #15231 from Security-Onion-Solutions/reyesj2/bond0 fix so-setup error duplicate bond0	2025-11-14 12:12:46 -06:00
reyesj2	6b525a2c21	fix so-setup error duplicate bond0	2025-11-14 11:19:32 -06:00
Jorge Reyes	a5d8385f07	Merge pull request #15230 from Security-Onion-Solutions/reyesj2/pipeline-upd suricata pipeline updates	2025-11-14 10:43:33 -06:00
reyesj2	211bf7e77b	ignore errors on tld script	2025-11-14 09:25:19 -06:00
reyesj2	1542b74133	move dns tld fields to its own pipeline	2025-11-14 09:24:58 -06:00
DefensiveDepth	431e5abf89	Extract ETPRO key if found	2025-11-14 09:39:33 -05:00
reyesj2	4314c79f85	bump suricata dns logging version	2025-11-14 08:24:31 -06:00
reyesj2	da9717bc79	don't attempt rename if field doesn't exist -- reducing pipeline stat errors	2025-11-14 08:15:40 -06:00
DefensiveDepth	f047677d8a	Check correct files	2025-11-14 09:03:08 -05:00
Jason Ertel	045cf7866c	Merge pull request #15225 from Security-Onion-Solutions/jertel/wip pcap annotations	2025-11-14 08:37:37 -05:00
reyesj2	431e0b0780	format suricata.alert json	2025-11-13 19:29:50 -06:00
reyesj2	e782266caa	suricata 8 dns v3	2025-11-13 19:21:31 -06:00
coreyogburn	a4666b2c08	Merge pull request #15229 from Security-Onion-Solutions/cogburn/toggle-models Add Enabled Flag to Models	2025-11-13 16:13:24 -07:00
Corey Ogburn	dcc3206e51	Add Enabled Flag to Models	2025-11-13 15:32:28 -07:00
Josh Patterson	8358b6ea6f	Merge pull request #15228 from Security-Onion-Solutions/bravo wait for 200 from registry before proceeding	2025-11-13 16:34:43 -05:00
coreyogburn	d1a66a91c6	Merge pull request #15221 from Security-Onion-Solutions/cogburn/compress-context CompressContextPrompt	2025-11-13 14:33:56 -07:00
Josh Patterson	7fdcb92614	wait for 200 from registry before proceeding	2025-11-13 16:30:58 -05:00
Jason Ertel	cec1890b6b	pcap annotations	2025-11-13 16:15:47 -05:00
DefensiveDepth	b2606b6094	fix perms	2025-11-13 14:10:51 -05:00
Corey Ogburn	b1b66045ea	Change in prompt wording	2025-11-13 12:08:47 -07:00
Corey Ogburn	33b22bf2e4	Shorten Prompt	2025-11-13 11:09:09 -07:00
Corey Ogburn	3a38886345	CompressContextPrompt	2025-11-13 11:09:08 -07:00
reyesj2	7be70faab6	format json	2025-11-13 10:49:37 -06:00
Josh Patterson	2729fdbea6	Merge pull request #15223 from Security-Onion-Solutions/bravo configure salt, then install. update bootstrap-salt. reduce salt install fail timeout	2025-11-13 11:35:43 -05:00
Jorge Reyes	bfd08d1d2e	Merge pull request #15204 from Security-Onion-Solutions/reyesj2/retention update so-elasticsearch-retention-estimate	2025-11-13 10:05:49 -06:00
DefensiveDepth	37b3fd9b7b	add detections backup	2025-11-13 10:41:12 -05:00
DefensiveDepth	573dded921	refactor to hash	2025-11-13 09:25:20 -05:00
Josh Patterson	fed75c7b39	use -r with bootstrap to disable script repo	2025-11-12 19:47:25 -05:00
Josh Patterson	3427df2a54	update bootstrap-salt to latest	2025-11-12 18:07:14 -05:00
Josh Patterson	be11c718f6	configure salt then install it	2025-11-12 18:06:55 -05:00
Josh Patterson	235dfd78f1	Revert "salt-minion service KillMode to control-group" This reverts commit `7c8b9b4374`.	2025-11-12 14:20:28 -05:00
Josh Patterson	7c8b9b4374	salt-minion service KillMode to control-group	2025-11-12 12:30:29 -05:00
DefensiveDepth	81d7c313af	remove dupe	2025-11-12 11:11:01 -05:00
DefensiveDepth	9a6ff75793	Merge remote-tracking branch 'origin/2.4/dev' into idstools-refactor	2025-11-12 08:51:51 -05:00
DefensiveDepth	1f24796eba	Fix ETPRO check	2025-11-12 08:48:47 -05:00
Jason Ertel	7762faf075	Merge pull request #15219 from Security-Onion-Solutions/jertel/wip add support to so-yaml for using yaml file content for values	2025-11-12 08:12:23 -05:00
Jason Ertel	80fbb31372	fix test	2025-11-11 17:04:19 -05:00
Jason Ertel	7c45db2295	add support to so-yaml for using yaml file content for values	2025-11-11 16:57:54 -05:00
Jason Ertel	0545e1d33b	add support to so-yaml for using yaml file content for values	2025-11-11 16:55:00 -05:00
DefensiveDepth	55bbbdb58d	idstools removal refactor	2025-11-11 14:34:28 -05:00
DefensiveDepth	3a8a6bf5ff	idstools removal refactor	2025-11-11 14:12:51 -05:00
DefensiveDepth	13789bc56f	idstools removal refactor	2025-11-11 13:45:37 -05:00
DefensiveDepth	11518f6eea	idstools removal refactor	2025-11-11 13:41:32 -05:00
Jason Ertel	08147e27b0	Merge pull request #15213 from Security-Onion-Solutions/jertel/wip reduce pcapMaxCount to fit better with max upload size	2025-11-10 19:08:58 -05:00
Josh Patterson	c9153617be	Merge pull request #15211 from Security-Onion-Solutions/bravo Suricata 8.0.2	2025-11-10 17:09:43 -05:00
Josh Patterson	245ceb2d49	suricata defaults and annotation	2025-11-10 16:40:11 -05:00
Jason Ertel	4c65975907	reduce pcapMaxCount to fit better with max upload size	2025-11-10 15:44:05 -05:00
Mike Reeves	dfef7036ce	Merge pull request #15209 from Security-Onion-Solutions/TOoSmOotH-patch-1 Update defaults.yaml	2025-11-10 14:53:00 -05:00
Mike Reeves	44594ba726	Update defaults.yaml	2025-11-10 14:24:27 -05:00
Josh Patterson	1876c4d9df	fix var name	2025-11-10 14:16:16 -05:00
Josh Patterson	a2ff66b5d0	update annotation	2025-11-10 14:12:20 -05:00
Josh Patterson	e3972dc5af	Merge remote-tracking branch 'origin/2.4/dev' into bravo	2025-11-10 13:28:42 -05:00
Josh Patterson	18c0f197b2	suricata bpf	2025-11-10 13:28:19 -05:00
Jorge Reyes	5b371c220c	Merge pull request #15207 from Security-Onion-Solutions/reyesj2/forwardnode-sensor	2025-11-10 08:46:12 -06:00
Josh Patterson	78c193f0a2	handle bpf for suricata 8 pcap	2025-11-07 17:40:24 -05:00
Josh Patterson	274295bc97	return exit codes	2025-11-07 17:39:13 -05:00
Josh Patterson	6c7ef622c1	spaces removed from expected output	2025-11-07 17:08:33 -05:00
Josh Patterson	da1cac0d53	tls-log, http-log and syslog outputs deprecated https://github.com/Security-Onion-Solutions/securityonion/issues/15203	2025-11-06 16:32:55 -05:00
reyesj2	a84df14137	rename forward node -> sensor node	2025-11-06 15:23:55 -06:00
Jorge Reyes	4a49f9d004	Merge branch '2.4/dev' into reyesj2/retention	2025-11-06 14:29:08 -06:00
reyesj2	1eb4b5379a	show 30d scheduled deletions or 7d scheduled deletions depending on what historical data is available	2025-11-06 14:25:25 -06:00
reyesj2	35c7fc06d7	fix bug showing duplicate backing indices in recommendations	2025-11-06 14:24:58 -06:00
reyesj2	b69d453a68	typo	2025-11-06 14:24:29 -06:00
Josh Patterson	b7e1989d45	resolve block-size not large enough for max fragmented IP packet size warning	2025-11-06 09:49:46 -05:00
Josh Patterson	4ffbb0bbd9	Merge remote-tracking branch 'origin/2.4/dev' into bravo	2025-11-05 15:22:11 -05:00
Josh Patterson	92be8df95d	Merge pull request #15122 from Security-Onion-Solutions/amv nsm virtual disk and new nsm_total grain	2025-10-08 14:15:51 -04:00
Josh Patterson	b1acbf3114	Merge pull request #15098 from Security-Onion-Solutions/byoh Byoh	2025-10-01 15:06:01 -04:00
Josh Patterson	8043e09ec1	Merge pull request #15076 from Security-Onion-Solutions/vlb2 Vlb2	2025-09-26 15:44:53 -04:00
Josh Patterson	25c746bb14	Merge pull request #15067 from Security-Onion-Solutions/vlb2 Vlb2	2025-09-25 16:12:52 -04:00
Josh Patterson	a91e8b26f6	Merge pull request #15066 from Security-Onion-Solutions/vlb2 set interface for network.ip_addrs for hypervisors	2025-09-24 16:51:07 -04:00
Josh Patterson	e826ea5d04	Merge pull request #15065 from Security-Onion-Solutions/vlb2 update service file, use salt.minion state to update mine_functions	2025-09-24 15:20:31 -04:00
Josh Patterson	23a9780ebb	Merge pull request #15061 from Security-Onion-Solutions/vlb2 only update mine for managerhype during setup	2025-09-23 15:56:47 -04:00
Josh Patterson	9cb8ebbaa7	Merge pull request #15056 from Security-Onion-Solutions/vlb2 Vlb2	2025-09-23 09:05:55 -04:00
Josh Patterson	03892bad5e	Merge pull request #15015 from Security-Onion-Solutions/vlb2 Vlb2	2025-09-10 14:58:41 -04:00
Josh Patterson	77fef02116	Merge pull request #14994 from Security-Onion-Solutions/vlb2 pass pillar properly	2025-09-04 11:06:31 -04:00
Josh Patterson	f3328c41fb	Merge pull request #14990 from Security-Onion-Solutions/vlb2 merge with 2.4/dev	2025-09-03 10:37:46 -04:00
Josh Patterson	23ae259c82	Merge pull request #14972 from Security-Onion-Solutions/vlb2 Vlb2	2025-08-28 10:41:23 -04:00
Josh Patterson	45f25ca62d	Merge pull request #14966 from Security-Onion-Solutions/vlb2 managerhype	2025-08-26 15:07:36 -04:00