Update Validation

Add trailing nl if it doesnt already exist

.github/workflows/pythontest.yml

@@ -4,7 +4,7 @@ on:
   pull_request:
     paths:
       - "salt/sensoroni/files/analyzers/**"
-      - "salt/manager/tools/sbin"
+      - "salt/manager/tools/sbin/**"
 
 jobs:
   build:

salt/_runners/setup_hypervisor.py

@@ -172,7 +172,15 @@ MANAGER_HOSTNAME = socket.gethostname()
 
 def _download_image():
     """
-    Download and validate the Oracle Linux KVM image.
+    Download and validate the Oracle Linux KVM image with retry logic and progress monitoring.
+    
+    Features:
+    - Detects stalled downloads (no progress for 30 seconds)
+    - Retries up to 3 times on failure
+    - Connection timeout of 30 seconds
+    - Read timeout of 60 seconds
+    - Cleans up partial downloads on failure
+    
     Returns:
         bool: True if successful or file exists with valid checksum, False on error
     """
@@ -185,45 +193,107 @@ def _download_image():
             os.unlink(IMAGE_PATH)
     
     log.info("Starting image download process")
+    
+    # Retry configuration
+    max_attempts = 3
+    retry_delay = 5  # seconds to wait between retry attempts
+    stall_timeout = 30  # seconds without progress before considering download stalled
+    connection_timeout = 30  # seconds to establish connection
+    read_timeout = 60  # seconds to wait for data chunks
+    
+    for attempt in range(1, max_attempts + 1):
+        log.info("Download attempt %d of %d", attempt, max_attempts)
+        
+        try:
+            # Download file with timeouts
+            log.info("Downloading Oracle Linux KVM image from %s to %s", IMAGE_URL, IMAGE_PATH)
+            response = requests.get(
+                IMAGE_URL,
+                stream=True,
+                timeout=(connection_timeout, read_timeout)
+            )
+            response.raise_for_status()
 
-    try:
-        # Download file
-        log.info("Downloading Oracle Linux KVM image from %s to %s", IMAGE_URL, IMAGE_PATH)
-        response = requests.get(IMAGE_URL, stream=True)
-        response.raise_for_status()
+            # Get total file size for progress tracking
+            total_size = int(response.headers.get('content-length', 0))
+            downloaded_size = 0
+            last_log_time = 0
+            last_progress_time = time.time()
+            last_downloaded_size = 0
 
-        # Get total file size for progress tracking
-        total_size = int(response.headers.get('content-length', 0))
-        downloaded_size = 0
-        last_log_time = 0
+            # Save file with progress logging and stall detection
+            with salt.utils.files.fopen(IMAGE_PATH, 'wb') as f:
+                for chunk in response.iter_content(chunk_size=8192):
+                    if chunk:  # filter out keep-alive new chunks
+                        f.write(chunk)
+                        downloaded_size += len(chunk)
+                        current_time = time.time()
+                        
+                        # Check for stalled download
+                        if downloaded_size > last_downloaded_size:
+                            # Progress made, reset stall timer
+                            last_progress_time = current_time
+                            last_downloaded_size = downloaded_size
+                        elif current_time - last_progress_time > stall_timeout:
+                            # No progress for stall_timeout seconds
+                            raise Exception(
+                                f"Download stalled: no progress for {stall_timeout} seconds "
+                                f"at {downloaded_size}/{total_size} bytes"
+                            )
+                        
+                        # Log progress every second
+                        if current_time - last_log_time >= 1:
+                            progress = (downloaded_size / total_size) * 100 if total_size > 0 else 0
+                            log.info("Progress - %.1f%% (%d/%d bytes)",
+                                    progress, downloaded_size, total_size)
+                            last_log_time = current_time
 
-        # Save file with progress logging
-        with salt.utils.files.fopen(IMAGE_PATH, 'wb') as f:
-            for chunk in response.iter_content(chunk_size=8192):
-                f.write(chunk)
-                downloaded_size += len(chunk)
+            # Validate downloaded file
+            log.info("Download complete, validating checksum...")
+            if not _validate_image_checksum(IMAGE_PATH, IMAGE_SHA256):
+                log.error("Checksum validation failed on attempt %d", attempt)
+                os.unlink(IMAGE_PATH)
+                if attempt < max_attempts:
+                    log.info("Will retry download...")
+                    continue
+                else:
+                    log.error("All download attempts failed due to checksum mismatch")
+                    return False
+
+            log.info("Successfully downloaded and validated Oracle Linux KVM image")
+            return True
+
+        except requests.exceptions.Timeout as e:
+            log.error("Download attempt %d failed: Timeout - %s", attempt, str(e))
+            if os.path.exists(IMAGE_PATH):
+                os.unlink(IMAGE_PATH)
+            if attempt < max_attempts:
+                log.info("Will retry download in %d seconds...", retry_delay)
+                time.sleep(retry_delay)
+            else:
+                log.error("All download attempts failed due to timeout")
                 
-                # Log progress every second
-                current_time = time.time()
-                if current_time - last_log_time >= 1:
-                    progress = (downloaded_size / total_size) * 100 if total_size > 0 else 0
-                    log.info("Progress - %.1f%% (%d/%d bytes)", 
-                            progress, downloaded_size, total_size)
-                    last_log_time = current_time
-
-        # Validate downloaded file
-        if not _validate_image_checksum(IMAGE_PATH, IMAGE_SHA256):
-            os.unlink(IMAGE_PATH)
-            return False
-
-        log.info("Successfully downloaded and validated Oracle Linux KVM image")
-        return True
-
-    except Exception as e:
-        log.error("Error downloading hypervisor image: %s", str(e))
-        if os.path.exists(IMAGE_PATH):
-            os.unlink(IMAGE_PATH)
-        return False
+        except requests.exceptions.RequestException as e:
+            log.error("Download attempt %d failed: Network error - %s", attempt, str(e))
+            if os.path.exists(IMAGE_PATH):
+                os.unlink(IMAGE_PATH)
+            if attempt < max_attempts:
+                log.info("Will retry download in %d seconds...", retry_delay)
+                time.sleep(retry_delay)
+            else:
+                log.error("All download attempts failed due to network errors")
+                
+        except Exception as e:
+            log.error("Download attempt %d failed: %s", attempt, str(e))
+            if os.path.exists(IMAGE_PATH):
+                os.unlink(IMAGE_PATH)
+            if attempt < max_attempts:
+                log.info("Will retry download in %d seconds...", retry_delay)
+                time.sleep(retry_delay)
+            else:
+                log.error("All download attempts failed")
+    
+    return False
 
 def _check_ssh_keys_exist():
     """
@@ -419,25 +489,28 @@ def _ensure_hypervisor_host_dir(minion_id: str = None):
         log.error(f"Error creating hypervisor host directory: {str(e)}")
         return False
 
-def _apply_dyanno_hypervisor_state():
+def _apply_dyanno_hypervisor_state(status):
     """
     Apply the soc.dyanno.hypervisor state on the salt master.
     
     This function applies the soc.dyanno.hypervisor state on the salt master
     to update the hypervisor annotation and ensure all hypervisor host directories exist.
     
+    Args:
+        status: Status passed to the hypervisor annotation state
+    
     Returns:
         bool: True if state was applied successfully, False otherwise
     """
     try:
-        log.info("Applying soc.dyanno.hypervisor state on salt master")
+        log.info(f"Applying soc.dyanno.hypervisor state on salt master with status: {status}")
         
         # Initialize the LocalClient
         local = salt.client.LocalClient()
         
         # Target the salt master to apply the soc.dyanno.hypervisor state
         target = MANAGER_HOSTNAME + '_*'
-        state_result = local.cmd(target, 'state.apply', ['soc.dyanno.hypervisor', "pillar={'baseDomain': {'status': 'PreInit'}}", 'concurrent=True'], tgt_type='glob')
+        state_result = local.cmd(target, 'state.apply', ['soc.dyanno.hypervisor', f"pillar={{'baseDomain': {{'status': '{status}'}}}}", 'concurrent=True'], tgt_type='glob')
         log.debug(f"state_result: {state_result}")
         # Check if state was applied successfully
         if state_result:
@@ -454,17 +527,17 @@ def _apply_dyanno_hypervisor_state():
                         success = False
             
             if success:
-                log.info("Successfully applied soc.dyanno.hypervisor state")
+                log.info(f"Successfully applied soc.dyanno.hypervisor state with status: {status}")
                 return True
             else:
-                log.error("Failed to apply soc.dyanno.hypervisor state")
+                log.error(f"Failed to apply soc.dyanno.hypervisor state with status: {status}")
                 return False
         else:
-            log.error("No response from salt master when applying soc.dyanno.hypervisor state")
+            log.error(f"No response from salt master when applying soc.dyanno.hypervisor state with status: {status}")
             return False
             
     except Exception as e:
-        log.error(f"Error applying soc.dyanno.hypervisor state: {str(e)}")
+        log.error(f"Error applying soc.dyanno.hypervisor state with status: {status}: {str(e)}")
         return False
 
 def _apply_cloud_config_state():
@@ -598,11 +671,6 @@ def setup_environment(vm_name: str = 'sool9', disk_size: str = '220G', minion_id
         log.warning("Failed to apply salt.cloud.config state, continuing with setup")
         # We don't return an error here as we want to continue with the setup process
 
-    # Apply the soc.dyanno.hypervisor state on the salt master
-    if not _apply_dyanno_hypervisor_state():
-        log.warning("Failed to apply soc.dyanno.hypervisor state, continuing with setup")
-        # We don't return an error here as we want to continue with the setup process
-
     log.info("Starting setup_environment in setup_hypervisor runner")
     
     # Check if environment is already set up
@@ -616,9 +684,12 @@ def setup_environment(vm_name: str = 'sool9', disk_size: str = '220G', minion_id
     
     # Handle image setup if needed
     if not image_valid:
+        _apply_dyanno_hypervisor_state('ImageDownloadStart')
         log.info("Starting image download/validation process")
         if not _download_image():
             log.error("Image download failed")
+            # Update hypervisor annotation with failure status
+            _apply_dyanno_hypervisor_state('ImageDownloadFailed')
             return {
                 'success': False,
                 'error': 'Image download failed',
@@ -631,6 +702,8 @@ def setup_environment(vm_name: str = 'sool9', disk_size: str = '220G', minion_id
         log.info("Setting up SSH keys")
         if not _setup_ssh_keys():
             log.error("SSH key setup failed")
+            # Update hypervisor annotation with failure status
+            _apply_dyanno_hypervisor_state('SSHKeySetupFailed')
             return {
                 'success': False,
                 'error': 'SSH key setup failed',
@@ -655,6 +728,12 @@ def setup_environment(vm_name: str = 'sool9', disk_size: str = '220G', minion_id
     success = vm_result.get('success', False)
     log.info("Setup environment completed with status: %s", "SUCCESS" if success else "FAILED")
     
+    # Update hypervisor annotation with success status
+    if success:
+        _apply_dyanno_hypervisor_state('PreInit')
+    else:
+        _apply_dyanno_hypervisor_state('SetupFailed')
+    
     # If setup was successful and we have a minion_id, run highstate
     if success and minion_id:
         log.info("Running highstate on hypervisor %s", minion_id)

salt/allowed_states.map.jinja

@@ -38,7 +38,6 @@
     'hydra',
     'elasticfleet',
     'elastic-fleet-package-registry',
-    'suricata.manager',
     'utility'
 ] %}
 

salt/bpf/pcap.map.jinja

@@ -1,4 +1,7 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
+{% set PCAP_BPF_STATUS = 0  %}
+{% set STENO_BPF_COMPILED = "" %}
+
 {% if GLOBALS.pcap_engine == "TRANSITION" %}
 {%   set PCAPBPF = ["ip and host 255.255.255.1 and port 1"] %}
 {% else %}
@@ -8,3 +11,11 @@
 {{   MACROS.remove_comments(BPFMERGED, 'pcap') }}
 {%   set PCAPBPF = BPFMERGED.pcap %}
 {% endif %}
+
+{% if PCAPBPF %}
+   {% set PCAP_BPF_CALC = salt['cmd.run_all']('/usr/sbin/so-bpf-compile ' ~ GLOBALS.sensor.interface ~ ' ' ~ PCAPBPF|join(" "), cwd='/root') %}
+   {% if PCAP_BPF_CALC['retcode'] == 0 %}
+      {% set PCAP_BPF_STATUS = 1 %}
+      {% set STENO_BPF_COMPILED =  ",\\\"--filter=" + PCAP_BPF_CALC['stdout'] + "\\\""  %}
+   {% endif %}
+{% endif %}

salt/bpf/soc_bpf.yaml

@@ -1,11 +1,11 @@
 bpf:
   pcap:
-    description: List of BPF filters to apply to Stenographer.
+    description: List of BPF filters to apply to the PCAP engine.
     multiline: True
     forcedType: "[]string"
     helpLink: bpf.html
   suricata:
-    description: List of BPF filters to apply to Suricata.
+    description: List of BPF filters to apply to Suricata. This will apply to alerts and, if enabled, to metadata and PCAP logs generated by Suricata.
     multiline: True
     forcedType: "[]string"
     helpLink: bpf.html

salt/bpf/suricata.map.jinja

@@ -1,7 +1,16 @@
+{% from 'vars/globals.map.jinja' import GLOBALS %}
 {% import_yaml 'bpf/defaults.yaml' as BPFDEFAULTS %}
 {% set BPFMERGED = salt['pillar.get']('bpf', BPFDEFAULTS.bpf, merge=True) %}
+{% set SURICATA_BPF_STATUS = 0 %}
 {% import 'bpf/macros.jinja' as MACROS %}
 
 {{ MACROS.remove_comments(BPFMERGED, 'suricata') }}
 
 {% set SURICATABPF = BPFMERGED.suricata %}
+
+{% if SURICATABPF %}
+   {% set SURICATA_BPF_CALC = salt['cmd.run_all']('/usr/sbin/so-bpf-compile ' ~ GLOBALS.sensor.interface ~ ' ' ~ SURICATABPF|join(" "), cwd='/root') %}
+   {% if SURICATA_BPF_CALC['retcode'] == 0 %}
+      {% set SURICATA_BPF_STATUS = 1 %}
+   {% endif %}
+{% endif %}

salt/bpf/zeek.map.jinja

@@ -1,7 +1,16 @@
+{% from 'vars/globals.map.jinja' import GLOBALS %}
 {% import_yaml 'bpf/defaults.yaml' as BPFDEFAULTS %}
 {% set BPFMERGED = salt['pillar.get']('bpf', BPFDEFAULTS.bpf, merge=True) %}
+{% set ZEEK_BPF_STATUS = 0 %}
 {% import 'bpf/macros.jinja' as MACROS %}
 
 {{ MACROS.remove_comments(BPFMERGED, 'zeek') }}
 
 {% set ZEEKBPF = BPFMERGED.zeek %}
+
+{% if ZEEKBPF %}
+   {% set ZEEK_BPF_CALC = salt['cmd.run_all']('/usr/sbin/so-bpf-compile ' ~ GLOBALS.sensor.interface ~ ' ' ~ ZEEKBPF|join(" "), cwd='/root') %}
+   {% if ZEEK_BPF_CALC['retcode'] == 0 %}
+      {% set ZEEK_BPF_STATUS = 1 %}
+   {% endif %}
+{% endif %}

salt/common/tools/sbin/so-bpf-compile

@@ -29,9 +29,26 @@ fi
 
 interface="$1"
 shift
-tcpdump -i $interface -ddd $@ | tail -n+2 |
-while read line; do
+
+# Capture tcpdump output and exit code
+tcpdump_output=$(tcpdump -i "$interface" -ddd "$@" 2>&1)
+tcpdump_exit=$?
+
+if [ $tcpdump_exit -ne 0 ]; then
+  echo "$tcpdump_output" >&2
+  exit $tcpdump_exit
+fi
+
+# Process the output, skipping the first line
+echo "$tcpdump_output" | tail -n+2 | while read -r line; do
   cols=( $line )
-  printf "%04x%02x%02x%08x" ${cols[0]} ${cols[1]} ${cols[2]} ${cols[3]}
+  printf "%04x%02x%02x%08x" "${cols[0]}" "${cols[1]}" "${cols[2]}" "${cols[3]}"
 done
+
+# Check if the pipeline succeeded
+if [ "${PIPESTATUS[0]}" -ne 0 ]; then
+  exit 1
+fi
+
 echo ""
+exit 0

salt/common/tools/sbin/so-common

@@ -395,7 +395,7 @@ is_manager_node() {
 }
 
 is_sensor_node() {
-	# Check to see if this is a sensor (forward) node
+	# Check to see if this is a sensor node
 	is_single_node_grid && return 0
 	grep "role: so-" /etc/salt/grains | grep -E "sensor|heavynode" &> /dev/null
 }

salt/common/tools/sbin_jinja/so-import-pcap

@@ -85,7 +85,7 @@ function suricata() {
   docker run --rm \
     -v /opt/so/conf/suricata/suricata.yaml:/etc/suricata/suricata.yaml:ro \
     -v /opt/so/conf/suricata/threshold.conf:/etc/suricata/threshold.conf:ro \
-    -v /opt/so/conf/suricata/rules:/etc/suricata/rules:ro \
+    -v /opt/so/rules/suricata/:/etc/suricata/rules:ro \
     -v ${LOG_PATH}:/var/log/suricata/:rw \
     -v ${NSM_PATH}/:/nsm/:rw \
     -v "$PCAP:/input.pcap:ro" \

salt/elasticfleet/config.map.jinja

@@ -0,0 +1,34 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+
+{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
+
+{# advanced config_yaml options for elasticfleet logstash output #}
+{% set ADV_OUTPUT_LOGSTASH_RAW = ELASTICFLEETMERGED.config.outputs.logstash %}
+{% set ADV_OUTPUT_LOGSTASH = {} %}
+{% for k, v in ADV_OUTPUT_LOGSTASH_RAW.items() %}
+{%   if v != "" and v is not none %}
+{%     if k == 'queue_mem_events' %}
+{#       rename queue_mem_events queue.mem.events #}
+{%       do ADV_OUTPUT_LOGSTASH.update({'queue.mem.events':v}) %}
+{%     elif k == 'loadbalance' %}
+{%       if v %}
+{#         only include loadbalance config when its True #}
+{%         do ADV_OUTPUT_LOGSTASH.update({k:v}) %}
+{%       endif %}
+{%     else %}
+{%       do ADV_OUTPUT_LOGSTASH.update({k:v}) %}
+{%     endif %}
+{%   endif %}
+{% endfor %}
+
+{% set LOGSTASH_CONFIG_YAML_RAW = [] %}
+{% if ADV_OUTPUT_LOGSTASH %}
+{%   for k, v in ADV_OUTPUT_LOGSTASH.items() %}
+{%     do LOGSTASH_CONFIG_YAML_RAW.append(k ~ ': ' ~ v) %}
+{%   endfor %}
+{% endif %}
+
+{% set LOGSTASH_CONFIG_YAML = LOGSTASH_CONFIG_YAML_RAW | join('\\n') if LOGSTASH_CONFIG_YAML_RAW else '' %}

salt/elasticfleet/defaults.yaml

@@ -10,12 +10,19 @@ elasticfleet:
       grid_enrollment: ''
     defend_filters:
       enable_auto_configuration: False
+    outputs:
+      logstash:
+        bulk_max_size: ''
+        worker: ''
+        queue_mem_events: ''
+        timeout: ''
+        loadbalance: False
+        compression_level: ''
     subscription_integrations: False
     auto_upgrade_integrations: False
   logging:
     zeek:
       excluded:
-        - analyzer
         - broker
         - capture_loss
         - cluster

salt/elasticfleet/enabled.sls

@@ -32,6 +32,17 @@ so-elastic-fleet-auto-configure-logstash-outputs:
     - retry:
         attempts: 4
         interval: 30
+
+{# Separate from above in order to catch elasticfleet-logstash.crt changes and force update to fleet output policy #}
+so-elastic-fleet-auto-configure-logstash-outputs-force:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-outputs-update --certs
+    - retry:
+        attempts: 4
+        interval: 30
+    - onchanges:
+        - x509: etc_elasticfleet_logstash_crt
+        - x509: elasticfleet_kafka_crt
 {% endif %}
 
 # If enabled, automatically update Fleet Server URLs & ES Connection

salt/elasticfleet/integration-defaults.map.jinja

@@ -121,6 +121,9 @@
                  "phases": {
                    "cold": {
                      "actions": {
+                        "allocate":{
+                          "number_of_replicas": ""
+                        },
                        "set_priority": {"priority": 0}
                      },
                      "min_age": "60d"
@@ -137,12 +140,31 @@
                          "max_age": "30d",
                          "max_primary_shard_size": "50gb"
                        },
+                       "forcemerge":{
+                         "max_num_segments": ""
+                       },
+                       "shrink":{
+                         "max_primary_shard_size": "",
+                         "method": "COUNT",
+                         "number_of_shards": ""
+                       },
                        "set_priority": {"priority": 100}
                       },
                       "min_age": "0ms"
                    },
                    "warm": {
                      "actions": {
+                        "allocate": {
+                          "number_of_replicas": ""
+                        },
+                        "forcemerge": {
+                          "max_num_segments": ""
+                        },
+                        "shrink":{
+                          "max_primary_shard_size": "",
+                          "method": "COUNT",
+                          "number_of_shards": ""
+                        },
                        "set_priority": {"priority": 50}
                      },
                      "min_age": "30d"

salt/elasticfleet/soc_elasticfleet.yaml

@@ -50,6 +50,46 @@ elasticfleet:
       global: True
       forcedType: bool
       helpLink: elastic-fleet.html
+    outputs:
+      logstash:
+        bulk_max_size:
+          description: The maximum number of events to bulk in a single Logstash request.
+          global: True
+          forcedType: int
+          advanced: True
+          helpLink: elastic-fleet.html
+        worker:
+          description: The number of workers per configured host publishing events.
+          global: True
+          forcedType: int
+          advanced: true
+          helpLink: elastic-fleet.html
+        queue_mem_events:
+          title: queued events
+          description: The number of events the queue can store. This value should be evenly divisible by the smaller of 'bulk_max_size' to avoid sending partial batches to the output.
+          global: True
+          forcedType: int
+          advanced: True
+          helpLink: elastic-fleet.html
+        timeout:
+          description: The number of seconds to wait for responses from the Logstash server before timing out. Eg 30s
+          regex: ^[0-9]+s$
+          advanced: True
+          global: True
+          helpLink: elastic-fleet.html
+        loadbalance:
+          description: If true and multiple Logstash hosts are configured, the output plugin load balances published events onto all Logstash hosts. If false, the output plugin sends all events to one host (determined at random) and switches to another host if the selected one becomes unresponsive.
+          forcedType: bool
+          advanced: True
+          global: True
+          helpLink: elastic-fleet.html
+        compression_level:
+          description: The gzip compression level. The compression level must be in the range of 1 (best speed) to 9 (best compression).
+          regex: ^[1-9]$
+          forcedType: int
+          advanced: True
+          global: True
+          helpLink: elastic-fleet.html
     server:
       custom_fqdn:
         description: Custom FQDN for Agents to connect to. One per line.

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update

@@ -3,11 +3,36 @@
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
 # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
 # this file except in compliance with the Elastic License 2.0.
-{% from 'vars/globals.map.jinja' import GLOBALS %}
-{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
+{%- from 'vars/globals.map.jinja' import GLOBALS %}
+{%- from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
+{%- from 'elasticfleet/config.map.jinja' import LOGSTASH_CONFIG_YAML %}
 
 . /usr/sbin/so-common
 
+FORCE_UPDATE=false
+UPDATE_CERTS=false
+LOGSTASH_PILLAR_CONFIG_YAML="{{ LOGSTASH_CONFIG_YAML }}"
+LOGSTASH_PILLAR_STATE_FILE="/opt/so/state/esfleet_logstash_config_pillar"
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -f|--force)
+      FORCE_UPDATE=true
+      shift
+      ;;
+    -c| --certs)
+      UPDATE_CERTS=true
+      FORCE_UPDATE=true
+      shift
+      ;;
+    *)
+      echo "Unknown option $1"
+      echo "Usage: $0 [-f|--force] [-c|--certs]"
+      exit 1
+      ;;
+  esac
+done
+
 # Only run on Managers
 if ! is_manager_node; then
     printf "Not a Manager Node... Exiting"
@@ -17,17 +42,49 @@ fi
 function update_logstash_outputs() {
     if logstash_policy=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs/so-manager_logstash" --retry 3 --retry-delay 10 --fail 2>/dev/null); then
         SSL_CONFIG=$(echo "$logstash_policy" | jq -r '.item.ssl')
+        LOGSTASHKEY=$(openssl rsa -in  /etc/pki/elasticfleet-logstash.key)
+        LOGSTASHCRT=$(openssl x509 -in /etc/pki/elasticfleet-logstash.crt)
+        LOGSTASHCA=$(openssl x509 -in  /etc/pki/tls/certs/intca.crt)
+        # Revert escaped \\n to \n for jq
+        LOGSTASH_PILLAR_CONFIG_YAML=$(printf '%b' "$LOGSTASH_PILLAR_CONFIG_YAML")
+
         if SECRETS=$(echo "$logstash_policy" | jq -er '.item.secrets' 2>/dev/null); then
-            JSON_STRING=$(jq -n \
-                --arg UPDATEDLIST "$NEW_LIST_JSON" \
-                --argjson SECRETS "$SECRETS" \
-                --argjson SSL_CONFIG "$SSL_CONFIG" \
-                '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl": $SSL_CONFIG,"secrets": $SECRETS}')
+            if [[ "$UPDATE_CERTS" != "true"  ]]; then
+                # Reuse existing secret
+                JSON_STRING=$(jq -n \
+                    --arg UPDATEDLIST "$NEW_LIST_JSON" \
+                    --arg CONFIG_YAML "$LOGSTASH_PILLAR_CONFIG_YAML" \
+                    --argjson SECRETS "$SECRETS" \
+                    --argjson SSL_CONFIG "$SSL_CONFIG" \
+                    '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":$CONFIG_YAML,"ssl": $SSL_CONFIG,"secrets": $SECRETS}')
+            else
+                # Update certs, creating new secret
+                JSON_STRING=$(jq -n \
+                    --arg UPDATEDLIST "$NEW_LIST_JSON" \
+                    --arg CONFIG_YAML "$LOGSTASH_PILLAR_CONFIG_YAML" \
+                    --arg LOGSTASHKEY "$LOGSTASHKEY" \
+                    --arg LOGSTASHCRT "$LOGSTASHCRT" \
+                    --arg LOGSTASHCA "$LOGSTASHCA" \
+                    '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":$CONFIG_YAML,"ssl": {"certificate": $LOGSTASHCRT,"certificate_authorities":[ $LOGSTASHCA ]},"secrets": {"ssl":{"key": $LOGSTASHKEY }}}')
+            fi
         else
-            JSON_STRING=$(jq -n \
-                --arg UPDATEDLIST "$NEW_LIST_JSON" \
-                --argjson SSL_CONFIG "$SSL_CONFIG" \
-                '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl": $SSL_CONFIG}')
+            if [[ "$UPDATE_CERTS" != "true" ]]; then
+                # Reuse existing ssl config
+                JSON_STRING=$(jq -n \
+                    --arg UPDATEDLIST "$NEW_LIST_JSON" \
+                    --arg CONFIG_YAML "$LOGSTASH_PILLAR_CONFIG_YAML" \
+                    --argjson SSL_CONFIG "$SSL_CONFIG" \
+                    '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":$CONFIG_YAML,"ssl": $SSL_CONFIG}')
+            else
+                # Update ssl config
+                JSON_STRING=$(jq -n \
+                    --arg UPDATEDLIST "$NEW_LIST_JSON" \
+                    --arg CONFIG_YAML "$LOGSTASH_PILLAR_CONFIG_YAML" \
+                    --arg LOGSTASHKEY "$LOGSTASHKEY" \
+                    --arg LOGSTASHCRT "$LOGSTASHCRT" \
+                    --arg LOGSTASHCA "$LOGSTASHCA" \
+                    '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":$CONFIG_YAML,"ssl": {"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]}}')
+            fi
         fi
     fi
 
@@ -38,19 +95,42 @@ function update_kafka_outputs() {
   # Make sure SSL configuration is included in policy updates for Kafka output. SSL is configured in so-elastic-fleet-setup
   if kafka_policy=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs/so-manager_kafka" --fail 2>/dev/null); then
     SSL_CONFIG=$(echo "$kafka_policy" | jq -r '.item.ssl')
+    KAFKAKEY=$(openssl rsa -in  /etc/pki/elasticfleet-kafka.key)
+    KAFKACRT=$(openssl x509 -in /etc/pki/elasticfleet-kafka.crt)
+    KAFKACA=$(openssl x509 -in  /etc/pki/tls/certs/intca.crt)
     if SECRETS=$(echo "$kafka_policy" | jq -er '.item.secrets' 2>/dev/null); then
-      # Update policy when fleet has secrets enabled
-      JSON_STRING=$(jq -n \
-        --arg UPDATEDLIST "$NEW_LIST_JSON" \
-        --argjson SSL_CONFIG "$SSL_CONFIG" \
-        --argjson SECRETS "$SECRETS" \
-        '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": $SSL_CONFIG,"secrets": $SECRETS}')
+        if [[ "$UPDATE_CERTS" != "true" ]]; then
+            # Update policy when fleet has secrets enabled
+            JSON_STRING=$(jq -n \
+              --arg UPDATEDLIST "$NEW_LIST_JSON" \
+              --argjson SSL_CONFIG "$SSL_CONFIG" \
+              --argjson SECRETS "$SECRETS" \
+              '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": $SSL_CONFIG,"secrets": $SECRETS}')
+        else
+            # Update certs, creating new secret
+            JSON_STRING=$(jq -n \
+              --arg UPDATEDLIST "$NEW_LIST_JSON" \
+              --arg KAFKAKEY "$KAFKAKEY" \
+              --arg KAFKACRT "$KAFKACRT" \
+              --arg KAFKACA "$KAFKACA" \
+              '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": {"certificate_authorities":[ $KAFKACA ],"certificate": $KAFKACRT ,"key":"","verification_mode":"full"},"secrets": {"ssl":{"key": $KAFKAKEY }}}')
+        fi
     else
-      # Update policy when fleet has secrets disabled or policy hasn't been force updated
-      JSON_STRING=$(jq -n \
-        --arg UPDATEDLIST "$NEW_LIST_JSON" \
-        --argjson SSL_CONFIG "$SSL_CONFIG" \
-        '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": $SSL_CONFIG}')
+        if [[ "$UPDATE_CERTS" != "true" ]]; then
+            # Update policy when fleet has secrets disabled or policy hasn't been force updated
+            JSON_STRING=$(jq -n \
+              --arg UPDATEDLIST "$NEW_LIST_JSON" \
+              --argjson SSL_CONFIG "$SSL_CONFIG" \
+              '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": $SSL_CONFIG}')
+        else
+            # Update ssl config
+            JSON_STRING=$(jq -n \
+              --arg UPDATEDLIST "$NEW_LIST_JSON" \
+              --arg KAFKAKEY "$KAFKAKEY" \
+              --arg KAFKACRT "$KAFKACRT" \
+              --arg KAFKACA "$KAFKACA" \
+              '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": { "certificate_authorities": [ $KAFKACA ], "certificate": $KAFKACRT, "key": $KAFKAKEY, "verification_mode": "full" }}')
+        fi
     fi
     # Update Kafka outputs
     curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_kafka" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" | jq
@@ -73,7 +153,7 @@ function update_kafka_outputs() {
 
   # Get the current list of kafka outputs & hash them
   CURRENT_LIST=$(jq -c -r '.item.hosts' <<<  "$RAW_JSON")
-  CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}')
+  CURRENT_HASH=$(sha256sum <<< "$CURRENT_LIST" | awk '{print $1}')
 
   declare -a NEW_LIST=()
 
@@ -96,10 +176,19 @@ function update_kafka_outputs() {
    printf "Failed to query for current Logstash Outputs..."
    exit 1
   fi
+  # logstash adv config - compare pillar to last state file value
+  if [[ -f "$LOGSTASH_PILLAR_STATE_FILE" ]]; then
+    PREVIOUS_LOGSTASH_PILLAR_CONFIG_YAML=$(cat "$LOGSTASH_PILLAR_STATE_FILE")
+    if [[ "$LOGSTASH_PILLAR_CONFIG_YAML" != "$PREVIOUS_LOGSTASH_PILLAR_CONFIG_YAML" ]]; then
+      echo "Logstash pillar config has changed - forcing update"
+      FORCE_UPDATE=true
+    fi
+    echo "$LOGSTASH_PILLAR_CONFIG_YAML" > "$LOGSTASH_PILLAR_STATE_FILE"
+  fi
 
   # Get the current list of Logstash outputs & hash them
   CURRENT_LIST=$(jq -c -r '.item.hosts' <<<  "$RAW_JSON")
-  CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}')
+  CURRENT_HASH=$(sha256sum <<< "$CURRENT_LIST" | awk '{print $1}')
 
   declare -a NEW_LIST=()
 
@@ -148,10 +237,10 @@ function update_kafka_outputs() {
 
 # Sort & hash the new list of Logstash Outputs
 NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${NEW_LIST[@]}")
-NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}')
+NEW_HASH=$(sha256sum <<< "$NEW_LIST_JSON" | awk '{print $1}')
 
 # Compare the current & new list of outputs - if different, update the Logstash outputs
-if [ "$NEW_HASH" = "$CURRENT_HASH" ]; then
+if [[ "$NEW_HASH" = "$CURRENT_HASH" ]] && [[ "$FORCE_UPDATE" != "true" ]]; then
     printf "\nHashes match - no update needed.\n"
     printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n"
 

salt/elasticsearch/defaults.yaml

@@ -72,6 +72,8 @@ elasticsearch:
             actions:
               set_priority:
                 priority: 0
+              allocate:
+                number_of_replicas: ""
             min_age: 60d
           delete:
             actions:
@@ -84,11 +86,25 @@ elasticsearch:
                 max_primary_shard_size: 50gb
               set_priority:
                 priority: 100
+              forcemerge:
+                max_num_segments: ""
+              shrink:
+                max_primary_shard_size: ""
+                method: COUNT
+                number_of_shards: ""
             min_age: 0ms
           warm:
             actions:
               set_priority:
                 priority: 50
+              forcemerge:
+                max_num_segments: ""
+              shrink:
+                max_primary_shard_size: ""
+                method: COUNT
+                number_of_shards: ""
+              allocate:
+                number_of_replicas: ""
             min_age: 30d
     so-case:
       index_sorting: false
@@ -245,7 +261,6 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
-      warm: 7
     so-detection:
       index_sorting: false
       index_template:
@@ -284,6 +299,19 @@ elasticsearch:
           hot:
             actions: {}
             min_age: 0ms
+    sos-backup:
+      index_sorting: false
+      index_template:
+        composed_of: []
+        ignore_missing_component_templates: []
+        index_patterns:
+        - sos-backup-*
+        priority: 501
+        template:
+          settings:
+            index:
+              number_of_replicas: 0
+              number_of_shards: 1
     so-assistant-chat:
       index_sorting: false
       index_template:
@@ -584,7 +612,6 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
-      warm: 7
     so-import:
       index_sorting: false
       index_template:
@@ -932,7 +959,6 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
-      warm: 7
     so-hydra:
       close: 30
       delete: 365
@@ -1043,7 +1069,6 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
-      warm: 7
     so-lists:
       index_sorting: false
       index_template:
@@ -1127,6 +1152,8 @@ elasticsearch:
             actions:
               set_priority:
                 priority: 0
+              allocate:
+                number_of_replicas: ""
             min_age: 60d
           delete:
             actions:
@@ -1139,11 +1166,25 @@ elasticsearch:
                 max_primary_shard_size: 50gb
               set_priority:
                 priority: 100
+              forcemerge:
+                max_num_segments: ""
+              shrink:
+                max_primary_shard_size: ""
+                method: COUNT
+                number_of_shards: ""
             min_age: 0ms
           warm:
             actions:
               set_priority:
                 priority: 50
+              allocate:
+                number_of_replicas: ""
+              forcemerge:
+                max_num_segments: ""
+              shrink:
+                max_primary_shard_size: ""
+                method: COUNT
+                number_of_shards: ""
             min_age: 30d
     so-logs-detections_x_alerts:
       index_sorting: false
@@ -3123,7 +3164,6 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
-      warm: 7
     so-logs-system_x_application:
       index_sorting: false
       index_template:

salt/elasticsearch/files/ingest/suricata.alert

@@ -1,15 +1,79 @@
 {
-  "description" : "suricata.alert",
-  "processors" : [
-    { "set":   { "if": "ctx.event?.imported != true", "field": "_index", "value": "logs-suricata.alerts-so" } },
-    { "set":   { "field": "tags","value": "alert" }},
-    { "rename":{ "field": "message2.alert",	"target_field": "rule",	"ignore_failure": true 	} },
-    { "rename":{ "field": "rule.signature",     "target_field": "rule.name", "ignore_failure": true  } },
-    { "rename":{ "field": "rule.ref",     "target_field": "rule.version", "ignore_failure": true  } },
-    { "rename":{ "field": "rule.signature_id",     "target_field": "rule.uuid", "ignore_failure": true  } },
-    { "rename":{ "field": "rule.signature_id",     "target_field": "rule.signature", "ignore_failure": true  } },
-    { "rename":{ "field": "message2.payload_printable",     "target_field": "network.data.decoded", "ignore_failure": true  } },
-    { "dissect": { "field": "rule.rule", "pattern": "%{?prefix}content:\"%{dns.query_name}\"%{?remainder}", "ignore_missing": true, "ignore_failure": true } },
-    { "pipeline": { "name": "common.nids" } }
-  ]
+    "description": "suricata.alert",
+    "processors": [
+        {
+            "set": {
+                "if": "ctx.event?.imported != true",
+                "field": "_index",
+                "value": "logs-suricata.alerts-so"
+            }
+        },
+        {
+            "set": {
+                "field": "tags",
+                "value": "alert"
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.alert",
+                "target_field": "rule",
+                "ignore_missing": true,
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "rule.signature",
+                "target_field": "rule.name",
+                "ignore_missing": true,
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "rule.ref",
+                "target_field": "rule.version",
+                "ignore_missing": true,
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "rule.signature_id",
+                "target_field": "rule.uuid",
+                "ignore_missing": true,
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "rule.signature_id",
+                "target_field": "rule.signature",
+                "ignore_missing": true,
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.payload_printable",
+                "target_field": "network.data.decoded",
+                "ignore_missing": true,
+                "ignore_failure": true
+            }
+        },
+        {
+            "dissect": {
+                "field": "rule.rule",
+                "pattern": "%{?prefix}content:\"%{dns.query_name}\"%{?remainder}",
+                "ignore_missing": true,
+                "ignore_failure": true
+            }
+        },
+        {
+            "pipeline": {
+                "name": "common.nids"
+            }
+        }
+    ]
 }

salt/elasticsearch/files/ingest/suricata.common

@@ -1,30 +1,155 @@
 {
-  "description" : "suricata.common",
-  "processors" : [
-    { "json":   { "field": "message",                   "target_field": "message2",             "ignore_failure": true  } },
-    { "rename": { "field": "message2.pkt_src",          "target_field": "network.packet_source","ignore_failure": true  } },
-    { "rename": { "field": "message2.proto",            "target_field": "network.transport",    "ignore_failure": true  } },
-    { "rename": { "field": "message2.in_iface",         "target_field": "observer.ingress.interface.name",    "ignore_failure": true  } },
-    { "rename": { "field": "message2.flow_id",          "target_field": "log.id.uid",           "ignore_failure": true  } },
-    { "rename": { "field": "message2.src_ip",           "target_field": "source.ip",            "ignore_failure": true  } },
-    { "rename": { "field": "message2.src_port",         "target_field": "source.port",          "ignore_failure": true  } },
-    { "rename": { "field": "message2.dest_ip",          "target_field": "destination.ip",       "ignore_failure": true  } },
-    { "rename": { "field": "message2.dest_port",        "target_field": "destination.port",     "ignore_failure": true  } },
-    { "rename": { "field": "message2.vlan",             "target_field": "network.vlan.id",      "ignore_failure": true  } },
-    { "rename": { "field": "message2.community_id",     "target_field": "network.community_id", "ignore_missing": true  } },
-    { "rename": { "field": "message2.xff",              "target_field": "xff.ip",               "ignore_missing": true  } },
-    { "set":    { "field": "event.dataset",             "value": "{{ message2.event_type }}"                            } },
-    { "set":    { "field": "observer.name",             "value": "{{agent.name}}"                                       } },
-    { "set":    { "field": "event.ingested",            "value": "{{@timestamp}}"                                       } },
-    { "date": { "field": "message2.timestamp", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "timezone": "UTC", "ignore_failure": true } },
-    { "remove":{ "field": "agent",                                                              "ignore_failure": true  } },
-    {"append":{"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"ignore_failure":true}},
-    {
-      "script": {
-        "source": "boolean isPrivate(def ip) { if (ip == null) return false; int dot1 = ip.indexOf('.'); if (dot1 == -1) return false; int dot2 = ip.indexOf('.', dot1 + 1); if (dot2 == -1) return false; int first = Integer.parseInt(ip.substring(0, dot1)); if (first == 10) return true; if (first == 192 && ip.startsWith('168.', dot1 + 1)) return true; if (first == 172) { int second = Integer.parseInt(ip.substring(dot1 + 1, dot2)); return second >= 16 && second <= 31; } return false; } String[] fields = new String[] {\"source\", \"destination\"}; for (int i = 0; i < fields.length; i++) { def field = fields[i]; def ip = ctx[field]?.ip; if (ip != null) { if (ctx.network == null) ctx.network = new HashMap(); if (isPrivate(ip)) { if (ctx.network.private_ip == null) ctx.network.private_ip = new ArrayList(); if (!ctx.network.private_ip.contains(ip)) ctx.network.private_ip.add(ip); } else { if (ctx.network.public_ip == null) ctx.network.public_ip = new ArrayList(); if (!ctx.network.public_ip.contains(ip)) ctx.network.public_ip.add(ip); } } }",
-        "ignore_failure": false
-      }
-    },
-    { "pipeline": { "if": "ctx?.event?.dataset != null", "name": "suricata.{{event.dataset}}" } }
-  ]
-}
+    "description": "suricata.common",
+    "processors": [
+        {
+            "json": {
+                "field": "message",
+                "target_field": "message2",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.pkt_src",
+                "target_field": "network.packet_source",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.proto",
+                "target_field": "network.transport",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.in_iface",
+                "target_field": "observer.ingress.interface.name",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.flow_id",
+                "target_field": "log.id.uid",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.src_ip",
+                "target_field": "source.ip",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.src_port",
+                "target_field": "source.port",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dest_ip",
+                "target_field": "destination.ip",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dest_port",
+                "target_field": "destination.port",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.vlan",
+                "target_field": "network.vlan.id",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.community_id",
+                "target_field": "network.community_id",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.xff",
+                "target_field": "xff.ip",
+                "ignore_missing": true
+            }
+        },
+        {
+            "set": {
+                "field": "event.dataset",
+                "value": "{{ message2.event_type }}"
+            }
+        },
+        {
+            "set": {
+                "field": "observer.name",
+                "value": "{{agent.name}}"
+            }
+        },
+        {
+            "set": {
+                "field": "event.ingested",
+                "value": "{{@timestamp}}"
+            }
+        },
+        {
+            "date": {
+                "field": "message2.timestamp",
+                "target_field": "@timestamp",
+                "formats": [
+                    "ISO8601",
+                    "UNIX"
+                ],
+                "timezone": "UTC",
+                "ignore_failure": true
+            }
+        },
+        {
+            "remove": {
+                "field": "agent",
+                "ignore_failure": true
+            }
+        },
+        {
+            "append": {
+                "field": "related.ip",
+                "value": [
+                    "{{source.ip}}",
+                    "{{destination.ip}}"
+                ],
+                "allow_duplicates": false,
+                "ignore_failure": true
+            }
+        },
+        {
+            "script": {
+                "source": "boolean isPrivate(def ip) { if (ip == null) return false; int dot1 = ip.indexOf('.'); if (dot1 == -1) return false; int dot2 = ip.indexOf('.', dot1 + 1); if (dot2 == -1) return false; int first = Integer.parseInt(ip.substring(0, dot1)); if (first == 10) return true; if (first == 192 && ip.startsWith('168.', dot1 + 1)) return true; if (first == 172) { int second = Integer.parseInt(ip.substring(dot1 + 1, dot2)); return second >= 16 && second <= 31; } return false; } String[] fields = new String[] {\"source\", \"destination\"}; for (int i = 0; i < fields.length; i++) { def field = fields[i]; def ip = ctx[field]?.ip; if (ip != null) { if (ctx.network == null) ctx.network = new HashMap(); if (isPrivate(ip)) { if (ctx.network.private_ip == null) ctx.network.private_ip = new ArrayList(); if (!ctx.network.private_ip.contains(ip)) ctx.network.private_ip.add(ip); } else { if (ctx.network.public_ip == null) ctx.network.public_ip = new ArrayList(); if (!ctx.network.public_ip.contains(ip)) ctx.network.public_ip.add(ip); } } }",
+                "ignore_failure": false
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.capture_file",
+                "target_field": "suricata.capture_file",
+                "ignore_missing": true
+            }
+        },
+        {
+            "pipeline": {
+                "if": "ctx?.event?.dataset != null",
+                "name": "suricata.{{event.dataset}}"
+            }
+        }
+    ]
+}

salt/elasticsearch/files/ingest/suricata.dns

@@ -1,21 +1,136 @@
 {
-  "description" : "suricata.dns",
-  "processors" : [
-    { "rename": 	{ "field": "message2.proto", 			"target_field": "network.transport",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.app_proto", 		"target_field": "network.protocol",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dns.type", 		"target_field": "dns.query.type",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dns.tx_id",		"target_field": "dns.id",			"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dns.version", 		"target_field": "dns.version",			"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dns.rrname", 		"target_field": "dns.query.name",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dns.rrtype", 		"target_field": "dns.query.type_name",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dns.flags", 		"target_field": "dns.flags",			"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dns.qr", 			"target_field": "dns.qr",			"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dns.rd", 			"target_field": "dns.recursion.desired",	"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dns.ra", 			"target_field": "dns.recursion.available",	"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dns.rcode", 		"target_field": "dns.response.code_name",	"ignore_missing": true 	} },
-    { "rename":		{ "field": "message2.dns.grouped.A", 		"target_field": "dns.answers.data",		"ignore_missing": true 	} },
-    { "rename":		{ "field": "message2.dns.grouped.CNAME",	"target_field": "dns.answers.name",		"ignore_missing": true 	} },
-    { "pipeline":	{ "if": "ctx.dns.query?.name != null && ctx.dns.query.name.contains('.')", "name": "dns.tld"				} },
-    { "pipeline": 	{ "name": "common"													} }
-  ]
-}
+    "description": "suricata.dns",
+    "processors": [
+        {
+            "rename": {
+                "field": "message2.proto",
+                "target_field": "network.transport",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.app_proto",
+                "target_field": "network.protocol",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dns.type",
+                "target_field": "dns.query.type",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dns.tx_id",
+                "target_field": "dns.tx_id",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dns.id",
+                "target_field": "dns.id",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dns.version",
+                "target_field": "dns.version",
+                "ignore_missing": true
+            }
+        },
+        {
+            "pipeline": {
+                "name": "suricata.dnsv3",
+                "ignore_missing_pipeline": true,
+                "if": "ctx?.dns?.version != null && ctx?.dns?.version == 3",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dns.rrname",
+                "target_field": "dns.query.name",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dns.rrtype",
+                "target_field": "dns.query.type_name",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dns.flags",
+                "target_field": "dns.flags",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dns.qr",
+                "target_field": "dns.qr",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dns.rd",
+                "target_field": "dns.recursion.desired",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dns.ra",
+                "target_field": "dns.recursion.available",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dns.opcode",
+                "target_field": "dns.opcode",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dns.rcode",
+                "target_field": "dns.response.code_name",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dns.grouped.A",
+                "target_field": "dns.answers.data",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dns.grouped.CNAME",
+                "target_field": "dns.answers.name",
+                "ignore_missing": true
+            }
+        },
+        {
+            "pipeline": {
+                "if": "ctx.dns.query?.name != null && ctx.dns.query.name.contains('.')",
+                "name": "dns.tld"
+            }
+        },
+        {
+            "pipeline": {
+                "name": "common"
+            }
+        }
+    ]
+}

salt/elasticsearch/files/ingest/suricata.dnsv3

@@ -0,0 +1,56 @@
+{
+  "processors": [
+    {
+      "rename": {
+        "field": "message2.dns.queries",
+        "target_field": "dns.queries",
+        "ignore_missing": true,
+        "ignore_failure": true
+      }
+    },
+    {
+      "script": {
+        "source": "if (ctx?.dns?.queries != null && ctx?.dns?.queries.length > 0) {\n    if (ctx.dns == null) {\n      ctx.dns = new HashMap();\n    }\n    if (ctx.dns.query == null) {\n      ctx.dns.query = new HashMap();\n    }\n    ctx.dns.query.name = ctx?.dns?.queries[0].rrname;\n}"
+      }
+    },
+    {
+      "script": {
+        "source": "if (ctx?.dns?.queries != null && ctx?.dns?.queries.length > 0) {\n    if (ctx.dns == null) {\n      ctx.dns = new HashMap();\n    }\n    if (ctx.dns.query == null) {\n      ctx.dns.query = new HashMap();\n    }\n    ctx.dns.query.type_name = ctx?.dns?.queries[0].rrtype;\n}"
+      }
+    },
+    {
+      "foreach": {
+        "field": "dns.queries",
+        "processor": {
+          "rename": {
+            "field": "_ingest._value.rrname",
+            "target_field": "_ingest._value.name",
+            "ignore_missing": true
+          }
+        },
+        "ignore_failure": true
+      }
+    },
+    {
+      "foreach": {
+        "field": "dns.queries",
+        "processor": {
+          "rename": {
+            "field": "_ingest._value.rrtype",
+            "target_field": "_ingest._value.type_name",
+            "ignore_missing": true
+          }
+        },
+        "ignore_failure": true
+      }
+    },
+    {
+      "pipeline": {
+        "name": "suricata.tld",
+        "ignore_missing_pipeline": true,
+        "if": "ctx?.dns?.queries != null && ctx?.dns?.queries.length > 0",
+        "ignore_failure": true
+      }
+    }
+  ]
+}

salt/elasticsearch/files/ingest/suricata.tld

@@ -0,0 +1,52 @@
+{
+    "processors": [
+        {
+            "script": {
+                "source": "if (ctx.dns != null && ctx.dns.queries != null) {\n  for (def q : ctx.dns.queries) {\n    if (q.name != null && q.name.contains('.')) {\n      q.top_level_domain = q.name.substring(q.name.lastIndexOf('.') + 1);\n    }\n  }\n}",
+                "ignore_failure": true
+            }
+        },
+        {
+            "script": {
+                "source": "if (ctx.dns != null && ctx.dns.queries != null) {\n  for (def q : ctx.dns.queries) {\n    if (q.name != null && q.name.contains('.')) {\n      q.query_without_tld = q.name.substring(0, q.name.lastIndexOf('.'));\n    }\n  }\n}",
+                "ignore_failure": true
+            }
+        },
+        {
+            "script": {
+                "source": "if (ctx.dns != null && ctx.dns.queries != null) {\n  for (def q : ctx.dns.queries) {\n    if (q.query_without_tld != null && q.query_without_tld.contains('.')) {\n      q.parent_domain = q.query_without_tld.substring(q.query_without_tld.lastIndexOf('.') + 1);\n    }\n  }\n}",
+                "ignore_failure": true
+            }
+        },
+        {
+            "script": {
+                "source": "if (ctx.dns != null && ctx.dns.queries != null) {\n  for (def q : ctx.dns.queries) {\n    if (q.query_without_tld != null && q.query_without_tld.contains('.')) {\n      q.subdomain = q.query_without_tld.substring(0, q.query_without_tld.lastIndexOf('.'));\n    }\n  }\n}",
+                "ignore_failure": true
+            }
+        },
+        {
+            "script": {
+                "source": "if (ctx.dns != null && ctx.dns.queries != null) {\n  for (def q : ctx.dns.queries) {\n    if (q.parent_domain != null && q.top_level_domain != null) {\n      q.highest_registered_domain = q.parent_domain + \".\" + q.top_level_domain;\n    }\n  }\n}",
+                "ignore_failure": true
+            }
+        },
+        {
+            "script": {
+                "source": "if (ctx.dns != null && ctx.dns.queries != null) {\n  for (def q : ctx.dns.queries) {\n    if (q.subdomain != null) {\n      q.subdomain_length = q.subdomain.length();\n    }\n  }\n}",
+                "ignore_failure": true
+            }
+        },
+        {
+            "script": {
+                "source": "if (ctx.dns != null && ctx.dns.queries != null) {\n  for (def q : ctx.dns.queries) {\n    if (q.parent_domain != null) {\n      q.parent_domain_length = q.parent_domain.length();\n    }\n  }\n}",
+                "ignore_failure": true
+            }
+        },
+        {
+            "script": {
+                "source": "if (ctx.dns != null && ctx.dns.queries != null) {\n  for (def q : ctx.dns.queries) {\n    q.remove('query_without_tld');\n  }\n}",
+                "ignore_failure": true
+            }
+        }
+    ]
+}

salt/elasticsearch/files/ingest/zeek.analyzer

@@ -0,0 +1,61 @@
+{
+    "description": "zeek.analyzer",
+    "processors": [
+        {
+            "set": {
+                "field": "event.dataset",
+                "value": "analyzer"
+            }
+        },
+        {
+            "remove": {
+                "field": [
+                    "host"
+                ],
+                "ignore_failure": true
+            }
+        },
+        {
+            "json": {
+                "field": "message",
+                "target_field": "message2",
+                "ignore_failure": true
+            }
+        },
+        {
+            "set": {
+                "field": "network.protocol",
+                "copy_from": "message2.analyzer_name",
+                "ignore_empty_value": true,
+                "if": "ctx?.message2?.analyzer_kind == 'protocol'"
+            }
+        },
+        {
+            "set": {
+                "field": "network.protocol",
+                "ignore_empty_value": true,
+                "if": "ctx?.message2?.analyzer_kind != 'protocol'",
+                "copy_from": "message2.proto"
+            }
+        },
+        {
+            "lowercase": {
+                "field": "network.protocol",
+                "ignore_missing": true,
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.failure_reason",
+                "target_field": "error.reason",
+                "ignore_missing": true
+            }
+        },
+        {
+            "pipeline": {
+                "name": "zeek.common"
+            }
+        }
+    ]
+}

salt/elasticsearch/files/ingest/zeek.dns

@@ -1,35 +1,227 @@
 {
-  "description" : "zeek.dns",
-  "processors" : [
-    { "set": { "field": "event.dataset", "value": "dns" } },
-    { "remove":         { "field": ["host"],     "ignore_failure": true                                                                  } },
-    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
-    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.proto", 		"target_field": "network.transport",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.trans_id",		"target_field": "dns.id",	"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.rtt",	 	"target_field": "event.duration",			"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.query", 		"target_field": "dns.query.name",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.qclass", 		"target_field": "dns.query.class",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.qclass_name", 	"target_field": "dns.query.class_name",	"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.qtype", 		"target_field": "dns.query.type",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.qtype_name", 	"target_field": "dns.query.type_name",	"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.rcode",	 	"target_field": "dns.response.code",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.rcode_name", 	"target_field": "dns.response.code_name",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.AA", 		"target_field": "dns.authoritative",			"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.TC", 		"target_field": "dns.truncated",			"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.RD", 		"target_field": "dns.recursion.desired",			"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.RA", 		"target_field": "dns.recursion.available",			"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.Z",		"target_field": "dns.reserved",			"ignore_missing": true 	} },
-    { "rename":  	{ "field": "message2.answers", 		"target_field": "dns.answers.name", 		"ignore_missing": true  	} },
-    { "foreach":  {"field": "dns.answers.name","processor": {"pipeline": {"name": "common.ip_validation"}},"if": "ctx.dns != null && ctx.dns.answers != null && ctx.dns.answers.name != null","ignore_failure": true}},
-    { "foreach":  {"field": "temp._valid_ips","processor": {"append": {"field": "dns.resolved_ip","allow_duplicates": false,"value": "{{{_ingest._value}}}","ignore_failure": true}},"ignore_failure": true}},
-    { "script":   { "source": "if (ctx.dns.resolved_ip != null && ctx.dns.resolved_ip instanceof List) {\n        ctx.dns.resolved_ip.removeIf(item -> item == null || item.toString().trim().isEmpty());\n      }","ignore_failure": true }},
-    { "remove": {"field": ["temp"], "ignore_missing": true ,"ignore_failure": true } },
-    { "rename": 	{ "field": "message2.TTLs", 		"target_field": "dns.ttls",			"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.rejected", 	"target_field": "dns.query.rejected",		"ignore_missing": true 	} },
-    { "script": 	{ "lang": "painless", "source": "ctx.dns.query.length = ctx.dns.query.name.length()",	"ignore_failure": true  } },
-    { "set":      { "if": "ctx._index == 'so-zeek'", "field": "_index",      "value": "so-zeek_dns", "override": true   } },
-    { "pipeline": { "if": "ctx.dns?.query?.name != null && ctx.dns.query.name.contains('.')", "name": "dns.tld" } },
-    { "pipeline":       { "name": "zeek.common"											} }
-  ]
+    "description": "zeek.dns",
+    "processors": [
+        {
+            "set": {
+                "field": "event.dataset",
+                "value": "dns"
+            }
+        },
+        {
+            "remove": {
+                "field": [
+                    "host"
+                ],
+                "ignore_failure": true
+            }
+        },
+        {
+            "json": {
+                "field": "message",
+                "target_field": "message2",
+                "ignore_failure": true
+            }
+        },
+        {
+            "dot_expander": {
+                "field": "id.orig_h",
+                "path": "message2",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.proto",
+                "target_field": "network.transport",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.trans_id",
+                "target_field": "dns.id",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.rtt",
+                "target_field": "event.duration",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.query",
+                "target_field": "dns.query.name",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.qclass",
+                "target_field": "dns.query.class",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.qclass_name",
+                "target_field": "dns.query.class_name",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.qtype",
+                "target_field": "dns.query.type",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.qtype_name",
+                "target_field": "dns.query.type_name",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.rcode",
+                "target_field": "dns.response.code",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.rcode_name",
+                "target_field": "dns.response.code_name",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.AA",
+                "target_field": "dns.authoritative",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.TC",
+                "target_field": "dns.truncated",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.RD",
+                "target_field": "dns.recursion.desired",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.RA",
+                "target_field": "dns.recursion.available",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.Z",
+                "target_field": "dns.reserved",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.answers",
+                "target_field": "dns.answers.name",
+                "ignore_missing": true
+            }
+        },
+        {
+            "foreach": {
+                "field": "dns.answers.name",
+                "processor": {
+                    "pipeline": {
+                        "name": "common.ip_validation"
+                    }
+                },
+                "if": "ctx.dns != null && ctx.dns.answers != null && ctx.dns.answers.name != null",
+                "ignore_failure": true
+            }
+        },
+        {
+            "foreach": {
+                "field": "temp._valid_ips",
+                "processor": {
+                    "append": {
+                        "field": "dns.resolved_ip",
+                        "allow_duplicates": false,
+                        "value": "{{{_ingest._value}}}",
+                        "ignore_failure": true
+                    }
+                },
+                "if": "ctx.dns != null && ctx.dns.answers != null && ctx.dns.answers.name != null",
+                "ignore_failure": true
+            }
+        },
+        {
+            "script": {
+                "source": "if (ctx.dns.resolved_ip != null && ctx.dns.resolved_ip instanceof List) {\n        ctx.dns.resolved_ip.removeIf(item -> item == null || item.toString().trim().isEmpty());\n      }",
+                "ignore_failure": true
+            }
+        },
+        {
+            "remove": {
+                "field": [
+                    "temp"
+                ],
+                "ignore_missing": true,
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.TTLs",
+                "target_field": "dns.ttls",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.rejected",
+                "target_field": "dns.query.rejected",
+                "ignore_missing": true
+            }
+        },
+        {
+            "script": {
+                "lang": "painless",
+                "source": "ctx.dns.query.length = ctx.dns.query.name.length()",
+                "ignore_failure": true
+            }
+        },
+        {
+            "set": {
+                "if": "ctx._index == 'so-zeek'",
+                "field": "_index",
+                "value": "so-zeek_dns",
+                "override": true
+            }
+        },
+        {
+            "pipeline": {
+                "if": "ctx.dns?.query?.name != null && ctx.dns.query.name.contains('.')",
+                "name": "dns.tld"
+            }
+        },
+        {
+            "pipeline": {
+                "name": "zeek.common"
+            }
+        }
+    ]
 }

salt/elasticsearch/files/ingest/zeek.dpd

@@ -1,20 +0,0 @@
-{
-  "description" : "zeek.dpd",
-  "processors" : [
-    { "set": { "field": "event.dataset", "value": "dpd" } },
-    { "remove":         { "field": ["host"],     "ignore_failure": true                                                                  } },
-    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
-    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source.ip",		"ignore_missing": true 	} },
-    { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source.port",		"ignore_missing": true 	} },
-    { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
-    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.proto", 		"target_field": "network.protocol",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.analyzer", 	"target_field": "observer.analyzer",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.failure_reason", 	"target_field": "error.reason",	"ignore_missing": true 	} },
-    { "pipeline":       { "name": "zeek.common"                                                                                   } }
-  ]
-}

salt/elasticsearch/soc_elasticsearch.yaml

@@ -131,6 +131,47 @@ elasticsearch:
                   description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index.
                   global: True
                   helpLink: elasticsearch.html
+              shrink:
+                method:
+                  description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
+                  options:
+                  - COUNT
+                  - SIZE
+                  global: True
+                  advanced: True
+                  forcedType: string
+                number_of_shards:
+                  title: shard count
+                  description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                max_primary_shard_size:
+                  title: max shard size
+                  description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
+                  regex: ^[0-9]+(?:gb|tb|pb)$
+                  global: True
+                  forcedType: string
+                  advanced: True
+                allow_write_after_shrink:
+                  description: Allow writes after shrink.
+                  global: True
+                  forcedType: bool
+                  default: False
+                  advanced: True
+              forcemerge:
+                max_num_segments:
+                  description: Reduce the number of segments in each index shard and clean up deleted documents.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                index_codec:
+                  title: compression
+                  description: Use higher compression for stored fields at the cost of slower performance.
+                  forcedType: bool
+                  global: True
+                  default: False
+                  advanced: True
           cold:
             min_age:
               description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and cold min_age set to 60 then there will be 30 days from index creation to rollover and then an additional 60 days before moving to cold tier.
@@ -144,6 +185,12 @@ elasticsearch:
                   description: Used for index recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
                   global: True
                   helpLink: elasticsearch.html
+              allocate:
+                number_of_replicas:
+                  description: Set the number of replicas. Remains the same as the previous phase by default.
+                  forcedType: int
+                  global: True
+                  advanced: True
           warm:
             min_age: 
               description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier.
@@ -158,6 +205,52 @@ elasticsearch:
                   forcedType: int
                   global: True
                   helpLink: elasticsearch.html
+              shrink:
+                method:
+                  description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
+                  options:
+                  - COUNT
+                  - SIZE
+                  global: True
+                  advanced: True
+                number_of_shards:
+                  title: shard count
+                  description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                max_primary_shard_size:
+                  title: max shard size
+                  description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
+                  regex: ^[0-9]+(?:gb|tb|pb)$
+                  global: True
+                  forcedType: string
+                  advanced: True
+                allow_write_after_shrink:
+                  description: Allow writes after shrink.
+                  global: True
+                  forcedType: bool
+                  default: False
+                  advanced: True
+              forcemerge:
+                max_num_segments:
+                  description: Reduce the number of segments in each index shard and clean up deleted documents.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                index_codec:
+                  title: compression
+                  description: Use higher compression for stored fields at the cost of slower performance.
+                  forcedType: bool
+                  global: True
+                  default: False
+                  advanced: True
+              allocate:
+                number_of_replicas:
+                  description: Set the number of replicas. Remains the same as the previous phase by default.
+                  forcedType: int
+                  global: True
+                  advanced: True
           delete:
             min_age:
               description: Minimum age of index. ex. 90d - This determines when the index should be deleted. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and delete min_age set to 90 then there will be 30 days from index creation to rollover and then an additional 90 days before deletion.
@@ -287,6 +380,47 @@ elasticsearch:
                   global: True
                   advanced: True
                   helpLink: elasticsearch.html
+              shrink:
+                method:
+                  description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
+                  options:
+                  - COUNT
+                  - SIZE
+                  global: True
+                  advanced: True
+                  forcedType: string
+                number_of_shards:
+                  title: shard count
+                  description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                max_primary_shard_size:
+                  title: max shard size
+                  description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
+                  regex: ^[0-9]+(?:gb|tb|pb)$
+                  global: True
+                  forcedType: string
+                  advanced: True
+                allow_write_after_shrink:
+                  description: Allow writes after shrink.
+                  global: True
+                  forcedType: bool
+                  default: False
+                  advanced: True
+              forcemerge:
+                max_num_segments:
+                  description: Reduce the number of segments in each index shard and clean up deleted documents.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                index_codec:
+                  title: compression
+                  description: Use higher compression for stored fields at the cost of slower performance.
+                  forcedType: bool
+                  global: True
+                  default: False
+                  advanced: True
           warm:
             min_age:
               description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier.
@@ -314,6 +448,52 @@ elasticsearch:
                   global: True
                   advanced: True
                   helpLink: elasticsearch.html
+              shrink:
+                method:
+                  description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
+                  options:
+                  - COUNT
+                  - SIZE
+                  global: True
+                  advanced: True
+                number_of_shards:
+                  title: shard count
+                  description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                max_primary_shard_size:
+                  title: max shard size
+                  description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
+                  regex: ^[0-9]+(?:gb|tb|pb)$
+                  global: True
+                  forcedType: string
+                  advanced: True
+                allow_write_after_shrink:
+                  description: Allow writes after shrink.
+                  global: True
+                  forcedType: bool
+                  default: False
+                  advanced: True
+              forcemerge:
+                max_num_segments:
+                  description: Reduce the number of segments in each index shard and clean up deleted documents.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                index_codec:
+                  title: compression
+                  description: Use higher compression for stored fields at the cost of slower performance.
+                  forcedType: bool
+                  global: True
+                  default: False
+                  advanced: True
+              allocate:
+                number_of_replicas:
+                  description: Set the number of replicas. Remains the same as the previous phase by default.
+                  forcedType: int
+                  global: True
+                  advanced: True
           cold:
             min_age:
               description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier.  While still searchable, this tier is typically optimized for lower storage costs rather than search speed. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and cold min_age set to 60 then there will be 30 days from index creation to rollover and then an additional 60 days before moving to cold tier.
@@ -330,6 +510,12 @@ elasticsearch:
                   global: True
                   advanced: True
                   helpLink: elasticsearch.html
+              allocate:
+                number_of_replicas:
+                  description: Set the number of replicas. Remains the same as the previous phase by default.
+                  forcedType: int
+                  global: True
+                  advanced: True
           delete:
             min_age:
               description: Minimum age of index. ex. 90d - This determines when the index should be deleted. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and delete min_age set to 90 then there will be 30 days from index creation to rollover and then an additional 90 days before deletion.

salt/elasticsearch/template.map.jinja

@@ -61,5 +61,55 @@
 {%       do settings.index_template.template.settings.index.pop('sort') %}
 {%     endif %}
 {%   endif %}
+
+{# advanced ilm actions #}
+{%   if settings.policy is defined and settings.policy.phases is defined %}
+{%     set PHASE_NAMES = ["hot", "warm", "cold"] %}
+{%     for P in PHASE_NAMES %}
+{%       if settings.policy.phases[P] is defined and settings.policy.phases[P].actions is defined %}
+{%         set PHASE = settings.policy.phases[P].actions %}
+{#         remove allocate action if number_of_replicas isn't configured #}
+{%         if PHASE.allocate is defined %}
+{%           if PHASE.allocate.number_of_replicas is not defined or PHASE.allocate.number_of_replicas == "" %}
+{%             do PHASE.pop('allocate', none) %}
+{%           endif %}
+{%         endif %}
+{#         start shrink action #}
+{%         if PHASE.shrink is defined %}
+{%           if PHASE.shrink.method is defined %}
+{%             if PHASE.shrink.method == 'COUNT' and PHASE.shrink.number_of_shards is defined and PHASE.shrink.number_of_shards %}
+{#               remove max_primary_shard_size value when doing shrink operation by count vs size #}
+{%               do PHASE.shrink.pop('max_primary_shard_size', none) %}
+{%             elif PHASE.shrink.method == 'SIZE' and PHASE.shrink.max_primary_shard_size is defined and PHASE.shrink.max_primary_shard_size %}
+{#               remove number_of_shards value when doing shrink operation by size vs count #}
+{%               do PHASE.shrink.pop('number_of_shards', none) %}
+{%             else %}
+{#               method isn't defined or missing a required config number_of_shards/max_primary_shard_size #}
+{%               do PHASE.pop('shrink', none) %}
+{%             endif %}
+{%           endif %}
+{%         endif %}
+{#         always remove shrink method since its only used for SOC config, not in the actual ilm policy #}
+{%         if PHASE.shrink is defined %}
+{%            do PHASE.shrink.pop('method', none) %}
+{%         endif %}
+{#         end shrink action #}
+{#         start force merge #}
+{%         if PHASE.forcemerge is defined %}
+{%           if PHASE.forcemerge.index_codec is defined and PHASE.forcemerge.index_codec %}
+{%             do PHASE.forcemerge.update({'index_codec': 'best_compression'}) %}
+{%           else %}
+{%             do PHASE.forcemerge.pop('index_codec', none) %}
+{%           endif %}
+{%           if PHASE.forcemerge.max_num_segments is not defined or not PHASE.forcemerge.max_num_segments %}
+{#             max_num_segments is empty, drop it #}
+{%             do PHASE.pop('forcemerge', none) %}
+{%           endif %}
+{%         endif %}
+{#         end force merge #}
+{%       endif %}
+{%     endfor %}
+{%   endif %}
+
 {%   do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index]}) %}
 {% endfor %}

salt/elasticsearch/templates/component/ecs/suricata.json

@@ -841,6 +841,10 @@
                   "type": "long"
                 }
               }
+            },
+            "capture_file": {
+                "type": "keyword",
+                "ignore_above": 1024
             }
           }
         }

salt/elasticsearch/tools/sbin/so-elasticsearch-retention-estimate

@@ -206,7 +206,7 @@ fail() {
     exit 1
 }
 
-echo -e "\nDISCLAIMER: Script output is based on current data patterns, but are approximations soley intended to assist with getting a general ILM policy configured."
+echo -e "\nDISCLAIMER: Script output is based on current data patterns, but are approximations solely intended to assist with getting a general ILM policy configured."
 
 ORG_ID=$(lookup_org_id)
 [ -n "$ORG_ID" ] || fail "Unable to resolve InfluxDB org id"
@@ -756,7 +756,7 @@ if [ "$should_trigger_recommendations" = true ]; then
 
             ilm_output=$(so-elasticsearch-query "${index}/_ilm/explain" --fail 2>/dev/null) || true
             if [ -n "$ilm_output" ]; then
-                policy=$(echo "$ilm_output" | jq --arg idx "$index" -r ".indices[$idx].policy // empty" 2>/dev/null)
+                policy=$(echo "$ilm_output" | jq -r '.indices | to_entries | .[0].value.policy // empty' 2>/dev/null)
             fi
             if [ -n "$policy" ] && [ -n "${policy_ages[$policy]:-}" ]; then
                 delete_min_age=${policy_ages[$policy]}
@@ -1024,8 +1024,12 @@ else
     if [ "$ilm_indices_immediate" -gt 0 ]; then
         echo -e "${BOLD}Deleting now:${NC}            $ilm_indices_immediate indices (~${ilm_delete_immediate_gb} GB, $ilm_shards_immediate shards)"
     fi
-    if [ "$ilm_indices_7d" -gt 0 ]; then
-        echo -e "${BOLD}Storage to be freed (7d):${NC}  $ilm_indices_7d indices (~${ilm_delete_7d_gb} GB, $ilm_shards_7d shards)"
+    if [ "$ilm_indices_30d" -gt 0 ]; then
+        if [ "$ilm_delete_scheduled_30d" -gt 0 ] && [ "$ilm_indices_scheduled_30d" -gt 0 ]; then
+            echo -e "${BOLD}Storage to be freed (30d):${NC} $ilm_indices_30d indices (~${ilm_delete_30d_gb} GB, $ilm_shards_30d shards)"
+        elif [ "$ilm_indices_7d" -gt 0 ]; then
+            echo -e "${BOLD}Storage to be freed (7d):${NC}  $ilm_indices_7d indices (~${ilm_delete_7d_gb} GB, $ilm_shards_7d shards)"
+        fi
     fi
 
     log_title "LOG" "Retention Projection"

salt/hypervisor/tools/sbin_jinja/so-kvm-create-volume

@@ -45,7 +45,7 @@ used during VM provisioning to add dedicated NSM storage volumes.
     This command creates and attaches a volume with the following settings:
     - VM Name: `vm1_sensor`
     - Volume Size: `500` GB
-    - Volume Path: `/nsm/libvirt/volumes/vm1_sensor-nsm.img`
+    - Volume Path: `/nsm/libvirt/volumes/vm1_sensor-nsm-<epoch_timestamp>.img`
     - Device: `/dev/vdb` (virtio-blk)
     - VM remains stopped after attachment
 
@@ -75,7 +75,8 @@ used during VM provisioning to add dedicated NSM storage volumes.
 
 - The script automatically stops the VM if it's running before creating and attaching the volume.
 - Volumes are created with full pre-allocation for optimal performance.
-- Volume files are stored in `/nsm/libvirt/volumes/` with naming pattern `<vm_name>-nsm.img`.
+- Volume files are stored in `/nsm/libvirt/volumes/` with naming pattern `<vm_name>-nsm-<epoch_timestamp>.img`.
+- The epoch timestamp ensures unique volume names and prevents conflicts.
 - Volumes are attached as `/dev/vdb` using virtio-blk for high performance.
 - The script checks available disk space before creating the volume.
 - Ownership is set to `qemu:qemu` with permissions `640`.
@@ -142,6 +143,7 @@ import socket
 import subprocess
 import pwd
 import grp
+import time
 import xml.etree.ElementTree as ET
 from io import StringIO
 from so_vm_utils import start_vm, stop_vm
@@ -242,10 +244,13 @@ def create_volume_file(vm_name, size_gb, logger):
     Raises:
         VolumeCreationError: If volume creation fails
     """
-    # Define volume path (directory already created in main())
-    volume_path = os.path.join(VOLUME_DIR, f"{vm_name}-nsm.img")
+    # Generate epoch timestamp for unique volume naming
+    epoch_timestamp = int(time.time())
     
-    # Check if volume already exists
+    # Define volume path with epoch timestamp for uniqueness
+    volume_path = os.path.join(VOLUME_DIR, f"{vm_name}-nsm-{epoch_timestamp}.img")
+    
+    # Check if volume already exists (shouldn't be possible with timestamp)
     if os.path.exists(volume_path):
         logger.error(f"VOLUME: Volume already exists: {volume_path}")
         raise VolumeCreationError(f"Volume already exists: {volume_path}")

salt/manager/init.sls

@@ -206,10 +206,33 @@ git_config_set_safe_dirs:
     - multivar:
       - /nsm/rules/custom-local-repos/local-sigma
       - /nsm/rules/custom-local-repos/local-yara
+      - /nsm/rules/custom-local-repos/local-suricata
       - /nsm/securityonion-resources
       - /opt/so/conf/soc/ai_summary_repos/securityonion-resources
       - /nsm/airgap-resources/playbooks
       - /opt/so/conf/soc/playbooks
+
+surinsmrulesdir:
+  file.directory:
+    - name: /nsm/rules/suricata/etopen
+    - user: 939
+    - group: 939
+    - makedirs: True
+
+suriextractionrules:
+  file.managed:
+    - name: /nsm/rules/suricata/so_extraction.rules
+    - source: salt://suricata/files/so_extraction.rules
+    - user: 939
+    - group: 939
+
+surifiltersrules:
+  file.managed:
+    - name: /nsm/rules/suricata/so_filters.rules
+    - source: salt://suricata/files/so_filters.rules
+    - user: 939
+    - group: 939
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/manager/managed_soc_annotations.sls

@@ -25,13 +25,11 @@
            {% set index_settings = es.get('index_settings', {}) %}
            {% set input = index_settings.get('so-logs', {}) %}
            {% for k in matched_integration_names %}
-           {%   if k not in index_settings %}
-           {%     set _ = index_settings.update({k: input}) %}
-           {%   endif %}
+           {%   do index_settings.update({k: input}) %}
            {% endfor %}
            {% for k in addon_integration_keys %}
            {%   if k not in matched_integration_names and k in index_settings %}
-           {%     set _ = index_settings.pop(k) %}
+           {%     do index_settings.pop(k) %}
            {%   endif %}
            {% endfor %}
            {{ data }}
@@ -45,14 +43,12 @@
            {% set es = data.get('elasticsearch', {}) %}
            {% set index_settings = es.get('index_settings', {}) %}
            {% for k in matched_integration_names %}
-           {%   if k not in index_settings %}
-           {%     set input = ADDON_INTEGRATION_DEFAULTS[k] %}
-           {%     set _ = index_settings.update({k: input})%}
-           {%   endif %}
+           {%   set input = ADDON_INTEGRATION_DEFAULTS[k] %}
+           {%     do index_settings.update({k: input})%}
            {% endfor %}
            {% for k in addon_integration_keys %}
            {%   if k not in matched_integration_names and k in index_settings %}
-           {%     set _ = index_settings.pop(k) %}
+           {%     do index_settings.pop(k) %}
            {%   endif %}
            {% endfor %}
            {{ data }}

salt/manager/tools/sbin/so-minion

@@ -126,16 +126,130 @@ ADVPILLARFILE=/opt/so/saltstack/local/pillar/minions/adv_$MINION_ID.sls
 
 function getinstallinfo() {
 	log "INFO" "Getting install info for minion $MINION_ID"
-	# Pull from file
-	INSTALLVARS=$(sudo salt "$MINION_ID" cp.get_file_str /opt/so/install.txt --out=newline_values_only)
-	if [ $? -ne 0 ]; then
+
+	local install_json
+	local install_text
+
+	# Reset any prior values so we fail closed if the file is missing keys
+	MAINIP=""
+	MNIC=""
+	NODE_DESCRIPTION=""
+	ES_HEAP_SIZE=""
+	PATCHSCHEDULENAME=""
+	INTERFACE=""
+	NODETYPE=""
+	CORECOUNT=""
+	LSHOSTNAME=""
+	LSHEAP=""
+	CPUCORES=""
+	IDH_MGTRESTRICT=""
+	IDH_SERVICES=""
+
+	# Pull from file (treat it as data, not code)
+	install_json=$(sudo salt "$MINION_ID" cp.get_file_str /opt/so/install.txt --out=json 2>/dev/null)
+	if [ $? -ne 0 ] || [ -z "$install_json" ]; then
 		log "ERROR" "Failed to get install info from $MINION_ID"
 		return 1
 	fi
-	
-	source <(echo $INSTALLVARS)
-	if [ $? -ne 0 ]; then
-		log "ERROR" "Failed to source install variables"
+
+	install_text=$(jq -r --arg id "$MINION_ID" '.[$id] // empty' <<<"$install_json" 2>/dev/null)
+	if [ $? -ne 0 ] || [ -z "$install_text" ] || [ "$install_text" == "null" ]; then
+		log "ERROR" "Failed to parse install info response for $MINION_ID"
+		return 1
+	fi
+
+	while IFS= read -r line; do
+		# Trim trailing CR (in case of CRLF files)
+		line=${line%$'\r'}
+
+		# Skip empty/comment lines
+		[[ -z "$line" || "$line" =~ ^[[:space:]]*# ]] && continue
+
+		if [[ "$line" =~ ^[[:space:]]*([A-Z0-9_]+)[[:space:]]*=(.*)$ ]]; then
+			local key="${BASH_REMATCH[1]}"
+			local value="${BASH_REMATCH[2]}"
+
+			# Trim leading whitespace from value (writers shouldn't include it, but tolerate it)
+			value="${value#"${value%%[![:space:]]*}"}"
+
+			# Strip a single layer of surrounding quotes
+			if [[ "$value" =~ ^\".*\"$ ]]; then
+				value="${value:1:${#value}-2}"
+			elif [[ "$value" =~ ^\'.*\'$ ]]; then
+				value="${value:1:${#value}-2}"
+			fi
+
+			case "$key" in
+				MAINIP)
+					[[ "$value" =~ ^[A-Za-z0-9.:-]+$ ]] || { log "ERROR" "Invalid MAINIP in install info"; return 1; }
+					MAINIP="$value"
+					;;
+				MNIC)
+					[[ "$value" =~ ^[A-Za-z0-9_.:-]+$ ]] || { log "ERROR" "Invalid MNIC in install info"; return 1; }
+					MNIC="$value"
+					;;
+				NODE_DESCRIPTION)
+					# Allow spaces and common punctuation, but reject control chars
+					[[ "$value" =~ ^[[:print:]]{0,256}$ ]] || { log "ERROR" "Invalid NODE_DESCRIPTION in install info"; return 1; }
+					NODE_DESCRIPTION="$value"
+					;;
+				ES_HEAP_SIZE)
+					[[ "$value" =~ ^[0-9]+[kKmMgGtTpPeE]?$ ]] || { log "ERROR" "Invalid ES_HEAP_SIZE in install info"; return 1; }
+					ES_HEAP_SIZE="$value"
+					;;
+				PATCHSCHEDULENAME)
+					[[ "$value" =~ ^[A-Za-z0-9._-]*$ ]] || { log "ERROR" "Invalid PATCHSCHEDULENAME in install info"; return 1; }
+					PATCHSCHEDULENAME="$value"
+					;;
+				INTERFACE)
+					[[ "$value" =~ ^[A-Za-z0-9._:,-]+$ ]] || { log "ERROR" "Invalid INTERFACE in install info"; return 1; }
+					INTERFACE="$value"
+					;;
+				NODETYPE)
+					[[ "$value" =~ ^[A-Z0-9_]+$ ]] || { log "ERROR" "Invalid NODETYPE in install info"; return 1; }
+					if ! declare -F "create${value}" >/dev/null; then
+						log "ERROR" "Unknown NODETYPE '$value' in install info"
+						return 1
+					fi
+					NODETYPE="$value"
+					;;
+				CORECOUNT)
+					[[ "$value" =~ ^[0-9]+$ ]] || { log "ERROR" "Invalid CORECOUNT in install info"; return 1; }
+					CORECOUNT="$value"
+					;;
+				LSHOSTNAME)
+					[[ "$value" =~ ^[A-Za-z0-9.-]+$ ]] || { log "ERROR" "Invalid LSHOSTNAME in install info"; return 1; }
+					LSHOSTNAME="$value"
+					;;
+				LSHEAP)
+					[[ "$value" =~ ^[0-9]+[kKmMgGtTpPeE]?$ ]] || { log "ERROR" "Invalid LSHEAP in install info"; return 1; }
+					LSHEAP="$value"
+					;;
+				CPUCORES)
+					[[ "$value" =~ ^[0-9]+$ ]] || { log "ERROR" "Invalid CPUCORES in install info"; return 1; }
+					CPUCORES="$value"
+					;;
+				IDH_MGTRESTRICT)
+					[[ "$value" == "True" || "$value" == "False" ]] || { log "ERROR" "Invalid IDH_MGTRESTRICT in install info"; return 1; }
+					IDH_MGTRESTRICT="$value"
+					;;
+				IDH_SERVICES)
+					[[ "$value" =~ ^[[:print:]]{0,512}$ ]] || { log "ERROR" "Invalid IDH_SERVICES in install info"; return 1; }
+					IDH_SERVICES="$value"
+					;;
+				*)
+					# Ignore unknown keys for forward compatibility
+					:
+					;;
+			esac
+		else
+			log "ERROR" "Invalid install info line from $MINION_ID"
+			return 1
+		fi
+	done <<<"$install_text"
+
+	if [[ -z "$NODETYPE" || -z "$MAINIP" || -z "$MNIC" ]]; then
+		log "ERROR" "Missing required install variables from $MINION_ID"
 		return 1
 	fi
 }
@@ -996,7 +1110,7 @@ function setupMinionFiles() {
 	log "INFO" "Setting up minion files for $MINION_ID"
 	
 	# Check to see if nodetype is set
-	if [ -z $NODETYPE ]; then
+	if [ -z "$NODETYPE" ]; then
 		error_msg="No node type specified"
 		log "ERROR" "$error_msg"
 		echo "$error_msg"
@@ -1018,7 +1132,12 @@ function setupMinionFiles() {
 	fi
 
 	# Create node-specific configuration
-	create$NODETYPE || return 1
+	local create_func="create${NODETYPE}"
+	if ! declare -F "$create_func" >/dev/null; then
+		log "ERROR" "Unknown node type '$NODETYPE'"
+		return 1
+	fi
+	"$create_func" || return 1
 	
 	# Ensure proper ownership after all content is written
 	ensure_socore_ownership || return 1

salt/manager/tools/sbin/so-yaml.py

@@ -17,6 +17,7 @@ def showUsage(args):
     print('Usage: {} <COMMAND> <YAML_FILE> [ARGS...]'.format(sys.argv[0]), file=sys.stderr)
     print('  General commands:', file=sys.stderr)
     print('    append         - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr)
+    print('    removelistitem - Remove a list item from a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr)
     print('    add            - Add a new key and set its value. Fails if key already exists. Requires KEY and VALUE args.', file=sys.stderr)
     print('    get            - Displays (to stdout) the value stored in the given key. Requires KEY arg.', file=sys.stderr)
     print('    remove         - Removes a yaml key, if it exists. Requires KEY arg.', file=sys.stderr)
@@ -26,8 +27,8 @@ def showUsage(args):
     print('  Where:', file=sys.stderr)
     print('   YAML_FILE       - Path to the file that will be modified. Ex: /opt/so/conf/service/conf.yaml', file=sys.stderr)
     print('   KEY             - YAML key, does not support \' or " characters at this time. Ex: level1.level2', file=sys.stderr)
-    print('   VALUE           - Value to set for a given key', file=sys.stderr)
-    print('   LISTITEM        - Item to append to a given key\'s list value', file=sys.stderr)
+    print('   VALUE           - Value to set for a given key. Can be a literal value or file:<path> to load from a YAML file.', file=sys.stderr)
+    print('   LISTITEM        - Item to append to a given key\'s list value. Can be a literal value or file:<path> to load from a YAML file.', file=sys.stderr)
     sys.exit(1)
 
 
@@ -57,8 +58,32 @@ def appendItem(content, key, listItem):
             return 1
 
 
+def removeListItem(content, key, listItem):
+    pieces = key.split(".", 1)
+    if len(pieces) > 1:
+        removeListItem(content[pieces[0]], pieces[1], listItem)
+    else:
+        try:
+            if not isinstance(content[key], list):
+                raise AttributeError("Value is not a list")
+            if listItem in content[key]:
+                content[key].remove(listItem)
+        except (AttributeError, TypeError):
+            print("The existing value for the given key is not a list. No action was taken on the file.", file=sys.stderr)
+            return 1
+        except KeyError:
+            print("The key provided does not exist. No action was taken on the file.", file=sys.stderr)
+            return 1
+
+
 def convertType(value):
-    if isinstance(value, str) and len(value) > 0 and (not value.startswith("0") or len(value) == 1):
+    if isinstance(value, str) and value.startswith("file:"):
+        path = value[5:]  # Remove "file:" prefix
+        if not os.path.exists(path):
+            print(f"File '{path}' does not exist.", file=sys.stderr)
+            sys.exit(1)
+        return loadYaml(path)
+    elif isinstance(value, str) and len(value) > 0 and (not value.startswith("0") or len(value) == 1):
         if "." in value:
             try:
                 value = float(value)
@@ -97,6 +122,23 @@ def append(args):
     return 0
 
 
+def removelistitem(args):
+    if len(args) != 3:
+        print('Missing filename, key arg, or list item to remove', file=sys.stderr)
+        showUsage(None)
+        return 1
+
+    filename = args[0]
+    key = args[1]
+    listItem = args[2]
+
+    content = loadYaml(filename)
+    removeListItem(content, key, convertType(listItem))
+    writeYaml(filename, content)
+
+    return 0
+
+
 def addKey(content, key, value):
     pieces = key.split(".", 1)
     if len(pieces) > 1:
@@ -205,6 +247,7 @@ def main():
         "help": showUsage,
         "add": add,
         "append": append,
+        "removelistitem": removelistitem,
         "get": get,
         "remove": remove,
         "replace": replace,

salt/manager/tools/sbin/so-yaml_test.py

@@ -361,6 +361,29 @@ class TestRemove(unittest.TestCase):
         self.assertEqual(soyaml.convertType("FALSE"), False)
         self.assertEqual(soyaml.convertType(""), "")
 
+    def test_convert_file(self):
+        import tempfile
+        import os
+
+        # Create a temporary YAML file
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f:
+            f.write("test:\n  - name: hi\n    color: blue\n")
+            temp_file = f.name
+
+        try:
+            result = soyaml.convertType(f"file:{temp_file}")
+            expected = {"test": [{"name": "hi", "color": "blue"}]}
+            self.assertEqual(result, expected)
+        finally:
+            os.unlink(temp_file)
+
+    def test_convert_file_nonexistent(self):
+        with self.assertRaises(SystemExit) as cm:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
+                soyaml.convertType("file:/nonexistent/file.yaml")
+                self.assertEqual(cm.exception.code, 1)
+                self.assertIn("File '/nonexistent/file.yaml' does not exist.", mock_stderr.getvalue())
+
     def test_get_int(self):
         with patch('sys.stdout', new=StringIO()) as mock_stdout:
             filename = "/tmp/so-yaml_test-get.yaml"
@@ -434,3 +457,126 @@ class TestRemove(unittest.TestCase):
                 self.assertEqual(result, 1)
                 self.assertIn("Missing filename or key arg", mock_stderr.getvalue())
                 sysmock.assert_called_once_with(1)
+
+
+class TestRemoveListItem(unittest.TestCase):
+
+    def test_removelistitem_missing_arg(self):
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
+                sys.argv = ["cmd", "help"]
+                soyaml.removelistitem(["file", "key"])
+                sysmock.assert_called()
+                self.assertIn("Missing filename, key arg, or list item to remove", mock_stderr.getvalue())
+
+    def test_removelistitem(self):
+        filename = "/tmp/so-yaml_test-removelistitem.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: abc }, key2: false, key3: [a,b,c]}")
+        file.close()
+
+        soyaml.removelistitem([filename, "key3", "b"])
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        expected = "key1:\n  child1: 123\n  child2: abc\nkey2: false\nkey3:\n- a\n- c\n"
+        self.assertEqual(actual, expected)
+
+    def test_removelistitem_nested(self):
+        filename = "/tmp/so-yaml_test-removelistitem.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: [a,b,c] }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        soyaml.removelistitem([filename, "key1.child2", "b"])
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        expected = "key1:\n  child1: 123\n  child2:\n  - a\n  - c\nkey2: false\nkey3:\n- e\n- f\n- g\n"
+        self.assertEqual(actual, expected)
+
+    def test_removelistitem_nested_deep(self):
+        filename = "/tmp/so-yaml_test-removelistitem.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        soyaml.removelistitem([filename, "key1.child2.deep2", "b"])
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        expected = "key1:\n  child1: 123\n  child2:\n    deep1: 45\n    deep2:\n    - a\n    - c\nkey2: false\nkey3:\n- e\n- f\n- g\n"
+        self.assertEqual(actual, expected)
+
+    def test_removelistitem_item_not_in_list(self):
+        filename = "/tmp/so-yaml_test-removelistitem.yaml"
+        file = open(filename, "w")
+        file.write("{key1: [a,b,c]}")
+        file.close()
+
+        soyaml.removelistitem([filename, "key1", "d"])
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        expected = "key1:\n- a\n- b\n- c\n"
+        self.assertEqual(actual, expected)
+
+    def test_removelistitem_key_noexist(self):
+        filename = "/tmp/so-yaml_test-removelistitem.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
+                sys.argv = ["cmd", "removelistitem", filename, "key4", "h"]
+                soyaml.main()
+                sysmock.assert_called()
+                self.assertEqual("The key provided does not exist. No action was taken on the file.\n", mock_stderr.getvalue())
+
+    def test_removelistitem_key_noexist_deep(self):
+        filename = "/tmp/so-yaml_test-removelistitem.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
+                sys.argv = ["cmd", "removelistitem", filename, "key1.child2.deep3", "h"]
+                soyaml.main()
+                sysmock.assert_called()
+                self.assertEqual("The key provided does not exist. No action was taken on the file.\n", mock_stderr.getvalue())
+
+    def test_removelistitem_key_nonlist(self):
+        filename = "/tmp/so-yaml_test-removelistitem.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
+                sys.argv = ["cmd", "removelistitem", filename, "key1", "h"]
+                soyaml.main()
+                sysmock.assert_called()
+                self.assertEqual("The existing value for the given key is not a list. No action was taken on the file.\n", mock_stderr.getvalue())
+
+    def test_removelistitem_key_nonlist_deep(self):
+        filename = "/tmp/so-yaml_test-removelistitem.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
+                sys.argv = ["cmd", "removelistitem", filename, "key1.child2.deep1", "h"]
+                soyaml.main()
+                sysmock.assert_called()
+                self.assertEqual("The existing value for the given key is not a list. No action was taken on the file.\n", mock_stderr.getvalue())

salt/manager/tools/sbin/soup

@@ -274,7 +274,7 @@ check_os_updates() {
         if [[ "$confirm" == [cC] ]]; then
           echo "Continuing without updating packages"
         elif [[ "$confirm" == [uU] ]]; then
-          echo "Applying Grid Updates"
+          echo "Applying Grid Updates. The following patch.os salt state may take a while depending on how many packages need to be updated."
           update_flag=true
         else
           echo "Exiting soup"
@@ -426,6 +426,7 @@ preupgrade_changes() {
     [[ "$INSTALLEDVERSION" == 2.4.160 ]] && up_to_2.4.170
     [[ "$INSTALLEDVERSION" == 2.4.170 ]] && up_to_2.4.180
     [[ "$INSTALLEDVERSION" == 2.4.180 ]] && up_to_2.4.190
+    [[ "$INSTALLEDVERSION" == 2.4.190 ]] && up_to_2.4.200
     true
 }
 
@@ -457,6 +458,7 @@ postupgrade_changes() {
     [[ "$POSTVERSION"  == 2.4.160 ]] && post_to_2.4.170
     [[ "$POSTVERSION"  == 2.4.170 ]] && post_to_2.4.180
     [[ "$POSTVERSION"  == 2.4.180 ]] && post_to_2.4.190
+    [[ "$POSTVERSION"  == 2.4.190 ]] && post_to_2.4.200
     true
 }
 
@@ -636,6 +638,13 @@ post_to_2.4.190() {
   POSTVERSION=2.4.190
 }
 
+post_to_2.4.200() {
+  echo "Initiating Suricata idstools migration..."
+  suricata_idstools_removal_post
+
+  POSTVERSION=2.4.200
+}
+
 repo_sync() {
   echo "Sync the local repo."
   su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync."
@@ -903,6 +912,15 @@ up_to_2.4.190() {
   INSTALLEDVERSION=2.4.190
 }
 
+up_to_2.4.200() {
+  echo "Backing up idstools config..."
+  suricata_idstools_removal_pre
+
+  touch /opt/so/state/esfleet_logstash_config_pillar
+
+  INSTALLEDVERSION=2.4.200
+}
+
 add_hydra_pillars() {
   mkdir -p /opt/so/saltstack/local/pillar/hydra
   touch /opt/so/saltstack/local/pillar/hydra/soc_hydra.sls
@@ -986,6 +1004,8 @@ rollover_index() {
 }
 
 suricata_idstools_migration() {
+  # For 2.4.70
+
   #Backup the pillars for idstools
   mkdir -p /nsm/backup/detections-migration/idstools
   rsync -av /opt/so/saltstack/local/pillar/idstools/* /nsm/backup/detections-migration/idstools
@@ -1086,6 +1106,217 @@ playbook_migration() {
   echo "Playbook Migration is complete...."
 }
 
+suricata_idstools_removal_pre() {
+# For SOUPs beginning with 2.4.200 - pre SOUP checks
+
+# Create syncBlock file
+install -d -o 939 -g 939 -m 755 /opt/so/conf/soc/fingerprints
+install -o 939 -g 939 -m 644 /dev/null /opt/so/conf/soc/fingerprints/suricataengine.syncBlock
+cat > /opt/so/conf/soc/fingerprints/suricataengine.syncBlock << EOF
+Suricata ruleset sync is blocked until this file is removed. **CRITICAL** Make sure that you have manually added any custom Suricata rulesets via SOC config before removing this file - review the documentation for more details: https://docs.securityonion.net/en/2.4/nids.html#sync-block
+EOF
+
+# Remove possible symlink & create salt local rules dir
+[ -L /opt/so/saltstack/local/salt/suricata/rules ] && rm -f /opt/so/saltstack/local/salt/suricata/rules
+install -d -o 939 -g 939 /opt/so/saltstack/local/salt/suricata/rules/ || echo "Failed to create Suricata local rules directory"
+
+# Backup custom rules & overrides
+mkdir -p /nsm/backup/detections-migration/2-4-200
+cp /usr/sbin/so-rule-update /nsm/backup/detections-migration/2-4-200
+cp /opt/so/conf/idstools/etc/rulecat.conf /nsm/backup/detections-migration/2-4-200
+
+# Backup so-detection index via reindex
+echo "Creating sos-backup index template..."
+template_result=$(/sbin/so-elasticsearch-query '_index_template/sos-backup' -X PUT \
+    --retry 5 --retry-delay 15 --retry-all-errors \
+    -d '{"index_patterns":["sos-backup-*"],"priority":501,"template":{"settings":{"index":{"number_of_replicas":0,"number_of_shards":1}}}}')
+
+if [[ -z "$template_result" ]] || ! echo "$template_result" | jq -e '.acknowledged == true' > /dev/null 2>&1; then
+    echo "Error: Failed to create sos-backup index template"
+    echo "$template_result"
+    exit 1
+fi
+
+BACKUP_INDEX="sos-backup-detection-$(date +%Y%m%d-%H%M%S)"
+echo "Backing up so-detection index to $BACKUP_INDEX..."
+reindex_result=$(/sbin/so-elasticsearch-query '_reindex?wait_for_completion=true' \
+    --retry 5 --retry-delay 15 --retry-all-errors \
+    -X POST -d "{\"source\": {\"index\": \"so-detection\"}, \"dest\": {\"index\": \"$BACKUP_INDEX\"}}")
+
+if [[ -z "$reindex_result" ]]; then
+    echo "Error: Backup of detections failed - no response from Elasticsearch"
+    exit 1
+elif echo "$reindex_result" | jq -e '.created >= 0' > /dev/null 2>&1; then
+    echo "Backup complete: $(echo "$reindex_result" | jq -r '.created') documents copied"
+elif echo "$reindex_result" | grep -q "index_not_found_exception"; then
+    echo "so-detection index does not exist, skipping backup"
+else
+    echo "Error: Backup of detections failed"
+    echo "$reindex_result"
+    exit 1
+fi
+
+}
+
+suricata_idstools_removal_post() {
+# For SOUPs beginning with 2.4.200 - post SOUP checks
+
+echo "Checking idstools configuration for custom modifications..."
+
+# Normalize and hash file content for consistent comparison
+# Args: $1 - file path
+# Outputs: SHA256 hash to stdout
+# Returns: 0 on success, 1 on failure
+hash_normalized_file() {
+    local file="$1"
+
+    if [[ ! -r "$file" ]]; then
+        return 1
+    fi
+
+    # Ensure trailing newline for consistent hashing regardless of source file
+    { sed -E \
+        -e 's/^[[:space:]]+//; s/[[:space:]]+$//' \
+        -e '/^$/d' \
+        -e 's|--url=http://[^:]+:7788|--url=http://MANAGER:7788|' \
+        "$file"; echo; } | sed '/^$/d' | sha256sum | awk '{print $1}'
+}
+
+# Known-default hashes for so-rule-update (ETOPEN ruleset)
+KNOWN_SO_RULE_UPDATE_HASHES=(
+    # 2.4.100+ (suricata 7.0.3, non-airgap)
+    "5fbd067ced86c8ec72ffb7e1798aa624123b536fb9d78f4b3ad8d3b45db1eae7"  # 2.4.100-2.4.190 non-Airgap
+    # 2.4.90+ airgap (same for 2.4.90 and 2.4.100+)
+    "61f632c55791338c438c071040f1490066769bcce808b595b5cc7974a90e653a"  # 2.4.90+ Airgap
+    # 2.4.90 (suricata 6.0, non-airgap, comment inside proxy block)
+    "0380ec52a05933244ab0f0bc506576e1d838483647b40612d5fe4b378e47aedd"  # 2.4.90 non-Airgap
+    # 2.4.10-2.4.80 (suricata 6.0, non-airgap, comment outside proxy block)
+    "b6e4d1b5a78d57880ad038a9cd2cc6978aeb2dd27d48ea1a44dd866a2aee7ff4"  # 2.4.10-2.4.80 non-Airgap
+    # 2.4.10-2.4.80 airgap
+    "b20146526ace2b142fde4664f1386a9a1defa319b3a1d113600ad33a1b037dad"  # 2.4.10-2.4.80 Airgap
+    # 2.4.5 and earlier (no pidof check, non-airgap)
+    "d04f5e4015c348133d28a7840839e82d60009781eaaa1c66f7f67747703590dc"  # 2.4.5 non-Airgap
+)
+
+# Known-default hashes for rulecat.conf
+KNOWN_RULECAT_CONF_HASHES=(
+    # 2.4.100+ (suricata 7.0.3)
+    "302e75dca9110807f09ade2eec3be1fcfc8b2bf6cf2252b0269bb72efeefe67e"  # 2.4.100-2.4.190 without SURICATA md_engine
+    "8029b7718c324a9afa06a5cf180afde703da1277af4bdd30310a6cfa3d6398cb"  # 2.4.100-2.4.190 with SURICATA md_engine
+    # 2.4.80-2.4.90 (suricata 6.0, with --suricata-version and --output)
+    "4d8b318e6950a6f60b02f307cf27c929efd39652990c1bd0c8820aa8a307e1e7"  # 2.4.80-2.4.90 without SURICATA md_engine
+    "a1ddf264c86c4e91c81c5a317f745a19466d4311e4533ec3a3c91fed04c11678"  # 2.4.80-2.4.90 with SURICATA md_engine
+    # 2.4.50-2.4.70 (/suri/ path, no --suricata-version)
+    "86e3afb8d0f00c62337195602636864c98580a13ca9cc85029661a539deae6ae"  # 2.4.50-2.4.70 without SURICATA md_engine
+    "5a97604ca5b820a10273a2d6546bb5e00c5122ca5a7dfe0ba0bfbce5fc026f4b"  # 2.4.50-2.4.70 with SURICATA md_engine
+    # 2.4.20-2.4.40 (/nids/ path without /suri/)
+    "d098ea9ecd94b5cca35bf33543f8ea8f48066a0785221fabda7fef43d2462c29"  # 2.4.20-2.4.40 without SURICATA md_engine
+    "9dbc60df22ae20d65738ba42e620392577857038ba92278e23ec182081d191cd"  # 2.4.20-2.4.40 with SURICATA md_engine
+    # 2.4.5-2.4.10 (/sorules/ path for extraction/filters)
+    "490f6843d9fca759ee74db3ada9c702e2440b8393f2cfaf07bbe41aaa6d955c3"  # 2.4.5-2.4.10 with SURICATA md_engine
+    # Note: 2.4.5-2.4.10 without SURICATA md_engine has same hash as 2.4.20-2.4.40 without SURICATA md_engine
+)
+
+# Check a config file against known hashes
+# Args: $1 - file path, $2 - array name of known hashes
+check_config_file() {
+    local file="$1"
+    local known_hashes_array="$2"
+    local file_display_name=$(basename "$file")
+
+    if [[ ! -f "$file" ]]; then
+        echo "Warning: $file not found"
+        echo "$file_display_name not found - manual verification required" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock
+        return 1
+    fi
+
+    echo "Hashing $file..."
+    local file_hash
+    if ! file_hash=$(hash_normalized_file "$file"); then
+        echo "Warning: Could not read $file"
+        echo "$file_display_name not readable - manual verification required" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock
+        return 1
+    fi
+
+    echo "  Hash: $file_hash"
+
+    # Check if hash matches any known default
+    local -n known_hashes=$known_hashes_array
+    for known_hash in "${known_hashes[@]}"; do
+        if [[ "$file_hash" == "$known_hash" ]]; then
+            echo "  Matches known default configuration"
+            return 0
+        fi
+    done
+
+    # No match - custom configuration detected
+    echo "Does not match known default - custom configuration detected"
+    echo "Custom $file_display_name detected (hash: $file_hash)" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock
+
+    # If this is so-rule-update, check for ETPRO license code and write out to the syncBlock file
+    # If ETPRO is enabled, the license code already exists in the so-rule-update script, this is just making it easier to migrate
+    if [[ "$file_display_name" == "so-rule-update" ]]; then
+        local etpro_code
+        etpro_code=$(grep -oP '\-\-etpro=\K[0-9a-fA-F]+' "$file" 2>/dev/null) || true
+        if [[ -n "$etpro_code" ]]; then
+            echo "ETPRO code found: $etpro_code" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock
+        fi
+    fi
+
+    return 1
+}
+
+# Check so-rule-update and rulecat.conf
+SO_RULE_UPDATE="/usr/sbin/so-rule-update"
+RULECAT_CONF="/opt/so/conf/idstools/etc/rulecat.conf"
+
+custom_found=0
+
+check_config_file "$SO_RULE_UPDATE" "KNOWN_SO_RULE_UPDATE_HASHES" || custom_found=1
+check_config_file "$RULECAT_CONF" "KNOWN_RULECAT_CONF_HASHES" || custom_found=1
+
+# Check for ETPRO rules on airgap systems
+if [[ $is_airgap -eq 0 ]] && grep -q 'ETPRO ' /nsm/rules/suricata/emerging-all.rules 2>/dev/null; then
+    echo "ETPRO rules detected on airgap system - custom configuration"
+    echo "ETPRO rules detected on Airgap in /nsm/rules/suricata/emerging-all.rules" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock
+    custom_found=1
+fi
+
+# If no custom configs found, remove syncBlock
+if [[ $custom_found -eq 0 ]]; then
+    echo "idstools migration completed successfully - removing Suricata engine syncBlock"
+    rm -f /opt/so/conf/soc/fingerprints/suricataengine.syncBlock
+else
+    echo "Custom idstools configuration detected - syncBlock remains in place"
+    echo "Review /opt/so/conf/soc/fingerprints/suricataengine.syncBlock for details"
+fi
+
+echo "Cleaning up idstools"
+echo "Stopping and removing the idstools container..."
+if [ -n "$(docker ps -q -f name=^so-idstools$)" ]; then
+    image_name=$(docker ps -a --filter name=^so-idstools$ --format '{{.Image}}' 2>/dev/null || true)
+    docker stop so-idstools || echo "Warning: failed to stop so-idstools container"
+    docker rm so-idstools || echo "Warning: failed to remove so-idstools container"
+
+    if [[ -n "$image_name" ]]; then
+        echo "Removing idstools image: $image_name"
+        docker rmi "$image_name" || echo "Warning: failed to remove image $image_name"
+    fi
+fi
+
+echo "Removing idstools symlink and scripts..."
+rm -rf /usr/sbin/so-idstools*
+sed -i '/^#\?so-idstools$/d' /opt/so/conf/so-status/so-status.conf
+crontab -l | grep -v 'so-rule-update' | crontab -
+
+# Backup the salt master config & manager pillar before editing it
+cp /opt/so/saltstack/local/pillar/minions/$MINIONID.sls /nsm/backup/detections-migration/2-4-200/
+cp /etc/salt/master  /nsm/backup/detections-migration/2-4-200/
+so-yaml.py remove /opt/so/saltstack/local/pillar/minions/$MINIONID.sls idstools
+so-yaml.py removelistitem /etc/salt/master file_roots.base /opt/so/rules/nids
+
+}
+
 determine_elastic_agent_upgrade() {
   if [[ $is_airgap -eq 0 ]]; then
     update_elastic_agent_airgap
@@ -1132,7 +1363,7 @@ unmount_update() {
 
 update_airgap_rules() {
   # Copy the rules over to update them for airgap.
-  rsync -a $UPDATE_DIR/agrules/suricata/* /nsm/rules/suricata/
+  rsync -a --delete $UPDATE_DIR/agrules/suricata/ /nsm/rules/suricata/etopen/
   rsync -a $UPDATE_DIR/agrules/detect-sigma/* /nsm/rules/detect-sigma/
   rsync -a $UPDATE_DIR/agrules/detect-yara/* /nsm/rules/detect-yara/
   # Copy the securityonion-resorces repo over for SOC Detection Summaries and checkout the published summaries branch
@@ -1318,6 +1549,8 @@ upgrade_salt() {
     fi
   # Else do Ubuntu things
   elif [[ $is_deb ]]; then
+    # ensure these files don't exist when upgrading from 3006.9 to 3006.16
+    rm -f /etc/apt/keyrings/salt-archive-keyring-2023.pgp /etc/apt/sources.list.d/salt.list
     echo "Removing apt hold for Salt."
     echo ""
     apt-mark unhold "salt-common"
@@ -1640,7 +1873,7 @@ main() {
       if [[ $is_airgap -eq 0 ]]; then
         echo ""
         echo "Cleaning repos on remote Security Onion nodes."
-        salt -C 'not *_eval and not *_manager and not *_managersearch and not *_standalone and G@os:CentOS' cmd.run "yum clean all"
+        salt -C 'not *_eval and not *_manager* and not *_standalone and G@os:OEL' cmd.run "dnf clean all"
         echo ""
       fi
     fi
@@ -1679,7 +1912,7 @@ This appears to be a distributed deployment. Other nodes should update themselve
 
 Each minion is on a random 15 minute check-in period and things like network bandwidth can be a factor in how long the actual upgrade takes. If you have a heavy node on a slow link, it is going to take a while to get the containers to it. Depending on what changes happened between the versions, Elasticsearch might not be able to talk to said heavy node until the update is complete.
 
-If it looks like you’re missing data after the upgrade, please avoid restarting services and instead make sure at least one search node has completed its upgrade. The best way to do this is to run 'sudo salt-call state.highstate' from a search node and make sure there are no errors. Typically if it works on one node it will work on the rest. Forward nodes are less complex and will update as they check in so you can monitor those from the Grid section of SOC.
+If it looks like you’re missing data after the upgrade, please avoid restarting services and instead make sure at least one search node has completed its upgrade. The best way to do this is to run 'sudo salt-call state.highstate' from a search node and make sure there are no errors. Typically if it works on one node it will work on the rest. Sensor nodes are less complex and will update as they check in so you can monitor those from the Grid section of SOC.
 
 For more information, please see $DOC_BASE_URL/soup.html#distributed-deployments.
 

salt/pcap/config.sls

@@ -8,12 +8,9 @@
 
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% from "pcap/config.map.jinja" import PCAPMERGED %}
-{% from 'bpf/pcap.map.jinja' import PCAPBPF %}
-
-{% set BPF_COMPILED = "" %}
+{% from 'bpf/pcap.map.jinja' import PCAPBPF, PCAP_BPF_STATUS, PCAP_BPF_CALC, STENO_BPF_COMPILED %}
 
 # PCAP Section
-
 stenographergroup:
   group.present:
     - name: stenographer
@@ -40,18 +37,12 @@ pcap_sbin:
     - group: 939
     - file_mode: 755
 
-{% if PCAPBPF %}
-   {% set BPF_CALC = salt['cmd.script']('salt://common/tools/sbin/so-bpf-compile', GLOBALS.sensor.interface + ' ' + PCAPBPF|join(" "),cwd='/root') %}
-   {% if BPF_CALC['stderr'] == "" %}
-      {% set BPF_COMPILED =  ",\\\"--filter=" + BPF_CALC['stdout'] + "\\\""  %}
-   {% else  %}
-
-bpfcompilationfailure:
+{% if PCAPBPF and not PCAP_BPF_STATUS %}
+stenoPCAPbpfcompilationfailure:
   test.configurable_test_state:
    - changes: False
    - result: False
-   - comment: "BPF Compilation Failed - Discarding Specified BPF"
-   {% endif %}
+  - comment: "BPF Syntax Error - Discarding Specified BPF. Error: {{ PCAP_BPF_CALC['stderr'] }}"
 {% endif %}
 
 stenoconf:
@@ -64,7 +55,7 @@ stenoconf:
     - template: jinja
     - defaults:
         PCAPMERGED: {{ PCAPMERGED }}
-        BPF_COMPILED: "{{ BPF_COMPILED }}"
+        STENO_BPF_COMPILED: "{{ STENO_BPF_COMPILED }}"
 
 stenoca:
   file.directory:

salt/pcap/files/config.jinja

@@ -6,6 +6,6 @@
   , "Interface": "{{ pillar.sensor.interface }}"
   , "Port": 1234
   , "Host": "127.0.0.1"
-  , "Flags": ["-v", "--blocks={{ PCAPMERGED.config.blocks }}", "--preallocate_file_mb={{ PCAPMERGED.config.preallocate_file_mb }}", "--aiops={{ PCAPMERGED.config.aiops }}", "--uid=stenographer", "--gid=stenographer"{{ BPF_COMPILED }}]
+  , "Flags": ["-v", "--blocks={{ PCAPMERGED.config.blocks }}", "--preallocate_file_mb={{ PCAPMERGED.config.preallocate_file_mb }}", "--aiops={{ PCAPMERGED.config.aiops }}", "--uid=stenographer", "--gid=stenographer"{{ STENO_BPF_COMPILED }}]
   , "CertPath": "/etc/stenographer/certs"
 }

salt/pcap/soc_pcap.yaml

@@ -7,7 +7,7 @@ pcap:
       description: By default, Stenographer limits the number of files in the pcap directory to 30000 to avoid limitations with the ext3 filesystem. However, if you're using the ext4 or xfs filesystems, then it is safe to increase this value. So if you have a large amount of storage and find that you only have 3 weeks worth of PCAP on disk while still having plenty of free space, then you may want to increase this default setting.
       helpLink: stenographer.html
     diskfreepercentage:
-      description: Stenographer will purge old PCAP on a regular basis to keep the disk free percentage at this level. If you have a distributed deployment with dedicated forward nodes, then the default value of 10 should be reasonable since Stenographer should be the main consumer of disk space in the /nsm partition. However, if you have systems that run both Stenographer and Elasticsearch at the same time (like eval and standalone installations), then you’ll want to make sure that this value is no lower than 21 so that you avoid Elasticsearch hitting its watermark setting at 80% disk usage. If you have an older standalone installation, then you may need to manually change this value to 21.
+      description: Stenographer will purge old PCAP on a regular basis to keep the disk free percentage at this level. If you have a distributed deployment with dedicated Sensor nodes, then the default value of 10 should be reasonable since Stenographer should be the main consumer of disk space in the /nsm partition. However, if you have systems that run both Stenographer and Elasticsearch at the same time (like eval and standalone installations), then you’ll want to make sure that this value is no lower than 21 so that you avoid Elasticsearch hitting its watermark setting at 80% disk usage. If you have an older standalone installation, then you may need to manually change this value to 21.
       helpLink: stenographer.html
     blocks:
       description: The number of 1MB packet blocks used by Stenographer and AF_PACKET to store packets in memory, per thread. You shouldn't need to change this. 

salt/registry/enabled.sls

@@ -5,6 +5,7 @@
 
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
+{%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKER %}
 
 include:
@@ -57,6 +58,17 @@ so-dockerregistry:
       - x509: registry_crt
       - x509: registry_key
 
+wait_for_so-dockerregistry:
+  http.wait_for_successful_query:
+    - name: 'https://{{ GLOBALS.registry_host }}:5000/v2/'
+    - ssl: True
+    - verify_ssl: False
+    - status: 200
+    - wait_for: 120
+    - request_interval: 5
+    - require:
+      -  docker_container: so-dockerregistry
+
 delete_so-dockerregistry_so-status.disabled:
   file.uncomment:
     - name: /opt/so/conf/so-status/so-status.conf

salt/salt/engines/master/virtual_node_manager.py

@@ -727,7 +727,8 @@ def check_hypervisor_disk_space(hypervisor: str, size_gb: int) -> Tuple[bool, Op
         result = local.cmd(
             hypervisor_minion,
             'cmd.run',
-            ["df -BG /nsm/libvirt/volumes | tail -1 | awk '{print $4}' | sed 's/G//'"]
+            ["df -BG /nsm/libvirt/volumes | tail -1 | awk '{print $4}' | sed 's/G//'"],
+            kwarg={'python_shell': True}
         )
         
         if not result or hypervisor_minion not in result:

salt/salt/scripts/bootstrap-salt.sh

@@ -26,7 +26,7 @@
 #======================================================================================================================
 set -o nounset                              # Treat unset variables as an error
 
-__ScriptVersion="2025.02.24"
+__ScriptVersion="2025.09.03"
 __ScriptName="bootstrap-salt.sh"
 
 __ScriptFullName="$0"
@@ -48,6 +48,7 @@ __ScriptArgs="$*"
 #   * BS_GENTOO_USE_BINHOST:    If 1 add `--getbinpkg` to gentoo's emerge
 #   * BS_SALT_MASTER_ADDRESS:   The IP or DNS name of the salt-master the minion should connect to
 #   * BS_SALT_GIT_CHECKOUT_DIR: The directory where to clone Salt on git installations
+#   * BS_TMP_DIR:               The directory to use for executing the installation (defaults to /tmp)
 #======================================================================================================================
 
 
@@ -171,12 +172,12 @@ __check_config_dir() {
 
     case "$CC_DIR_NAME" in
         http://*|https://*)
-            __fetch_url "/tmp/${CC_DIR_BASE}" "${CC_DIR_NAME}"
-            CC_DIR_NAME="/tmp/${CC_DIR_BASE}"
+            __fetch_url "${_TMP_DIR}/${CC_DIR_BASE}" "${CC_DIR_NAME}"
+            CC_DIR_NAME="${_TMP_DIR}/${CC_DIR_BASE}"
             ;;
         ftp://*)
-            __fetch_url "/tmp/${CC_DIR_BASE}" "${CC_DIR_NAME}"
-            CC_DIR_NAME="/tmp/${CC_DIR_BASE}"
+            __fetch_url "${_TMP_DIR}/${CC_DIR_BASE}" "${CC_DIR_NAME}"
+            CC_DIR_NAME="${_TMP_DIR}/${CC_DIR_BASE}"
             ;;
         *://*)
             echoerror "Unsupported URI scheme for $CC_DIR_NAME"
@@ -194,22 +195,22 @@ __check_config_dir() {
 
     case "$CC_DIR_NAME" in
         *.tgz|*.tar.gz)
-            tar -zxf "${CC_DIR_NAME}" -C /tmp
+            tar -zxf "${CC_DIR_NAME}" -C ${_TMP_DIR}
             CC_DIR_BASE=$(basename "${CC_DIR_BASE}" ".tgz")
             CC_DIR_BASE=$(basename "${CC_DIR_BASE}" ".tar.gz")
-            CC_DIR_NAME="/tmp/${CC_DIR_BASE}"
+            CC_DIR_NAME="${_TMP_DIR}/${CC_DIR_BASE}"
             ;;
         *.tbz|*.tar.bz2)
-            tar -xjf "${CC_DIR_NAME}" -C /tmp
+            tar -xjf "${CC_DIR_NAME}" -C ${_TMP_DIR}
             CC_DIR_BASE=$(basename "${CC_DIR_BASE}" ".tbz")
             CC_DIR_BASE=$(basename "${CC_DIR_BASE}" ".tar.bz2")
-            CC_DIR_NAME="/tmp/${CC_DIR_BASE}"
+            CC_DIR_NAME="${_TMP_DIR}/${CC_DIR_BASE}"
             ;;
         *.txz|*.tar.xz)
-            tar -xJf "${CC_DIR_NAME}" -C /tmp
+            tar -xJf "${CC_DIR_NAME}" -C ${_TMP_DIR}
             CC_DIR_BASE=$(basename "${CC_DIR_BASE}" ".txz")
             CC_DIR_BASE=$(basename "${CC_DIR_BASE}" ".tar.xz")
-            CC_DIR_NAME="/tmp/${CC_DIR_BASE}"
+            CC_DIR_NAME="${_TMP_DIR}/${CC_DIR_BASE}"
             ;;
     esac
 
@@ -245,6 +246,7 @@ __check_unparsed_options() {
 #----------------------------------------------------------------------------------------------------------------------
 _KEEP_TEMP_FILES=${BS_KEEP_TEMP_FILES:-$BS_FALSE}
 _TEMP_CONFIG_DIR="null"
+_TMP_DIR=${BS_TMP_DIR:-"/tmp"}
 _SALTSTACK_REPO_URL="https://github.com/saltstack/salt.git"
 _SALT_REPO_URL=${_SALTSTACK_REPO_URL}
 _TEMP_KEYS_DIR="null"
@@ -281,7 +283,7 @@ _SIMPLIFY_VERSION=$BS_TRUE
 _LIBCLOUD_MIN_VERSION="0.14.0"
 _EXTRA_PACKAGES=""
 _HTTP_PROXY=""
-_SALT_GIT_CHECKOUT_DIR=${BS_SALT_GIT_CHECKOUT_DIR:-/tmp/git/salt}
+_SALT_GIT_CHECKOUT_DIR=${BS_SALT_GIT_CHECKOUT_DIR:-${_TMP_DIR}/git/salt}
 _NO_DEPS=$BS_FALSE
 _FORCE_SHALLOW_CLONE=$BS_FALSE
 _DISABLE_SSL=$BS_FALSE
@@ -367,7 +369,7 @@ __usage() {
         also be specified. Salt installation will be ommitted, but some of the
         dependencies could be installed to write configuration with -j or -J.
     -d  Disables checking if Salt services are enabled to start on system boot.
-        You can also do this by touching /tmp/disable_salt_checks on the target
+        You can also do this by touching ${BS_TMP_DIR}/disable_salt_checks on the target
         host. Default: \${BS_FALSE}
     -D  Show debug output
     -f  Force shallow cloning for git installations.
@@ -424,6 +426,9 @@ __usage() {
     -r  Disable all repository configuration performed by this script. This
         option assumes all necessary repository configuration is already present
         on the system.
+    -T  If set this overrides the use of /tmp for script execution. This is
+        to allow for systems in which noexec is applied to temp filesystem mounts
+        for security reasons
     -U  If set, fully upgrade the system prior to bootstrapping Salt
     -v  Display script version
     -V  Install Salt into virtualenv
@@ -436,7 +441,7 @@ __usage() {
 EOT
 }   # ----------  end of function __usage  ----------
 
-while getopts ':hvnDc:g:Gx:k:s:MSWNXCPFUKIA:i:Lp:dH:bflV:J:j:rR:aqQ' opt
+while getopts ':hvnDc:g:Gx:k:s:MSWNXCPFUKIA:i:Lp:dH:bflV:J:j:rR:T:aqQ' opt
 do
   case "${opt}" in
 
@@ -478,6 +483,7 @@ do
     a )  _PIP_ALL=$BS_TRUE                              ;;
     r )  _DISABLE_REPOS=$BS_TRUE                        ;;
     R )  _CUSTOM_REPO_URL=$OPTARG                       ;;
+    T )  _TMP_DIR="$OPTARG"                             ;;
     J )  _CUSTOM_MASTER_CONFIG=$OPTARG                  ;;
     j )  _CUSTOM_MINION_CONFIG=$OPTARG                  ;;
     q )  _QUIET_GIT_INSTALLATION=$BS_TRUE               ;;
@@ -495,10 +501,10 @@ done
 shift $((OPTIND-1))
 
 # Define our logging file and pipe paths
-LOGFILE="/tmp/$( echo "$__ScriptName" | sed s/.sh/.log/g )"
-LOGPIPE="/tmp/$( echo "$__ScriptName" | sed s/.sh/.logpipe/g )"
+LOGFILE="${_TMP_DIR}/$( echo "$__ScriptName" | sed s/.sh/.log/g )"
+LOGPIPE="${_TMP_DIR}/$( echo "$__ScriptName" | sed s/.sh/.logpipe/g )"
 # Ensure no residual pipe exists
-rm "$LOGPIPE" 2>/dev/null
+rm -f "$LOGPIPE" 2>/dev/null
 
 # Create our logging pipe
 # On FreeBSD we have to use mkfifo instead of mknod
@@ -534,7 +540,7 @@ exec 2>"$LOGPIPE"
 #              14               SIGALRM
 #              15               SIGTERM
 #----------------------------------------------------------------------------------------------------------------------
-APT_ERR=$(mktemp /tmp/apt_error.XXXXXX)
+APT_ERR=$(mktemp ${_TMP_DIR}/apt_error.XXXXXX)
 __exit_cleanup() {
     EXIT_CODE=$?
 
@@ -927,6 +933,11 @@ if [ -d "${_VIRTUALENV_DIR}" ]; then
     exit 1
 fi
 
+# Make sure the designated temp directory exists
+if [ ! -d "${_TMP_DIR}" ]; then
+    mkdir -p "${_TMP_DIR}"
+fi
+
 #---  FUNCTION  -------------------------------------------------------------------------------------------------------
 #         NAME:  __fetch_url
 #  DESCRIPTION:  Retrieves a URL and writes it to a given path
@@ -1941,11 +1952,6 @@ __wait_for_apt(){
     # Timeout set at 15 minutes
     WAIT_TIMEOUT=900
 
-    ## see if sync'ing the clocks helps
-    if [ -f /usr/sbin/hwclock ]; then
-        /usr/sbin/hwclock -s
-    fi
-
     # Run our passed in apt command
     "${@}" 2>"$APT_ERR"
     APT_RETURN=$?
@@ -1996,14 +2002,14 @@ __apt_get_upgrade_noinput() {
 #----------------------------------------------------------------------------------------------------------------------
 __temp_gpg_pub() {
     if __check_command_exists mktemp; then
-        tempfile="$(mktemp /tmp/salt-gpg-XXXXXXXX.pub 2>/dev/null)"
+        tempfile="$(mktemp ${_TMP_DIR}/salt-gpg-XXXXXXXX.pub 2>/dev/null)"
 
         if [ -z "$tempfile" ]; then
-            echoerror "Failed to create temporary file in /tmp"
+            echoerror "Failed to create temporary file in ${_TMP_DIR}"
             return 1
         fi
     else
-        tempfile="/tmp/salt-gpg-$$.pub"
+        tempfile="${_TMP_DIR}/salt-gpg-$$.pub"
     fi
 
     echo $tempfile
@@ -2043,7 +2049,7 @@ __rpm_import_gpg() {
     __fetch_url "$tempfile" "$url" || return 1
 
     # At least on CentOS 8, a missing newline at the end causes:
-    #   error: /tmp/salt-gpg-n1gKUb1u.pub: key 1 not an armored public key.
+    #   error: ${_TMP_DIR}/salt-gpg-n1gKUb1u.pub: key 1 not an armored public key.
     # shellcheck disable=SC1003,SC2086
     sed -i -e '$a\' $tempfile
 
@@ -2109,7 +2115,7 @@ __git_clone_and_checkout() {
     fi
 
     __SALT_GIT_CHECKOUT_PARENT_DIR=$(dirname "${_SALT_GIT_CHECKOUT_DIR}" 2>/dev/null)
-    __SALT_GIT_CHECKOUT_PARENT_DIR="${__SALT_GIT_CHECKOUT_PARENT_DIR:-/tmp/git}"
+    __SALT_GIT_CHECKOUT_PARENT_DIR="${__SALT_GIT_CHECKOUT_PARENT_DIR:-${_TMP_DIR}/git}"
     __SALT_CHECKOUT_REPONAME="$(basename "${_SALT_GIT_CHECKOUT_DIR}" 2>/dev/null)"
     __SALT_CHECKOUT_REPONAME="${__SALT_CHECKOUT_REPONAME:-salt}"
     [ -d "${__SALT_GIT_CHECKOUT_PARENT_DIR}" ] || mkdir "${__SALT_GIT_CHECKOUT_PARENT_DIR}"
@@ -2162,7 +2168,7 @@ __git_clone_and_checkout() {
 
         if [ "$__SHALLOW_CLONE" -eq $BS_TRUE ]; then
             # Let's try 'treeless' cloning to speed up. Treeless cloning omits trees and blobs ('files')
-	    # but includes metadata (commit history, tags, branches etc.
+            # but includes metadata (commit history, tags, branches etc.
             # Test for "--filter" option introduced in git 2.19, the minimal version of git where the treeless
             # cloning we need actually works
             if [ "$(git clone 2>&1 | grep 'filter')" != "" ]; then
@@ -2390,14 +2396,14 @@ __overwriteconfig() {
 
     # Make a tempfile to dump any python errors into.
     if __check_command_exists mktemp; then
-        tempfile="$(mktemp /tmp/salt-config-XXXXXXXX 2>/dev/null)"
+        tempfile="$(mktemp ${_TMP_DIR}/salt-config-XXXXXXXX 2>/dev/null)"
 
         if [ -z "$tempfile" ]; then
-            echoerror "Failed to create temporary file in /tmp"
+            echoerror "Failed to create temporary file in ${_TMP_DIR}"
             return 1
         fi
     else
-        tempfile="/tmp/salt-config-$$"
+        tempfile="${_TMP_DIR}/salt-config-$$"
     fi
 
     if [ -n "$_PY_EXE" ]; then
@@ -2760,8 +2766,8 @@ __install_salt_from_repo() {
     echoinfo "Installing salt using ${_py_exe}, $(${_py_exe} --version)"
     cd "${_SALT_GIT_CHECKOUT_DIR}" || return 1
 
-    mkdir -p /tmp/git/deps
-    echodebug "Created directory /tmp/git/deps"
+    mkdir -p ${_TMP_DIR}/git/deps
+    echodebug "Created directory ${_TMP_DIR}/git/deps"
 
     if [ ${DISTRO_NAME_L} = "ubuntu" ] && [ "$DISTRO_MAJOR_VERSION" -eq 22 ]; then
         echodebug "Ubuntu 22.04 has problem with base.txt requirements file, not parsing sys_platform == 'win32', upgrading from default pip works"
@@ -2774,7 +2780,7 @@ __install_salt_from_repo() {
         fi
     fi
 
-    rm -f /tmp/git/deps/*
+    rm -f ${_TMP_DIR}/git/deps/*
 
     echodebug "Installing Salt requirements from PyPi, ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed ${_PIP_INSTALL_ARGS} -r requirements/static/ci/py${_py_version}/linux.txt"
     ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed ${_PIP_INSTALL_ARGS} -r "requirements/static/ci/py${_py_version}/linux.txt"
@@ -2799,7 +2805,7 @@ __install_salt_from_repo() {
 
     echodebug "Running '${_py_exe} setup.py --salt-config-dir=$_SALT_ETC_DIR --salt-cache-dir=${_SALT_CACHE_DIR} ${SETUP_PY_INSTALL_ARGS} bdist_wheel'"
     ${_py_exe} setup.py --salt-config-dir="$_SALT_ETC_DIR" --salt-cache-dir="${_SALT_CACHE_DIR} ${SETUP_PY_INSTALL_ARGS}" bdist_wheel || return 1
-    mv dist/salt*.whl /tmp/git/deps/ || return 1
+    mv dist/salt*.whl ${_TMP_DIR}/git/deps/ || return 1
 
     cd "${__SALT_GIT_CHECKOUT_PARENT_DIR}" || return 1
 
@@ -2813,14 +2819,14 @@ __install_salt_from_repo() {
         ${_pip_cmd} install --force-reinstall --break-system-packages "${_arch_dep}"
     fi
 
-    echodebug "Running '${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --no-deps --force-reinstall ${_PIP_INSTALL_ARGS} /tmp/git/deps/salt*.whl'"
+    echodebug "Running '${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --no-deps --force-reinstall ${_PIP_INSTALL_ARGS} ${_TMP_DIR}/git/deps/salt*.whl'"
 
-    echodebug "Running ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --no-deps --force-reinstall ${_PIP_INSTALL_ARGS} --global-option=--salt-config-dir=$_SALT_ETC_DIR --salt-cache-dir=${_SALT_CACHE_DIR} ${SETUP_PY_INSTALL_ARGS} /tmp/git/deps/salt*.whl"
+    echodebug "Running ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --no-deps --force-reinstall ${_PIP_INSTALL_ARGS} --global-option=--salt-config-dir=$_SALT_ETC_DIR --salt-cache-dir=${_SALT_CACHE_DIR} ${SETUP_PY_INSTALL_ARGS} ${_TMP_DIR}/git/deps/salt*.whl"
 
     ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --no-deps --force-reinstall \
         ${_PIP_INSTALL_ARGS} \
         --global-option="--salt-config-dir=$_SALT_ETC_DIR --salt-cache-dir=${_SALT_CACHE_DIR} ${SETUP_PY_INSTALL_ARGS}" \
-        /tmp/git/deps/salt*.whl || return 1
+        ${_TMP_DIR}/git/deps/salt*.whl || return 1
 
     echoinfo "Checking if Salt can be imported using ${_py_exe}"
     CHECK_SALT_SCRIPT=$(cat << EOM
@@ -6295,8 +6301,8 @@ __get_packagesite_onedir_latest() {
 }
 
 
-__install_saltstack_photon_onedir_repository() {
-    echodebug "__install_saltstack_photon_onedir_repository() entry"
+__install_saltstack_vmware_photon_os_onedir_repository() {
+    echodebug "__install_saltstack_vmware_photon_os_onedir_repository() entry"
 
     if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -ne 3 ]; then
         echoerror "Python version is no longer supported, only Python 3"
@@ -6376,8 +6382,8 @@ __install_saltstack_photon_onedir_repository() {
     return 0
 }
 
-install_photon_deps() {
-    echodebug "install_photon_deps() entry"
+install_vmware_photon_os_deps() {
+    echodebug "install_vmware_photon_os_deps() entry"
 
     if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -ne 3 ]; then
         echoerror "Python version is no longer supported, only Python 3"
@@ -6406,8 +6412,8 @@ install_photon_deps() {
     return 0
 }
 
-install_photon_stable_post() {
-    echodebug "install_photon_stable_post() entry"
+install_vmware_photon_os_stable_post() {
+    echodebug "install_vmware_photon_os_stable_post() entry"
 
     for fname in api master minion syndic; do
         # Skip salt-api since the service should be opt-in and not necessarily started on boot
@@ -6424,8 +6430,8 @@ install_photon_stable_post() {
     done
 }
 
-install_photon_git_deps() {
-    echodebug "install_photon_git_deps() entry"
+install_vmware_photon_os_git_deps() {
+    echodebug "install_vmware_photon_os_git_deps() entry"
 
     if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -ne 3 ]; then
         echoerror "Python version is no longer supported, only Python 3"
@@ -6463,7 +6469,7 @@ install_photon_git_deps() {
 
     __PACKAGES="python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc glibc-devel linux-devel.x86_64 cython${PY_PKG_VER}"
 
-    echodebug "install_photon_git_deps() distro major version, ${DISTRO_MAJOR_VERSION}"
+    echodebug "install_vmware_photon_os_git_deps() distro major version, ${DISTRO_MAJOR_VERSION}"
 
     ## Photon 5 container is missing systemd on default installation
     if [ "${DISTRO_MAJOR_VERSION}" -lt 5  ]; then
@@ -6489,8 +6495,8 @@ install_photon_git_deps() {
     return 0
 }
 
-install_photon_git() {
-    echodebug "install_photon_git() entry"
+install_vmware_photon_os_git() {
+    echodebug "install_vmware_photon_os_git() entry"
 
     if [ "${_PY_EXE}" != "" ]; then
         _PYEXE=${_PY_EXE}
@@ -6500,7 +6506,7 @@ install_photon_git() {
         return 1
     fi
 
-    install_photon_git_deps
+    install_vmware_photon_os_git_deps
 
     if [ -f "${_SALT_GIT_CHECKOUT_DIR}/salt/syspaths.py" ]; then
         ${_PYEXE} setup.py --salt-config-dir="$_SALT_ETC_DIR" --salt-cache-dir="${_SALT_CACHE_DIR}" ${SETUP_PY_INSTALL_ARGS} install --prefix=/usr || return 1
@@ -6510,8 +6516,8 @@ install_photon_git() {
     return 0
 }
 
-install_photon_git_post() {
-    echodebug "install_photon_git_post() entry"
+install_vmware_photon_os_git_post() {
+    echodebug "install_vmware_photon_os_git_post() entry"
 
     for fname in api master minion syndic; do
         # Skip if not meant to be installed
@@ -6543,9 +6549,9 @@ install_photon_git_post() {
     done
 }
 
-install_photon_restart_daemons() {
+install_vmware_photon_os_restart_daemons() {
     [ "$_START_DAEMONS" -eq $BS_FALSE ] && return
-    echodebug "install_photon_restart_daemons() entry"
+    echodebug "install_vmware_photon_os_restart_daemons() entry"
 
 
     for fname in api master minion syndic; do
@@ -6567,8 +6573,8 @@ install_photon_restart_daemons() {
     done
 }
 
-install_photon_check_services() {
-    echodebug "install_photon_check_services() entry"
+install_vmware_photon_os_check_services() {
+    echodebug "install_vmware_photon_os_check_services() entry"
 
     for fname in api master minion syndic; do
         # Skip salt-api since the service should be opt-in and not necessarily started on boot
@@ -6585,8 +6591,8 @@ install_photon_check_services() {
     return 0
 }
 
-install_photon_onedir_deps() {
-    echodebug "install_photon_onedir_deps() entry"
+install_vmware_photon_os_onedir_deps() {
+    echodebug "install_vmware_photon_os_onedir_deps() entry"
 
 
     if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then
@@ -6600,17 +6606,17 @@ install_photon_onedir_deps() {
     fi
 
     if [ "$_DISABLE_REPOS" -eq "$BS_FALSE" ]; then
-        __install_saltstack_photon_onedir_repository || return 1
+        __install_saltstack_vmware_photon_os_onedir_repository || return 1
     fi
 
     # If -R was passed, we need to configure custom repo url with rsync-ed packages
     # Which was handled in __install_saltstack_rhel_repository buu that hanlded old-stable which is for
     # releases which are End-Of-Life. This call has its own check in case -r was passed without -R.
     if [ "$_CUSTOM_REPO_URL" != "null" ]; then
-        __install_saltstack_photon_onedir_repository || return 1
+        __install_saltstack_vmware_photon_os_onedir_repository || return 1
     fi
 
-    __PACKAGES="procps-ng sudo shadow"
+    __PACKAGES="procps-ng sudo shadow wget"
 
     # shellcheck disable=SC2086
     __tdnf_install_noinput ${__PACKAGES} || return 1
@@ -6626,9 +6632,9 @@ install_photon_onedir_deps() {
 }
 
 
-install_photon_onedir() {
+install_vmware_photon_os_onedir() {
 
-    echodebug "install_photon_onedir() entry"
+    echodebug "install_vmware_photon_os_onedir() entry"
 
     STABLE_REV=$ONEDIR_REV
     _GENERIC_PKG_VERSION=""
@@ -6672,9 +6678,9 @@ install_photon_onedir() {
     return 0
 }
 
-install_photon_onedir_post() {
+install_vmware_photon_os_onedir_post() {
     STABLE_REV=$ONEDIR_REV
-    install_photon_stable_post || return 1
+    install_vmware_photon_os_stable_post || return 1
 
     return 0
 }
@@ -7797,7 +7803,7 @@ install_macosx_git_deps() {
         export PATH=/usr/local/bin:$PATH
     fi
 
-    __fetch_url "/tmp/get-pip.py" "https://bootstrap.pypa.io/get-pip.py" || return 1
+    __fetch_url "${_TMP_DIR}/get-pip.py" "https://bootstrap.pypa.io/get-pip.py" || return 1
 
     if [ -n "$_PY_EXE" ]; then
         _PYEXE="${_PY_EXE}"
@@ -7807,7 +7813,7 @@ install_macosx_git_deps() {
     fi
 
     # Install PIP
-    $_PYEXE /tmp/get-pip.py || return 1
+    $_PYEXE ${_TMP_DIR}/get-pip.py || return 1
 
     # shellcheck disable=SC2119
     __git_clone_and_checkout || return 1
@@ -7819,9 +7825,9 @@ install_macosx_stable() {
 
     install_macosx_stable_deps || return 1
 
-    __fetch_url "/tmp/${PKG}" "${SALTPKGCONFURL}" || return 1
+    __fetch_url "${_TMP_DIR}/${PKG}" "${SALTPKGCONFURL}" || return 1
 
-    /usr/sbin/installer -pkg "/tmp/${PKG}" -target / || return 1
+    /usr/sbin/installer -pkg "${_TMP_DIR}/${PKG}" -target / || return 1
 
     return 0
 }
@@ -7830,9 +7836,9 @@ install_macosx_onedir() {
 
     install_macosx_onedir_deps || return 1
 
-    __fetch_url "/tmp/${PKG}" "${SALTPKGCONFURL}" || return 1
+    __fetch_url "${_TMP_DIR}/${PKG}" "${SALTPKGCONFURL}" || return 1
 
-    /usr/sbin/installer -pkg "/tmp/${PKG}" -target / || return 1
+    /usr/sbin/installer -pkg "${_TMP_DIR}/${PKG}" -target / || return 1
 
     return 0
 }

salt/sensoroni/defaults.yaml

@@ -15,7 +15,7 @@ sensoroni:
     sensoronikey:
     soc_host:
     suripcap:
-      pcapMaxCount: 999999
+      pcapMaxCount: 100000
   analyzers:
       echotrail:
         base_url: https://api.echotrail.io/insights/

salt/soc/config.sls

@@ -215,7 +215,6 @@ socsensoronirepos:
     - mode: 775
     - makedirs: True
 
-
 create_custom_local_yara_repo_template:
   git.present:
     - name: /nsm/rules/custom-local-repos/local-yara
@@ -249,6 +248,39 @@ add_readme_custom_local_sigma_repo_template:
     - context:
         repo_type: "sigma"
 
+create_custom_local_suricata_repo_template:
+  git.present:
+    - name: /nsm/rules/custom-local-repos/local-suricata
+    - bare: False
+    - force: True
+
+add_readme_custom_local_suricata_repo_template:
+  file.managed:
+    - name: /nsm/rules/custom-local-repos/local-suricata/README
+    - source: salt://soc/files/soc/detections_custom_repo_template_readme.jinja
+    - user: 939
+    - group: 939
+    - template: jinja
+    - context:
+        repo_type: "suricata"
+
+etpro_airgap_folder:
+  file.directory:
+    - name: /nsm/rules/custom-local-repos/local-etpro-suricata
+    - user: 939
+    - group: 939
+    - makedirs: True
+
+add_readme_etpro_airgap_template:
+  file.managed:
+    - name: /nsm/rules/custom-local-repos/local-etpro-suricata/README
+    - source: salt://soc/files/soc/detections_custom_repo_template_readme.jinja
+    - user: 939
+    - group: 939
+    - template: jinja
+    - context:
+        repo_type: "suricata-etpro"
+
 socore_own_custom_repos:
   file.directory:
     - name: /nsm/rules/custom-local-repos/

salt/soc/defaults.yaml

@@ -1364,6 +1364,8 @@ soc:
         cases: soc
         filedatastore:
           jobDir: jobs
+          retryFailureIntervalMs: 600000
+          retryFailureMaxAttempts: 5
         kratos:
           hostUrl:
         hydra:
@@ -1586,25 +1588,45 @@ soc:
                 insecureSkipVerify: false
                 readOnly: true
                 deleteUnreferenced: true
+              - name: ABUSECH-SSLBL
+                deleteUnreferenced: true
+                description: 'Abuse.ch SSL Blacklist'
+                enabled: false
+                license: CC0-1.0
+                readOnly: true
+                sourcePath: https://sslbl.abuse.ch/blacklist/sslblacklist_tls_cert.tar.gz
+                sourceType: url
               - name: local-rules
-                id: local-rules
-                description: "Local custom rules from files (*.rules) in a directory on the filesystem"
+                description: "Local rules from files (*.rules) in a directory on the filesystem"
                 license: "custom"
                 sourceType: directory
-                sourcePath: /nsm/rules/local/
+                sourcePath: /nsm/rules/custom-local-repos/local-suricata
                 readOnly: false
                 deleteUnreferenced: false
+                enabled: true
+              - name: SO_FILTERS
+                deleteUnreferenced: true
+                description: Filter rules for when Suricata is set as the metadata engine
                 enabled: false
-                excludeFiles:
-                  - "*backup*"
+                license: Elastic-2.0
+                readOnly: true
+                sourcePath: /nsm/rules/suricata/so_filters.rules
+                sourceType: directory
+              - name: SO_EXTRACTIONS
+                description: Extraction rules for when Suricata is set as the metadata engine
+                deleteUnreferenced: true
+                enabled: false
+                license: Elastic-2.0
+                readOnly: true
+                sourcePath: /nsm/rules/suricata/so_extraction.rules
+                sourceType: directory
             airgap:
               - name: Emerging-Threats
-                description: "Emerging Threats ruleset - To enable ET Pro, enter your license key below. Leave empty for ET Open (free) rules."
+                description: "Emerging Threats ruleset - To enable ET Pro on Airgap, review the documentation at https://docs.securityonion.net/suricata"
                 licenseKey: ""
                 enabled: true
-                sourceType: url
-                sourcePath: 'https://rules.emergingthreats.net/open/suricata/emerging.rules.tar.gz'
-                urlHash: "https://rules.emergingthreats.net/open/suricata/emerging.rules.tar.gz.md5"
+                sourceType: directory
+                sourcePath: /nsm/rules/suricata/etopen/
                 license: "BSD"
                 excludeFiles:
                   - "*deleted*"
@@ -1617,16 +1639,29 @@ soc:
                 readOnly: true
                 deleteUnreferenced: true
               - name: local-rules
-                id: local-rules
-                description: "Local custom rules from files (*.rules) in a directory on the filesystem"
+                description: "Local rules from files (*.rules) in a directory on the filesystem"
                 license: "custom"
                 sourceType: directory
-                sourcePath: /nsm/rules/local/
+                sourcePath: /nsm/rules/custom-local-repos/local-suricata
                 readOnly: false
                 deleteUnreferenced: false
+                enabled: true
+              - name: SO_FILTERS
+                deleteUnreferenced: true
+                description: Filter rules for when Suricata is set as the metadata engine
                 enabled: false
-                excludeFiles:
-                  - "*backup*"
+                license: Elastic-2.0
+                readOnly: true
+                sourcePath: /nsm/rules/suricata/so_filters.rules
+                sourceType: directory
+              - name: SO_EXTRACTIONS
+                description: Extraction rules for when Suricata is set as the metadata engine
+                deleteUnreferenced: true
+                enabled: false
+                license: Elastic-2.0
+                readOnly: true
+                sourcePath: /nsm/rules/suricata/so_extraction.rules
+                sourceType: directory
         navigator:
           intervalMinutes: 30
           outputPath: /opt/sensoroni/navigator
@@ -1804,7 +1839,7 @@ soc:
               showSubtitle: true
             - name: DPD
               description: Dynamic Protocol Detection errors
-              query: 'tags:dpd | groupby error.reason'
+              query: '(tags:dpd OR tags:analyzer) | groupby error.reason'
               showSubtitle: true
             - name: Files
               description: Files grouped by mimetype
@@ -2070,7 +2105,7 @@ soc:
               query: 'tags:dns | groupby dns.query.name | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby dns.highest_registered_domain | groupby dns.parent_domain | groupby dns.query.type_name | groupby dns.response.code_name | groupby dns.answers.name | groupby destination.as.organization.name'
             - name: DPD
               description: DPD (Dynamic Protocol Detection) errors
-              query: 'tags:dpd | groupby error.reason | groupby -sankey error.reason source.ip | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby network.protocol | groupby destination.as.organization.name'
+              query: '(tags:dpd OR tags:analyzer) | groupby error.reason | groupby -sankey error.reason source.ip | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby network.protocol | groupby destination.as.organization.name'
             - name: Files
               description: Files seen in network traffic
               query: 'tags:file | groupby file.mime_type | groupby -sankey file.mime_type file.source | groupby file.source | groupby file.bytes.total | groupby source.ip | groupby destination.ip | groupby destination.as.organization.name'
@@ -2612,22 +2647,22 @@ soc:
         assistant:
           enabled: false
           investigationPrompt: Investigate Alert ID {socId}
+          compressContextPrompt: Summarize the conversation for context compaction
           thresholdColorRatioLow: 0.5
           thresholdColorRatioMed: 0.75
           thresholdColorRatioMax: 1
           availableModels:
-            - id: sonnet-4
-              displayName: Claude Sonnet 4
-              contextLimitSmall: 200000
-              contextLimitLarge: 1000000
-              lowBalanceColorAlert: 500000
             - id: sonnet-4.5
-              displayName: Claude Sonnet 4.5
+              displayName: Claude Sonnet 4.5 ($$$)
+              origin: USA
               contextLimitSmall: 200000
               contextLimitLarge: 1000000
               lowBalanceColorAlert: 500000
-            - id: gptoss-120b
-              displayName: GPT-OSS 120B
-              contextLimitSmall: 128000
-              contextLimitLarge: 128000
-              lowBalanceColorAlert: 500000
+              enabled: true
+            - id: qwen-235b
+              displayName: QWEN 235B ($)
+              origin: China
+              contextLimitSmall: 256000
+              contextLimitLarge: 256000
+              lowBalanceColorAlert: 500000
+              enabled: true

salt/soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja

@@ -43,10 +43,26 @@
 
 No Virtual Machines Found
 {%-   endif %}
-{%- else %}
+{%- elif baseDomainStatus == 'ImageDownloadStart' %}
+#### INFO
+
+Base domain image download started.
+{%- elif baseDomainStatus == 'ImageDownloadFailed' %}
+#### ERROR
+
+Base domain image download failed. Please check the salt-master log for details and verify network connectivity.
+{%- elif baseDomainStatus == 'SSHKeySetupFailed' %}
+#### ERROR
+
+SSH key setup failed. Please check the salt-master log for details.
+{%- elif baseDomainStatus == 'SetupFailed' %}
 #### WARNING
 
-Base domain has not been initialized.
+Setup failed. Please check the salt-master log for details.
+{%- elif baseDomainStatus == 'PreInit' %}
+#### WARNING
+
+Base domain has not been initialized. Waiting for hypervisor to highstate.
 {%- endif %}
 {%- endmacro -%}
 

salt/soc/enabled.sls

@@ -27,7 +27,8 @@ so-soc:
       - /opt/so/conf/strelka:/opt/sensoroni/yara:rw
       - /opt/so/conf/sigma:/opt/sensoroni/sigma:rw
       - /opt/so/rules/elastalert/rules:/opt/sensoroni/elastalert:rw
-      - /opt/so/rules/nids/suri:/opt/sensoroni/nids:rw
+      - /opt/so/saltstack/local/salt/suricata/rules:/opt/sensoroni/suricata/rules:rw
+      - /opt/so/saltstack/local/salt/suricata/files:/opt/sensoroni/suricata/threshold:rw
       - /opt/so/conf/soc/fingerprints:/opt/sensoroni/fingerprints:rw
       - /nsm/soc/jobs:/opt/sensoroni/jobs:rw
       - /nsm/soc/uploads:/nsm/soc/uploads:rw

salt/soc/files/soc/detections_custom_repo_template_readme.jinja

@@ -45,6 +45,61 @@ Finally, commit it:
 The next time the Strelka / YARA engine syncs, the new rule should be imported
 If there are errors, review the sync log to troubleshoot further.
 
+{% elif repo_type == 'suricata' %}
+# Suricata Local Custom Rules Repository
+
+This folder has already been initialized as a git repo 
+and your Security Onion grid is configured to import any Suricata rule files found here. 
+
+Just add your rule file and commit it.
+
+For example:
+
+** Note: If this is your first time making changes to this repo, you may run into the following error:
+
+fatal: detected dubious ownership in repository at '/nsm/rules/custom-local-repos/local-suricata'
+To add an exception for this directory, call:
+	git config --global --add safe.directory /nsm/rules/custom-local-repos/local-suricata
+
+This means that the user you are running commands as does not match the user that is used for this git repo (socore).
+You will need to make sure your rule files are accessible to the socore user, so either su to socore 
+or add the exception and then chown the rule files later.
+
+Also, you will be asked to set some configuration:
+```
+Author identity unknown
+*** Please tell me who you are.
+Run
+  git config --global user.email "you@example.com"
+  git config --global user.name "Your Name"
+to set your account's default identity.
+Omit --global to set the identity only in this repository.
+```
+
+Run these commands, ommitting the `--global`.
+
+With that out of the way:
+
+First, create the rule file with a .rules extension:
+`vi my_custom_rules.rules`
+
+Next, use git to stage the new rule to be committed:
+`git add my_custom_rules.rules`
+
+Finally, commit it:
+`git commit -m "Initial commit of my_custom_rule.rules"`
+
+The next time the Suricata engine syncs, the new rule/s should be imported
+If there are errors, review the sync log to troubleshoot further.
+
+{% elif repo_type == 'suricata-etpro' %}
+# Suricata ETPRO - Airgap
+
+This folder has been initialized for use with ETPRO during Airgap deployment.
+
+Just add your ETPRO rule/s file to this folder and the Suricata engine will import them.
+
+If there are errors, review the sync log to troubleshoot further.
 {% elif repo_type == 'sigma' %}
 # Sigma Local Custom Rules Repository
 

salt/soc/files/soc/so-detections-backup.py

@@ -6,6 +6,7 @@
 # This script queries Elasticsearch for Custom Detections and all Overrides,
 # and git commits them to disk at $OUTPUT_DIR
 
+import argparse
 import os
 import subprocess
 import json
@@ -18,10 +19,10 @@ from datetime import datetime
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 # Constants
-ES_URL = "https://localhost:9200/so-detection/_search"
+DEFAULT_INDEX = "so-detection"
+DEFAULT_OUTPUT_DIR = "/nsm/backup/detections/repo"
 QUERY_DETECTIONS = '{"query": {"bool": {"must": [{"match_all": {}}, {"term": {"so_detection.ruleset": "__custom__"}}]}},"size": 10000}'
 QUERY_OVERRIDES = '{"query": {"bool": {"must": [{"exists": {"field": "so_detection.overrides"}}]}},"size": 10000}'
-OUTPUT_DIR = "/nsm/backup/detections/repo"
 AUTH_FILE = "/opt/so/conf/elasticsearch/curl.config"
 
 def get_auth_credentials(auth_file):
@@ -30,9 +31,10 @@ def get_auth_credentials(auth_file):
             if line.startswith('user ='):
                 return line.split('=', 1)[1].strip().replace('"', '')
 
-def query_elasticsearch(query, auth):
+def query_elasticsearch(query, auth, index):
+    url = f"https://localhost:9200/{index}/_search"
     headers = {"Content-Type": "application/json"}
-    response = requests.get(ES_URL, headers=headers, data=query, auth=auth, verify=False)
+    response = requests.get(url, headers=headers, data=query, auth=auth, verify=False)
     response.raise_for_status()
     return response.json()
 
@@ -47,12 +49,12 @@ def save_content(hit, base_folder, subfolder="", extension="txt"):
         f.write(content)
     return file_path
 
-def save_overrides(hit):
+def save_overrides(hit, output_dir):
     so_detection = hit["_source"]["so_detection"]
     public_id = so_detection["publicId"]
     overrides = so_detection["overrides"]
     language = so_detection["language"]
-    folder = os.path.join(OUTPUT_DIR, language, "overrides")
+    folder = os.path.join(output_dir, language, "overrides")
     os.makedirs(folder, exist_ok=True)
     extension = "yaml" if language == "sigma" else "txt"
     file_path = os.path.join(folder, f"{public_id}.{extension}")
@@ -60,20 +62,20 @@ def save_overrides(hit):
         f.write('\n'.join(json.dumps(override) for override in overrides) if isinstance(overrides, list) else overrides)
     return file_path
 
-def ensure_git_repo():
-    if not os.path.isdir(os.path.join(OUTPUT_DIR, '.git')):
+def ensure_git_repo(output_dir):
+    if not os.path.isdir(os.path.join(output_dir, '.git')):
         subprocess.run(["git", "config", "--global", "init.defaultBranch", "main"], check=True)
-        subprocess.run(["git", "-C", OUTPUT_DIR, "init"], check=True)
-        subprocess.run(["git", "-C", OUTPUT_DIR, "remote", "add", "origin", "default"], check=True)
+        subprocess.run(["git", "-C", output_dir, "init"], check=True)
+        subprocess.run(["git", "-C", output_dir, "remote", "add", "origin", "default"], check=True)
 
-def commit_changes():
-    ensure_git_repo()
-    subprocess.run(["git", "-C", OUTPUT_DIR, "config", "user.email", "securityonion@local.invalid"], check=True)
-    subprocess.run(["git", "-C", OUTPUT_DIR, "config", "user.name", "securityonion"], check=True)
-    subprocess.run(["git", "-C", OUTPUT_DIR, "add", "."], check=True)
-    status_result = subprocess.run(["git", "-C", OUTPUT_DIR, "status"], capture_output=True, text=True)
+def commit_changes(output_dir):
+    ensure_git_repo(output_dir)
+    subprocess.run(["git", "-C", output_dir, "config", "user.email", "securityonion@local.invalid"], check=True)
+    subprocess.run(["git", "-C", output_dir, "config", "user.name", "securityonion"], check=True)
+    subprocess.run(["git", "-C", output_dir, "add", "."], check=True)
+    status_result = subprocess.run(["git", "-C", output_dir, "status"], capture_output=True, text=True)
     print(status_result.stdout)
-    commit_result = subprocess.run(["git", "-C", OUTPUT_DIR, "commit", "-m", "Update detections and overrides"], check=False, capture_output=True)
+    commit_result = subprocess.run(["git", "-C", output_dir, "commit", "-m", "Update detections and overrides"], check=False, capture_output=True)
     if commit_result.returncode == 1:
         print("No changes to commit.")
     elif commit_result.returncode == 0:
@@ -81,29 +83,41 @@ def commit_changes():
     else:
         commit_result.check_returncode()
 
+def parse_args():
+    parser = argparse.ArgumentParser(description="Backup custom detections and overrides from Elasticsearch")
+    parser.add_argument("--output", "-o", default=DEFAULT_OUTPUT_DIR,
+                        help=f"Output directory for backups (default: {DEFAULT_OUTPUT_DIR})")
+    parser.add_argument("--index", "-i", default=DEFAULT_INDEX,
+                        help=f"Elasticsearch index to query (default: {DEFAULT_INDEX})")
+    return parser.parse_args()
+
 def main():
+    args = parse_args()
+    output_dir = args.output
+    index = args.index
+
     try:
         timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f"Backing up Custom Detections and all Overrides to {OUTPUT_DIR} - {timestamp}\n")
-        
-        os.makedirs(OUTPUT_DIR, exist_ok=True)
+        print(f"Backing up Custom Detections and all Overrides to {output_dir} - {timestamp}\n")
+
+        os.makedirs(output_dir, exist_ok=True)
 
         auth_credentials = get_auth_credentials(AUTH_FILE)
         username, password = auth_credentials.split(':', 1)
         auth = HTTPBasicAuth(username, password)
-        
+
         # Query and save custom detections
-        detections = query_elasticsearch(QUERY_DETECTIONS, auth)["hits"]["hits"]
+        detections = query_elasticsearch(QUERY_DETECTIONS, auth, index)["hits"]["hits"]
         for hit in detections:
-            save_content(hit, OUTPUT_DIR, hit["_source"]["so_detection"]["language"], "yaml" if hit["_source"]["so_detection"]["language"] == "sigma" else "txt")
-        
+            save_content(hit, output_dir, hit["_source"]["so_detection"]["language"], "yaml" if hit["_source"]["so_detection"]["language"] == "sigma" else "txt")
+
         # Query and save overrides
-        overrides = query_elasticsearch(QUERY_OVERRIDES, auth)["hits"]["hits"]
+        overrides = query_elasticsearch(QUERY_OVERRIDES, auth, index)["hits"]["hits"]
         for hit in overrides:
-            save_overrides(hit)
-        
-        commit_changes()
-        
+            save_overrides(hit, output_dir)
+
+        commit_changes(output_dir)
+
         timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
         print(f"Backup Completed - {timestamp}")
     except Exception as e:

salt/soc/files/soc/so-detections-backup_test.py

@@ -57,12 +57,12 @@ class TestBackupScript(unittest.TestCase):
         mock_response.json.return_value = {'hits': {'hits': []}}
         mock_response.raise_for_status = MagicMock()
         mock_get.return_value = mock_response
-        
-        response = ds.query_elasticsearch(ds.QUERY_DETECTIONS, self.auth)
-        
+
+        response = ds.query_elasticsearch(ds.QUERY_DETECTIONS, self.auth, ds.DEFAULT_INDEX)
+
         self.assertEqual(response, {'hits': {'hits': []}})
         mock_get.assert_called_once_with(
-            ds.ES_URL,
+            f"https://localhost:9200/{ds.DEFAULT_INDEX}/_search",
             headers={"Content-Type": "application/json"},
             data=ds.QUERY_DETECTIONS,
             auth=self.auth,
@@ -81,7 +81,7 @@ class TestBackupScript(unittest.TestCase):
     @patch('os.makedirs')
     @patch('builtins.open', new_callable=mock_open)
     def test_save_overrides(self, mock_file, mock_makedirs):
-        file_path = ds.save_overrides(self.mock_override_hit)
+        file_path = ds.save_overrides(self.mock_override_hit, self.output_dir)
         expected_path = f'{self.output_dir}/sigma/overrides/test_id.yaml'
         self.assertEqual(file_path, expected_path)
         mock_makedirs.assert_called_once_with(f'{self.output_dir}/sigma/overrides', exist_ok=True)
@@ -90,9 +90,9 @@ class TestBackupScript(unittest.TestCase):
     @patch('subprocess.run')
     def test_ensure_git_repo(self, mock_run):
         mock_run.return_value = MagicMock(returncode=0)
-        
-        ds.ensure_git_repo()
-        
+
+        ds.ensure_git_repo(self.output_dir)
+
         mock_run.assert_has_calls([
             call(["git", "config", "--global", "init.defaultBranch", "main"], check=True),
             call(["git", "-C", self.output_dir, "init"], check=True),
@@ -106,9 +106,9 @@ class TestBackupScript(unittest.TestCase):
         mock_commit_result = MagicMock(returncode=1)
         # Ensure sufficient number of MagicMock instances for each subprocess.run call
         mock_run.side_effect = [mock_status_result, mock_commit_result, MagicMock(returncode=0), MagicMock(returncode=0), MagicMock(returncode=0), MagicMock(returncode=0), MagicMock(returncode=0), MagicMock(returncode=0)]
-        
+
         print("Running test_commit_changes...")
-        ds.commit_changes()
+        ds.commit_changes(self.output_dir)
         print("Finished test_commit_changes.")
 
         mock_run.assert_has_calls([
@@ -120,39 +120,45 @@ class TestBackupScript(unittest.TestCase):
         ])
 
     @patch('builtins.print')
-    @patch('so-detections-backup.commit_changes')
-    @patch('so-detections-backup.save_overrides')
-    @patch('so-detections-backup.save_content')
-    @patch('so-detections-backup.query_elasticsearch')
-    @patch('so-detections-backup.get_auth_credentials')
+    @patch.object(ds, 'commit_changes')
+    @patch.object(ds, 'save_overrides')
+    @patch.object(ds, 'save_content')
+    @patch.object(ds, 'query_elasticsearch')
+    @patch.object(ds, 'get_auth_credentials')
     @patch('os.makedirs')
-    def test_main(self, mock_makedirs, mock_get_auth, mock_query, mock_save_content, mock_save_overrides, mock_commit, mock_print):
+    @patch.object(ds, 'parse_args')
+    def test_main(self, mock_parse_args, mock_makedirs, mock_get_auth, mock_query, mock_save_content, mock_save_overrides, mock_commit, mock_print):
+        mock_args = MagicMock()
+        mock_args.output = self.output_dir
+        mock_args.index = ds.DEFAULT_INDEX
+        mock_parse_args.return_value = mock_args
         mock_get_auth.return_value = self.auth_credentials
         mock_query.side_effect = [
             {'hits': {'hits': [{"_source": {"so_detection": {"publicId": "1", "content": "content1", "language": "sigma"}}}]}},
             {'hits': {'hits': [{"_source": {"so_detection": {"publicId": "2", "overrides": [{"key": "value"}], "language": "suricata"}}}]}}
         ]
-        
+
         with patch('datetime.datetime') as mock_datetime:
             mock_datetime.now.return_value.strftime.return_value = "2024-05-23 20:49:44"
             ds.main()
-        
+
         mock_makedirs.assert_called_once_with(self.output_dir, exist_ok=True)
         mock_get_auth.assert_called_once_with(ds.AUTH_FILE)
         mock_query.assert_has_calls([
-            call(ds.QUERY_DETECTIONS, self.auth),
-            call(ds.QUERY_OVERRIDES, self.auth)
+            call(ds.QUERY_DETECTIONS, self.auth, ds.DEFAULT_INDEX),
+            call(ds.QUERY_OVERRIDES, self.auth, ds.DEFAULT_INDEX)
         ])
         mock_save_content.assert_called_once_with(
-            {"_source": {"so_detection": {"publicId": "1", "content": "content1", "language": "sigma"}}}, 
-            self.output_dir, 
-            "sigma", 
+            {"_source": {"so_detection": {"publicId": "1", "content": "content1", "language": "sigma"}}},
+            self.output_dir,
+            "sigma",
             "yaml"
         )
         mock_save_overrides.assert_called_once_with(
-            {"_source": {"so_detection": {"publicId": "2", "overrides": [{"key": "value"}], "language": "suricata"}}}
+            {"_source": {"so_detection": {"publicId": "2", "overrides": [{"key": "value"}], "language": "suricata"}}},
+            self.output_dir
         )
-        mock_commit.assert_called_once()
+        mock_commit.assert_called_once_with(self.output_dir)
         mock_print.assert_called()
 
 if __name__ == '__main__':

salt/soc/merged.map.jinja

@@ -70,7 +70,7 @@
 
 {# Define the Detections custom ruleset that should always be present #}
 {% set CUSTOM_RULESET = {
-  'name': 'custom',
+  'name': '__custom__',
   'description': 'User-created custom rules created via the Detections module in the SOC UI',
   'sourceType': 'elasticsearch',
   'sourcePath': 'so_detection.ruleset:__custom__',
@@ -83,13 +83,24 @@
 {# Always append the custom ruleset to suricataengine.rulesetSources if not already present #}
 {% if SOCMERGED.config.server.modules.suricataengine is defined and SOCMERGED.config.server.modules.suricataengine.rulesetSources is defined %}
 {%   if SOCMERGED.config.server.modules.suricataengine.rulesetSources is not mapping %}
-{%     set custom_names = SOCMERGED.config.server.modules.suricataengine.rulesetSources | selectattr('name', 'equalto', 'custom') | list %}
+{%     set custom_names = SOCMERGED.config.server.modules.suricataengine.rulesetSources | selectattr('name', 'equalto', '__custom__') | list %}
 {%     if custom_names | length == 0 %}
 {%       do SOCMERGED.config.server.modules.suricataengine.rulesetSources.append(CUSTOM_RULESET) %}
 {%     endif %}
 {%   endif %}
 {% endif %}
 
+{# Enable SO_FILTERS and SO_EXTRACTIONS when Suricata is the metadata engine #}
+{% if SOCMERGED.config.server.modules.suricataengine is defined and SOCMERGED.config.server.modules.suricataengine.rulesetSources is defined %}
+{%   if SOCMERGED.config.server.modules.suricataengine.rulesetSources is not mapping %}
+{%     for ruleset in SOCMERGED.config.server.modules.suricataengine.rulesetSources %}
+{%       if ruleset.name in ['SO_FILTERS', 'SO_EXTRACTIONS'] and GLOBALS.md_engine == 'SURICATA' %}
+{%         do ruleset.update({'enabled': true}) %}
+{%       endif %}
+{%     endfor %}
+{%   endif %}
+{% endif %}
+
 {# Transform Emerging-Threats ruleset based on license key #}
 {% if SOCMERGED.config.server.modules.suricataengine is defined and SOCMERGED.config.server.modules.suricataengine.rulesetSources is defined %}
 {%   if SOCMERGED.config.server.modules.suricataengine.rulesetSources is not mapping %}
@@ -97,20 +108,39 @@
 {%     if ruleset.name == 'Emerging-Threats' %}
 {%       if ruleset.licenseKey and ruleset.licenseKey != '' %}
 {#         License key is defined - transform to ETPRO #}
-{%         do ruleset.update({
-             'name': 'ETPRO',
-             'sourcePath': 'https://rules.emergingthreatspro.com/' ~ ruleset.licenseKey ~ '/suricata-7.0.3/etpro.rules.tar.gz',
-             'urlHash': 'https://rules.emergingthreatspro.com/' ~ ruleset.licenseKey ~ '/suricata-7.0.3/etpro.rules.tar.gz.md5',
-             'license': 'Commercial'
-           }) %}
+{%         if ruleset.sourceType == 'directory' %}
+{#           Airgap mode - update directory path #}
+{%           do ruleset.update({
+               'name': 'ETPRO',
+               'sourcePath': '/nsm/rules/custom-local-repos/local-etpro-suricata/etpro.rules.tar.gz',
+               'license': 'Commercial'
+             }) %}
+{%         else %}
+{#           Engine Version is hardcoded in the URL - this does not change often: https://community.emergingthreats.net/t/supported-engines/71 #}
+{%           do ruleset.update({
+               'name': 'ETPRO',
+               'sourcePath': 'https://rules.emergingthreatspro.com/' ~ ruleset.licenseKey ~ '/suricata-7.0.3/etpro.rules.tar.gz',
+               'urlHash': 'https://rules.emergingthreatspro.com/' ~ ruleset.licenseKey ~ '/suricata-7.0.3/etpro.rules.tar.gz.md5',
+               'license': 'Commercial'
+             }) %}
+{%         endif %}
 {%       else %}
 {#         No license key - explicitly set to ETOPEN #}
-{%         do ruleset.update({
-             'name': 'ETOPEN',
-             'sourcePath': 'https://rules.emergingthreats.net/open/suricata-7.0.3/emerging.rules.tar.gz',
-             'urlHash': 'https://rules.emergingthreats.net/open/suricata-7.0.3/emerging.rules.tar.gz.md5',
-             'license': 'BSD'
-           }) %}
+{%         if ruleset.sourceType == 'directory' %}
+{#           Airgap mode - update directory path #}
+{%           do ruleset.update({
+               'name': 'ETOPEN',
+               'sourcePath': '/nsm/rules/suricata/etopen/',
+               'license': 'BSD'
+             }) %}
+{%         else %}
+{%           do ruleset.update({
+               'name': 'ETOPEN',
+               'sourcePath': 'https://rules.emergingthreats.net/open/suricata-7.0.3/emerging.rules.tar.gz',
+               'urlHash': 'https://rules.emergingthreats.net/open/suricata-7.0.3/emerging.rules.tar.gz.md5',
+               'license': 'BSD'
+             }) %}
+{%         endif %}
 {%       endif %}
 {%     endif %}
 {%     endfor %}

salt/soc/soc_soc.yaml

@@ -424,6 +424,17 @@ soc:
             description: The maximum number of documents to request in a single Elasticsearch scroll request.
           bulkIndexWorkerCount:
             description: The number of worker threads to use when bulk indexing data into Elasticsearch. A value below 1 will default to the number of CPUs available.
+        filedatastore:
+          jobDir:
+            description: The location where local job files are stored on the manager.
+            global: True
+            advanced: True
+          retryFailureIntervalMs:
+            description: The interval, in milliseconds, to wait before attempting to reprocess a failed job.
+            global: True
+          retryFailureMaxAttempts:
+            description: The max number of attempts to process a job, in the event the job fails to complete.
+            global: True
         sostatus:
           refreshIntervalMs:
             description: Duration (in milliseconds) between refreshes of the grid status. Shortening this duration may not have expected results, as the backend systems feeding this sostatus data will continue their updates as scheduled.
@@ -590,13 +601,25 @@ soc:
                 label: Ruleset License
                 required: True
               - field: readOnly
-                label: Read Only
+                label: Read Only (Prevents changes to the rule itself - can still be enabled/disabled/tuned)
                 forcedType: bool
                 required: False
               - field: deleteUnreferenced
-                label: Delete Unreferenced
+                label: Delete Unreferenced (Deletes rules that are no longer referenced by ruleset source)
                 forcedType: bool
                 required: False
+              - field: proxyURL
+                label: HTTP/HTTPS proxy URL for downloading the ruleset.
+                required: False
+              - field: proxyUsername
+                label: Proxy authentication username.
+                required: False
+              - field: proxyPassword
+                label: Proxy authentication password.
+                required: False
+              - field: proxyCACert
+                label: Path to CA certificate file for MITM proxy verification.
+                required: False
             airgap: *serulesetSources
         navigator:
           intervalMinutes:
@@ -652,6 +675,9 @@ soc:
           investigationPrompt: 
             description: Prompt given to Onion AI when beginning an investigation.
             global: True
+          compressContextPrompt:
+            description: Prompt given to Onion AI when summarizing a conversation in order to compress context.
+            global: True
           thresholdColorRatioLow:
             description: Lower visual context color change threshold.
             global: True
@@ -682,6 +708,9 @@ soc:
             - field: displayName
               label: Display Name
               required: True
+            - field: origin
+              label: Country of Origin for the Model Training
+              required: false
             - field: contextLimitSmall
               label: Context Limit (Small)
               forcedType: int
@@ -694,6 +723,9 @@ soc:
               label: Low Balance Color Alert
               forcedType: int
               required: True
+            - field: enabled
+              label: Enabled
+              forcedType: bool
         apiTimeoutMs:
           description: Duration (in milliseconds) to wait for a response from the SOC server API before giving up and showing an error on the SOC UI.
           global: True

salt/suricata/config.sls

@@ -7,9 +7,49 @@
 {% if sls.split('.')[0] in allowed_states %}
 
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
-{%   from 'bpf/suricata.map.jinja' import SURICATABPF %}
 {%   from 'suricata/map.jinja' import SURICATAMERGED %}
-{%   set BPF_STATUS = 0  %}
+{%   from 'bpf/suricata.map.jinja' import SURICATABPF, SURICATA_BPF_STATUS, SURICATA_BPF_CALC %}
+
+{%   if GLOBALS.pcap_engine in ["SURICATA", "TRANSITION"] %}
+{%     from 'bpf/pcap.map.jinja' import PCAPBPF, PCAP_BPF_STATUS, PCAP_BPF_CALC %}
+# BPF compilation and configuration
+{%     if PCAPBPF and not PCAP_BPF_STATUS %}
+suriPCAPbpfcompilationfailure:
+  test.configurable_test_state:
+    - changes: False
+    - result: False
+    - comment: "BPF Syntax Error - Discarding Specified BPF. Error: {{ PCAP_BPF_CALC['stderr'] }}"
+{%     endif %}
+{%   endif %}
+
+suridir:
+  file.directory:
+    - name: /opt/so/conf/suricata
+    - user: 940
+    - group: 939
+    - mode: 775
+    - makedirs: True
+
+# BPF applied to all of Suricata - alerts/metadata/pcap
+suribpf:
+  file.managed:
+    - name: /opt/so/conf/suricata/bpf
+    - user: 940
+    - group: 940
+   {% if SURICATA_BPF_STATUS %}
+    - contents: {{ SURICATABPF }}
+   {% else %}
+    - contents:
+      - ""
+   {% endif %}
+
+{%   if SURICATABPF and not SURICATA_BPF_STATUS %}
+suribpfcompilationfailure:
+  test.configurable_test_state:
+    - changes: False
+    - result: False
+    - comment: "BPF Syntax Error - Discarding Specified BPF. Error: {{ SURICATA_BPF_CALC['stderr'] }}"
+{%   endif %}
 
 # Add Suricata Group
 suricatagroup:
@@ -49,17 +89,12 @@ suricata_sbin_jinja:
     - file_mode: 755
     - template: jinja
 
-suridir:
-  file.directory:
-    - name: /opt/so/conf/suricata
-    - user: 940
-    - group: 940
-
 suriruledir:
   file.directory:
-    - name: /opt/so/conf/suricata/rules
+    - name: /opt/so/rules/suricata
     - user: 940
-    - group: 940
+    - group: 939
+    - mode: 775
     - makedirs: True
 
 surilogdir:
@@ -84,14 +119,12 @@ suridatadir:
     - mode: 770
     - makedirs: True
 
-# salt:// would resolve to /opt/so/rules/nids because of the defined file_roots and
-#  not existing under /opt/so/saltstack/local/salt or /opt/so/saltstack/default/salt
 surirulesync:
   file.recurse:
-    - name: /opt/so/conf/suricata/rules/
-    - source: salt://suri/
+    - name: /opt/so/rules/suricata/
+    - source: salt://suricata/rules/
     - user: 940
-    - group: 940
+    - group: 939
     - show_changes: False
 
 surilogscript:
@@ -124,10 +157,9 @@ suriconfig:
 surithresholding:
   file.managed:
     - name: /opt/so/conf/suricata/threshold.conf
-    - source: salt://suricata/files/threshold.conf.jinja
+    - source: salt://suricata/files/threshold.conf
     - user: 940
     - group: 940
-    - template: jinja
 
 suriclassifications:
   file.managed:
@@ -136,32 +168,6 @@ suriclassifications:
     - user: 940
     - group: 940
 
-# BPF compilation and configuration
-{% if SURICATABPF %}
-   {% set BPF_CALC = salt['cmd.script']('salt://common/tools/sbin/so-bpf-compile', GLOBALS.sensor.interface + ' ' + SURICATABPF|join(" "),cwd='/root') %}
-   {% if BPF_CALC['stderr'] == "" %}
-      {% set BPF_STATUS = 1  %}
-   {% else  %}
-suribpfcompilationfailure:
-  test.configurable_test_state:
-   - changes: False
-   - result: False
-   - comment: "BPF Syntax Error - Discarding Specified BPF"
-   {% endif %}
-{% endif %}
-
-suribpf:
-  file.managed:
-    - name: /opt/so/conf/suricata/bpf
-    - user: 940
-    - group: 940
-   {% if BPF_STATUS %}
-    - contents: {{ SURICATABPF }}
-   {% else %}
-    - contents:
-      - ""
-   {% endif %}
-
 so-suricata-eve-clean:
   file.managed:
     - name: /usr/sbin/so-suricata-eve-clean
@@ -171,6 +177,14 @@ so-suricata-eve-clean:
     - template: jinja
     - source: salt://suricata/cron/so-suricata-eve-clean
 
+so-suricata-rulestats:
+  file.managed:
+    - name: /usr/sbin/so-suricata-rulestats
+    - user: root
+    - group: root
+    - mode: 755
+    - source: salt://suricata/cron/so-suricata-rulestats
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/suricata/cron/so-suricata-rulestats

@@ -0,0 +1,39 @@
+#!/bin/bash
+#
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+# Query Suricata for ruleset stats and reload time, write to JSON file for Telegraf to consume
+
+OUTFILE="/opt/so/log/suricata/rulestats.json"
+SURICATASC="docker exec so-suricata /opt/suricata/bin/suricatasc"
+SOCKET="/var/run/suricata/suricata-command.socket"
+
+query() {
+    timeout 10 $SURICATASC -c "$1" "$SOCKET" 2>/dev/null
+}
+
+STATS=$(query "ruleset-stats")
+RELOAD=$(query "ruleset-reload-time")
+[ -z "$RELOAD" ] && RELOAD='{}'
+
+# Outputs valid JSON on success, empty on failure
+OUTPUT=$(jq -n \
+    --argjson stats "$STATS" \
+    --argjson reload "$RELOAD" \
+    'if $stats.return == "OK" and ($stats.message[0].rules_loaded | type) == "number" and ($stats.message[0].rules_failed | type) == "number" then
+        {
+            rules_loaded: $stats.message[0].rules_loaded,
+            rules_failed: $stats.message[0].rules_failed,
+            last_reload: ($reload.message[0].last_reload // ""),
+            return: "OK"
+        }
+    else empty end' 2>/dev/null)
+
+if [ -n "$OUTPUT" ]; then
+    echo "$OUTPUT" > "$OUTFILE"
+else
+    echo '{"return":"FAIL"}' > "$OUTFILE"
+fi

salt/suricata/defaults.yaml

@@ -34,7 +34,7 @@ suricata:
       threads: 1
       tpacket-v3: "yes"
       ring-size: 5000
-      block-size: 32768
+      block-size: 69632
       block-timeout: 10
       use-emergency-flush: "yes"
       buffer-size: 32768
@@ -97,6 +97,11 @@ suricata:
           - 4789
         TEREDO_PORTS:
           - 3544
+        SIP_PORTS:
+          - 5060
+          - 5061
+        GENEVE_PORTS:
+          - 6081
     default-log-dir: /var/log/suricata/
     stats:
       enabled: "yes"
@@ -134,14 +139,6 @@ suricata:
               header: X-Forwarded-For
       unified2-alert:
         enabled: "no"
-      http-log:
-        enabled: "no"
-        filename: http.log
-        append: "yes"
-      tls-log:
-        enabled: "no"
-        filename: tls.log
-        append: "yes"
       tls-store:
         enabled: "no"
       pcap-log:
@@ -157,9 +154,6 @@ suricata:
         totals: "yes"
         threads: "no"
         null-values: "yes"
-      syslog:
-        enabled: "no"
-        facility: local5
       drop:
         enabled: "no"
       file-store:
@@ -206,6 +200,9 @@ suricata:
           enabled: "yes"
           detection-ports:
             dp: 443
+          ja3-fingerprints: auto
+          ja4-fingerprints: auto
+          encryption-handling: track-only
         dcerpc:
           enabled: "yes"
         ftp:
@@ -255,19 +252,21 @@ suricata:
           libhtp:
              default-config:
                personality: IDS
-               request-body-limit: 100kb
-               response-body-limit: 100kb
-               request-body-minimal-inspect-size: 32kb
-               request-body-inspect-window: 4kb
-               response-body-minimal-inspect-size: 40kb
-               response-body-inspect-window: 16kb
+               request-body-limit: 100 KiB
+               response-body-limit: 100 KiB
+               request-body-minimal-inspect-size: 32 KiB
+               request-body-inspect-window: 4 KiB
+               response-body-minimal-inspect-size: 40 KiB
+               response-body-inspect-window: 16 KiB
                response-body-decompress-layer-limit: 2
                http-body-inline: auto
                swf-decompression:
-                 enabled: "yes"
+                 enabled: "no"
                  type: both
-                 compress-depth: 0
-                 decompress-depth: 0
+                 compress-depth: 100 KiB
+                 decompress-depth: 100 KiB
+               randomize-inspection-sizes: "yes"
+               randomize-inspection-range: 10
                double-decode-path: "no"
                double-decode-query: "no"
              server-config:
@@ -401,8 +400,12 @@ suricata:
       vxlan:
         enabled: true
         ports: $VXLAN_PORTS
-      erspan:
+      geneve:
         enabled: true
+        ports: $GENEVE_PORTS
+        max-layers: 16
+      recursion-level:
+        use-for-tracking: true
     detect:
       profile: medium
       custom-values:
@@ -422,7 +425,12 @@ suricata:
     spm-algo: auto
     luajit:
       states: 128
-
+    security:
+      lua:
+        allow-rules: false
+        max-bytes: 500000 
+        max-instructions: 500000
+        allow-restricted-functions: false
     profiling:
       rules:
         enabled: "yes"

salt/suricata/disabled.sls

@@ -23,6 +23,11 @@ clean_suricata_eve_files:
   cron.absent:
     - identifier: clean_suricata_eve_files
 
+# Remove rulestats cron
+rulestats:
+  cron.absent:
+    - identifier: suricata_rulestats
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/suricata/enabled.sls

@@ -36,7 +36,7 @@ so-suricata:
       - /opt/so/conf/suricata/suricata.yaml:/etc/suricata/suricata.yaml:ro
       - /opt/so/conf/suricata/threshold.conf:/etc/suricata/threshold.conf:ro
       - /opt/so/conf/suricata/classification.config:/etc/suricata/classification.config:ro
-      - /opt/so/conf/suricata/rules:/etc/suricata/rules:ro
+      - /opt/so/rules/suricata:/etc/suricata/rules:ro
       - /opt/so/log/suricata/:/var/log/suricata/:rw
       - /nsm/suricata/:/nsm/:rw
       - /nsm/suricata/extracted:/var/log/suricata//filestore:rw
@@ -90,6 +90,18 @@ clean_suricata_eve_files:
     - month: '*'
     - dayweek: '*'
 
+# Add rulestats cron - runs every minute to query Suricata for rule load status
+suricata_rulestats:
+  cron.present:
+    - name: /usr/sbin/so-suricata-rulestats > /dev/null 2>&1
+    - identifier: suricata_rulestats
+    - user: root
+    - minute: '*'
+    - hour: '*'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/suricata/files/so_extraction.rules

@@ -0,0 +1,26 @@
+# Extract all PDF mime type
+alert http any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100000; rev:1;)
+alert smtp any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100001; rev:1;)
+alert nfs any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100002; rev:1;)
+alert smb any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100003; rev:1;)
+# Extract EXE/DLL file types
+alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100004; rev:1;)
+alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100005; rev:1;)
+alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100006; rev:1;)
+alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100007; rev:1;)
+alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100008; rev:1;)
+alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100009; rev:1;)
+alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100010; rev:1;)
+alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100011; rev:1;)
+
+# Extract all Zip files
+alert http any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100012; rev:1;)
+alert smtp any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100013; rev:1;)
+alert nfs any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100014; rev:1;)
+alert smb any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100015; rev:1;)
+
+# Extract Word Docs
+alert http any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100016; rev:1;)
+alert smtp any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100017; rev:1;)
+alert nfs any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100018; rev:1;)
+alert smb any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100019; rev:1;)

salt/suricata/files/so_filters.rules

@@ -0,0 +1,12 @@
+# Start the filters at sid 1200000
+# Example of filtering out *google.com from being in the dns log.
+#config dns any any -> any any (dns.query; content:"google.com"; config: logging disable, type tx, scope tx; sid:1200000;)
+# Example of filtering out  *google.com from being in the http log.
+#config http any any -> any any (http.host; content:"google.com"; config: logging disable, type tx, scope tx; sid:1200001;)
+# Example of filtering out  someuseragent from being in the http log.
+#config http any any -> any any (http.user_agent; content:"someuseragent"; config: logging disable, type tx, scope tx; sid:1200002;)
+# Example of filtering out  Google's certificate from being in the ssl log.
+#config tls any any -> any any (tls.fingerprint; content:"4f:a4:5e:58:7e:d9:db:20:09:d7:b6:c7:ff:58:c4:7b:dc:3f:55:b4"; config: logging disable, type tx, scope tx; sid:1200003;)
+# Example of filtering out a md5 of a file from being in the files log.
+#config fileinfo any any -> any any (fileinfo.filemd5; content:"7a125dc69c82d5caf94d3913eecde4b5"; config: logging disable, type tx, scope tx; sid:1200004;)
+

salt/suricata/files/threshold.conf

@@ -0,0 +1,2 @@
+# Threshold configuration generated by Security Onion
+# This file is automatically generated - do not edit manually

salt/suricata/files/threshold.conf.jinja

@@ -1,35 +0,0 @@
-{% import_yaml 'suricata/thresholding/sids.yaml' as THRESHOLDING %}
-{% if THRESHOLDING -%}
-  
-  {% for EACH_SID in THRESHOLDING -%}
-    {% for ACTIONS_LIST in THRESHOLDING[EACH_SID] -%}
-      {% for EACH_ACTION in ACTIONS_LIST -%}
-        
-        {%- if EACH_ACTION == 'threshold' %}
-{{ EACH_ACTION }} gen_id {{ ACTIONS_LIST[EACH_ACTION].gen_id }}, sig_id {{ EACH_SID }}, type {{ ACTIONS_LIST[EACH_ACTION].type }}, track {{ ACTIONS_LIST[EACH_ACTION].track }}, count {{ ACTIONS_LIST[EACH_ACTION].count }}, seconds {{ ACTIONS_LIST[EACH_ACTION].seconds }}
-        
-        {%- elif EACH_ACTION == 'rate_filter' %}
-          {%- if ACTIONS_LIST[EACH_ACTION].new_action not in ['drop','reject'] %}
-{{ EACH_ACTION }} gen_id {{ ACTIONS_LIST[EACH_ACTION].gen_id }}, sig_id {{ EACH_SID }}, track {{ ACTIONS_LIST[EACH_ACTION].track }}, count {{ ACTIONS_LIST[EACH_ACTION].count }}, seconds {{ ACTIONS_LIST[EACH_ACTION].seconds }}, new_action {{ ACTIONS_LIST[EACH_ACTION].new_action }}, timeout {{ ACTIONS_LIST[EACH_ACTION].timeout }}
-          {%- else %}
-##### Security Onion does not support drop or reject actions for rate_filter
-##### {{ EACH_ACTION }} gen_id {{ ACTIONS_LIST[EACH_ACTION].gen_id }}, sig_id {{ EACH_SID }}, track {{ ACTIONS_LIST[EACH_ACTION].track }}, count {{ ACTIONS_LIST[EACH_ACTION].count }}, seconds {{ ACTIONS_LIST[EACH_ACTION].seconds }}, new_action {{ ACTIONS_LIST[EACH_ACTION].new_action }}, timeout {{ ACTIONS_LIST[EACH_ACTION].timeout }}
-          {%- endif %}
-        
-        {%- elif EACH_ACTION == 'suppress' %}
-          {%- if ACTIONS_LIST[EACH_ACTION].track is defined %}
-{{ EACH_ACTION }} gen_id {{ ACTIONS_LIST[EACH_ACTION].gen_id }}, sig_id {{ EACH_SID }}, track {{ ACTIONS_LIST[EACH_ACTION].track }}, ip {{ ACTIONS_LIST[EACH_ACTION].ip }}
-          {%- else %}
-{{ EACH_ACTION }} gen_id {{ ACTIONS_LIST[EACH_ACTION].gen_id }}, sig_id {{ EACH_SID }}
-          {%- endif %}
-        
-        {%- endif %}
-      
-      {%- endfor %}
-    {%- endfor %}
-  {%- endfor %}
-
-{%- else %}
-##### Navigate to suricata > thresholding > SIDS in SOC to define thresholding
-
-{%- endif %}

salt/suricata/manager.sls

@@ -1,30 +0,0 @@
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls in allowed_states %}
-
-surilocaldir:
-  file.directory:
-    - name: /opt/so/saltstack/local/salt/suricata
-    - user: socore
-    - group: socore
-    - makedirs: True
-
-ruleslink:
-  file.symlink:
-    - name: /opt/so/saltstack/local/salt/suricata/rules
-    - user: socore
-    - group: socore
-    - target: /opt/so/rules/nids/suri
-
-refresh_salt_master_fileserver_suricata_ruleslink:
-  salt.runner:
-    - name: fileserver.update
-    - onchanges:
-      - file: ruleslink
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

salt/suricata/map.jinja

@@ -10,6 +10,12 @@
 
 {# before we change outputs back to list, enable pcap-log if suricata is the pcapengine #}
 {% if GLOBALS.pcap_engine in ["SURICATA", "TRANSITION"] %}
+
+{%   from 'bpf/pcap.map.jinja' import PCAPBPF, PCAP_BPF_STATUS %}
+{%   if PCAPBPF and PCAP_BPF_STATUS %}
+{%     do SURICATAMERGED.config.outputs['pcap-log'].update({'bpf-filter': PCAPBPF|join(" ")}) %}
+{%   endif %}
+
 {%   do SURICATAMERGED.config.outputs['pcap-log'].update({'enabled': 'yes'}) %}
 {#   move the items in suricata.pcap into suricata.config.outputs.pcap-log. these items were placed under suricata.config for ease of access in SOC #}
 {%   do SURICATAMERGED.config.outputs['pcap-log'].update({'compression': SURICATAMERGED.pcap.compression}) %}

salt/suricata/soc_suricata.yaml

@@ -190,6 +190,8 @@ suricata:
         FTP_PORTS: *suriportgroup
         VXLAN_PORTS: *suriportgroup
         TEREDO_PORTS: *suriportgroup
+        SIP_PORTS: *suriportgroup
+        GENEVE_PORTS: *suriportgroup
     outputs:
       eve-log:
         types:
@@ -209,7 +211,7 @@ suricata:
                 helpLink: suricata.html
       pcap-log:
         enabled:
-          description: This value is ignored by SO. pcapengine in globals takes precidence.
+          description: This value is ignored by SO. pcapengine in globals takes precedence.
           readonly: True
           helpLink: suricata.html
           advanced: True        
@@ -297,3 +299,10 @@ suricata:
         ports: 
           description: Ports to listen for. This should be a variable.    
           helpLink: suricata.html
+      geneve:
+        enabled:
+          description: Enable VXLAN capabilities.
+          helpLink: suricata.html
+        ports: 
+          description: Ports to listen for. This should be a variable.    
+          helpLink: suricata.html

salt/suricata/suricata_mdengine.yaml

@@ -29,7 +29,7 @@ suricata:
                 #custom: [Accept-Encoding, Accept-Language, Authorization]
                 # dump-all-headers: none
             - dns:
-                version: 2
+                version: 3
                 enabled: "yes"
                 #requests: "no"
                 #responses: "no"

salt/suricata/tools/sbin/so-suricata-reload-rules

@@ -7,5 +7,5 @@
 
 . /usr/sbin/so-common
 
-retry 60 3 'docker exec so-suricata /opt/suricata/bin/suricatasc -c reload-rules /var/run/suricata/suricata-command.socket' '{"message": "done", "return": "OK"}' || fail "The Suricata container was not ready in time."
-retry 60 3 'docker exec so-suricata /opt/suricata/bin/suricatasc -c ruleset-reload-nonblocking /var/run/suricata/suricata-command.socket' '{"message": "done", "return": "OK"}' || fail "The Suricata container was not ready in time."
+retry 60 3 'docker exec so-suricata /opt/suricata/bin/suricatasc -c reload-rules /var/run/suricata/suricata-command.socket' '{"message":"done","return":"OK"}' || fail "The Suricata container was not ready in time."
+retry 60 3 'docker exec so-suricata /opt/suricata/bin/suricatasc -c ruleset-reload-nonblocking /var/run/suricata/suricata-command.socket' '{"message":"done","return":"OK"}' || fail "The Suricata container was not ready in time."

salt/telegraf/defaults.yaml

@@ -21,6 +21,7 @@ telegraf:
       - sostatus.sh
       - stenoloss.sh
       - suriloss.sh
+      - surirules.sh
       - zeekcaptureloss.sh
       - zeekloss.sh
     standalone:
@@ -36,6 +37,7 @@ telegraf:
       - sostatus.sh
       - stenoloss.sh
       - suriloss.sh
+      - surirules.sh
       - zeekcaptureloss.sh
       - zeekloss.sh
       - features.sh
@@ -81,6 +83,7 @@ telegraf:
       - sostatus.sh
       - stenoloss.sh
       - suriloss.sh
+      - surirules.sh
       - zeekcaptureloss.sh
       - zeekloss.sh
       - features.sh
@@ -95,6 +98,7 @@ telegraf:
       - sostatus.sh
       - stenoloss.sh
       - suriloss.sh
+      - surirules.sh
       - zeekcaptureloss.sh
       - zeekloss.sh
     idh:

salt/telegraf/scripts/surirules.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+#
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+# Read Suricata ruleset stats from JSON file written by so-suricata-rulestats cron job
+# JSON format: {"rules_loaded":45879,"rules_failed":1,"last_reload":"2025-12-04T14:10:57+0000","return":"OK"}
+# or on failure: {"return":"FAIL"}
+
+# if this script isn't already running
+if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
+
+    STATSFILE="/var/log/suricata/rulestats.json"
+
+    # Check file exists, is less than 90 seconds old, and has valid data
+    if [ -f "$STATSFILE" ] && [ $(($(date +%s) - $(stat -c %Y "$STATSFILE"))) -lt 90 ] && jq -e '.return == "OK" and .rules_loaded != null and .rules_failed != null' "$STATSFILE" > /dev/null 2>&1; then
+        LOADED=$(jq -r '.rules_loaded' "$STATSFILE")
+        FAILED=$(jq -r '.rules_failed' "$STATSFILE")
+        RELOAD_TIME=$(jq -r 'if .last_reload then .last_reload else "" end' "$STATSFILE")
+
+        if [ -n "$RELOAD_TIME" ]; then
+            echo "surirules loaded=${LOADED}i,failed=${FAILED}i,reload_time=\"${RELOAD_TIME}\",status=\"ok\""
+        else
+            echo "surirules loaded=${LOADED}i,failed=${FAILED}i,status=\"ok\""
+        fi
+    else
+        echo "surirules loaded=0i,failed=0i,status=\"unknown\""
+    fi
+
+fi
+
+exit 0

salt/top.sls

@@ -74,7 +74,6 @@ base:
     - sensoroni
     - telegraf
     - firewall
-    - suricata.manager
     - healthcheck
     - elasticsearch
     - elastic-fleet-package-registry
@@ -105,7 +104,6 @@ base:
     - firewall
     - sensoroni
     - telegraf
-    - suricata.manager
     - healthcheck
     - elasticsearch
     - logstash
@@ -140,7 +138,6 @@ base:
     - sensoroni
     - telegraf
     - backup.config_backup
-    - suricata.manager
     - elasticsearch
     - logstash
     - redis
@@ -174,7 +171,6 @@ base:
     - sensoroni
     - telegraf
     - backup.config_backup
-    - suricata.manager
     - elasticsearch
     - logstash
     - redis
@@ -204,7 +200,6 @@ base:
     - sensoroni
     - telegraf
     - firewall
-    - suricata.manager
     - pcap
     - elasticsearch
     - elastic-fleet-package-registry

salt/zeek/config.sls

@@ -8,8 +8,7 @@
 
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% from "zeek/config.map.jinja" import ZEEKMERGED %}
-{% from 'bpf/zeek.map.jinja' import ZEEKBPF %}
-{% set BPF_STATUS = 0  %}
+{% from 'bpf/zeek.map.jinja' import ZEEKBPF, ZEEK_BPF_STATUS, ZEEK_BPF_CALC %}
 
 # Add Zeek group
 zeekgroup:
@@ -158,18 +157,13 @@ zeekja4cfg:
     - user: 937
     - group: 939
 
-# BPF compilation and configuration
-{% if ZEEKBPF %}
-   {% set BPF_CALC = salt['cmd.script']('salt://common/tools/sbin/so-bpf-compile', GLOBALS.sensor.interface + ' ' + ZEEKBPF|join(" "),cwd='/root') %}
-   {% if BPF_CALC['stderr'] == "" %}
-       {% set BPF_STATUS = 1  %}
-  {% else  %}
+# BPF compilation failed
+{% if ZEEKBPF and not ZEEK_BPF_STATUS %}
 zeekbpfcompilationfailure:
   test.configurable_test_state:
     - changes: False
     - result: False
-    - comment: "BPF Syntax Error - Discarding Specified BPF"
-   {% endif %}
+    - comment: "BPF Syntax Error - Discarding Specified BPF. Error: {{ ZEEK_BPF_CALC['stderr'] }}"
 {% endif %}
 
 zeekbpf:
@@ -177,7 +171,7 @@ zeekbpf:
     - name: /opt/so/conf/zeek/bpf
     - user: 940
     - group: 940
-{% if BPF_STATUS %}
+{% if ZEEK_BPF_STATUS %}
     - contents: {{ ZEEKBPF }}
 {% else %}
     - contents:

salt/zeek/defaults.yaml

@@ -45,7 +45,7 @@ zeek:
         - protocols/ssh/geo-data
         - protocols/ssh/detect-bruteforcing
         - protocols/ssh/interesting-hostnames
-        - protocols/http/detect-sqli
+        - protocols/http/detect-sql-injection
         - frameworks/files/hash-all-files
         - frameworks/files/detect-MHR
         - policy/frameworks/notice/extend-email/hostnames

salt/zeek/files/config.zeek.ja4

@@ -11,6 +11,8 @@ export {
   option JA4S_enabled:   bool = F;
   option JA4S_raw:   bool = F;
 
+  option JA4D_enabled:  bool = F;
+
   option JA4H_enabled:   bool = F;
   option JA4H_raw:   bool = F;
 

setup/so-functions

@@ -502,6 +502,7 @@ configure_minion() {
 		minion_type=desktop
 	fi
 	info "Configuring minion type as $minion_type" 
+	logCmd "mkdir -p /etc/salt/minion.d"
 	echo "role: so-$minion_type" > /etc/salt/grains
 	
 	local minion_config=/etc/salt/minion
@@ -541,20 +542,6 @@ configure_minion() {
 		"log_file: /opt/so/log/salt/minion"\
 		"#startup_states: highstate" >> "$minion_config"
 
-	# At the time the so-managerhype node does not yet have the bridge configured.
-	# The so-hypervisor node doesn't either, but it doesn't cause issues here.
-	local usebr0=false
-	if [ "$minion_type" == 'hypervisor' ]; then
-    	usebr0=true
-	fi
-	local pillar_json="{\"host\": {\"mainint\": \"$MNIC\"}, \"usebr0\": $usebr0}"
-	info "Running: salt-call state.apply salt.mine_functions --local --file-root=../salt/ -l info pillar='$pillar_json'"
-	salt-call state.apply salt.mine_functions --local --file-root=../salt/ -l info pillar="$pillar_json"
-
-	{
-		logCmd "systemctl enable salt-minion";
-		logCmd "systemctl restart salt-minion";
-	} >> "$setup_log" 2>&1
 }
 
 checkin_at_boot() {
@@ -669,11 +656,11 @@ check_requirements() {
 	fi
 	
 	if [[ $total_mem_hr -lt $req_mem ]]; then
-		whiptail_requirements_error "memory" "${total_mem_hr} GB" "${req_mem} GB"
 		if [[ $is_standalone || $is_heavynode ]]; then
 			echo "This install type will fail with less than $req_mem GB of memory. Exiting setup."
 			exit 0
 		fi
+		whiptail_requirements_error "memory" "${total_mem_hr} GB" "${req_mem} GB"
 	fi
 	if [[ $is_standalone || $is_heavynode ]]; then
 		if [[ $total_mem_hr -gt 15 && $total_mem_hr -lt 24 ]]; then
@@ -729,7 +716,7 @@ configure_network_sensor() {
 	fi
 
 	# Create the bond interface only if it doesn't already exist
-	nmcli -f name,uuid -p con | grep -q '$INTERFACE'
+	nmcli -f name,uuid -p con | grep -q "$INTERFACE"
 	local found_int=$?
 
 	if [[ $found_int != 0 ]]; then
@@ -798,25 +785,18 @@ configure_hyper_bridge() {
 }
 
 copy_salt_master_config() {
-
+	logCmd "mkdir /etc/salt"
 	title "Copy the Salt master config template to the proper directory"
 	if [ "$setup_type" = 'iso' ]; then
 		logCmd "cp /root/SecurityOnion/files/salt/master/master /etc/salt/master"
-		#logCmd "cp /root/SecurityOnion/files/salt/master/salt-master.service /usr/lib/systemd/system/salt-master.service"
 	else
 		logCmd "cp ../files/salt/master/master /etc/salt/master"
-		#logCmd "cp ../files/salt/master/salt-master.service /usr/lib/systemd/system/salt-master.service"
 	fi
 	info "Copying pillar and salt files in $temp_install_dir to $local_salt_dir"
 	logCmd "cp -R $temp_install_dir/pillar/ $local_salt_dir/"
 	if [ -d "$temp_install_dir"/salt ] ; then
 		logCmd "cp -R $temp_install_dir/salt/ $local_salt_dir/"
 	fi
-
-	# Restart the service so it picks up the changes
-	logCmd "systemctl daemon-reload"
-	logCmd "systemctl enable salt-master"
-	logCmd "systemctl restart salt-master"
 }
 
 create_local_nids_rules() {
@@ -1524,7 +1504,13 @@ networking_needful() {
 	if [[ $reinit_networking ]] || ! [[ -f $net_init_file ]]; then
 		collect_hostname
 	fi
-	[[ ! ( $is_eval || $is_import ) ]] && whiptail_node_description
+	if [[ ! ( $is_eval || $is_import ) ]]; then
+		whiptail_node_description
+		while [[ "$NODE_DESCRIPTION" =~ [[:cntrl:]] ]]; do
+			whiptail_error_message "Node description cannot contain control characters. Please enter a new description."
+			whiptail_node_description
+		done
+	fi
 	if [[ $reinit_networking ]] || ! [[ -f $net_init_file ]]; then
 		network_init_whiptail
 	else
@@ -1618,16 +1604,21 @@ proxy_validate() {
 
 reserve_group_ids() {
 	# This is a hack to fix OS from taking group IDs that we need
+	logCmd "groupadd -g 920 docker"
 	logCmd "groupadd -g 928 kratos"
 	logCmd "groupadd -g 930 elasticsearch"
 	logCmd "groupadd -g 931 logstash"
 	logCmd "groupadd -g 932 kibana"
 	logCmd "groupadd -g 933 elastalert"
 	logCmd "groupadd -g 937 zeek"
+	logCmd "groupadd -g 938 salt"
+	logCmd "groupadd -g 939 socore"
 	logCmd "groupadd -g 940 suricata"
+	logCmd "groupadd -g 948 elastic-agent-pr"
+	logCmd "groupadd -g 949 elastic-agent"
 	logCmd "groupadd -g 941 stenographer"
-	logCmd "groupadd -g 945 ossec"
-	logCmd "groupadd -g 946 cyberchef"
+	logCmd "groupadd -g 947 elastic-fleet"
+	logCmd "groupadd -g 960 kafka"
 }
 
 reserve_ports() {
@@ -1771,6 +1762,50 @@ backup_dir() {
 }
 
 drop_install_options() {
+	# Ensure values written to install.txt won't later fail manager-side parsing in so-minion
+	strip_control_chars() {
+		# bash: remove ASCII control characters (incl. newlines/tabs/ESC)
+		printf '%s' "$1" | tr -d '[:cntrl:]'
+	}
+
+	validate_install_txt_vars() {
+		# Sanitize first (fail closed if still invalid)
+		MAINIP="$(strip_control_chars "$MAINIP")"
+		MNIC="$(strip_control_chars "$MNIC")"
+		NODE_DESCRIPTION="$(strip_control_chars "$NODE_DESCRIPTION")"
+		ES_HEAP_SIZE="$(strip_control_chars "$ES_HEAP_SIZE")"
+		PATCHSCHEDULENAME="$(strip_control_chars "$PATCHSCHEDULENAME")"
+		INTERFACE="$(strip_control_chars "$INTERFACE")"
+		HOSTNAME="$(strip_control_chars "$HOSTNAME")"
+		LS_HEAP_SIZE="$(strip_control_chars "$LS_HEAP_SIZE")"
+		IDH_MGTRESTRICT="$(strip_control_chars "$IDH_MGTRESTRICT")"
+		IDH_SERVICES="$(strip_control_chars "$IDH_SERVICES")"
+
+		valid_ip4 "$MAINIP" || return 1
+		[[ "$MNIC" =~ ^[A-Za-z0-9_.:-]+$ ]] || return 1
+		[[ "$NODE_DESCRIPTION" =~ ^[[:print:]]{0,256}$ ]] || return 1
+		[[ "$ES_HEAP_SIZE" =~ ^[0-9]+[kKmMgGtTpPeE]?$ ]] || return 1
+		[[ "$PATCHSCHEDULENAME" =~ ^[A-Za-z0-9._-]*$ ]] || return 1
+		[[ "$INTERFACE" =~ ^[A-Za-z0-9._:,-]+$ ]] || return 1
+		valid_hostname "$HOSTNAME" || return 1
+		[[ "$LS_HEAP_SIZE" =~ ^[0-9]+[kKmMgGtTpPeE]?$ ]] || return 1
+		[[ "$lb_procs" =~ ^[0-9]+$ ]] || return 1
+		[[ "$num_cpu_cores" =~ ^[0-9]+$ ]] || return 1
+		[[ -z "$IDH_MGTRESTRICT" || "$IDH_MGTRESTRICT" == "True" || "$IDH_MGTRESTRICT" == "False" ]] || return 1
+		[[ -z "$IDH_SERVICES" || "$IDH_SERVICES" =~ ^[[:print:]]{0,512}$ ]] || return 1
+
+		return 0
+	}
+
+	if ! validate_install_txt_vars; then
+		if declare -F whiptail_error_message >/dev/null; then
+			whiptail_error_message "One or more setup values were invalid and would cause the manager to reject this node when adding it. Please re-run setup and verify hostname, management IP/interface, and node description."
+		else
+			echo "Error: invalid setup values detected; refusing to write /opt/so/install.txt"
+		fi
+		return 1
+	fi
+
 	# Drop the install Variable
 	echo "MAINIP=$MAINIP" > /opt/so/install.txt
 	echo "MNIC=$MNIC" >> /opt/so/install.txt
@@ -1935,11 +1970,12 @@ repo_sync_local() {
 }
 
 saltify() {
-	info "Installing Salt"
 	SALTVERSION=$(grep "version:" ../salt/salt/master.defaults.yaml | grep -o "[0-9]\+\.[0-9]\+")
+	info "Installing Salt $SALTVERSION"
+	chmod u+x ../salt/salt/scripts/bootstrap-salt.sh
 	if [[ $is_deb ]]; then
 
-		DEBIAN_FRONTEND=noninteractive retry 150 20 "apt-get -y -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" upgrade" >> "$setup_log" 2>&1 || fail_setup
+		DEBIAN_FRONTEND=noninteractive retry 30 10 "apt-get -y -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" upgrade" >> "$setup_log" 2>&1 || fail_setup
 		if [ $OSVER == "focal" ]; then update-alternatives --install /usr/bin/python python /usr/bin/python3.10 10; fi
 		local pkg_arr=(
 			'apache2-utils'
@@ -1952,16 +1988,11 @@ saltify() {
 			'jq'
 			'gnupg'
 		)
-		retry 150 20 "apt-get -y install ${pkg_arr[*]}" || fail_setup
+		retry 30 10 "apt-get -y install ${pkg_arr[*]}" || fail_setup
 
 		logCmd "mkdir -vp /etc/apt/keyrings"
 		logCmd "wget -q --inet4-only -O /etc/apt/keyrings/docker.pub https://download.docker.com/linux/ubuntu/gpg"
 
-		# Download public key
-		logCmd "curl -fsSL -o /etc/apt/keyrings/salt-archive-keyring-2023.pgp https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public"
-		# Create apt repo target configuration
-		echo "deb [signed-by=/etc/apt/keyrings/salt-archive-keyring-2023.pgp arch=amd64] https://packages.broadcom.com/artifactory/saltproject-deb/ stable main" | sudo tee /etc/apt/sources.list.d/salt.list
-
 		if [[ $is_ubuntu ]]; then
 			# Add Docker Repo
 			add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" 
@@ -1972,45 +2003,50 @@ saltify() {
 			echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian $OSVER stable" > /etc/apt/sources.list.d/docker.list 
 		fi
 
-		logCmd "apt-key add /etc/apt/keyrings/salt-archive-keyring-2023.pgp"
-
-		#logCmd "apt-key add /opt/so/gpg/SALTSTACK-GPG-KEY.pub"
 		logCmd "apt-key add /etc/apt/keyrings/docker.pub"
 
-		# Add SO Saltstack Repo
-		#echo "deb https://repo.securityonion.net/file/securityonion-repo/ubuntu/20.04/amd64/salt3004.2/ focal main" > /etc/apt/sources.list.d/saltstack.list
-
-		# Ain't nothing but a GPG 
-
-		retry 150 20 "apt-get update" "" "Err:" || fail_setup
+		retry 30 10 "apt-get update" "" "Err:" || fail_setup
 		if [[ $waitforstate ]]; then
-			retry 150 20 "apt-get -y install salt-common=$SALTVERSION salt-minion=$SALTVERSION salt-master=$SALTVERSION" || fail_setup
-			retry 150 20 "apt-mark hold salt-minion salt-common salt-master" || fail_setup
-			retry 150 20 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-packaging python3-influxdb python3-lxml" || exit 1
+			retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -M -X stable $SALTVERSION" || fail_setup
+			retry 30 10 "apt-mark hold salt-minion salt-common salt-master" || fail_setup
+			retry 30 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-packaging python3-influxdb python3-lxml" || exit 1
 		else
-			retry 150 20 "apt-get -y install salt-common=$SALTVERSION salt-minion=$SALTVERSION" || fail_setup
-			retry 150 20 "apt-mark hold salt-minion salt-common" || fail_setup
+			retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -X stable $SALTVERSION" || fail_setup
+			retry 30 10 "apt-mark hold salt-minion salt-common" || fail_setup
 		fi
 	fi
 		
 	if [[ $is_rpm ]]; then
 		if [[ $waitforstate ]]; then 
 			# install all for a manager
-			logCmd "dnf -y install salt-$SALTVERSION salt-master-$SALTVERSION salt-minion-$SALTVERSION"
+			retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -M -X stable $SALTVERSION" || fail_setup
 		else
-			# We just need the minion
-			if [[ $is_airgap ]]; then
-			    logCmd "dnf -y install salt salt-minion"
-			else
-			    logCmd "dnf -y install salt-$SALTVERSION salt-minion-$SALTVERSION"
-			fi
+			# just a minion
+			retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -X stable $SALTVERSION" || fail_setup
 		fi
 	fi
 
-	logCmd "mkdir -p /etc/salt/minion.d"
 	salt_install_module_deps
 	salt_patch_x509_v2
 
+	# At the time the so-managerhype node does not yet have the bridge configured.
+	# The so-hypervisor node doesn't either, but it doesn't cause issues here.
+	local usebr0=false
+	if [ "$minion_type" == 'hypervisor' ]; then
+    	usebr0=true
+	fi
+	local pillar_json="{\"host\": {\"mainint\": \"$MNIC\"}, \"usebr0\": $usebr0}"
+	info "Running: salt-call state.apply salt.mine_functions --local --file-root=../salt/ -l info pillar='$pillar_json'"
+	salt-call state.apply salt.mine_functions --local --file-root=../salt/ -l info pillar="$pillar_json"
+
+	if [[ $waitforstate ]]; then 
+		logCmd "systemctl enable salt-master";
+		logCmd "systemctl start salt-master";
+	fi
+
+	logCmd "systemctl enable salt-minion";
+	logCmd "systemctl restart salt-minion";
+
 }
 
 salt_install_module_deps() {

setup/so-setup

@@ -682,6 +682,8 @@ if ! [[ -f $install_opt_file ]]; then
 		fi
 		info "Reserving ports"
 		reserve_ports
+		info "Reserving group ids"
+		reserve_group_ids
 		info "Setting Paths"
 		# Set the paths
 		set_path
@@ -745,13 +747,12 @@ if ! [[ -f $install_opt_file ]]; then
 		securityonion_repo
 		# Update existing packages
 		update_packages
-		# Install salt
-		saltify
-		# Start the master service
+		# Put salt-master config in place
 		copy_salt_master_config
 		configure_minion "$minion_type"
+		# Install salt
+		saltify
 		check_sos_appliance
-
 		logCmd "salt-key -yd $MINION_ID"
 		sleep 2 # Debug RSA Key format errors
 		logCmd "salt-call state.show_top"
@@ -841,7 +842,10 @@ if ! [[ -f $install_opt_file ]]; then
 		if [[ $monints ]]; then
 			configure_network_sensor
 		fi
+		info "Reserving ports"
 		reserve_ports
+		info "Reserving group ids"
+		reserve_group_ids
 		# Set the version
 		mark_version
 		# Disable the setup from prompting at login
@@ -852,8 +856,8 @@ if ! [[ -f $install_opt_file ]]; then
 		gpg_rpm_import
 		securityonion_repo
 		update_packages
-		saltify
 		configure_minion "$minion_type"
+		saltify
 		check_sos_appliance
 		drop_install_options
 		hypervisor_local_states

setup/so-whiptail

@@ -676,8 +676,8 @@ whiptail_install_type_dist_existing() {
 	EOM
 	
 	install_type=$(whiptail --title "$whiptail_title" --menu "$node_msg" 19 75 7 \
-		"SENSOR" "Create a forward only sensor " \
-		"SEARCHNODE" "Add a search node with parsing " \
+		"SENSOR" "Add a Sensor Node for monitoring network traffic " \
+		"SEARCHNODE" "Add a Search Node with parsing " \
 		"FLEET" "Dedicated Elastic Fleet Node " \
 		"HEAVYNODE" "Sensor + Search Node " \
 		"IDH" "Intrusion Detection Honeypot Node " \
@@ -903,8 +903,9 @@ whiptail_management_nic() {
 	local exitstatus=$?
 	whiptail_check_exitstatus $exitstatus
 
-	while [ -z "$MNIC" ]
+	while [ -z "$MNIC" ] || [[ "$MNIC" =~ [[:cntrl:]] ]] || [[ ! "$MNIC" =~ ^[A-Za-z0-9_.:-]+$ ]]
 	do
+		whiptail_invalid_input
 		MNIC=$(whiptail --title "$whiptail_title" --menu "Please select the NIC you would like to use for management.\n\nUse the arrow keys to move around and the Enter key to select." 22 75 12 "${nic_list_management[@]}" 3>&1 1>&2 2>&3 )	
 		local exitstatus=$?
 		whiptail_check_exitstatus $exitstatus
@@ -1098,6 +1099,14 @@ whiptail_node_description() {
 
 	local exitstatus=$?
 	whiptail_check_exitstatus $exitstatus
+
+	while [[ "$NODE_DESCRIPTION" =~ [[:cntrl:]] ]]; do
+		whiptail_error_message "Node description cannot contain control characters. Please enter a new description."
+		NODE_DESCRIPTION=$(whiptail --title "$whiptail_title" \
+			--inputbox "Enter a short description for the node or press ENTER to leave blank:" 10 75 3>&1 1>&2 2>&3)
+		local exitstatus=$?
+		whiptail_check_exitstatus $exitstatus
+	done
 }
 
 whiptail_ntp_ask() {