From 5b6a7035af9148c9ba77fb6d3277b728f09ee4df Mon Sep 17 00:00:00 2001
From: Josh Patterson <josh.patterson@securityonionsolutions.com>
Date: Wed, 19 Nov 2025 10:22:58 -0500
Subject: [PATCH 01/18] need python_shell for pipes

---
 salt/salt/engines/master/virtual_node_manager.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/salt/salt/engines/master/virtual_node_manager.py b/salt/salt/engines/master/virtual_node_manager.py
index 6d88bd688..ccc063d64 100644
--- a/salt/salt/engines/master/virtual_node_manager.py
+++ b/salt/salt/engines/master/virtual_node_manager.py
@@ -727,7 +727,8 @@ def check_hypervisor_disk_space(hypervisor: str, size_gb: int) -> Tuple[bool, Op
         result = local.cmd(
             hypervisor_minion,
             'cmd.run',
-            ["df -BG /nsm/libvirt/volumes | tail -1 | awk '{print $4}' | sed 's/G//'"]
+            ["df -BG /nsm/libvirt/volumes | tail -1 | awk '{print $4}' | sed 's/G//'"],
+            kwarg={'python_shell': True}
         )
         
         if not result or hypervisor_minion not in result:

From dd0b4c38201dea4d050f50043fcdb93be87b98b2 Mon Sep 17 00:00:00 2001
From: Josh Patterson <josh.patterson@securityonionsolutions.com>
Date: Wed, 19 Nov 2025 15:48:53 -0500
Subject: [PATCH 02/18] fix failed or hung qcow2 image download

---
 salt/_runners/setup_hypervisor.py | 138 ++++++++++++++++++++++--------
 1 file changed, 102 insertions(+), 36 deletions(-)

diff --git a/salt/_runners/setup_hypervisor.py b/salt/_runners/setup_hypervisor.py
index 929801783..b23734654 100644
--- a/salt/_runners/setup_hypervisor.py
+++ b/salt/_runners/setup_hypervisor.py
@@ -172,7 +172,15 @@ MANAGER_HOSTNAME = socket.gethostname()
 
 def _download_image():
     """
-    Download and validate the Oracle Linux KVM image.
+    Download and validate the Oracle Linux KVM image with retry logic and progress monitoring.
+    
+    Features:
+    - Detects stalled downloads (no progress for 30 seconds)
+    - Retries up to 3 times on failure
+    - Connection timeout of 30 seconds
+    - Read timeout of 60 seconds
+    - Cleans up partial downloads on failure
+    
     Returns:
         bool: True if successful or file exists with valid checksum, False on error
     """
@@ -185,45 +193,103 @@ def _download_image():
             os.unlink(IMAGE_PATH)
     
     log.info("Starting image download process")
+    
+    # Retry configuration
+    max_attempts = 3
+    stall_timeout = 30  # seconds without progress before considering download stalled
+    connection_timeout = 30  # seconds to establish connection
+    read_timeout = 60  # seconds to wait for data chunks
+    
+    for attempt in range(1, max_attempts + 1):
+        log.info("Download attempt %d of %d", attempt, max_attempts)
+        
+        try:
+            # Download file with timeouts
+            log.info("Downloading Oracle Linux KVM image from %s to %s", IMAGE_URL, IMAGE_PATH)
+            response = requests.get(
+                IMAGE_URL,
+                stream=True,
+                timeout=(connection_timeout, read_timeout)
+            )
+            response.raise_for_status()
 
-    try:
-        # Download file
-        log.info("Downloading Oracle Linux KVM image from %s to %s", IMAGE_URL, IMAGE_PATH)
-        response = requests.get(IMAGE_URL, stream=True)
-        response.raise_for_status()
+            # Get total file size for progress tracking
+            total_size = int(response.headers.get('content-length', 0))
+            downloaded_size = 0
+            last_log_time = 0
+            last_progress_time = time.time()
+            last_downloaded_size = 0
 
-        # Get total file size for progress tracking
-        total_size = int(response.headers.get('content-length', 0))
-        downloaded_size = 0
-        last_log_time = 0
+            # Save file with progress logging and stall detection
+            with salt.utils.files.fopen(IMAGE_PATH, 'wb') as f:
+                for chunk in response.iter_content(chunk_size=8192):
+                    if chunk:  # filter out keep-alive new chunks
+                        f.write(chunk)
+                        downloaded_size += len(chunk)
+                        current_time = time.time()
+                        
+                        # Check for stalled download
+                        if downloaded_size > last_downloaded_size:
+                            # Progress made, reset stall timer
+                            last_progress_time = current_time
+                            last_downloaded_size = downloaded_size
+                        elif current_time - last_progress_time > stall_timeout:
+                            # No progress for stall_timeout seconds
+                            raise Exception(
+                                f"Download stalled: no progress for {stall_timeout} seconds "
+                                f"at {downloaded_size}/{total_size} bytes"
+                            )
+                        
+                        # Log progress every second
+                        if current_time - last_log_time >= 1:
+                            progress = (downloaded_size / total_size) * 100 if total_size > 0 else 0
+                            log.info("Progress - %.1f%% (%d/%d bytes)",
+                                    progress, downloaded_size, total_size)
+                            last_log_time = current_time
 
-        # Save file with progress logging
-        with salt.utils.files.fopen(IMAGE_PATH, 'wb') as f:
-            for chunk in response.iter_content(chunk_size=8192):
-                f.write(chunk)
-                downloaded_size += len(chunk)
+            # Validate downloaded file
+            log.info("Download complete, validating checksum...")
+            if not _validate_image_checksum(IMAGE_PATH, IMAGE_SHA256):
+                log.error("Checksum validation failed on attempt %d", attempt)
+                os.unlink(IMAGE_PATH)
+                if attempt < max_attempts:
+                    log.info("Will retry download...")
+                    continue
+                else:
+                    log.error("All download attempts failed due to checksum mismatch")
+                    return False
+
+            log.info("Successfully downloaded and validated Oracle Linux KVM image")
+            return True
+
+        except requests.exceptions.Timeout as e:
+            log.error("Download attempt %d failed: Timeout - %s", attempt, str(e))
+            if os.path.exists(IMAGE_PATH):
+                os.unlink(IMAGE_PATH)
+            if attempt < max_attempts:
+                log.info("Will retry download...")
+            else:
+                log.error("All download attempts failed due to timeout")
                 
-                # Log progress every second
-                current_time = time.time()
-                if current_time - last_log_time >= 1:
-                    progress = (downloaded_size / total_size) * 100 if total_size > 0 else 0
-                    log.info("Progress - %.1f%% (%d/%d bytes)", 
-                            progress, downloaded_size, total_size)
-                    last_log_time = current_time
-
-        # Validate downloaded file
-        if not _validate_image_checksum(IMAGE_PATH, IMAGE_SHA256):
-            os.unlink(IMAGE_PATH)
-            return False
-
-        log.info("Successfully downloaded and validated Oracle Linux KVM image")
-        return True
-
-    except Exception as e:
-        log.error("Error downloading hypervisor image: %s", str(e))
-        if os.path.exists(IMAGE_PATH):
-            os.unlink(IMAGE_PATH)
-        return False
+        except requests.exceptions.RequestException as e:
+            log.error("Download attempt %d failed: Network error - %s", attempt, str(e))
+            if os.path.exists(IMAGE_PATH):
+                os.unlink(IMAGE_PATH)
+            if attempt < max_attempts:
+                log.info("Will retry download...")
+            else:
+                log.error("All download attempts failed due to network errors")
+                
+        except Exception as e:
+            log.error("Download attempt %d failed: %s", attempt, str(e))
+            if os.path.exists(IMAGE_PATH):
+                os.unlink(IMAGE_PATH)
+            if attempt < max_attempts:
+                log.info("Will retry download...")
+            else:
+                log.error("All download attempts failed")
+    
+    return False
 
 def _check_ssh_keys_exist():
     """

From 841ce6b6ec1de5bc06fd91fdcce47ce5ac51db07 Mon Sep 17 00:00:00 2001
From: Josh Patterson <josh.patterson@securityonionsolutions.com>
Date: Thu, 20 Nov 2025 13:55:22 -0500
Subject: [PATCH 03/18] update hypervisor annotation for image download or ssh
 key creation failure

---
 salt/_runners/setup_hypervisor.py             | 36 ++++++++++++-------
 .../hypervisor/soc_hypervisor.yaml.jinja      |  8 +++++
 2 files changed, 32 insertions(+), 12 deletions(-)

diff --git a/salt/_runners/setup_hypervisor.py b/salt/_runners/setup_hypervisor.py
index b23734654..5efdca021 100644
--- a/salt/_runners/setup_hypervisor.py
+++ b/salt/_runners/setup_hypervisor.py
@@ -196,6 +196,7 @@ def _download_image():
     
     # Retry configuration
     max_attempts = 3
+    retry_delay = 5  # seconds to wait between retry attempts
     stall_timeout = 30  # seconds without progress before considering download stalled
     connection_timeout = 30  # seconds to establish connection
     read_timeout = 60  # seconds to wait for data chunks
@@ -267,7 +268,8 @@ def _download_image():
             if os.path.exists(IMAGE_PATH):
                 os.unlink(IMAGE_PATH)
             if attempt < max_attempts:
-                log.info("Will retry download...")
+                log.info("Will retry download in %d seconds...", retry_delay)
+                time.sleep(retry_delay)
             else:
                 log.error("All download attempts failed due to timeout")
                 
@@ -276,7 +278,8 @@ def _download_image():
             if os.path.exists(IMAGE_PATH):
                 os.unlink(IMAGE_PATH)
             if attempt < max_attempts:
-                log.info("Will retry download...")
+                log.info("Will retry download in %d seconds...", retry_delay)
+                time.sleep(retry_delay)
             else:
                 log.error("All download attempts failed due to network errors")
                 
@@ -285,7 +288,8 @@ def _download_image():
             if os.path.exists(IMAGE_PATH):
                 os.unlink(IMAGE_PATH)
             if attempt < max_attempts:
-                log.info("Will retry download...")
+                log.info("Will retry download in %d seconds...", retry_delay)
+                time.sleep(retry_delay)
             else:
                 log.error("All download attempts failed")
     
@@ -485,25 +489,29 @@ def _ensure_hypervisor_host_dir(minion_id: str = None):
         log.error(f"Error creating hypervisor host directory: {str(e)}")
         return False
 
-def _apply_dyanno_hypervisor_state():
+def _apply_dyanno_hypervisor_state(status='Initialized'):
     """
     Apply the soc.dyanno.hypervisor state on the salt master.
     
     This function applies the soc.dyanno.hypervisor state on the salt master
     to update the hypervisor annotation and ensure all hypervisor host directories exist.
     
+    Args:
+        status: Status to set for the base domain (default: 'Initialized')
+                Valid values: 'PreInit', 'Initialized', 'ImageDownloadFailed', 'SSHKeySetupFailed'
+    
     Returns:
         bool: True if state was applied successfully, False otherwise
     """
     try:
-        log.info("Applying soc.dyanno.hypervisor state on salt master")
+        log.info(f"Applying soc.dyanno.hypervisor state on salt master with status: {status}")
         
         # Initialize the LocalClient
         local = salt.client.LocalClient()
         
         # Target the salt master to apply the soc.dyanno.hypervisor state
         target = MANAGER_HOSTNAME + '_*'
-        state_result = local.cmd(target, 'state.apply', ['soc.dyanno.hypervisor', "pillar={'baseDomain': {'status': 'PreInit'}}", 'concurrent=True'], tgt_type='glob')
+        state_result = local.cmd(target, 'state.apply', ['soc.dyanno.hypervisor', f"pillar={{'baseDomain': {{'status': '{status}'}}}}", 'concurrent=True'], tgt_type='glob')
         log.debug(f"state_result: {state_result}")
         # Check if state was applied successfully
         if state_result:
@@ -520,17 +528,17 @@ def _apply_dyanno_hypervisor_state():
                         success = False
             
             if success:
-                log.info("Successfully applied soc.dyanno.hypervisor state")
+                log.info(f"Successfully applied soc.dyanno.hypervisor state with status: {status}")
                 return True
             else:
-                log.error("Failed to apply soc.dyanno.hypervisor state")
+                log.error(f"Failed to apply soc.dyanno.hypervisor state with status: {status}")
                 return False
         else:
-            log.error("No response from salt master when applying soc.dyanno.hypervisor state")
+            log.error(f"No response from salt master when applying soc.dyanno.hypervisor state with status: {status}")
             return False
             
     except Exception as e:
-        log.error(f"Error applying soc.dyanno.hypervisor state: {str(e)}")
+        log.error(f"Error applying soc.dyanno.hypervisor state with status: {status}: {str(e)}")
         return False
 
 def _apply_cloud_config_state():
@@ -664,8 +672,8 @@ def setup_environment(vm_name: str = 'sool9', disk_size: str = '220G', minion_id
         log.warning("Failed to apply salt.cloud.config state, continuing with setup")
         # We don't return an error here as we want to continue with the setup process
 
-    # Apply the soc.dyanno.hypervisor state on the salt master
-    if not _apply_dyanno_hypervisor_state():
+    # Apply the soc.dyanno.hypervisor state on the salt master with PreInit status
+    if not _apply_dyanno_hypervisor_state('PreInit'):
         log.warning("Failed to apply soc.dyanno.hypervisor state, continuing with setup")
         # We don't return an error here as we want to continue with the setup process
 
@@ -685,6 +693,8 @@ def setup_environment(vm_name: str = 'sool9', disk_size: str = '220G', minion_id
         log.info("Starting image download/validation process")
         if not _download_image():
             log.error("Image download failed")
+            # Update hypervisor annotation with failure status
+            _apply_dyanno_hypervisor_state('ImageDownloadFailed')
             return {
                 'success': False,
                 'error': 'Image download failed',
@@ -697,6 +707,8 @@ def setup_environment(vm_name: str = 'sool9', disk_size: str = '220G', minion_id
         log.info("Setting up SSH keys")
         if not _setup_ssh_keys():
             log.error("SSH key setup failed")
+            # Update hypervisor annotation with failure status
+            _apply_dyanno_hypervisor_state('SSHKeySetupFailed')
             return {
                 'success': False,
                 'error': 'SSH key setup failed',
diff --git a/salt/soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja b/salt/soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja
index ac2fd6fea..d4b88b091 100644
--- a/salt/soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja
+++ b/salt/soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja
@@ -43,6 +43,14 @@
 
 No Virtual Machines Found
 {%-   endif %}
+{%- elif baseDomainStatus == 'ImageDownloadFailed' %}
+#### ERROR
+
+Base domain image download failed. Please check the salt-master log for details and verify network connectivity.
+{%- elif baseDomainStatus == 'SSHKeySetupFailed' %}
+#### ERROR
+
+SSH key setup failed. Please check the salt-master log for details.
 {%- else %}
 #### WARNING
 

From fbe97221bbc4919069ad8bbf2569dcd1adb2639a Mon Sep 17 00:00:00 2001
From: Josh Patterson <josh.patterson@securityonionsolutions.com>
Date: Thu, 20 Nov 2025 14:43:09 -0500
Subject: [PATCH 04/18] set initialized status

---
 salt/_runners/setup_hypervisor.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/salt/_runners/setup_hypervisor.py b/salt/_runners/setup_hypervisor.py
index 5efdca021..e1bf0a45a 100644
--- a/salt/_runners/setup_hypervisor.py
+++ b/salt/_runners/setup_hypervisor.py
@@ -733,6 +733,10 @@ def setup_environment(vm_name: str = 'sool9', disk_size: str = '220G', minion_id
     success = vm_result.get('success', False)
     log.info("Setup environment completed with status: %s", "SUCCESS" if success else "FAILED")
     
+    # Update hypervisor annotation with success status
+    if success:
+        _apply_dyanno_hypervisor_state('Initialized')
+    
     # If setup was successful and we have a minion_id, run highstate
     if success and minion_id:
         log.info("Running highstate on hypervisor %s", minion_id)

From 97c1a460133ddac1e5e6f4c7461e207680068009 Mon Sep 17 00:00:00 2001
From: Josh Patterson <josh.patterson@securityonionsolutions.com>
Date: Thu, 20 Nov 2025 15:08:04 -0500
Subject: [PATCH 05/18] update annotation for general failure

---
 salt/_runners/setup_hypervisor.py                    | 2 ++
 salt/soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja | 4 ++++
 2 files changed, 6 insertions(+)

diff --git a/salt/_runners/setup_hypervisor.py b/salt/_runners/setup_hypervisor.py
index e1bf0a45a..b30f9c6d6 100644
--- a/salt/_runners/setup_hypervisor.py
+++ b/salt/_runners/setup_hypervisor.py
@@ -736,6 +736,8 @@ def setup_environment(vm_name: str = 'sool9', disk_size: str = '220G', minion_id
     # Update hypervisor annotation with success status
     if success:
         _apply_dyanno_hypervisor_state('Initialized')
+    else:
+        _apply_dyanno_hypervisor_state('SetupFailed')
     
     # If setup was successful and we have a minion_id, run highstate
     if success and minion_id:
diff --git a/salt/soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja b/salt/soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja
index d4b88b091..97170a55f 100644
--- a/salt/soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja
+++ b/salt/soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja
@@ -51,6 +51,10 @@ Base domain image download failed. Please check the salt-master log for details
 #### ERROR
 
 SSH key setup failed. Please check the salt-master log for details.
+{%- elif baseDomainStatus == 'SetupFailed' %}
+#### WARNING
+
+Setup failed. Please check the salt-master log for details.
 {%- else %}
 #### WARNING
 

From 433dab7376dc02d79f628102ecd5c010a5632153 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Thu, 20 Nov 2025 14:16:10 -0600
Subject: [PATCH 06/18] format json

---
 .../files/ingest/suricata.common              | 176 +++++++++++++++---
 1 file changed, 147 insertions(+), 29 deletions(-)

diff --git a/salt/elasticsearch/files/ingest/suricata.common b/salt/elasticsearch/files/ingest/suricata.common
index 102b5dac8..5af35dc37 100644
--- a/salt/elasticsearch/files/ingest/suricata.common
+++ b/salt/elasticsearch/files/ingest/suricata.common
@@ -1,30 +1,148 @@
 {
-  "description" : "suricata.common",
-  "processors" : [
-    { "json":   { "field": "message",                   "target_field": "message2",             "ignore_failure": true  } },
-    { "rename": { "field": "message2.pkt_src",          "target_field": "network.packet_source","ignore_failure": true  } },
-    { "rename": { "field": "message2.proto",            "target_field": "network.transport",    "ignore_failure": true  } },
-    { "rename": { "field": "message2.in_iface",         "target_field": "observer.ingress.interface.name",    "ignore_failure": true  } },
-    { "rename": { "field": "message2.flow_id",          "target_field": "log.id.uid",           "ignore_failure": true  } },
-    { "rename": { "field": "message2.src_ip",           "target_field": "source.ip",            "ignore_failure": true  } },
-    { "rename": { "field": "message2.src_port",         "target_field": "source.port",          "ignore_failure": true  } },
-    { "rename": { "field": "message2.dest_ip",          "target_field": "destination.ip",       "ignore_failure": true  } },
-    { "rename": { "field": "message2.dest_port",        "target_field": "destination.port",     "ignore_failure": true  } },
-    { "rename": { "field": "message2.vlan",             "target_field": "network.vlan.id",      "ignore_failure": true  } },
-    { "rename": { "field": "message2.community_id",     "target_field": "network.community_id", "ignore_missing": true  } },
-    { "rename": { "field": "message2.xff",              "target_field": "xff.ip",               "ignore_missing": true  } },
-    { "set":    { "field": "event.dataset",             "value": "{{ message2.event_type }}"                            } },
-    { "set":    { "field": "observer.name",             "value": "{{agent.name}}"                                       } },
-    { "set":    { "field": "event.ingested",            "value": "{{@timestamp}}"                                       } },
-    { "date": { "field": "message2.timestamp", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "timezone": "UTC", "ignore_failure": true } },
-    { "remove":{ "field": "agent",                                                              "ignore_failure": true  } },
-    {"append":{"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"ignore_failure":true}},
-    {
-      "script": {
-        "source": "boolean isPrivate(def ip) { if (ip == null) return false; int dot1 = ip.indexOf('.'); if (dot1 == -1) return false; int dot2 = ip.indexOf('.', dot1 + 1); if (dot2 == -1) return false; int first = Integer.parseInt(ip.substring(0, dot1)); if (first == 10) return true; if (first == 192 && ip.startsWith('168.', dot1 + 1)) return true; if (first == 172) { int second = Integer.parseInt(ip.substring(dot1 + 1, dot2)); return second >= 16 && second <= 31; } return false; } String[] fields = new String[] {\"source\", \"destination\"}; for (int i = 0; i < fields.length; i++) { def field = fields[i]; def ip = ctx[field]?.ip; if (ip != null) { if (ctx.network == null) ctx.network = new HashMap(); if (isPrivate(ip)) { if (ctx.network.private_ip == null) ctx.network.private_ip = new ArrayList(); if (!ctx.network.private_ip.contains(ip)) ctx.network.private_ip.add(ip); } else { if (ctx.network.public_ip == null) ctx.network.public_ip = new ArrayList(); if (!ctx.network.public_ip.contains(ip)) ctx.network.public_ip.add(ip); } } }",
-        "ignore_failure": false
-      }
-    },
-    { "pipeline": { "if": "ctx?.event?.dataset != null", "name": "suricata.{{event.dataset}}" } }
-  ]
-}
+    "description": "suricata.common",
+    "processors": [
+        {
+            "json": {
+                "field": "message",
+                "target_field": "message2",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.pkt_src",
+                "target_field": "network.packet_source",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.proto",
+                "target_field": "network.transport",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.in_iface",
+                "target_field": "observer.ingress.interface.name",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.flow_id",
+                "target_field": "log.id.uid",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.src_ip",
+                "target_field": "source.ip",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.src_port",
+                "target_field": "source.port",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dest_ip",
+                "target_field": "destination.ip",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dest_port",
+                "target_field": "destination.port",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.vlan",
+                "target_field": "network.vlan.id",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.community_id",
+                "target_field": "network.community_id",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.xff",
+                "target_field": "xff.ip",
+                "ignore_missing": true
+            }
+        },
+        {
+            "set": {
+                "field": "event.dataset",
+                "value": "{{ message2.event_type }}"
+            }
+        },
+        {
+            "set": {
+                "field": "observer.name",
+                "value": "{{agent.name}}"
+            }
+        },
+        {
+            "set": {
+                "field": "event.ingested",
+                "value": "{{@timestamp}}"
+            }
+        },
+        {
+            "date": {
+                "field": "message2.timestamp",
+                "target_field": "@timestamp",
+                "formats": [
+                    "ISO8601",
+                    "UNIX"
+                ],
+                "timezone": "UTC",
+                "ignore_failure": true
+            }
+        },
+        {
+            "remove": {
+                "field": "agent",
+                "ignore_failure": true
+            }
+        },
+        {
+            "append": {
+                "field": "related.ip",
+                "value": [
+                    "{{source.ip}}",
+                    "{{destination.ip}}"
+                ],
+                "allow_duplicates": false,
+                "ignore_failure": true
+            }
+        },
+        {
+            "script": {
+                "source": "boolean isPrivate(def ip) { if (ip == null) return false; int dot1 = ip.indexOf('.'); if (dot1 == -1) return false; int dot2 = ip.indexOf('.', dot1 + 1); if (dot2 == -1) return false; int first = Integer.parseInt(ip.substring(0, dot1)); if (first == 10) return true; if (first == 192 && ip.startsWith('168.', dot1 + 1)) return true; if (first == 172) { int second = Integer.parseInt(ip.substring(dot1 + 1, dot2)); return second >= 16 && second <= 31; } return false; } String[] fields = new String[] {\"source\", \"destination\"}; for (int i = 0; i < fields.length; i++) { def field = fields[i]; def ip = ctx[field]?.ip; if (ip != null) { if (ctx.network == null) ctx.network = new HashMap(); if (isPrivate(ip)) { if (ctx.network.private_ip == null) ctx.network.private_ip = new ArrayList(); if (!ctx.network.private_ip.contains(ip)) ctx.network.private_ip.add(ip); } else { if (ctx.network.public_ip == null) ctx.network.public_ip = new ArrayList(); if (!ctx.network.public_ip.contains(ip)) ctx.network.public_ip.add(ip); } } }",
+                "ignore_failure": false
+            }
+        },
+        {
+            "pipeline": {
+                "if": "ctx?.event?.dataset != null",
+                "name": "suricata.{{event.dataset}}"
+            }
+        }
+    ]
+}
\ No newline at end of file

From 6f42ff34422ba1bf7e8c560a1cd7fb7da1fe63c9 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Thu, 20 Nov 2025 14:16:49 -0600
Subject: [PATCH 07/18] suricata capture_file

---
 salt/elasticsearch/files/ingest/suricata.common | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/salt/elasticsearch/files/ingest/suricata.common b/salt/elasticsearch/files/ingest/suricata.common
index 5af35dc37..7b2dc7eeb 100644
--- a/salt/elasticsearch/files/ingest/suricata.common
+++ b/salt/elasticsearch/files/ingest/suricata.common
@@ -138,6 +138,13 @@
                 "ignore_failure": false
             }
         },
+        {
+            "rename": {
+                "field": "message2.capture_file",
+                "target_field": "suricata.capture_file",
+                "ignore_missing": true
+            }
+        },
         {
             "pipeline": {
                 "if": "ctx?.event?.dataset != null",

From c5db7c87525c1670a109e2e8a91bdc7587f72cb8 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Thu, 20 Nov 2025 14:26:12 -0600
Subject: [PATCH 08/18] suricata.capture_file keyword

---
 salt/elasticsearch/templates/component/ecs/suricata.json | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/salt/elasticsearch/templates/component/ecs/suricata.json b/salt/elasticsearch/templates/component/ecs/suricata.json
index 1eb06d266..3f393ff6a 100644
--- a/salt/elasticsearch/templates/component/ecs/suricata.json
+++ b/salt/elasticsearch/templates/component/ecs/suricata.json
@@ -841,6 +841,10 @@
                   "type": "long"
                 }
               }
+            },
+            "capture_file": {
+                "type": "keyword",
+                "ignore_above": 1024
             }
           }
         }

From 2d716b44a8665514f3720ef73d564986062ee102 Mon Sep 17 00:00:00 2001
From: Josh Patterson <josh.patterson@securityonionsolutions.com>
Date: Thu, 20 Nov 2025 15:52:21 -0500
Subject: [PATCH 09/18] update comment

---
 salt/_runners/setup_hypervisor.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/salt/_runners/setup_hypervisor.py b/salt/_runners/setup_hypervisor.py
index b30f9c6d6..7d521bd62 100644
--- a/salt/_runners/setup_hypervisor.py
+++ b/salt/_runners/setup_hypervisor.py
@@ -497,8 +497,7 @@ def _apply_dyanno_hypervisor_state(status='Initialized'):
     to update the hypervisor annotation and ensure all hypervisor host directories exist.
     
     Args:
-        status: Status to set for the base domain (default: 'Initialized')
-                Valid values: 'PreInit', 'Initialized', 'ImageDownloadFailed', 'SSHKeySetupFailed'
+        status: Status passed to the hypervisor annotation state
     
     Returns:
         bool: True if state was applied successfully, False otherwise

From 1f5f283c06579765fa2ffbfc92faedf0cd6b385e Mon Sep 17 00:00:00 2001
From: Josh Patterson <josh.patterson@securityonionsolutions.com>
Date: Thu, 20 Nov 2025 16:53:55 -0500
Subject: [PATCH 10/18] update hypervisor annotaion. preinit instead of
 initialized

---
 salt/_runners/setup_hypervisor.py                    | 4 ++--
 salt/soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/salt/_runners/setup_hypervisor.py b/salt/_runners/setup_hypervisor.py
index 7d521bd62..a123a1501 100644
--- a/salt/_runners/setup_hypervisor.py
+++ b/salt/_runners/setup_hypervisor.py
@@ -489,7 +489,7 @@ def _ensure_hypervisor_host_dir(minion_id: str = None):
         log.error(f"Error creating hypervisor host directory: {str(e)}")
         return False
 
-def _apply_dyanno_hypervisor_state(status='Initialized'):
+def _apply_dyanno_hypervisor_state(status):
     """
     Apply the soc.dyanno.hypervisor state on the salt master.
     
@@ -734,7 +734,7 @@ def setup_environment(vm_name: str = 'sool9', disk_size: str = '220G', minion_id
     
     # Update hypervisor annotation with success status
     if success:
-        _apply_dyanno_hypervisor_state('Initialized')
+        _apply_dyanno_hypervisor_state('PreInit')
     else:
         _apply_dyanno_hypervisor_state('SetupFailed')
     
diff --git a/salt/soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja b/salt/soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja
index 97170a55f..966f8838d 100644
--- a/salt/soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja
+++ b/salt/soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja
@@ -55,10 +55,10 @@ SSH key setup failed. Please check the salt-master log for details.
 #### WARNING
 
 Setup failed. Please check the salt-master log for details.
-{%- else %}
+{%- elif baseDomainStatus == 'PreInit' %}
 #### WARNING
 
-Base domain has not been initialized.
+Base domain has not been initialized. Waiting for hypervisor to highstate.
 {%- endif %}
 {%- endmacro -%}
 

From fb5ad4193d8bc4fe09ec256a0f7cfa86cf36ee45 Mon Sep 17 00:00:00 2001
From: Josh Patterson <josh.patterson@securityonionsolutions.com>
Date: Thu, 20 Nov 2025 17:13:36 -0500
Subject: [PATCH 11/18] indicate base image download start

---
 salt/_runners/setup_hypervisor.py                    | 6 +-----
 salt/soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja | 4 ++++
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/salt/_runners/setup_hypervisor.py b/salt/_runners/setup_hypervisor.py
index a123a1501..182a9b2c8 100644
--- a/salt/_runners/setup_hypervisor.py
+++ b/salt/_runners/setup_hypervisor.py
@@ -671,11 +671,6 @@ def setup_environment(vm_name: str = 'sool9', disk_size: str = '220G', minion_id
         log.warning("Failed to apply salt.cloud.config state, continuing with setup")
         # We don't return an error here as we want to continue with the setup process
 
-    # Apply the soc.dyanno.hypervisor state on the salt master with PreInit status
-    if not _apply_dyanno_hypervisor_state('PreInit'):
-        log.warning("Failed to apply soc.dyanno.hypervisor state, continuing with setup")
-        # We don't return an error here as we want to continue with the setup process
-
     log.info("Starting setup_environment in setup_hypervisor runner")
     
     # Check if environment is already set up
@@ -689,6 +684,7 @@ def setup_environment(vm_name: str = 'sool9', disk_size: str = '220G', minion_id
     
     # Handle image setup if needed
     if not image_valid:
+        _apply_dyanno_hypervisor_state('ImageDownloadStart')
         log.info("Starting image download/validation process")
         if not _download_image():
             log.error("Image download failed")
diff --git a/salt/soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja b/salt/soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja
index 966f8838d..f23fdb5d9 100644
--- a/salt/soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja
+++ b/salt/soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja
@@ -43,6 +43,10 @@
 
 No Virtual Machines Found
 {%-   endif %}
+{%- elif baseDomainStatus == 'ImageDownloadStart' %}
+#### INFO
+
+Base domain image download started.
 {%- elif baseDomainStatus == 'ImageDownloadFailed' %}
 #### ERROR
 

From 23da0d4ba0ca59993f65b3097dd52b8c4957ecac Mon Sep 17 00:00:00 2001
From: Josh Patterson <josh.patterson@securityonionsolutions.com>
Date: Fri, 21 Nov 2025 14:49:03 -0500
Subject: [PATCH 12/18] use timestamp in filename to prevent duplicates

---
 .../tools/sbin_jinja/so-kvm-create-volume         | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/salt/hypervisor/tools/sbin_jinja/so-kvm-create-volume b/salt/hypervisor/tools/sbin_jinja/so-kvm-create-volume
index 2322c3a94..601de643f 100644
--- a/salt/hypervisor/tools/sbin_jinja/so-kvm-create-volume
+++ b/salt/hypervisor/tools/sbin_jinja/so-kvm-create-volume
@@ -45,7 +45,7 @@ used during VM provisioning to add dedicated NSM storage volumes.
     This command creates and attaches a volume with the following settings:
     - VM Name: `vm1_sensor`
     - Volume Size: `500` GB
-    - Volume Path: `/nsm/libvirt/volumes/vm1_sensor-nsm.img`
+    - Volume Path: `/nsm/libvirt/volumes/vm1_sensor-nsm-<epoch_timestamp>.img`
     - Device: `/dev/vdb` (virtio-blk)
     - VM remains stopped after attachment
 
@@ -75,7 +75,8 @@ used during VM provisioning to add dedicated NSM storage volumes.
 
 - The script automatically stops the VM if it's running before creating and attaching the volume.
 - Volumes are created with full pre-allocation for optimal performance.
-- Volume files are stored in `/nsm/libvirt/volumes/` with naming pattern `<vm_name>-nsm.img`.
+- Volume files are stored in `/nsm/libvirt/volumes/` with naming pattern `<vm_name>-nsm-<epoch_timestamp>.img`.
+- The epoch timestamp ensures unique volume names and prevents conflicts.
 - Volumes are attached as `/dev/vdb` using virtio-blk for high performance.
 - The script checks available disk space before creating the volume.
 - Ownership is set to `qemu:qemu` with permissions `640`.
@@ -142,6 +143,7 @@ import socket
 import subprocess
 import pwd
 import grp
+import time
 import xml.etree.ElementTree as ET
 from io import StringIO
 from so_vm_utils import start_vm, stop_vm
@@ -242,10 +244,13 @@ def create_volume_file(vm_name, size_gb, logger):
     Raises:
         VolumeCreationError: If volume creation fails
     """
-    # Define volume path (directory already created in main())
-    volume_path = os.path.join(VOLUME_DIR, f"{vm_name}-nsm.img")
+    # Generate epoch timestamp for unique volume naming
+    epoch_timestamp = int(time.time())
     
-    # Check if volume already exists
+    # Define volume path with epoch timestamp for uniqueness
+    volume_path = os.path.join(VOLUME_DIR, f"{vm_name}-nsm-{epoch_timestamp}.img")
+    
+    # Check if volume already exists (shouldn't be possible with timestamp)
     if os.path.exists(volume_path):
         logger.error(f"VOLUME: Volume already exists: {volume_path}")
         raise VolumeCreationError(f"Volume already exists: {volume_path}")

From edf3c9464f794dbcb5fd96b94bdc4298969a282a Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Tue, 25 Nov 2025 16:16:19 -0600
Subject: [PATCH 13/18] add --certs flag to update certs. Used with --force, to
 ensure certs are updated even if hosts update isn't needed

---
 salt/elasticfleet/enabled.sls                 | 10 +++
 .../so-elastic-fleet-outputs-update           | 66 ++++++++++++++++---
 2 files changed, 66 insertions(+), 10 deletions(-)

diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls
index cef47168f..ec8c8337e 100644
--- a/salt/elasticfleet/enabled.sls
+++ b/salt/elasticfleet/enabled.sls
@@ -32,6 +32,16 @@ so-elastic-fleet-auto-configure-logstash-outputs:
     - retry:
         attempts: 4
         interval: 30
+
+{# Separate from above in order to catch elasticfleet-logstash.crt changes and force update to fleet output policy #}
+so-elastic-fleet-auto-configure-logstash-outputs-force:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-outputs-update --force --certs
+    - retry:
+        attempts: 4
+        interval: 30
+    - onchanges:
+        - x509: etc_elasticfleet_logstash_crt
 {% endif %}
 
 # If enabled, automatically update Fleet Server URLs & ES Connection
diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update
index 9efe8a19d..4fa68298c 100644
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update
@@ -8,6 +8,27 @@
 
 . /usr/sbin/so-common
 
+FORCE_UPDATE=false
+UPDATE_CERTS=false
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -f|--force)
+      FORCE_UPDATE=true
+      shift
+      ;;
+    -c| --certs)
+      UPDATE_CERTS=true
+      shift
+      ;;
+    *)
+      echo "Unknown option $1"
+      echo "Usage: $0 [-f|--force] [-c|--certs]"
+      exit 1
+      ;;
+  esac
+done
+
 # Only run on Managers
 if ! is_manager_node; then
     printf "Not a Manager Node... Exiting"
@@ -17,17 +38,42 @@ fi
 function update_logstash_outputs() {
     if logstash_policy=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs/so-manager_logstash" --retry 3 --retry-delay 10 --fail 2>/dev/null); then
         SSL_CONFIG=$(echo "$logstash_policy" | jq -r '.item.ssl')
+        LOGSTASHKEY=$(openssl rsa -in  /etc/pki/elasticfleet-logstash.key)
+        LOGSTASHCRT=$(openssl x509 -in /etc/pki/elasticfleet-logstash.crt)
+        LOGSTASHCA=$(openssl x509 -in  /etc/pki/tls/certs/intca.crt)
         if SECRETS=$(echo "$logstash_policy" | jq -er '.item.secrets' 2>/dev/null); then
-            JSON_STRING=$(jq -n \
-                --arg UPDATEDLIST "$NEW_LIST_JSON" \
-                --argjson SECRETS "$SECRETS" \
-                --argjson SSL_CONFIG "$SSL_CONFIG" \
-                '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl": $SSL_CONFIG,"secrets": $SECRETS}')
+            if [[ "$UPDATE_CERTS" != "true"  ]]; then
+                # Reuse existing secret
+                JSON_STRING=$(jq -n \
+                    --arg UPDATEDLIST "$NEW_LIST_JSON" \
+                    --argjson SECRETS "$SECRETS" \
+                    --argjson SSL_CONFIG "$SSL_CONFIG" \
+                    '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl": $SSL_CONFIG,"secrets": $SECRETS}')
+            else
+                # Update certs, creating new secret
+                JSON_STRING=$(jq -n \
+                    --arg UPDATEDLIST "$NEW_LIST_JSON" \
+                    --arg LOGSTASHKEY "$LOGSTASHKEY" \
+                    --arg LOGSTASHCRT "$LOGSTASHCRT" \
+                    --arg LOGSTASHCA "$LOGSTASHCA" \
+                    '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl": {"certificate": $LOGSTASHCRT,"certificate_authorities":[ $LOGSTASHCA ]},"secrets": {"ssl":{"key": $LOGSTASHKEY }}}')
+            fi
         else
-            JSON_STRING=$(jq -n \
-                --arg UPDATEDLIST "$NEW_LIST_JSON" \
-                --argjson SSL_CONFIG "$SSL_CONFIG" \
-                '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl": $SSL_CONFIG}')
+            if [[ "$UPDATE_CERTS" != "true" ]]; then
+                # Reuse existing ssl config
+                JSON_STRING=$(jq -n \
+                    --arg UPDATEDLIST "$NEW_LIST_JSON" \
+                    --argjson SSL_CONFIG "$SSL_CONFIG" \
+                    '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl": $SSL_CONFIG}')
+            else
+                # Update ssl config
+                JSON_STRING=$(jq -n \
+                    --arg UPDATEDLIST "$NEW_LIST_JSON" \
+                    --arg LOGSTASHKEY "$LOGSTASHKEY" \
+                    --arg LOGSTASHCRT "$LOGSTASHCRT" \
+                    --arg LOGSTASHCA "$LOGSTASHCA" \
+                    '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl": {"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]}}')
+            fi
         fi
     fi
 
@@ -151,7 +197,7 @@ NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "$
 NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}')
 
 # Compare the current & new list of outputs - if different, update the Logstash outputs
-if [ "$NEW_HASH" = "$CURRENT_HASH" ]; then
+if [[ "$NEW_HASH" = "$CURRENT_HASH" ]] && [[ "$FORCE_UPDATE" != "true" ]]; then
     printf "\nHashes match - no update needed.\n"
     printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n"
 

From 63bb44886ef19fb36e4a418108269f1d085c4914 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Mon, 1 Dec 2025 10:00:42 -0500
Subject: [PATCH 14/18] Add JA4D option to config.zeek.ja4

---
 salt/zeek/files/config.zeek.ja4 | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/salt/zeek/files/config.zeek.ja4 b/salt/zeek/files/config.zeek.ja4
index e3dd08a48..3d0035481 100644
--- a/salt/zeek/files/config.zeek.ja4
+++ b/salt/zeek/files/config.zeek.ja4
@@ -11,6 +11,8 @@ export {
   option JA4S_enabled:   bool = F;
   option JA4S_raw:   bool = F;
 
+  option JA4D_enabled:  bool = F;
+
   option JA4H_enabled:   bool = F;
   option JA4H_raw:   bool = F;
 

From d6bd951c377b07dcb1453d9ead760da560a3d9cd Mon Sep 17 00:00:00 2001
From: Josh Patterson <josh.patterson@securityonionsolutions.com>
Date: Tue, 2 Dec 2025 14:31:57 -0500
Subject: [PATCH 15/18] add new so-yaml_test for removefromlist

---
 salt/manager/tools/sbin/so-yaml_test.py | 122 ++++++++++++++++++++++++
 1 file changed, 122 insertions(+)

diff --git a/salt/manager/tools/sbin/so-yaml_test.py b/salt/manager/tools/sbin/so-yaml_test.py
index f33c1300a..9b4055841 100644
--- a/salt/manager/tools/sbin/so-yaml_test.py
+++ b/salt/manager/tools/sbin/so-yaml_test.py
@@ -457,3 +457,125 @@ class TestRemove(unittest.TestCase):
                 self.assertEqual(result, 1)
                 self.assertIn("Missing filename or key arg", mock_stderr.getvalue())
                 sysmock.assert_called_once_with(1)
+
+class TestRemoveFromList(unittest.TestCase):
+
+    def test_removefromlist_missing_arg(self):
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
+                sys.argv = ["cmd", "help"]
+                soyaml.removefromlist(["file", "key"])
+                sysmock.assert_called()
+                self.assertIn("Missing filename, key arg, or list item to remove", mock_stderr.getvalue())
+
+    def test_removefromlist(self):
+        filename = "/tmp/so-yaml_test-removefromlist.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: abc }, key2: false, key3: [a,b,c]}")
+        file.close()
+
+        soyaml.removefromlist([filename, "key3", "b"])
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        expected = "key1:\n  child1: 123\n  child2: abc\nkey2: false\nkey3:\n- a\n- c\n"
+        self.assertEqual(actual, expected)
+
+    def test_removefromlist_nested(self):
+        filename = "/tmp/so-yaml_test-removefromlist.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: [a,b,c] }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        soyaml.removefromlist([filename, "key1.child2", "b"])
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        expected = "key1:\n  child1: 123\n  child2:\n  - a\n  - c\nkey2: false\nkey3:\n- e\n- f\n- g\n"
+        self.assertEqual(actual, expected)
+
+    def test_removefromlist_nested_deep(self):
+        filename = "/tmp/so-yaml_test-removefromlist.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        soyaml.removefromlist([filename, "key1.child2.deep2", "b"])
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        expected = "key1:\n  child1: 123\n  child2:\n    deep1: 45\n    deep2:\n    - a\n    - c\nkey2: false\nkey3:\n- e\n- f\n- g\n"
+        self.assertEqual(actual, expected)
+
+    def test_removefromlist_item_not_in_list(self):
+        filename = "/tmp/so-yaml_test-removefromlist.yaml"
+        file = open(filename, "w")
+        file.write("{key1: [a,b,c]}")
+        file.close()
+
+        soyaml.removefromlist([filename, "key1", "d"])
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        expected = "key1:\n- a\n- b\n- c\n"
+        self.assertEqual(actual, expected)
+
+    def test_removefromlist_key_noexist(self):
+        filename = "/tmp/so-yaml_test-removefromlist.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
+                sys.argv = ["cmd", "removefromlist", filename, "key4", "h"]
+                soyaml.main()
+                sysmock.assert_called()
+                self.assertEqual("The key provided does not exist. No action was taken on the file.\n", mock_stderr.getvalue())
+
+    def test_removefromlist_key_noexist_deep(self):
+        filename = "/tmp/so-yaml_test-removefromlist.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
+                sys.argv = ["cmd", "removefromlist", filename, "key1.child2.deep3", "h"]
+                soyaml.main()
+                sysmock.assert_called()
+                self.assertEqual("The key provided does not exist. No action was taken on the file.\n", mock_stderr.getvalue())
+
+    def test_removefromlist_key_nonlist(self):
+        filename = "/tmp/so-yaml_test-removefromlist.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
+                sys.argv = ["cmd", "removefromlist", filename, "key1", "h"]
+                soyaml.main()
+                sysmock.assert_called()
+                self.assertEqual("The existing value for the given key is not a list. No action was taken on the file.\n", mock_stderr.getvalue())
+
+    def test_removefromlist_key_nonlist_deep(self):
+        filename = "/tmp/so-yaml_test-removefromlist.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
+                sys.argv = ["cmd", "removefromlist", filename, "key1.child2.deep1", "h"]
+                soyaml.main()
+                sysmock.assert_called()
+                self.assertEqual("The existing value for the given key is not a list. No action was taken on the file.\n", mock_stderr.getvalue())

From e871ec358ed6f0e0f4b881e4acb50eb261d37f02 Mon Sep 17 00:00:00 2001
From: Josh Patterson <josh.patterson@securityonionsolutions.com>
Date: Tue, 2 Dec 2025 14:43:33 -0500
Subject: [PATCH 16/18] need additional line bw class

---
 salt/manager/tools/sbin/so-yaml_test.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/salt/manager/tools/sbin/so-yaml_test.py b/salt/manager/tools/sbin/so-yaml_test.py
index 9b4055841..3d936e0f5 100644
--- a/salt/manager/tools/sbin/so-yaml_test.py
+++ b/salt/manager/tools/sbin/so-yaml_test.py
@@ -458,6 +458,7 @@ class TestRemove(unittest.TestCase):
                 self.assertIn("Missing filename or key arg", mock_stderr.getvalue())
                 sysmock.assert_called_once_with(1)
 
+
 class TestRemoveFromList(unittest.TestCase):
 
     def test_removefromlist_missing_arg(self):

From 89eb95c077e0d16f8349bb02c7b8fd06e103755a Mon Sep 17 00:00:00 2001
From: Josh Patterson <josh.patterson@securityonionsolutions.com>
Date: Tue, 2 Dec 2025 14:46:24 -0500
Subject: [PATCH 17/18] add removefromlist

---
 salt/manager/tools/sbin/so-yaml.py | 37 ++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/salt/manager/tools/sbin/so-yaml.py b/salt/manager/tools/sbin/so-yaml.py
index 4c8544893..c25c71095 100755
--- a/salt/manager/tools/sbin/so-yaml.py
+++ b/salt/manager/tools/sbin/so-yaml.py
@@ -17,6 +17,7 @@ def showUsage(args):
     print('Usage: {} <COMMAND> <YAML_FILE> [ARGS...]'.format(sys.argv[0]), file=sys.stderr)
     print('  General commands:', file=sys.stderr)
     print('    append         - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr)
+    print('    removefromlist - Remove a list item from a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr)
     print('    add            - Add a new key and set its value. Fails if key already exists. Requires KEY and VALUE args.', file=sys.stderr)
     print('    get            - Displays (to stdout) the value stored in the given key. Requires KEY arg.', file=sys.stderr)
     print('    remove         - Removes a yaml key, if it exists. Requires KEY arg.', file=sys.stderr)
@@ -57,6 +58,24 @@ def appendItem(content, key, listItem):
             return 1
 
 
+def removeFromList(content, key, listItem):
+    pieces = key.split(".", 1)
+    if len(pieces) > 1:
+        removeFromList(content[pieces[0]], pieces[1], listItem)
+    else:
+        try:
+            if not isinstance(content[key], list):
+                raise AttributeError("Value is not a list")
+            if listItem in content[key]:
+                content[key].remove(listItem)
+        except (AttributeError, TypeError):
+            print("The existing value for the given key is not a list. No action was taken on the file.", file=sys.stderr)
+            return 1
+        except KeyError:
+            print("The key provided does not exist. No action was taken on the file.", file=sys.stderr)
+            return 1
+
+
 def convertType(value):
     if isinstance(value, str) and value.startswith("file:"):
         path = value[5:]  # Remove "file:" prefix
@@ -103,6 +122,23 @@ def append(args):
     return 0
 
 
+def removefromlist(args):
+    if len(args) != 3:
+        print('Missing filename, key arg, or list item to remove', file=sys.stderr)
+        showUsage(None)
+        return 1
+
+    filename = args[0]
+    key = args[1]
+    listItem = args[2]
+
+    content = loadYaml(filename)
+    removeFromList(content, key, convertType(listItem))
+    writeYaml(filename, content)
+
+    return 0
+
+
 def addKey(content, key, value):
     pieces = key.split(".", 1)
     if len(pieces) > 1:
@@ -211,6 +247,7 @@ def main():
         "help": showUsage,
         "add": add,
         "append": append,
+        "removefromlist": removefromlist,
         "get": get,
         "remove": remove,
         "replace": replace,

From ef092e28937e27deb6f04ba9c54707c0691bc736 Mon Sep 17 00:00:00 2001
From: Josh Patterson <josh.patterson@securityonionsolutions.com>
Date: Tue, 2 Dec 2025 15:01:32 -0500
Subject: [PATCH 18/18] rename to removelistitem

---
 salt/manager/tools/sbin/so-yaml.py      | 12 +++---
 salt/manager/tools/sbin/so-yaml_test.py | 54 ++++++++++++-------------
 2 files changed, 33 insertions(+), 33 deletions(-)

diff --git a/salt/manager/tools/sbin/so-yaml.py b/salt/manager/tools/sbin/so-yaml.py
index c25c71095..00290f18b 100755
--- a/salt/manager/tools/sbin/so-yaml.py
+++ b/salt/manager/tools/sbin/so-yaml.py
@@ -17,7 +17,7 @@ def showUsage(args):
     print('Usage: {} <COMMAND> <YAML_FILE> [ARGS...]'.format(sys.argv[0]), file=sys.stderr)
     print('  General commands:', file=sys.stderr)
     print('    append         - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr)
-    print('    removefromlist - Remove a list item from a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr)
+    print('    removelistitem - Remove a list item from a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr)
     print('    add            - Add a new key and set its value. Fails if key already exists. Requires KEY and VALUE args.', file=sys.stderr)
     print('    get            - Displays (to stdout) the value stored in the given key. Requires KEY arg.', file=sys.stderr)
     print('    remove         - Removes a yaml key, if it exists. Requires KEY arg.', file=sys.stderr)
@@ -58,10 +58,10 @@ def appendItem(content, key, listItem):
             return 1
 
 
-def removeFromList(content, key, listItem):
+def removeListItem(content, key, listItem):
     pieces = key.split(".", 1)
     if len(pieces) > 1:
-        removeFromList(content[pieces[0]], pieces[1], listItem)
+        removeListItem(content[pieces[0]], pieces[1], listItem)
     else:
         try:
             if not isinstance(content[key], list):
@@ -122,7 +122,7 @@ def append(args):
     return 0
 
 
-def removefromlist(args):
+def removelistitem(args):
     if len(args) != 3:
         print('Missing filename, key arg, or list item to remove', file=sys.stderr)
         showUsage(None)
@@ -133,7 +133,7 @@ def removefromlist(args):
     listItem = args[2]
 
     content = loadYaml(filename)
-    removeFromList(content, key, convertType(listItem))
+    removeListItem(content, key, convertType(listItem))
     writeYaml(filename, content)
 
     return 0
@@ -247,7 +247,7 @@ def main():
         "help": showUsage,
         "add": add,
         "append": append,
-        "removefromlist": removefromlist,
+        "removelistitem": removelistitem,
         "get": get,
         "remove": remove,
         "replace": replace,
diff --git a/salt/manager/tools/sbin/so-yaml_test.py b/salt/manager/tools/sbin/so-yaml_test.py
index 3d936e0f5..3b5ec498e 100644
--- a/salt/manager/tools/sbin/so-yaml_test.py
+++ b/salt/manager/tools/sbin/so-yaml_test.py
@@ -459,23 +459,23 @@ class TestRemove(unittest.TestCase):
                 sysmock.assert_called_once_with(1)
 
 
-class TestRemoveFromList(unittest.TestCase):
+class TestRemoveListItem(unittest.TestCase):
 
-    def test_removefromlist_missing_arg(self):
+    def test_removelistitem_missing_arg(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
             with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 sys.argv = ["cmd", "help"]
-                soyaml.removefromlist(["file", "key"])
+                soyaml.removelistitem(["file", "key"])
                 sysmock.assert_called()
                 self.assertIn("Missing filename, key arg, or list item to remove", mock_stderr.getvalue())
 
-    def test_removefromlist(self):
-        filename = "/tmp/so-yaml_test-removefromlist.yaml"
+    def test_removelistitem(self):
+        filename = "/tmp/so-yaml_test-removelistitem.yaml"
         file = open(filename, "w")
         file.write("{key1: { child1: 123, child2: abc }, key2: false, key3: [a,b,c]}")
         file.close()
 
-        soyaml.removefromlist([filename, "key3", "b"])
+        soyaml.removelistitem([filename, "key3", "b"])
 
         file = open(filename, "r")
         actual = file.read()
@@ -484,13 +484,13 @@ class TestRemoveFromList(unittest.TestCase):
         expected = "key1:\n  child1: 123\n  child2: abc\nkey2: false\nkey3:\n- a\n- c\n"
         self.assertEqual(actual, expected)
 
-    def test_removefromlist_nested(self):
-        filename = "/tmp/so-yaml_test-removefromlist.yaml"
+    def test_removelistitem_nested(self):
+        filename = "/tmp/so-yaml_test-removelistitem.yaml"
         file = open(filename, "w")
         file.write("{key1: { child1: 123, child2: [a,b,c] }, key2: false, key3: [e,f,g]}")
         file.close()
 
-        soyaml.removefromlist([filename, "key1.child2", "b"])
+        soyaml.removelistitem([filename, "key1.child2", "b"])
 
         file = open(filename, "r")
         actual = file.read()
@@ -499,13 +499,13 @@ class TestRemoveFromList(unittest.TestCase):
         expected = "key1:\n  child1: 123\n  child2:\n  - a\n  - c\nkey2: false\nkey3:\n- e\n- f\n- g\n"
         self.assertEqual(actual, expected)
 
-    def test_removefromlist_nested_deep(self):
-        filename = "/tmp/so-yaml_test-removefromlist.yaml"
+    def test_removelistitem_nested_deep(self):
+        filename = "/tmp/so-yaml_test-removelistitem.yaml"
         file = open(filename, "w")
         file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
         file.close()
 
-        soyaml.removefromlist([filename, "key1.child2.deep2", "b"])
+        soyaml.removelistitem([filename, "key1.child2.deep2", "b"])
 
         file = open(filename, "r")
         actual = file.read()
@@ -514,13 +514,13 @@ class TestRemoveFromList(unittest.TestCase):
         expected = "key1:\n  child1: 123\n  child2:\n    deep1: 45\n    deep2:\n    - a\n    - c\nkey2: false\nkey3:\n- e\n- f\n- g\n"
         self.assertEqual(actual, expected)
 
-    def test_removefromlist_item_not_in_list(self):
-        filename = "/tmp/so-yaml_test-removefromlist.yaml"
+    def test_removelistitem_item_not_in_list(self):
+        filename = "/tmp/so-yaml_test-removelistitem.yaml"
         file = open(filename, "w")
         file.write("{key1: [a,b,c]}")
         file.close()
 
-        soyaml.removefromlist([filename, "key1", "d"])
+        soyaml.removelistitem([filename, "key1", "d"])
 
         file = open(filename, "r")
         actual = file.read()
@@ -529,54 +529,54 @@ class TestRemoveFromList(unittest.TestCase):
         expected = "key1:\n- a\n- b\n- c\n"
         self.assertEqual(actual, expected)
 
-    def test_removefromlist_key_noexist(self):
-        filename = "/tmp/so-yaml_test-removefromlist.yaml"
+    def test_removelistitem_key_noexist(self):
+        filename = "/tmp/so-yaml_test-removelistitem.yaml"
         file = open(filename, "w")
         file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
         file.close()
 
         with patch('sys.exit', new=MagicMock()) as sysmock:
             with patch('sys.stderr', new=StringIO()) as mock_stderr:
-                sys.argv = ["cmd", "removefromlist", filename, "key4", "h"]
+                sys.argv = ["cmd", "removelistitem", filename, "key4", "h"]
                 soyaml.main()
                 sysmock.assert_called()
                 self.assertEqual("The key provided does not exist. No action was taken on the file.\n", mock_stderr.getvalue())
 
-    def test_removefromlist_key_noexist_deep(self):
-        filename = "/tmp/so-yaml_test-removefromlist.yaml"
+    def test_removelistitem_key_noexist_deep(self):
+        filename = "/tmp/so-yaml_test-removelistitem.yaml"
         file = open(filename, "w")
         file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
         file.close()
 
         with patch('sys.exit', new=MagicMock()) as sysmock:
             with patch('sys.stderr', new=StringIO()) as mock_stderr:
-                sys.argv = ["cmd", "removefromlist", filename, "key1.child2.deep3", "h"]
+                sys.argv = ["cmd", "removelistitem", filename, "key1.child2.deep3", "h"]
                 soyaml.main()
                 sysmock.assert_called()
                 self.assertEqual("The key provided does not exist. No action was taken on the file.\n", mock_stderr.getvalue())
 
-    def test_removefromlist_key_nonlist(self):
-        filename = "/tmp/so-yaml_test-removefromlist.yaml"
+    def test_removelistitem_key_nonlist(self):
+        filename = "/tmp/so-yaml_test-removelistitem.yaml"
         file = open(filename, "w")
         file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
         file.close()
 
         with patch('sys.exit', new=MagicMock()) as sysmock:
             with patch('sys.stderr', new=StringIO()) as mock_stderr:
-                sys.argv = ["cmd", "removefromlist", filename, "key1", "h"]
+                sys.argv = ["cmd", "removelistitem", filename, "key1", "h"]
                 soyaml.main()
                 sysmock.assert_called()
                 self.assertEqual("The existing value for the given key is not a list. No action was taken on the file.\n", mock_stderr.getvalue())
 
-    def test_removefromlist_key_nonlist_deep(self):
-        filename = "/tmp/so-yaml_test-removefromlist.yaml"
+    def test_removelistitem_key_nonlist_deep(self):
+        filename = "/tmp/so-yaml_test-removelistitem.yaml"
         file = open(filename, "w")
         file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
         file.close()
 
         with patch('sys.exit', new=MagicMock()) as sysmock:
             with patch('sys.stderr', new=StringIO()) as mock_stderr:
-                sys.argv = ["cmd", "removefromlist", filename, "key1.child2.deep1", "h"]
+                sys.argv = ["cmd", "removelistitem", filename, "key1.child2.deep1", "h"]
                 soyaml.main()
                 sysmock.assert_called()
                 self.assertEqual("The existing value for the given key is not a list. No action was taken on the file.\n", mock_stderr.getvalue())