Merge pull request #14917 from Security-Onion-Solutions/2.4/dev

2.4.170

.github/.gitleaks.toml

@@ -541,5 +541,6 @@ paths = [
     '''gitleaks.toml''',
     '''(.*?)(jpg|gif|doc|pdf|bin|svg|socket)$''',
     '''(go.mod|go.sum)$''',
-    '''salt/nginx/files/enterprise-attack.json'''
+    '''salt/nginx/files/enterprise-attack.json''',
+    '''(.*?)whl$'''
 ]

.github/DISCUSSION_TEMPLATE/2-4.yml

@@ -28,6 +28,8 @@ body:
         - 2.4.140
         - 2.4.141
         - 2.4.150
+        - 2.4.160
+        - 2.4.170
         - Other (please provide detail below)
     validations:
       required: true

.gitignore

@@ -1,4 +1,3 @@
-
 # Created by https://www.gitignore.io/api/macos,windows
 # Edit at https://www.gitignore.io/?templates=macos,windows
 
@@ -67,4 +66,4 @@ __pycache__
 
 # Analyzer dev/test config files
 *_dev.yaml
-site-packages
+site-packages

DOWNLOAD_AND_VERIFY_ISO.md

@@ -1,17 +1,17 @@
-### 2.4.150-20250522 ISO image released on 2025/05/22
+### 2.4.170-20250812 ISO image released on 2025/08/12
 
 
 ### Download and Verify
 
-2.4.150-20250522 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.4.150-20250522.iso
+2.4.170-20250812 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-2.4.170-20250812.iso
  
-MD5: 239E69B83072BBF2602D4043FE53A160  
-SHA1: C62893D3C7F5592665BFDCBC9A45BB20A926F9A8  
-SHA256: 2ADE037C7FD34591030B1FAC10392C4E6613F152DD24BFBD897E57EE300895B9  
+MD5: 50ECAAD05736298452DECEAE074FA773  
+SHA1: 1B1EB520DE61ECC4BF34E512DAFE307317D7666A  
+SHA256: 87D176A48A58BAD1C2D57196F999BED23DE9B526226E3754F0C166C866CCDC1A  
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.150-20250522.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.170-20250812.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS  
@@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.150-20250522.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.170-20250812.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.4.150-20250522.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.4.170-20250812.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.4.150-20250522.iso.sig securityonion-2.4.150-20250522.iso
+gpg --verify securityonion-2.4.170-20250812.iso.sig securityonion-2.4.170-20250812.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Thu 22 May 2025 11:15:06 AM EDT using RSA key ID FE507013
+gpg: Signature made Fri 08 Aug 2025 06:24:56 PM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

HOTFIX

@@ -1 +0,0 @@
-20250522

VERSION

@@ -1 +1 @@
-2.4.150
+2.4.170

pillar/hypervisor/nodes.sls

@@ -0,0 +1,34 @@
+{% set node_types = {} %}
+{% for minionid, ip in salt.saltutil.runner(
+    'mine.get',
+    tgt='G@role:so-hypervisor or G@role:so-managerhype',
+    fun='network.ip_addrs',
+    tgt_type='compound') | dictsort()
+%}
+
+# only add a node to the pillar if it returned an ip from the mine
+{%   if ip | length > 0%}
+{%     set hostname = minionid.split('_') | first %}
+{%     set node_type = minionid.split('_') | last %}
+{%     if node_type not in node_types.keys() %}
+{%       do node_types.update({node_type: {hostname: ip[0]}}) %}
+{%     else %}
+{%       if hostname not in node_types[node_type] %}
+{%         do node_types[node_type].update({hostname: ip[0]}) %}
+{%       else %}
+{%         do node_types[node_type][hostname].update(ip[0]) %}
+{%       endif %}
+{%     endif %}
+{%   endif %}
+{% endfor %}
+
+
+hypervisor:
+  nodes:
+{% for node_type, values in node_types.items() %}
+    {{node_type}}:
+{%   for hostname, ip in values.items() %}
+      {{hostname}}:
+        ip: {{ip}}
+{%   endfor %}
+{% endfor %}

pillar/top.sls

@@ -18,6 +18,7 @@ base:
     - telegraf.adv_telegraf
     - versionlock.soc_versionlock
     - versionlock.adv_versionlock
+    - soc.license
 
   '* and not *_desktop':
     - firewall.soc_firewall
@@ -25,7 +26,12 @@ base:
     - nginx.soc_nginx
     - nginx.adv_nginx
 
-  '*_manager or *_managersearch':
+  'salt-cloud:driver:libvirt':
+    - match: grain
+    - vm.soc_vm
+    - vm.adv_vm
+
+  '*_manager or *_managersearch or *_managerhype':
     - match: compound
     - node_data.ips
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
@@ -44,7 +50,6 @@ base:
     - logstash.adv_logstash
     - soc.soc_soc
     - soc.adv_soc
-    - soc.license
     - kibana.soc_kibana
     - kibana.adv_kibana
     - kratos.soc_kratos
@@ -70,6 +75,9 @@ base:
     - kafka.nodes
     - kafka.soc_kafka
     - kafka.adv_kafka
+    - hypervisor.nodes
+    - hypervisor.soc_hypervisor
+    - hypervisor.adv_hypervisor
     - stig.soc_stig
 
   '*_sensor':
@@ -87,7 +95,6 @@ base:
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
     - stig.soc_stig
-    - soc.license
 
   '*_eval':
     - node_data.ips
@@ -114,7 +121,6 @@ base:
     - idstools.adv_idstools
     - soc.soc_soc
     - soc.adv_soc
-    - soc.license
     - kibana.soc_kibana
     - kibana.adv_kibana
     - strelka.soc_strelka
@@ -174,7 +180,6 @@ base:
     - manager.adv_manager
     - soc.soc_soc
     - soc.adv_soc
-    - soc.license
     - kibana.soc_kibana
     - kibana.adv_kibana
     - strelka.soc_strelka
@@ -240,7 +245,6 @@ base:
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
     - stig.soc_stig
-    - soc.license
     - kafka.nodes
     - kafka.soc_kafka
     - kafka.adv_kafka
@@ -258,8 +262,6 @@ base:
     - minions.adv_{{ grains.id }}
     - kafka.nodes
     - kafka.soc_kafka
-    - kafka.adv_kafka
-    - soc.license
 
   '*_import':
     - node_data.ips
@@ -283,7 +285,6 @@ base:
     - manager.adv_manager
     - soc.soc_soc
     - soc.adv_soc
-    - soc.license
     - kibana.soc_kibana
     - kibana.adv_kibana
     - backup.soc_backup
@@ -319,8 +320,12 @@ base:
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
 
+  '*_hypervisor':
+    - minions.{{ grains.id }}
+    - minions.adv_{{ grains.id }}
+
   '*_desktop':
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
     - stig.soc_stig
-    - soc.license
+

salt/_modules/qcow2.py

@@ -0,0 +1,246 @@
+#!py
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+"""
+Salt module for managing QCOW2 image configurations and VM hardware settings. This module provides functions
+for modifying network configurations within QCOW2 images and adjusting virtual machine hardware settings.
+It serves as a Salt interface to the so-qcow2-modify-network and so-kvm-modify-hardware scripts.
+
+The module offers two main capabilities:
+1. Network Configuration: Modify network settings (DHCP/static IP) within QCOW2 images
+2. Hardware Configuration: Adjust VM hardware settings (CPU, memory, PCI passthrough)
+
+This module is intended to work with Security Onion's virtualization infrastructure and is typically
+used in conjunction with salt-cloud for VM provisioning and management.
+"""
+
+import logging
+import subprocess
+import shlex
+
+log = logging.getLogger(__name__)
+
+__virtualname__ = 'qcow2'
+
+def __virtual__():
+    return __virtualname__
+
+def modify_network_config(image, interface, mode, vm_name, ip4=None, gw4=None, dns4=None, search4=None):
+    '''
+    Usage:
+        salt '*' qcow2.modify_network_config image=<path> interface=<iface> mode=<mode> vm_name=<name> [ip4=<addr>] [gw4=<addr>] [dns4=<servers>] [search4=<domain>]
+
+    Options:
+        image
+            Path to the QCOW2 image file that will be modified
+        interface
+            Network interface name to configure (e.g., 'enp1s0')
+        mode
+            Network configuration mode, either 'dhcp4' or 'static4'
+        vm_name
+            Full name of the VM (hostname_role)
+        ip4
+            IPv4 address with CIDR notation (e.g., '192.168.1.10/24')
+            Required when mode='static4'
+        gw4
+            IPv4 gateway address (e.g., '192.168.1.1')
+            Required when mode='static4'
+        dns4
+            Comma-separated list of IPv4 DNS servers (e.g., '8.8.8.8,8.8.4.4')
+            Optional for both DHCP and static configurations
+        search4
+            DNS search domain for IPv4 (e.g., 'example.local')
+            Optional for both DHCP and static configurations
+
+    Examples:
+        1. **Configure DHCP:**
+            ```bash
+            salt '*' qcow2.modify_network_config image='/nsm/libvirt/images/sool9/sool9.qcow2' interface='enp1s0' mode='dhcp4'
+            ```
+            This configures enp1s0 to use DHCP for IP assignment
+
+        2. **Configure Static IP:**
+            ```bash
+            salt '*' qcow2.modify_network_config image='/nsm/libvirt/images/sool9/sool9.qcow2' interface='enp1s0' mode='static4' ip4='192.168.1.10/24' gw4='192.168.1.1' dns4='192.168.1.1,8.8.8.8' search4='example.local'
+            ```
+            This sets a static IP configuration with DNS servers and search domain
+
+    Notes:
+        - The QCOW2 image must be accessible and writable by the salt minion
+        - The image should not be in use by a running VM when modified
+        - Network changes take effect on next VM boot
+        - Requires so-qcow2-modify-network script to be installed
+
+    Description:
+        This function modifies network configuration within a QCOW2 image file by executing
+        the so-qcow2-modify-network script. It supports both DHCP and static IPv4 configuration.
+        The script mounts the image, modifies the network configuration files, and unmounts
+        safely. All operations are logged for troubleshooting purposes.
+
+    Exit Codes:
+        0: Success
+        1: Invalid parameters or configuration
+        2: Image access or mounting error
+        3: Network configuration error
+        4: System command error
+        255: Unexpected error
+
+    Logging:
+        - All operations are logged to the salt minion log
+        - Log entries are prefixed with 'qcow2 module:'
+        - Error conditions include detailed error messages and stack traces
+        - Success/failure status is logged for verification
+    '''
+
+    cmd = ['/usr/sbin/so-qcow2-modify-network', '-I', image, '-i', interface, '-n', vm_name]
+
+    if mode.lower() == 'dhcp4':
+        cmd.append('--dhcp4')
+    elif mode.lower() == 'static4':
+        cmd.append('--static4')
+        if not ip4 or not gw4:
+            raise ValueError('Both ip4 and gw4 are required for static configuration.')
+        cmd.extend(['--ip4', ip4, '--gw4', gw4])
+        if dns4:
+            cmd.extend(['--dns4', dns4])
+        if search4:
+            cmd.extend(['--search4', search4])
+    else:
+        raise ValueError("Invalid mode '{}'. Expected 'dhcp4' or 'static4'.".format(mode))
+
+    log.info('qcow2 module: Executing command: {}'.format(' '.join(shlex.quote(arg) for arg in cmd)))
+
+    try:
+        result = subprocess.run(cmd, capture_output=True, text=True, check=False)
+        ret = {
+            'retcode': result.returncode,
+            'stdout': result.stdout,
+            'stderr': result.stderr
+        }
+        if result.returncode != 0:
+            log.error('qcow2 module: Script execution failed with return code {}: {}'.format(result.returncode, result.stderr))
+        else:
+            log.info('qcow2 module: Script executed successfully.')
+        return ret
+    except Exception as e:
+        log.error('qcow2 module: An error occurred while executing the script: {}'.format(e))
+        raise
+
+def modify_hardware_config(vm_name, cpu=None, memory=None, pci=None, start=False):
+    '''
+    Usage:
+        salt '*' qcow2.modify_hardware_config vm_name=<name> [cpu=<count>] [memory=<size>] [pci=<id>] [pci=<id>] [start=<bool>]
+
+    Options:
+        vm_name
+            Name of the virtual machine to modify
+        cpu
+            Number of virtual CPUs to assign (positive integer)
+            Optional - VM's current CPU count retained if not specified
+        memory
+            Amount of memory to assign in MiB (positive integer)
+            Optional - VM's current memory size retained if not specified
+        pci
+            PCI hardware ID(s) to passthrough to the VM (e.g., '0000:c7:00.0')
+            Can be specified multiple times for multiple devices
+            Optional - no PCI passthrough if not specified
+        start
+            Boolean flag to start the VM after modification
+            Optional - defaults to False
+
+    Examples:
+        1. **Modify CPU and Memory:**
+            ```bash
+            salt '*' qcow2.modify_hardware_config vm_name='sensor1' cpu=4 memory=8192
+            ```
+            This assigns 4 CPUs and 8GB memory to the VM
+
+        2. **Enable PCI Passthrough:**
+            ```bash
+            salt '*' qcow2.modify_hardware_config vm_name='sensor1' pci='0000:c7:00.0' pci='0000:c4:00.0' start=True
+            ```
+            This configures PCI passthrough and starts the VM
+
+        3. **Complete Hardware Configuration:**
+            ```bash
+            salt '*' qcow2.modify_hardware_config vm_name='sensor1' cpu=8 memory=16384 pci='0000:c7:00.0' start=True
+            ```
+            This sets CPU, memory, PCI passthrough, and starts the VM
+
+    Notes:
+        - VM must be stopped before modification unless only the start flag is set
+        - Memory is specified in MiB (1024 = 1GB)
+        - PCI devices must be available and not in use by the host
+        - CPU count should align with host capabilities
+        - Requires so-kvm-modify-hardware script to be installed
+
+    Description:
+        This function modifies the hardware configuration of a KVM virtual machine using
+        the so-kvm-modify-hardware script. It can adjust CPU count, memory allocation,
+        and PCI device passthrough. Changes are applied to the VM's libvirt configuration.
+        The VM can optionally be started after modifications are complete.
+
+    Exit Codes:
+        0: Success
+        1: Invalid parameters
+        2: VM state error (running when should be stopped)
+        3: Hardware configuration error
+        4: System command error
+        255: Unexpected error
+
+    Logging:
+        - All operations are logged to the salt minion log
+        - Log entries are prefixed with 'qcow2 module:'
+        - Hardware configuration changes are logged
+        - Errors include detailed messages and stack traces
+        - Final status of modification is logged
+    '''
+
+    cmd = ['/usr/sbin/so-kvm-modify-hardware', '-v', vm_name]
+
+    if cpu is not None:
+        if isinstance(cpu, int) and cpu > 0:
+            cmd.extend(['-c', str(cpu)])
+        else:
+            raise ValueError('cpu must be a positive integer.')
+    if memory is not None:
+        if isinstance(memory, int) and memory > 0:
+            cmd.extend(['-m', str(memory)])
+        else:
+            raise ValueError('memory must be a positive integer.')
+    if pci:
+        # Handle PCI IDs (can be a single device or comma-separated list)
+        if isinstance(pci, str):
+            devices = [dev.strip() for dev in pci.split(',') if dev.strip()]
+        elif isinstance(pci, list):
+            devices = pci
+        else:
+            devices = [pci]
+
+        # Add each device with its own -p flag
+        for device in devices:
+            cmd.extend(['-p', str(device)])
+    if start:
+        cmd.append('-s')
+
+    log.info('qcow2 module: Executing command: {}'.format(' '.join(shlex.quote(arg) for arg in cmd)))
+
+    try:
+        result = subprocess.run(cmd, capture_output=True, text=True, check=False)
+        ret = {
+            'retcode': result.returncode,
+            'stdout': result.stdout,
+            'stderr': result.stderr
+        }
+        if result.returncode != 0:
+            log.error('qcow2 module: Script execution failed with return code {}: {}'.format(result.returncode, result.stderr))
+        else:
+            log.info('qcow2 module: Script executed successfully.')
+        return ret
+    except Exception as e:
+        log.error('qcow2 module: An error occurred while executing the script: {}'.format(e))
+        raise

salt/allowed_states.map.jinja

@@ -1,264 +1,179 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
 
 {% set ISAIRGAP = salt['pillar.get']('global:airgap', False) %}
 {% import_yaml 'salt/minion.defaults.yaml' as saltversion %}
 {% set saltversion = saltversion.salt.minion.version %}
 
-{# this is the list we are returning from this map file, it gets built below #}
-{% set allowed_states= [] %}
+{# Define common state groups to reduce redundancy #}
+{% set base_states = [
+    'common',
+    'patch.os.schedule',
+    'motd',
+    'salt.minion-check',
+    'sensoroni',
+    'salt.lasthighstate',
+    'salt.minion'
+] %}
+
+{% set ssl_states = [
+    'ssl',
+    'telegraf',
+    'firewall',
+    'schedule',
+    'docker_clean'
+] %}
+
+{% set manager_states = [
+    'salt.master',
+    'ca',
+    'registry',
+    'manager',
+    'nginx',
+    'influxdb',
+    'soc',
+    'kratos',
+    'hydra',
+    'elasticfleet',
+    'elastic-fleet-package-registry',
+    'idstools',
+    'suricata.manager',
+    'utility'
+] %}
+
+{% set sensor_states = [
+    'pcap',
+    'suricata',
+    'healthcheck',
+    'tcpreplay',
+    'zeek',
+    'strelka'
+] %}
+
+{% set kafka_states = [
+    'kafka'
+] %}
+
+{% set stig_states = [
+    'stig'
+] %}
+
+{% set elastic_stack_states = [
+    'elasticsearch',
+    'elasticsearch.auth',
+    'kibana',
+    'kibana.secrets',
+    'elastalert',
+    'logstash',
+    'redis'
+] %}
+
+{# Initialize the allowed_states list #}
+{% set allowed_states = [] %}
 
 {% if grains.saltversion | string == saltversion | string %}
+    {# Map role-specific states #}
+    {% set role_states = {
+        'so-eval': (
+            ssl_states +
+            manager_states +
+            sensor_states +
+            elastic_stack_states | reject('equalto', 'logstash') | list
+        ),
+        'so-heavynode': (
+            ssl_states +
+            sensor_states +
+            ['elasticagent', 'elasticsearch', 'logstash', 'redis', 'nginx']
+        ),
+        'so-idh': (
+            ssl_states +
+            ['idh']
+        ),
+        'so-import': (
+            ssl_states +
+            manager_states +
+            sensor_states | reject('equalto', 'strelka') | reject('equalto', 'healthcheck') | list +
+            ['elasticsearch', 'elasticsearch.auth', 'kibana', 'kibana.secrets', 'strelka.manager']
+        ),
+        'so-manager': (
+            ssl_states +
+            manager_states +
+            ['salt.cloud', 'libvirt.packages', 'libvirt.ssh.users', 'strelka.manager'] +
+            stig_states +
+            kafka_states +
+            elastic_stack_states
+        ),
+        'so-managerhype': (
+            ssl_states +
+            manager_states +
+            ['salt.cloud', 'strelka.manager', 'hypervisor', 'libvirt'] +
+            stig_states +
+            kafka_states +
+            elastic_stack_states
+        ),
+        'so-managersearch': (
+            ssl_states +
+            manager_states +
+            ['salt.cloud', 'libvirt.packages', 'libvirt.ssh.users', 'strelka.manager'] +
+            stig_states +
+            kafka_states +
+            elastic_stack_states
+        ),
+        'so-searchnode': (
+            ssl_states +
+            ['kafka.ca', 'kafka.ssl', 'elasticsearch', 'logstash', 'nginx'] +
+            stig_states
+        ),
+        'so-standalone': (
+            ssl_states +
+            manager_states +
+            ['salt.cloud', 'libvirt.packages', 'libvirt.ssh.users'] +
+            sensor_states +
+            stig_states +
+            kafka_states +
+            elastic_stack_states
+        ),
+        'so-sensor': (
+            ssl_states +
+            sensor_states +
+            ['nginx'] +
+            stig_states
+        ),
+        'so-fleet': (
+            ssl_states +
+            ['logstash', 'nginx', 'healthcheck', 'elasticfleet']
+        ),
+        'so-receiver': (
+            ssl_states +
+            kafka_states +
+            stig_states +
+            ['logstash', 'redis']
+        ),
+        'so-hypervisor': (
+            ssl_states +
+            stig_states +
+            ['hypervisor', 'libvirt']
+        ),
+        'so-desktop': (
+            ['ssl', 'docker_clean', 'telegraf'] +
+            stig_states
+        )
+    } %}
 
-  {% set allowed_states= salt['grains.filter_by']({
-      'so-eval': [
-          'salt.master',
-          'ca',
-          'ssl',
-          'registry',
-          'manager',
-          'nginx',
-          'telegraf',
-          'influxdb',
-          'soc',
-          'kratos',
-          'hydra',
-          'elasticfleet',
-          'elastic-fleet-package-registry',
-          'firewall',
-          'idstools',
-          'suricata.manager',
-          'healthcheck',
-          'pcap',
-          'suricata',
-          'utility',
-          'schedule',
-          'tcpreplay',
-          'docker_clean'
-          ],
-      'so-heavynode': [
-          'ssl',
-          'nginx',
-          'telegraf',
-          'firewall',
-          'pcap',
-          'suricata',
-          'healthcheck',
-          'elasticagent',
-          'schedule',
-          'tcpreplay',
-          'docker_clean'
-          ],
-     'so-idh': [
-          'ssl',
-          'telegraf',
-          'firewall',
-          'idh',
-          'schedule',
-          'docker_clean'
-          ],
-      'so-import': [
-          'salt.master',
-          'ca',
-          'ssl',
-          'registry',
-          'manager',
-          'nginx',
-          'strelka.manager',
-          'soc',
-          'kratos',
-          'hydra',
-          'influxdb',
-          'telegraf',
-          'firewall',
-          'idstools',
-          'suricata.manager',
-          'pcap',
-          'utility',
-          'suricata',
-          'zeek',
-          'schedule',
-          'tcpreplay',
-          'docker_clean',
-          'elasticfleet',
-          'elastic-fleet-package-registry'
-          ],
-      'so-manager': [
-          'salt.master',
-          'ca',
-          'ssl',
-          'registry',
-          'manager',
-          'nginx',
-          'telegraf',
-          'influxdb',
-          'strelka.manager',
-          'soc',
-          'kratos',
-          'hydra',
-          'elasticfleet',
-          'elastic-fleet-package-registry',
-          'firewall',
-          'idstools',
-          'suricata.manager',
-          'utility',
-          'schedule',
-          'docker_clean',
-          'stig',
-          'kafka'
-          ],
-      'so-managersearch': [
-          'salt.master',
-          'ca',
-          'ssl',
-          'registry',
-          'nginx',
-          'telegraf',
-          'influxdb',
-          'strelka.manager',
-          'soc',
-          'kratos',
-          'hydra',
-          'elastic-fleet-package-registry',
-          'elasticfleet',
-          'firewall',
-          'manager',
-          'idstools',
-          'suricata.manager',
-          'utility',
-          'schedule',
-          'docker_clean',
-          'stig',
-          'kafka'
-          ],
-      'so-searchnode': [
-          'ssl',
-          'nginx',
-          'telegraf',
-          'firewall',
-          'schedule',
-          'docker_clean',
-          'stig',
-          'kafka.ca',
-          'kafka.ssl'
-          ],
-      'so-standalone': [
-          'salt.master',
-          'ca',
-          'ssl',
-          'registry',
-          'manager',
-          'nginx',
-          'telegraf',
-          'influxdb',
-          'soc',
-          'kratos',
-          'hydra',
-          'elastic-fleet-package-registry',
-          'elasticfleet',
-          'firewall',
-          'idstools',
-          'suricata.manager',
-          'pcap',
-          'suricata',
-          'healthcheck',
-          'utility',
-          'schedule',
-          'tcpreplay',
-          'docker_clean',
-          'stig',
-          'kafka'
-          ],
-      'so-sensor': [
-          'ssl',
-          'telegraf',
-          'firewall',
-          'nginx',
-          'pcap',
-          'suricata',
-          'healthcheck',
-          'schedule',
-          'tcpreplay',
-          'docker_clean',
-          'stig'
-          ],
-      'so-fleet': [
-          'ssl',
-          'telegraf',
-          'firewall',
-          'logstash',
-          'nginx',
-          'healthcheck',
-          'schedule',
-          'elasticfleet',
-          'docker_clean'
-          ],
-      'so-receiver': [
-          'ssl',
-          'telegraf',
-          'firewall',
-          'schedule',
-          'docker_clean',
-          'kafka',
-          'stig'
-          ],
-      'so-desktop': [
-          'ssl',
-          'docker_clean',
-          'telegraf',
-          'stig'
-          ],
-  }, grain='role') %}
-
-  {%- if grains.role in ['so-sensor', 'so-eval', 'so-standalone', 'so-heavynode'] %}
-    {% do allowed_states.append('zeek') %}
-  {%- endif %}
-
-  {% if grains.role in ['so-sensor', 'so-eval', 'so-standalone', 'so-heavynode'] %}
-    {% do allowed_states.append('strelka') %}
-  {% endif %}
-
-  {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-searchnode', 'so-managersearch', 'so-heavynode', 'so-import'] %}
-    {% do allowed_states.append('elasticsearch') %}
-  {% endif %}
-
-  {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %}
-    {% do allowed_states.append('elasticsearch.auth') %}
-  {% endif %}
-
-  {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %}
-    {% do allowed_states.append('kibana') %}
-    {% do allowed_states.append('kibana.secrets') %}
-  {% endif %}
-
-  {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch'] %}
-    {% do allowed_states.append('elastalert') %}
-  {% endif %}
-
-  {% if grains.role in ['so-manager', 'so-standalone', 'so-searchnode', 'so-managersearch', 'so-heavynode', 'so-receiver'] %}
-    {% do allowed_states.append('logstash') %}
-  {% endif %}
-
-  {% if grains.role in ['so-manager', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-receiver', 'so-eval'] %}
-    {% do allowed_states.append('redis') %}
-  {% endif %}
-  
-  {# all nodes on the right salt version can run the following states #}
-  {% do allowed_states.append('common') %}
-  {% do allowed_states.append('patch.os.schedule') %}
-  {% do allowed_states.append('motd') %}
-  {% do allowed_states.append('salt.minion-check') %}
-  {% do allowed_states.append('sensoroni') %}
-  {% do allowed_states.append('salt.lasthighstate') %}
+    {# Get states for the current role #}
+    {% if grains.role in role_states %}
+        {% set allowed_states = role_states[grains.role] %}
+    {% endif %}
 
+    {# Add base states that apply to all roles #}
+    {% for state in base_states %}
+        {% do allowed_states.append(state) %}
+    {% endfor %}
 {% endif %}
 
-
+{# Add airgap state if needed #}
 {% if ISAIRGAP %}
-  {% do allowed_states.append('airgap') %}
+    {% do allowed_states.append('airgap') %}
 {% endif %}
-
-{# all nodes can always run salt.minion state #}
-{% do allowed_states.append('salt.minion') %}

salt/backup/tools/sbin/so-config-backup.jinja

@@ -11,6 +11,10 @@ TODAY=$(date '+%Y_%m_%d')
 BACKUPDIR={{ DESTINATION }}
 BACKUPFILE="$BACKUPDIR/so-config-backup-$TODAY.tar"
 MAXBACKUPS=7
+EXCLUSIONS=(
+  "--exclude=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers"
+)
+
 
 # Create backup dir if it does not exist
 mkdir -p /nsm/backup
@@ -23,7 +27,7 @@ if [ ! -f $BACKUPFILE ]; then
 
   # Loop through all paths defined in global.sls, and append them to backup file
   {%- for LOCATION in BACKUPLOCATIONS %}
-  tar -rf $BACKUPFILE {{ LOCATION }}
+  tar -rf $BACKUPFILE "${EXCLUSIONS[@]}" {{ LOCATION }}
   {%- endfor %}
 
 fi

salt/common/init.sls

@@ -106,7 +106,7 @@ Etc/UTC:
   timezone.system
 
 # Sync curl configuration for Elasticsearch authentication
-{% if GLOBALS.role in ['so-eval', 'so-heavynode', 'so-import', 'so-manager', 'so-managersearch', 'so-searchnode', 'so-standalone'] %}
+{% if GLOBALS.is_manager or GLOBALS.role in ['so-heavynode', 'so-searchnode'] %}
 elastic_curl_config:
   file.managed:
     - name: /opt/so/conf/elasticsearch/curl.config

salt/common/packages.sls

@@ -1,6 +1,6 @@
-{% from 'vars/globals.map.jinja' import GLOBALS %}
-
-{% if GLOBALS.os_family == 'Debian' %}
+# we cannot import GLOBALS from vars/globals.map.jinja in this state since it is called in setup.virt.init
+# since it is early in setup of a new VM, the pillars imported in GLOBALS are not yet defined
+{% if grains.os_family == 'Debian' %}
 commonpkgs:
   pkg.installed:
     - skip_suggestions: True
@@ -46,7 +46,7 @@ python-rich:
 {%     endif %}
 {% endif %}
 
-{% if GLOBALS.os_family == 'RedHat' %}
+{% if grains.os_family == 'RedHat' %}
 
 remove_mariadb:
   pkg.removed:

salt/common/tools/sbin/so-common

@@ -99,6 +99,17 @@ add_interface_bond0() {
 	fi
 }
 
+airgap_playbooks() {
+	SRC_DIR=$1
+	# Copy playbooks if using airgap
+	mkdir -p /nsm/airgap-resources
+	# Purge old airgap playbooks to ensure SO only uses the latest released playbooks
+	rm -fr /nsm/airgap-resources/playbooks
+	tar xf $SRC_DIR/airgap-resources/playbooks.tgz -C /nsm/airgap-resources/
+	chown -R socore:socore /nsm/airgap-resources/playbooks
+	git config --global --add safe.directory /nsm/airgap-resources/playbooks
+}
+
 check_container() {
 	docker ps | grep "$1:" > /dev/null 2>&1
 	return $?
@@ -299,7 +310,8 @@ fail() {
 
 get_agent_count() {
 	if [ -f /opt/so/log/agents/agentstatus.log ]; then
-		AGENTCOUNT=$(cat /opt/so/log/agents/agentstatus.log | grep -wF active | awk '{print $2}')
+		AGENTCOUNT=$(cat /opt/so/log/agents/agentstatus.log | grep -wF active | awk '{print $2}' | sed 's/,//')
+		[[ -z "$AGENTCOUNT" ]] && AGENTCOUNT="0"
 	else
 		AGENTCOUNT=0
 	fi

salt/common/tools/sbin/so-common-status-check

@@ -45,7 +45,7 @@ def check_for_fps():
         result = subprocess.run([feat_full + '-mode-setup', '--is-enabled'], stdout=subprocess.PIPE)
         if result.returncode == 0:
             fps = 1
-    except FileNotFoundError:
+    except:
         fn = '/proc/sys/crypto/' + feat_full + '_enabled'
         try:
           with open(fn, 'r') as f:

salt/common/tools/sbin/so-log-check

@@ -158,6 +158,8 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|adding index lifecycle policy" # false positive (elasticsearch policy names contain 'error') 
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|adding ingest pipeline"       # false positive (elasticsearch ingest pipeline names contain 'error') 
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|updating index template"      # false positive (elasticsearch index or template names contain 'error') 
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|updating component template"  # false positive (elasticsearch index or template names contain 'error') 
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|upgrading composable template" # false positive (elasticsearch composable template names contain 'error')
 fi
 
 if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then

salt/common/tools/sbin/so_logging_utils.py

@@ -0,0 +1,53 @@
+#!/usr/bin/python3
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+import logging
+import os 
+import sys
+
+def setup_logging(logger_name, log_file_path, log_level=logging.INFO, format_str='%(asctime)s - %(levelname)s - %(message)s'):
+    """
+    Sets up logging for a script.
+
+    Parameters:
+        logger_name (str): The name of the logger.
+        log_file_path (str): The file path for the log file.
+        log_level (int): The logging level (e.g., logging.INFO, logging.DEBUG).
+        format_str (str): The format string for log messages.
+
+    Returns:
+        logging.Logger: Configured logger object.
+    """
+    logger = logging.getLogger(logger_name)
+    logger.setLevel(log_level)
+
+    # Create directory for log file if it doesn't exist
+    log_file_dir = os.path.dirname(log_file_path)
+    if log_file_dir and not os.path.exists(log_file_dir):
+        try:
+            os.makedirs(log_file_dir)
+        except OSError as e:
+            print(f"Error creating directory {log_file_dir}: {e}")
+            sys.exit(1)
+
+    # Create handlers
+    c_handler = logging.StreamHandler()
+    f_handler = logging.FileHandler(log_file_path)
+    c_handler.setLevel(log_level)
+    f_handler.setLevel(log_level)
+
+    # Create formatter and add it to handlers
+    formatter = logging.Formatter(format_str)
+    c_handler.setFormatter(formatter)
+    f_handler.setFormatter(formatter)
+
+    # Add handlers to the logger if they are not already added
+    if not logger.hasHandlers():
+        logger.addHandler(c_handler)
+        logger.addHandler(f_handler)
+
+    return logger

salt/common/tools/sbin_jinja/so-import-pcap

@@ -248,7 +248,7 @@ fi
 START_OLDEST_SLASH=$(echo $START_OLDEST | sed -e 's/-/%2F/g')
 END_NEWEST_SLASH=$(echo $END_NEWEST | sed -e 's/-/%2F/g')
 if [[ $VALID_PCAPS_COUNT -gt 0 ]] || [[ $SKIPPED_PCAPS_COUNT -gt 0 ]]; then
-  URL="https://{{ URLBASE }}/#/dashboards?q=$HASH_FILTERS%20%7C%20groupby%20event.module*%20%7C%20groupby%20-sankey%20event.module*%20event.dataset%20%7C%20groupby%20event.dataset%20%7C%20groupby%20source.ip%20%7C%20groupby%20destination.ip%20%7C%20groupby%20destination.port%20%7C%20groupby%20network.protocol%20%7C%20groupby%20rule.name%20rule.category%20event.severity_label%20%7C%20groupby%20dns.query.name%20%7C%20groupby%20file.mime_type%20%7C%20groupby%20http.virtual_host%20http.uri%20%7C%20groupby%20notice.note%20notice.message%20notice.sub_message%20%7C%20groupby%20ssl.server_name%20%7C%20groupby%20source_geo.organization_name%20source.geo.country_name%20%7C%20groupby%20destination_geo.organization_name%20destination.geo.country_name&t=${START_OLDEST_SLASH}%2000%3A00%3A00%20AM%20-%20${END_NEWEST_SLASH}%2000%3A00%3A00%20AM&z=UTC"
+  URL="https://{{ URLBASE }}/#/dashboards?q=$HASH_FILTERS%20%7C%20groupby%20event.module*%20%7C%20groupby%20-sankey%20event.module*%20event.dataset%20%7C%20groupby%20event.dataset%20%7C%20groupby%20source.ip%20%7C%20groupby%20destination.ip%20%7C%20groupby%20destination.port%20%7C%20groupby%20network.protocol%20%7C%20groupby%20rule.name%20rule.category%20event.severity_label%20%7C%20groupby%20dns.query.name%20%7C%20groupby%20file.mime_type%20%7C%20groupby%20http.virtual_host%20http.uri%20%7C%20groupby%20notice.note%20notice.message%20notice.sub_message%20%7C%20groupby%20ssl.server_name%20%7C%20groupby%20source.as.organization.name%20source.geo.country_name%20%7C%20groupby%20destination.as.organization.name%20destination.geo.country_name&t=${START_OLDEST_SLASH}%2000%3A00%3A00%20AM%20-%20${END_NEWEST_SLASH}%2000%3A00%3A00%20AM&z=UTC"
 
   status "Import complete!"
   status

salt/common/tools/sbin_jinja/so-salt-emit-vm-deployment-status-event

@@ -0,0 +1,132 @@
+#!/opt/saltstack/salt/bin/python3
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+#
+# Note: Per the Elastic License 2.0, the second limitation states:
+#
+#   "You may not move, change, disable, or circumvent the license key functionality
+#    in the software, and you may not remove or obscure any functionality in the
+#    software that is protected by the license key."
+
+{%   if 'vrt' in salt['pillar.get']('features', []) -%}
+
+"""
+Script for emitting VM deployment status events to the Salt event bus.
+
+This script provides functionality to emit status events for VM deployment operations,
+used by various Security Onion VM management tools.
+
+Usage:
+    so-salt-emit-vm-deployment-status-event -v <vm_name> -H <hypervisor> -s <status>
+
+Arguments:
+    -v, --vm-name     Name of the VM (hostname_role)
+    -H, --hypervisor  Name of the hypervisor
+    -s, --status      Current deployment status of the VM
+
+Example:
+    so-salt-emit-vm-deployment-status-event -v sensor1_sensor -H hypervisor1 -s "Creating"
+"""
+
+import sys
+import argparse
+import logging
+import salt.client
+from typing import Dict, Any
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s'
+)
+log = logging.getLogger(__name__)
+
+def emit_event(vm_name: str, hypervisor: str, status: str) -> bool:
+    """
+    Emit a VM deployment status event to the salt event bus.
+    
+    Args:
+        vm_name: Name of the VM (hostname_role)
+        hypervisor: Name of the hypervisor
+        status: Current deployment status of the VM
+        
+    Returns:
+        bool: True if event was sent successfully, False otherwise
+    
+    Raises:
+        ValueError: If status is not a valid deployment status
+    """
+    log.info("Attempting to emit deployment event...")
+    
+    try:
+        caller = salt.client.Caller()
+        event_data = {
+            'vm_name': vm_name,
+            'hypervisor': hypervisor,
+            'status': status
+        }
+        
+        # Use consistent event tag structure
+        event_tag = f'soc/dyanno/hypervisor/{status.lower()}'
+        
+        ret = caller.cmd(
+            'event.send',
+            event_tag,
+            event_data
+        )
+        
+        if not ret:
+            log.error("Failed to emit VM deployment status event: %s", event_data)
+            return False
+            
+        log.info("Successfully emitted VM deployment status event: %s", event_data)
+        return True
+        
+    except Exception as e:
+        log.error("Error emitting VM deployment status event: %s", str(e))
+        return False
+
+def parse_args():
+    """Parse command line arguments."""
+    parser = argparse.ArgumentParser(
+        description='Emit VM deployment status events to the Salt event bus.'
+    )
+    parser.add_argument('-v', '--vm-name', required=True,
+                        help='Name of the VM (hostname_role)')
+    parser.add_argument('-H', '--hypervisor', required=True,
+                        help='Name of the hypervisor')
+    parser.add_argument('-s', '--status', required=True,
+                        help='Current deployment status of the VM')
+    return parser.parse_args()
+
+def main():
+    """Main entry point for the script."""
+    try:
+        args = parse_args()
+        
+        success = emit_event(
+            vm_name=args.vm_name,
+            hypervisor=args.hypervisor,
+            status=args.status
+        )
+        
+        if not success:
+            sys.exit(1)
+            
+    except Exception as e:
+        log.error("Failed to emit status event: %s", str(e))
+        sys.exit(1)
+
+if __name__ == '__main__':
+    main()
+
+{%- else -%}
+
+echo "Hypervisor nodes are a feature supported only for customers with a valid license. \
+      Contact Security Onion Solutions, LLC via our website at https://securityonionsolutions.com \
+      for more information about purchasing a license to enable this feature."
+
+{% endif -%}

salt/elasticfleet/config.sls

@@ -166,7 +166,7 @@ eaoptionalintegrationsdir:
 
 {% for minion in node_data %}
 {% set role = node_data[minion]["role"] %}
-{% if role in [ "eval","fleet","heavynode","import","manager","managersearch","standalone" ] %}
+{% if role in [ "eval","fleet","heavynode","import","manager", "managerhype", "managersearch","standalone" ] %}
 {% set optional_integrations = ELASTICFLEETMERGED.optional_integrations %}
 {% set integration_keys = optional_integrations.keys() %}
 fleet_server_integrations_{{ minion }}:

salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json

@@ -1,32 +1,33 @@
 {
-	"name": "elastic-defend-endpoints",
-	"namespace": "default",
-	"description": "",
-	"package": {
-	  "name": "endpoint",
-	  "title": "Elastic Defend",
-	  "version": "8.17.0",
-	  "requires_root": true
-	},
-	"enabled": true,
-	"policy_id": "endpoints-initial",
-	"vars": {},
-	"inputs": [
-	  {
-		"type": "endpoint",
-		"enabled": true,
-		"config": {
-		  "integration_config": {
-			"value": {
-			  "type": "endpoint",
-			  "endpointConfig": {
-				"preset": "DataCollection"
-			  }
-			}
-		  }
-		},
-		"streams": []
-	  }
-	]
-  }
-  
+  "name": "elastic-defend-endpoints",
+  "namespace": "default",
+  "description": "",
+  "package": {
+    "name": "endpoint",
+    "title": "Elastic Defend",
+    "version": "8.18.1",
+    "requires_root": true
+  },
+  "enabled": true,
+  "policy_ids": [
+    "endpoints-initial"
+  ],
+  "vars": {},
+  "inputs": [
+    {
+      "type": "ENDPOINT_INTEGRATION_CONFIG",
+      "enabled": true,
+      "config": {
+        "_config": {
+          "value": {
+            "type": "endpoint",
+            "endpointConfig": {
+              "preset": "DataCollection"
+            }
+          }
+        }
+      },
+      "streams": []
+    }
+  ]
+}

salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json

@@ -19,7 +19,7 @@
             ],
             "data_stream.dataset": "idh",
             "tags": [],
-            "processors": "\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true\n- drop_fields:\n    when:\n      equals:\n        logtype: \"1001\"\n    fields: [\"src_host\", \"src_port\", \"dst_host\", \"dst_port\" ]\n    ignore_missing: true\n- rename:\n    fields:\n      - from: \"src_host\"\n        to: \"source.ip\"\n      - from: \"src_port\"\n        to: \"source.port\"\n      - from: \"dst_host\"\n        to: \"destination.host\"\n      - from: \"dst_port\"\n        to: \"destination.port\"\n    ignore_missing: true\n- convert:\n    fields:\n      - {from: \"logtype\", to: \"event.code\", type: \"string\"}\n    ignore_missing: true\n- drop_fields:\n    fields: '[\"prospector\", \"input\", \"offset\", \"beat\"]'\n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: opencanary",
+            "processors": "\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true\n- convert:\n    fields:\n      - {from: \"logtype\", to: \"event.code\", type: \"string\"}\n- drop_fields:\n    when:\n      equals:\n        event.code: \"1001\"\n    fields: [\"src_host\", \"src_port\", \"dst_host\", \"dst_port\" ]\n    ignore_missing: true\n- rename:\n    fields:\n      - from: \"src_host\"\n        to: \"source.ip\"\n      - from: \"src_port\"\n        to: \"source.port\"\n      - from: \"dst_host\"\n        to: \"destination.host\"\n      - from: \"dst_port\"\n        to: \"destination.port\"\n    ignore_missing: true\n- drop_fields:\n    fields: '[\"prospector\", \"input\", \"offset\", \"beat\"]'\n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: opencanary",
             "custom": "pipeline: common"
           }
         }

salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json

@@ -20,7 +20,7 @@
             ],
             "data_stream.dataset": "import",
             "custom": "",
-            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-1.67.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-2.5.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-1.67.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-1.67.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-2.5.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.3.3\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.1.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.3.3\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.3.3\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.1.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
             "tags": [
               "import"
             ]

salt/elasticfleet/files/integrations/grid-nodes_general/syslog-tcp-514.json

@@ -11,7 +11,7 @@
     "tcp-tcp": {
       "enabled": true,
       "streams": {
-        "tcp.generic": {
+        "tcp.tcp": {
           "enabled": true,
           "vars": {
             "listen_address": "0.0.0.0",
@@ -23,7 +23,8 @@
               "syslog"
             ],
             "syslog_options": "field: message\n#format: auto\n#timezone: Local",
-            "ssl": ""
+            "ssl": "",
+            "custom": ""
           }
         }
       }

salt/elasticfleet/integration-defaults.map.jinja

@@ -4,6 +4,7 @@
 
 
 {% import_json '/opt/so/state/esfleet_package_components.json' as ADDON_PACKAGE_COMPONENTS %}
+{% import_json '/opt/so/state/esfleet_component_templates.json' as INSTALLED_COMPONENT_TEMPLATES %}
 {% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
 
 {% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
@@ -14,6 +15,7 @@
     'awsfirehose.logs': 'awsfirehose',
     'awsfirehose.metrics': 'aws.cloudwatch',
     'cribl.logs': 'cribl',
+    'cribl.metrics': 'cribl',
     'sentinel_one_cloud_funnel.logins': 'sentinel_one_cloud_funnel.login',
     'azure_application_insights.app_insights': 'azure.app_insights',
     'azure_application_insights.app_state': 'azure.app_state',
@@ -45,7 +47,10 @@
     'synthetics.browser_screenshot': 'synthetics-browser.screenshot',
     'synthetics.http': 'synthetics-http',
     'synthetics.icmp': 'synthetics-icmp',
-    'synthetics.tcp': 'synthetics-tcp'
+    'synthetics.tcp': 'synthetics-tcp',
+    'swimlane.swimlane_api': 'swimlane.api',
+    'swimlane.tenant_api': 'swimlane.tenant',
+    'swimlane.turbine_api': 'turbine.api'
    } %}
 
 {% for pkg in ADDON_PACKAGE_COMPONENTS %}
@@ -62,70 +67,90 @@
 {%         else %}
 {%           set integration_type = "" %}
 {%         endif %}
-{%           set component_name = pkg.name ~ "." ~ pattern.title %}
-{#           fix weirdly named components #}
-{%           if component_name in WEIRD_INTEGRATIONS %}
-{%             set component_name = WEIRD_INTEGRATIONS[component_name] %}
-{%           endif %}
+{%         set component_name = pkg.name ~ "." ~ pattern.title %}
+{%         set index_pattern = pattern.name %}
+
+{#         fix weirdly named components #}
+{%         if component_name in WEIRD_INTEGRATIONS %}
+{%           set component_name = WEIRD_INTEGRATIONS[component_name] %}
+{%         endif %}
+
+{#         create duplicate of component_name, so we can split generics from @custom component templates in the index template below and overwrite the default @package when needed
+           eg. having to replace unifiedlogs.generic@package with filestream.generic@package, but keep the ability to customize unifiedlogs.generic@custom and its ILM policy #}
+{%         set custom_component_name = component_name %}
+
+{#         duplicate integration_type to assist with sometimes needing to overwrite component templates with 'logs-filestream.generic@package' (there is no metrics-filestream.generic@package) #}
+{%         set generic_integration_type = integration_type %}
+
 {#           component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
 {%           set component_name_x = component_name.replace(".","_x_") %}
 {#           pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
 {%           set integration_key = "so-" ~ integration_type ~ component_name_x %}
 
+{#         if its a .generic template make sure that a .generic@package for the integration exists. Else default to logs-filestream.generic@package #}
+{%         if ".generic" in component_name and integration_type ~ component_name ~ "@package" not in INSTALLED_COMPONENT_TEMPLATES %}
+{#           these generic templates by default are directed to index_pattern of 'logs-generic-*', overwrite that here to point to eg gcp_pubsub.generic-* #}
+{%           set index_pattern = integration_type ~ component_name ~ "-*" %}
+{#           includes use of .generic component template, but it doesn't exist in installed component templates. Redirect it to filestream.generic@package #}
+{%           set component_name = "filestream.generic" %}
+{%           set generic_integration_type = "logs-" %}
+{%         endif %}
+
 {#           Default integration settings #}
 {%           set integration_defaults = {
-              "index_sorting": false,
-              "index_template": {
-                  "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
-                  "data_stream": {
-                      "allow_custom_routing": false,
-                      "hidden": false
-                  },
-                  "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
-                  "index_patterns": [pattern.name],
-                  "priority": 501,
-                  "template": {
-                      "settings": {
-                          "index": {
-                              "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
-                              "number_of_replicas": 0
-                          }
-                      }
-                  }
-              },
-              "policy": {
-                  "phases": {
-                      "cold": {
-                          "actions": {
-                              "set_priority": {"priority": 0}
-                          },
-                          "min_age": "60d"
+               "index_sorting": false,
+               "index_template": {
+                 "composed_of": [generic_integration_type ~ component_name ~ "@package", integration_type ~ custom_component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
+                 "data_stream": {
+                   "allow_custom_routing": false,
+                   "hidden": false
+                 },
+                 "ignore_missing_component_templates": [integration_type ~ custom_component_name ~ "@custom"],
+                 "index_patterns": [index_pattern],
+                 "priority": 501,
+                 "template": {
+                   "settings": {
+                     "index": {
+                       "lifecycle": {"name": "so-" ~ integration_type ~ custom_component_name ~ "-logs"},
+                         "number_of_replicas": 0
+                     }
+                   }
+                 }
+               },
+               "policy": {
+                 "phases": {
+                   "cold": {
+                     "actions": {
+                       "set_priority": {"priority": 0}
+                     },
+                     "min_age": "60d"
+                   },
+                   "delete": {
+                     "actions": {
+                       "delete": {}
+                     },
+                     "min_age": "365d"
+                   },
+                   "hot": {
+                     "actions": {
+                       "rollover": {
+                         "max_age": "30d",
+                         "max_primary_shard_size": "50gb"
+                       },
+                       "set_priority": {"priority": 100}
                       },
-                      "delete": {
-                          "actions": {
-                              "delete": {}
-                          },
-                          "min_age": "365d"
-                      },
-                      "hot": {
-                          "actions": {
-                              "rollover": {
-                                  "max_age": "30d",
-                                  "max_primary_shard_size": "50gb"
-                              },
-                              "set_priority": {"priority": 100}
-                          },
-                          "min_age": "0ms"
-                      },
-                      "warm": {
-                          "actions": {
-                              "set_priority": {"priority": 50}
-                          },
-                          "min_age": "30d"
-                      }
-                  }
-              }
-          } %}
+                      "min_age": "0ms"
+                   },
+                   "warm": {
+                     "actions": {
+                       "set_priority": {"priority": 50}
+                     },
+                     "min_age": "30d"
+                   }
+                 }
+               }
+             } %}
+
 {%         do ADDON_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %}
 {%       endfor %}
 {%     endif %}

salt/elasticfleet/tools/sbin/so-elastic-fleet-common

@@ -88,7 +88,13 @@ elastic_fleet_package_version_check() {
 
 elastic_fleet_package_latest_version_check() {
     PACKAGE=$1
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages/$PACKAGE" | jq -r '.item.latestVersion'
+    if output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages/$PACKAGE" --fail); then
+        if version=$(jq -e -r '.item.latestVersion' <<< $output); then
+            echo "$version"
+        fi
+    else
+        echo "Error: Failed to get latest version for $PACKAGE"
+    fi
 }
 
 elastic_fleet_package_install() {
@@ -149,9 +155,13 @@ elastic_fleet_integration_policy_package_name() {
 elastic_fleet_integration_policy_package_version() {
     AGENT_POLICY=$1
     INTEGRATION=$2
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" | jq -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .package.version'
-    if [ $? -ne 0 ]; then
-        echo "Error: Failed to retrieve package version for '$INTEGRATION' in '$AGENT_POLICY'."
+
+    if output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" --fail); then
+        if version=$(jq -e -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .package.version' <<< $output); then
+            echo "$version"
+        fi
+    else
+        echo "Error: Failed to retrieve agent policy $AGENT_POLICY"
         exit 1
     fi
 }

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade

@@ -34,10 +34,18 @@ for AGENT_POLICY in $agent_policies; do
             if [[ " ${default_packages[@]} " =~ " $PACKAGE_NAME " ]]; then
             {%- endif %}
                 # Get currently installed version of package
-                PACKAGE_VERSION=$(elastic_fleet_integration_policy_package_version "$AGENT_POLICY" "$INTEGRATION")
-
-                # Get latest available version of package
-                AVAILABLE_VERSION=$(elastic_fleet_package_latest_version_check "$PACKAGE_NAME")
+                attempt=0
+                max_attempts=3
+                while [ $attempt -lt $max_attempts ]; do
+                    if PACKAGE_VERSION=$(elastic_fleet_integration_policy_package_version "$AGENT_POLICY" "$INTEGRATION") && AVAILABLE_VERSION=$(elastic_fleet_package_latest_version_check "$PACKAGE_NAME"); then
+                        break
+                    fi
+                    attempt=$((attempt + 1))
+                done
+                if [ $attempt -eq $max_attempts ]; then
+                    echo "Error: Failed getting $PACKAGE_VERSION or $AVAILABLE_VERSION"
+                    exit 1
+                fi
 
                 # Get integration ID
                 INTEGRATION_ID=$(elastic_fleet_integration_id "$AGENT_POLICY" "$INTEGRATION")

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load

@@ -19,6 +19,7 @@ BULK_INSTALL_PACKAGE_LIST=/tmp/esfleet_bulk_install.json
 BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json
 BULK_INSTALL_OUTPUT=/opt/so/state/esfleet_bulk_install_results.json
 PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json
+COMPONENT_TEMPLATES=/opt/so/state/esfleet_component_templates.json
 
 PENDING_UPDATE=false
 
@@ -147,14 +148,33 @@ if [[ -f $STATE_FILE_SUCCESS  ]]; then
         done <<< "$(jq -c '.packages[]' "$INSTALLED_PACKAGE_LIST")"
 
         if [ "$PENDING_UPDATE" = true ]; then
-            # Run bulk install of packages
-            elastic_fleet_bulk_package_install $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_OUTPUT
+            # Run chunked install of packages
+            echo "" > $BULK_INSTALL_OUTPUT
+            pkg_group=1
+            pkg_filename="${BULK_INSTALL_PACKAGE_LIST%.json}"
+
+            jq -c '.packages | _nwise(25)' $BULK_INSTALL_PACKAGE_LIST | while read -r line; do
+                echo "$line" | jq '{ "packages": . }' > "${pkg_filename}_${pkg_group}.json"
+                pkg_group=$((pkg_group + 1))
+            done
+
+            for file in "${pkg_filename}_"*.json; do
+                [ -e "$file" ] || continue
+                elastic_fleet_bulk_package_install $file >> $BULK_INSTALL_OUTPUT
+            done
+            # cleanup any temp files for chunked package install
+            rm -f ${pkg_filename}_*.json $BULK_INSTALL_PACKAGE_LIST
         else
             echo "Elastic integrations don't appear to need installation/updating..."
         fi
         # Write out file for generating index/component/ilm templates
         latest_installed_package_list=$(elastic_fleet_installed_packages)
         echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS
+        if retry 3 1 "so-elasticsearch-query / --fail --output /dev/null"; then
+            # Refresh installed component template list
+            latest_component_templates_list=$(so-elasticsearch-query _component_template | jq '.component_templates[] | .name' | jq -s '.')
+            echo $latest_component_templates_list > $COMPONENT_TEMPLATES
+        fi
 
     else
         # This is the installation of add-on integrations and upgrade of existing integrations. Exiting without error, next highstate will attempt to re-run.

salt/elasticsearch/config.map.jinja

@@ -28,7 +28,7 @@
 {%   endfor %}
 {% endfor %}
 
-{% if grains.id.split('_') | last in ['manager','managersearch','standalone'] %}
+{% if grains.id.split('_') | last in ['manager','managerhype','managersearch','standalone'] %}
     {% if ELASTICSEARCH_SEED_HOSTS | length > 1 %}
       {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.update({'discovery': {'seed_hosts': []}}) %}
       {% for NODE in ELASTICSEARCH_SEED_HOSTS %}

salt/elasticsearch/defaults.yaml

@@ -1,6 +1,6 @@
 elasticsearch:
   enabled: false
-  version: 8.17.3
+  version: 8.18.4
   index_clean: true
   config:
     action:
@@ -567,6 +567,7 @@ elasticsearch:
         - common-settings
         - common-dynamic-mappings
         - winlog-mappings
+        - hash-mappings
         data_stream: {}
         ignore_missing_component_templates: []
         index_patterns:
@@ -3874,6 +3875,7 @@ elasticsearch:
         - vulnerability-mappings
         - common-settings
         - common-dynamic-mappings
+        - hash-mappings
         data_stream: {}
         ignore_missing_component_templates: []
         index_patterns:
@@ -3987,6 +3989,7 @@ elasticsearch:
         - vulnerability-mappings
         - common-settings
         - common-dynamic-mappings
+        - hash-mappings
         data_stream: {}
         ignore_missing_component_templates: []
         index_patterns:
@@ -4100,6 +4103,7 @@ elasticsearch:
         - vulnerability-mappings
         - common-settings
         - common-dynamic-mappings
+        - hash-mappings
         data_stream: {}
         ignore_missing_component_templates: []
         index_patterns:
@@ -4329,6 +4333,7 @@ elasticsearch:
         - zeek-mappings
         - common-settings
         - common-dynamic-mappings
+        - hash-mappings
         data_stream: {}
         ignore_missing_component_templates: []
         index_patterns:
@@ -4501,6 +4506,14 @@ elasticsearch:
           - data
           - remote_cluster_client
           - transform
+    so-managerhype:
+      config:
+        node:
+          roles:
+          - master
+          - data
+          - remote_cluster_client
+          - transform
     so-managersearch:
       config:
         node:

salt/elasticsearch/enabled.sls

@@ -204,7 +204,7 @@ so-elasticsearch-roles-load:
       - docker_container: so-elasticsearch
       - file: elasticsearch_sbin_jinja
 
-{%     if grains.role in ['so-managersearch', 'so-manager'] %}
+{%     if grains.role in ['so-managersearch', 'so-manager', 'so-managerhype'] %}
 {%       set ap = "absent" %}
 {%     endif %}
 {%     if grains.role in ['so-eval', 'so-standalone', 'so-heavynode'] %}

salt/elasticsearch/files/ingest-dynamic/common

@@ -26,7 +26,7 @@
     {
         "geoip":  {
                 "field": "destination.ip",
-                "target_field": "destination_geo",
+                "target_field": "destination.as",
                 "database_file": "GeoLite2-ASN.mmdb",
                 "ignore_missing": true,
                 "ignore_failure": true,
@@ -36,13 +36,17 @@
     {
         "geoip":  {
                 "field": "source.ip",
-                "target_field": "source_geo",
+                "target_field": "source.as",
                 "database_file": "GeoLite2-ASN.mmdb",
                 "ignore_missing": true,
                 "ignore_failure": true,
                 "properties": ["ip", "asn", "organization_name", "network"]
         }
     },
+    { "rename":          { "field": "destination.as.organization_name", "target_field": "destination.as.organization.name", "ignore_failure": true, "ignore_missing": true  } },
+    { "rename":          { "field": "source.as.organization_name", "target_field": "source.as.organization.name", "ignore_failure": true, "ignore_missing": true  } },
+    { "rename":          { "field": "destination.as.asn", "target_field": "destination.as.number", "ignore_failure": true, "ignore_missing": true  } },
+    { "rename":          { "field": "source.as.asn", "target_field": "source.as.number", "ignore_failure": true, "ignore_missing": true  } },
     { "set":             { "if": "ctx.event?.severity == 1",   "field": "event.severity_label", "value": "low", "override": true }  },
     { "set":             { "if": "ctx.event?.severity == 2",   "field": "event.severity_label", "value": "medium", "override": true }  },
     { "set":             { "if": "ctx.event?.severity == 3",   "field": "event.severity_label", "value": "high", "override": true }  },

salt/elasticsearch/files/ingest/global@custom

@@ -19,11 +19,12 @@
     { "set":        { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } },
     { "set":        { "if": "ctx.tags != null && ctx.tags.contains('import')", "override": true, "field": "data_stream.dataset", "value": "import" } },
     { "set":        { "if": "ctx.tags != null && ctx.tags.contains('import')", "override": true, "field": "data_stream.namespace", "value": "so" } },
-    { "date":       { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp","ignore_failure": true, "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSX","yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } },
     { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true }  },
     { "set":        { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } },
-    { "rename":        { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } },
+    { "rename":       { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } },
     { "set":        { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } },
+    {"append":      {"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"if":"ctx?.event?.dataset == 'endpoint.events.network' && ctx?.source?.ip != null","ignore_failure":true}},
+    {"foreach":     {"field":"host.ip","processor":{"append":{"field":"related.ip","value":"{{_ingest._value}}","allow_duplicates":false}},"if":"ctx?.event?.module == 'endpoint'","description":"Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip"}},
     { "remove":     { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp", "datastream_dataset_temp" ], "ignore_missing": true, "ignore_failure": true } }
   ]
 }

salt/elasticsearch/files/ingest/import.wel

@@ -1,11 +0,0 @@
-{
-  "description" : "import.wel",
-  "processors" : [
-    { "set":           { "field": "event.ingested", "value": "{{ @timestamp }}"  } },
-    { "set" :          { "field" : "@timestamp", "value" : "{{ event.created }}" } },
-    { "remove":        { "field": [ "event_record_id", "event.created" , "timestamp" , "winlog.event_data.UtcTime" ], "ignore_failure": true } },
-    { "pipeline":      { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'",   "name": "sysmon" } },
-    { "pipeline":      { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational'",  "name":"win.eventlogs" } },
-    { "pipeline":      { "name": "common" } }
-  ]
-}

salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.0

@@ -107,61 +107,61 @@
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.21.0-firewall",
+        "name": "logs-pfsense.log-1.23.0-firewall",
         "if": "ctx.event.provider == 'filterlog'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.21.0-openvpn",
+        "name": "logs-pfsense.log-1.23.0-openvpn",
         "if": "ctx.event.provider == 'openvpn'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.21.0-ipsec",
+        "name": "logs-pfsense.log-1.23.0-ipsec",
         "if": "ctx.event.provider == 'charon'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.21.0-dhcp",
+        "name": "logs-pfsense.log-1.23.0-dhcp",
         "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\"].contains(ctx.event.provider)"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.21.0-unbound",
+        "name": "logs-pfsense.log-1.23.0-unbound",
         "if": "ctx.event.provider == 'unbound'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.21.0-haproxy",
+        "name": "logs-pfsense.log-1.23.0-haproxy",
         "if": "ctx.event.provider == 'haproxy'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.21.0-php-fpm",
+        "name": "logs-pfsense.log-1.23.0-php-fpm",
         "if": "ctx.event.provider == 'php-fpm'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.21.0-squid",
+        "name": "logs-pfsense.log-1.23.0-squid",
         "if": "ctx.event.provider == 'squid'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.21.0-snort",
+        "name": "logs-pfsense.log-1.23.0-snort",
         "if": "ctx.event.provider == 'snort'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.21.0-suricata",
+        "name": "logs-pfsense.log-1.23.0-suricata",
         "if": "ctx.event.provider == 'suricata'"
       }
     },
@@ -358,14 +358,6 @@
         "source": "void handleMap(Map map) {\n  for (def x : map.values()) {\n    if (x instanceof Map) {\n        handleMap(x);\n    } else if (x instanceof List) {\n        handleList(x);\n    }\n  }\n  map.values().removeIf(v -> v == null || (v instanceof String && v == \"-\"));\n}\nvoid handleList(List list) {\n  for (def x : list) {\n      if (x instanceof Map) {\n          handleMap(x);\n      } else if (x instanceof List) {\n          handleList(x);\n      }\n  }\n}\nhandleMap(ctx);\n"
       }
     },
-    {
-      "remove": {
-        "field": "event.original",
-        "if": "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))",
-        "ignore_failure": true,
-        "ignore_missing": true
-      }
-    },
     {
       "pipeline": {
         "name": "global@custom",

salt/elasticsearch/files/ingest/suricata.alert

@@ -9,6 +9,7 @@
     { "rename":{ "field": "rule.signature_id",     "target_field": "rule.uuid", "ignore_failure": true  } },
     { "rename":{ "field": "rule.signature_id",     "target_field": "rule.signature", "ignore_failure": true  } },
     { "rename":{ "field": "message2.payload_printable",     "target_field": "network.data.decoded", "ignore_failure": true  } },
+    { "dissect": { "field": "rule.rule", "pattern": "%{?prefix}content:\"%{dns.query_name}\"%{?remainder}", "ignore_missing": true, "ignore_failure": true } },
     { "pipeline": { "name": "common.nids" } }
   ]
-}
+}

salt/elasticsearch/files/ingest/suricata.common

@@ -18,6 +18,13 @@
     { "set":    { "field": "event.ingested",            "value": "{{@timestamp}}"                                       } },
     { "date": { "field": "message2.timestamp", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "timezone": "UTC", "ignore_failure": true } },
     { "remove":{ "field": "agent",                                                              "ignore_failure": true  } },
+    {"append":{"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"ignore_failure":true}},
+    {
+      "script": {
+        "source": "boolean isPrivate(def ip) { if (ip == null) return false; int dot1 = ip.indexOf('.'); if (dot1 == -1) return false; int dot2 = ip.indexOf('.', dot1 + 1); if (dot2 == -1) return false; int first = Integer.parseInt(ip.substring(0, dot1)); if (first == 10) return true; if (first == 192 && ip.startsWith('168.', dot1 + 1)) return true; if (first == 172) { int second = Integer.parseInt(ip.substring(dot1 + 1, dot2)); return second >= 16 && second <= 31; } return false; } String[] fields = new String[] {\"source\", \"destination\"}; for (int i = 0; i < fields.length; i++) { def field = fields[i]; def ip = ctx[field]?.ip; if (ip != null) { if (ctx.network == null) ctx.network = new HashMap(); if (isPrivate(ip)) { if (ctx.network.private_ip == null) ctx.network.private_ip = new ArrayList(); if (!ctx.network.private_ip.contains(ip)) ctx.network.private_ip.add(ip); } else { if (ctx.network.public_ip == null) ctx.network.public_ip = new ArrayList(); if (!ctx.network.public_ip.contains(ip)) ctx.network.public_ip.add(ip); } } }",
+        "ignore_failure": false
+      }
+    },
     { "pipeline": { "if": "ctx?.event?.dataset != null", "name": "suricata.{{event.dataset}}" } }
   ]
 }

salt/elasticsearch/files/ingest/zeek.common

@@ -12,7 +12,8 @@
     { "rename":         { "field": "message2.id.orig_p",        "target_field": "source.port",          "ignore_missing": true  } },
     { "rename":         { "field": "message2.id.resp_h",        "target_field": "destination.ip",       "ignore_missing": true  } },
     { "rename":         { "field": "message2.id.resp_p",        "target_field": "destination.port",     "ignore_missing": true  } },
-    { "community_id": {} },
+    { "rename":         { "field": "message2.community_id",     "target_field": "network.community_id", "ignore_missing": true  } },
+    { "community_id":   { "if":    "ctx.network?.community_id == null"     } },
     { "set":         { "if": "ctx.source?.ip != null", "field": "client.ip",        "value": "{{source.ip}}"           } },
     { "set":         { "if": "ctx.source?.port != null", "field": "client.port",        "value": "{{source.port}}"       } },
     { "set":         { "if": "ctx.destination?.ip != null", "field": "server.ip",        "value": "{{destination.ip}}"  } },

salt/elasticsearch/files/ingest/zeek.conn

@@ -24,6 +24,10 @@
     { "rename": 	{ "field": "message2.resp_cc", 		"target_field": "server.country_code",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.sensorname", 	"target_field": "observer.name",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.vlan", 		"target_field": "network.vlan.id",		"ignore_missing": true 	} },
+    { "rename":   { "field": "message2.ja4l",      "target_field": "hash.ja4l",         "ignore_missing" : true,        "if": "ctx.message2?.ja4l != null && ctx.message2.ja4l.length() > 0" }},
+    { "rename":   { "field": "message2.ja4ls",     "target_field": "hash.ja4ls",        "ignore_missing" : true,        "if": "ctx.message2?.ja4ls != null && ctx.message2.ja4ls.length() > 0" }},
+    { "rename":   { "field": "message2.ja4t",      "target_field": "hash.ja4t",         "ignore_missing" : true,        "if": "ctx.message2?.ja4t != null && ctx.message2.ja4t.length() > 0" }},
+    { "rename":   { "field": "message2.ja4ts",      "target_field": "hash.ja4ts",         "ignore_missing" : true,      "if": "ctx.message2?.ja4ts != null && ctx.message2.ja4ts.length() > 0" }},
     { "script":         { "lang": "painless", "source": "ctx.network.bytes = (ctx.client.bytes + ctx.server.bytes)", "ignore_failure": true } },
     { "set": { "if": "ctx.connection?.state == 'S0'",    "field": "connection.state_description", "value": "Connection attempt seen, no reply" } },
     { "set": { "if": "ctx.connection?.state == 'S1'",    "field": "connection.state_description", "value": "Connection established, not terminated" } },

salt/elasticsearch/files/ingest/zeek.dns

@@ -20,7 +20,8 @@
     { "rename": 	{ "field": "message2.RD", 		"target_field": "dns.recursion.desired",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.RA", 		"target_field": "dns.recursion.available",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.Z",		"target_field": "dns.reserved",			"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.answers", 		"target_field": "dns.answers.name",		"ignore_missing": true 	} },
+    { "rename":  	{ "field": "message2.answers", 		"target_field": "dns.answers.name", 		"ignore_missing": true  	} },
+    { "script": { "lang": "painless", "if": "ctx.dns != null && ctx.dns.answers != null && ctx.dns.answers.name != null", "source": "def ips = []; for (item in ctx.dns.answers.name) { if (item =~ /^(?:[0-9]{1,3}\\.){3}[0-9]{1,3}$/ || item =~ /^([a-fA-F0-9:]+:+)+[a-fA-F0-9]+$/) { ips.add(item); } } ctx.dns.resolved_ip = ips;" } },
     { "rename": 	{ "field": "message2.TTLs", 		"target_field": "dns.ttls",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.rejected", 	"target_field": "dns.query.rejected",		"ignore_missing": true 	} },
     { "script": 	{ "lang": "painless", "source": "ctx.dns.query.length = ctx.dns.query.name.length()",	"ignore_failure": true  } },
@@ -28,4 +29,4 @@
     { "pipeline": { "if": "ctx.dns?.query?.name != null && ctx.dns.query.name.contains('.')", "name": "dns.tld" } },
     { "pipeline":       { "name": "zeek.common"											} }
   ]
-}
+}

salt/elasticsearch/files/ingest/zeek.http

@@ -27,6 +27,7 @@
     { "rename": 	{ "field": "message2.resp_fuids",	"target_field": "log.id.resp_fuids",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.resp_filenames",	"target_field": "file.resp_filenames",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.resp_mime_types",	"target_field": "file.resp_mime_types",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.ja4h",	"target_field": "hash.ja4h",	"ignore_missing": true, "if": "ctx?.message2?.ja4h != null && ctx.message2.ja4h.length() > 0" 	} },
     { "script":	{ "lang": "painless", "source": "ctx.uri_length = ctx.uri.length()", 			"ignore_failure": true 	} },
     { "script":	{ "lang": "painless", "source": "ctx.useragent_length = ctx.useragent.length()",	"ignore_failure": true 	} },
     { "script":	{ "lang": "painless", "source": "ctx.virtual_host_length = ctx.virtual_host.length()", 	"ignore_failure": true	} },

salt/elasticsearch/files/ingest/zeek.http2

@@ -27,6 +27,7 @@
     { "rename": 	{ "field": "message2.resp_filenames",	        "target_field": "file.resp_filenames",	      "ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.resp_mime_types",	      "target_field": "file.resp_mime_types",	      "ignore_missing": true 	} },
     { "rename":   { "field": "message2.stream_id",              "target_field": "http2.stream_id",            "ignore_missing": true  } },
+    { "rename":   { "field": "message2.ja4h",                   "target_field": "hash.ja4h",                  "ignore_missing": true,   "if": "ctx?.message2?.ja4h != null && ctx.message2.ja4h.length() > 0"  } },
     { "remove":		{ "field": "message2.tags",		                                                                  "ignore_failure": true } },
     { "remove":   { "field": ["host"],                                                                            "ignore_failure": true } },
     { "script":	{ "lang": "painless", "source": "ctx.uri_length = ctx.uri.length()", 			                        "ignore_failure": true 	} },

salt/elasticsearch/files/ingest/zeek.ja4ssh

@@ -0,0 +1,10 @@
+{
+  "description": "zeek.ja4ssh",
+  "processors": [
+    {"set": {"field": "event.dataset","value": "ja4ssh"}},
+    {"remove": {"field": "host","ignore_missing": true,"ignore_failure": true}},
+    {"json": {"field": "message","target_field": "message2","ignore_failure": true}},
+    {"rename": {"field": "message2.ja4ssh", "target_field": "hash.ja4ssh", "ignore_missing": true, "if": "ctx?.message2?.ja4ssh != null && ctx.message2.ja4ssh.length() > 0" }},
+    {"pipeline": {"name": "zeek.common"}}
+  ]
+}

salt/elasticsearch/files/ingest/zeek.ssl

@@ -23,6 +23,8 @@
     { "rename":         { "field": "message2.validation_status","target_field": "ssl.validation_status",        "ignore_missing": true  } },
     { "rename":         { "field": "message2.ja3",              "target_field": "hash.ja3",                     "ignore_missing": true  } },
     { "rename":         { "field": "message2.ja3s",             "target_field": "hash.ja3s",                    "ignore_missing": true  } },
+    { "rename":         { "field": "message2.ja4",             "target_field": "hash.ja4",                    "ignore_missing": true,         "if": "ctx?.message2?.ja4 != null && ctx.message2.ja4.length() > 0"  } },
+    { "rename":         { "field": "message2.ja4s",             "target_field": "hash.ja4s",                    "ignore_missing": true,       "if": "ctx?.message2?.ja4s != null && ctx.message2.ja4s.length() > 0"  } },
     { "foreach":
       {
         "if": "ctx?.tls?.client?.hash?.sha256 !=null",

salt/elasticsearch/files/ingest/zeek.x509

@@ -42,6 +42,7 @@
     { "dot_expander":   { "field": "basic_constraints.path_length",             "path": "message2",                                     "ignore_failure": true  } },
     { "rename":         { "field": "message2.basic_constraints.path_length",    "target_field": "x509.basic_constraints.path_length",   "ignore_missing": true  } },
     { "rename":         { "field": "message2.fingerprint",                      "target_field": "hash.sha256",   "ignore_missing": true  } },
+    { "rename":         { "field": "message2.ja4x",                             "target_field": "hash.ja4x",   "ignore_missing": true, "if": "ctx?.message2?.ja4x != null && ctx.message2.ja4x.length() > 0"  } },
     { "pipeline":       { "name": "zeek.common_ssl"                                                                                                             } }
   ]
 }

salt/elasticsearch/soc_elasticsearch.yaml

@@ -12,7 +12,7 @@ elasticsearch:
     description: Specify the memory heap size in (m)egabytes for Elasticsearch.
     helpLink: elasticsearch.html
   index_clean:
-    description: Determines if indices should be considered for deletion by available disk space in the cluster. Otherwise, indices will only be deleted by the age defined in the ILM settings.
+    description: Determines if indices should be considered for deletion by available disk space in the cluster. Otherwise, indices will only be deleted by the age defined in the ILM settings. This setting only applies to EVAL, STANDALONE, and HEAVY NODE installations. Other installations can only use ILM settings.
     forcedType: bool
     helpLink: elasticsearch.html
   retention: 

salt/elasticsearch/template.map.jinja

@@ -15,7 +15,7 @@
 {% set ES_INDEX_SETTINGS_ORIG = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings %}
 
 {# start generation of integration default index_settings #}
-{% if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') %}
+{% if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') and salt['file.file_exists']('/opt/so/state/esfleet_component_templates.json') %}
 {%   set check_package_components = salt['file.stats']('/opt/so/state/esfleet_package_components.json') %}
 {%   if check_package_components.size > 1 %}
 {%    from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %}

salt/elasticsearch/templates/component/ecs/hash.json

@@ -0,0 +1,69 @@
+{
+  "template": {
+    "mappings": {
+      "properties": {
+        "hash": {
+          "type": "object",
+          "properties": {
+            "ja3": {
+              "type": "keyword",
+              "ignore_above": 1024
+            },
+            "ja3s": {
+              "type": "keyword",
+              "ignore_above": 1024
+            },
+            "hassh": {
+              "type": "keyword",
+              "ignore_above": 1024
+            },
+            "md5": {
+              "type": "keyword",
+              "ignore_above": 1024
+            },
+            "sha1": {
+              "type": "keyword",
+              "ignore_above": 1024
+            },
+            "sha256": {
+              "type": "keyword",
+              "ignore_above": 1024
+            },
+            "ja4": {
+              "type": "keyword",
+              "ignore_above": 1024
+            },
+            "ja4l": {
+              "type": "keyword",
+              "ignore_above": 1024
+            },
+            "ja4ls": {
+              "type": "keyword",
+              "ignore_above": 1024
+            },
+            "ja4t": {
+              "type": "keyword",
+              "ignore_above": 1024
+            },
+            "ja4ts": {
+              "type": "keyword",
+              "ignore_above": 1024
+            },
+            "ja4ssh": {
+              "type": "keyword",
+              "ignore_above": 1024
+            },
+            "ja4h": {
+              "type": "keyword",
+              "ignore_above": 1024
+            },
+            "ja4x": {
+              "type": "keyword",
+              "ignore_above": 1024
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-start

@@ -1,4 +1,4 @@
-/bin/bash
+#!/bin/bash
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
 # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
 # https://securityonion.net/license; you may not use this file except in compliance with the
@@ -6,6 +6,6 @@
 
 . /usr/sbin/so-common
 
-
 echo "Starting ILM..."
 curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -X POST https://localhost:9200/_ilm/start
+echo

salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-stop

@@ -8,3 +8,4 @@
 
 echo "Stopping ILM..."
 curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -X POST https://localhost:9200/_ilm/stop
+echo

salt/elasticsearch/tools/sbin/so-elasticsearch-indices-growth

@@ -0,0 +1,113 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+INFLUX_URL="https://localhost:8086/api/v2"
+
+. /usr/sbin/so-common
+
+request() {
+    curl -skK /opt/so/conf/influxdb/curl.config "$INFLUX_URL/$@"
+}
+
+lookup_org_id() {
+  response=$(request orgs?org=Security+Onion)
+  echo "$response" | jq -r ".orgs[] | select(.name == \"Security Onion\").id"
+}
+
+ORG_ID=$(lookup_org_id)
+
+run_flux_query() {
+    local query=$1
+    request "query?org=$ORG_ID" -H 'Accept:application/csv' -H 'Content-type:application/vnd.flux' -d "$query" -XPOST 2>/dev/null
+}
+
+read_csv_result() {
+    local result="$1"
+    echo "$result" | grep '^,_result,' | head -1 | awk -F',' '{print $NF}' | tr -d '\r\n\t '
+}
+
+bytes_to_gb() {
+    local bytes="${1:-0}"
+    if [[ "$bytes" =~ ^-?[0-9]+$ ]]; then
+        echo "$bytes" | awk '{printf "%.2f", $1 / 1024 / 1024 / 1024}'
+    else
+        echo "0.00"
+    fi
+}
+
+indexes_query='from(bucket: "telegraf/so_long_term")
+|> range(start: -7d)
+|> filter(fn: (r) => r._measurement == "elasticsearch_index_size")
+|> distinct(column: "_field")
+|> keep(columns: ["_field"])'
+
+indexes_result=$(run_flux_query "$indexes_query")
+indexes=$(echo "$indexes_result" | tail -n +2 | cut -d',' -f4 | grep -v '^$' | grep -v '^_field$' | sed 's/\r$//' | sort -u)
+
+printf "%-50s %15s %15s %15s\n" "Index Name" "Last 24hr (GB)" "Last 7d (GB)" "Last 30d (GB)"
+printf "%-50s %15s %15s %15s\n" "$(printf '%.0s-' {1..50})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})"
+
+for index in $indexes; do
+  [[ -z "$index" ]] && continue
+  current_query="from(bucket: \"telegraf/so_long_term\")
+    |> range(start: -4h)
+    |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
+    |> last()
+    |> keep(columns: [\"_value\"])"
+  current_result=$(run_flux_query "$current_query")
+  current_size=$(read_csv_result "$current_result")
+  current_size=${current_size:-0}
+
+  size_24h_query="from(bucket: \"telegraf/so_long_term\")
+    |> range(start: -25h, stop: -23h)
+    |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
+    |> last()
+    |> keep(columns: [\"_value\"])"
+  size_24h_result=$(run_flux_query "$size_24h_query")
+  size_24h_ago=$(read_csv_result "$size_24h_result")
+  size_24h_ago=${size_24h_ago:-$current_size}
+
+  size_7d_query="from(bucket: \"telegraf/so_long_term\")
+    |> range(start: -7d8h, stop: -7d)
+    |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
+    |> last()
+    |> keep(columns: [\"_value\"])"
+  size_7d_result=$(run_flux_query "$size_7d_query")
+  size_7d_ago=$(read_csv_result "$size_7d_result")
+  size_7d_ago=${size_7d_ago:-$current_size}
+
+  size_30d_query="from(bucket: \"telegraf/so_long_term\")
+    |> range(start: -30d8h, stop: -30d)
+    |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
+    |> last()
+    |> keep(columns: [\"_value\"])"
+  size_30d_result=$(run_flux_query "$size_30d_query")
+  size_30d_ago=$(read_csv_result "$size_30d_result")
+  size_30d_ago=${size_30d_ago:-$current_size}
+
+  # if an index was recently cleaned up by ilm it will result in a negative number for 'index growth'.
+  growth_24h=$(( current_size > size_24h_ago ? current_size - size_24h_ago : 0 ))
+
+  growth_7d=$(( current_size > size_7d_ago ? current_size - size_7d_ago : 0 ))
+
+  growth_30d=$(( current_size > size_30d_ago ? current_size - size_30d_ago : 0 ))
+
+  growth_24h_gb=$(bytes_to_gb "$growth_24h")
+  growth_7d_gb=$(bytes_to_gb "$growth_7d")
+  growth_30d_gb=$(bytes_to_gb "$growth_30d")
+
+  # Only results for indices with atleast 1 metric above 0.00
+  if [[ "$growth_24h_gb" != "0.00" ]] || [[ "$growth_7d_gb" != "0.00" ]] || [[ "$growth_30d_gb" != "0.00" ]]; then
+    printf "%020.2f|%-50s %15s %15s %15s\n" \
+      "$growth_24h" \
+      "$index" \
+      "$growth_24h_gb" \
+      "$growth_7d_gb" \
+      "$growth_30d_gb"
+  fi
+done | sort -t'|' -k1,1nr | cut -d'|' -f2-
+

salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load

@@ -21,7 +21,7 @@ while [[ "$COUNT" -le 240 ]]; do
 		ELASTICSEARCH_CONNECTED="yes"
 		echo "connected!"
 		# Check cluster health once connected
-		so-elasticsearch-query _cluster/health?wait_for_status=yellow > /dev/null 2>&1
+		so-elasticsearch-query _cluster/health?wait_for_status=yellow\&timeout=120s > /dev/null 2>&1
 		break
 	else
 		((COUNT+=1))

salt/elasticsearch/tools/sbin/so-elasticsearch-troubleshoot

@@ -0,0 +1,195 @@
+#!/bin/bash
+
+. /usr/sbin/so-common
+
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+BOLD='\033[1;37m'
+NC='\033[0m'
+
+log_title() {
+  if [ $1 == "LOG" ]; then
+    echo -e "\n${BOLD}================ $2 ================${NC}\n"
+  elif [ $1 == "OK" ]; then
+    echo -e "${GREEN} $2 ${NC}"
+  elif [ $1 == "WARN" ]; then
+    echo -e "${YELLOW} $2 ${NC}"
+  elif [ $1 == "ERROR" ]; then
+    echo -e "${RED} $2 ${NC}"
+  fi
+}
+
+health_report() {
+  if ! health_report_output=$(so-elasticsearch-query _health_report?format=json --fail 2>/dev/null); then
+    log_title "ERROR" "Failed to retrieve health report from Elasticsearch"
+    return 1
+  fi
+  non_green_count=$(echo "$health_report_output" | jq '[.indicators | to_entries[] | select(.value.status != "green")] | length')
+
+  if [ "$non_green_count" -gt 0 ]; then
+    echo "$health_report_output" | jq -r '.indicators | to_entries[] | select(.value.status != "green") | .key' | while read -r indicator_name; do
+      indicator=$(echo "$health_report_output" | jq -r ".indicators.\"$indicator_name\"")
+      status=$(echo "$indicator" | jq -r '.status')
+      symptom=$(echo "$indicator" | jq -r '.symptom // "No symptom available"')
+
+      # reormat indicator name
+      display_name=$(echo "$indicator_name" | tr '_' ' ' | sed 's/\b\(.\)/\u\1/g')
+
+      if [ "$status" = "yellow" ]; then
+        log_title "WARN" "$display_name: $symptom"
+      else
+        log_title "ERROR" "$display_name: $symptom"
+      fi
+
+      # diagnosis if available
+      echo "$indicator" | jq -c '.diagnosis[]? // empty' | while read -r diagnosis; do
+        cause=$(echo "$diagnosis" | jq -r '.cause // "Unknown"')
+        action=$(echo "$diagnosis" | jq -r '.action // "No action specified"')
+
+        echo -e "  ${BOLD}Cause:${NC} $cause\n"
+        echo -e "  ${BOLD}Action:${NC} $action\n"
+
+        # Check for affected indices
+        affected_indices=$(echo "$diagnosis" | jq -r '.affected_resources.indices[]? // empty')
+        if [ -n "$affected_indices" ]; then
+          echo -e "  ${BOLD}Affected indices:${NC}"
+          total_indices=$(echo "$affected_indices" | wc -l)
+          echo "$affected_indices" | head -10 | while read -r index; do
+            echo "    - $index"
+          done
+          if [ "$total_indices" -gt 10 ]; then
+            remaining=$((total_indices - 10))
+            echo "    ... and $remaining more indices (truncated for readability)"
+          fi
+        fi
+        echo
+      done
+    done
+  else
+    log_title "OK" "All health indicators are green"
+  fi
+}
+
+elasticsearch_status() {
+  log_title "LOG" "Elasticsearch Status"
+  if so-elasticsearch-query / --fail --output /dev/null; then
+    health_report
+  else
+    log_title "ERROR" "Elasticsearch API is not accessible"
+    so-status
+    log_title "ERROR" "Make sure Elasticsearch is running. Addtionally, check for startup errors in /opt/so/log/elasticsearch/securityonion.log${NC}\n"
+
+    exit 1
+  fi
+
+}
+
+indices_by_age() {
+  log_title "LOG" "Indices by Creation Date - Size > 1KB"
+  log_title "WARN" "Since high/flood watermark has been reached consider updating ILM policies.\n"
+  if ! indices_output=$(so-elasticsearch-query '_cat/indices?v&s=creation.date:asc&h=creation.date.string,index,status,health,docs.count,pri.store.size&bytes=b&format=json' --fail 2>/dev/null); then
+    log_title "ERROR" "Failed to retrieve indices list from Elasticsearch"
+    return 1
+  fi
+
+  # Filter for indices with size > 1KB (1024 bytes) and format output
+  echo -e "${BOLD}Creation Date                   Name                                                                        Size${NC}"
+  echo -e "${BOLD}--------------------------------------------------------------------------------------------------------------${NC}"
+
+  #  Create list of indices excluding .internal, so-detection*, so-case*
+  echo "$indices_output" | jq -r '.[] | select((."pri.store.size" | tonumber) > 1024) | select(.index | (startswith(".internal") or startswith("so-detection") or startswith("so-case")) | not ) | "\(."creation.date.string") | \(.index) | \(."pri.store.size")"' | while IFS='|' read -r creation_date index_name size_bytes; do
+    # Convert bytes to GB / MB
+    if [ "$size_bytes" -gt 1073741824 ]; then
+      size_human=$(echo "scale=2; $size_bytes / 1073741824" | bc)GB
+    else
+      size_human=$(echo "scale=2; $size_bytes / 1048576" | bc)MB
+    fi
+
+    creation_date=$(date -d "$creation_date" '+%Y-%m-%dT%H:%MZ' )
+
+    # Format output with spacing
+    printf "%-19s %-76s %10s\n" "$creation_date" "$index_name" "$size_human"
+  done
+}
+
+watermark_settings() {
+  watermark_path=".defaults.cluster.routing.allocation.disk.watermark"
+  if ! watermark_output=$(so-elasticsearch-query _cluster/settings?include_defaults=true\&filter_path=*.cluster.routing.allocation.disk.* --fail 2>/dev/null); then
+    log_title "ERROR" "Failed to retrieve watermark settings from Elasticsearch"
+    return 1
+  fi
+
+  if ! disk_allocation_output=$(so-elasticsearch-query _cat/nodes?v\&h=name,ip,disk.used_percent,disk.avail,disk.total,node.role\&format=json --fail 2>/dev/null); then
+    log_title "ERROR" "Failed to retrieve disk allocation data from Elasticsearch"
+    return 1
+  fi
+
+  flood=$(echo $watermark_output | jq -r "$watermark_path.flood_stage" )
+  high=$(echo $watermark_output | jq -r "$watermark_path.high" )
+  low=$(echo $watermark_output | jq -r "$watermark_path.low" )
+
+  # Strip percentage signs for comparison
+  flood_num=${flood%\%}
+  high_num=${high%\%}
+  low_num=${low%\%}
+
+  # Check each nodes disk usage
+  log_title "LOG" "Disk Usage Check"
+  echo -e "${BOLD}LOW:${GREEN}$low${NC}${BOLD} HIGH:${YELLOW}${high}${NC}${BOLD} FLOOD:${RED}${flood}${NC}\n"
+
+  # Only show data nodes (d=data, h=hot, w=warm, c=cold, f=frozen, s=content)
+  echo "$disk_allocation_output" | jq -r '.[] | select(.["node.role"] | test("[dhwcfs]")) | "\(.name)|\(.["disk.used_percent"])"' | while IFS='|' read -r node_name disk_used; do
+    disk_used_num=$(echo $disk_used | bc)
+
+    if (( $(echo "$disk_used_num >= $flood_num" | bc -l) )); then
+      log_title "ERROR" "$node_name is at or above the flood watermark ($flood)! Disk usage: ${disk_used}%"
+      touch /tmp/watermark_reached
+    elif (( $(echo "$disk_used_num >= $high_num" | bc -l) )); then
+      log_title "ERROR" "$node_name is at or above the high watermark ($high)! Disk usage: ${disk_used}%"
+      touch /tmp/watermark_reached
+    else
+      log_title "OK" "$node_name disk usage: ${disk_used}%"
+    fi
+  done
+
+   # Check if we need to show indices by age
+  if [ -f /tmp/watermark_reached ]; then
+    indices_by_age
+    rm -f /tmp/watermark_reached
+  fi
+
+}
+
+unassigned_shards() {
+
+  if ! unassigned_shards_output=$(so-elasticsearch-query _cat/shards?v\&h=index,shard,prirep,state,unassigned.reason,unassigned.details\&s=state\&format=json --fail 2>/dev/null); then
+    log_title "ERROR" "Failed to retrieve shard data from Elasticsearch"
+    return 1
+  fi
+
+  log_title "LOG" "Unassigned Shards Check"
+  # Check if there are any UNASSIGNED shards
+  unassigned_count=$(echo "$unassigned_shards_output" | jq '[.[] | select(.state == "UNASSIGNED")] | length')
+
+  if [ "$unassigned_count" -gt 0 ]; then
+    echo "$unassigned_shards_output" | jq -r '.[] | select(.state == "UNASSIGNED") | "\(.index)|\(.shard)|\(.prirep)|\(."unassigned.reason")"' | while IFS='|' read -r index shard prirep reason; do
+      if [ "$prirep" = "r" ]; then
+        log_title "WARN" "Replica shard for index $index is unassigned. Reason: $reason"
+      elif [ "$prirep" = "p" ]; then
+        log_title "ERROR" "Primary shard for index $index is unassigned. Reason: $reason"
+      fi
+    done
+  else
+    log_title "OK" "All shards are assigned"
+  fi
+}
+
+main() {
+  elasticsearch_status
+  watermark_settings
+  unassigned_shards
+}
+
+main

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load

@@ -136,7 +136,7 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then
       TEMPLATE=${i::-14}
       COMPONENT_PATTERN=${TEMPLATE:3}
       MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -vE "detections|osquery")
-      if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ logs-http_endpoint\.generic|logs-winlog\.winlog ]]; then
+      if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ \.generic|logs-winlog\.winlog ]]; then
         load_failures=$((load_failures+1))
         echo "Component template does not exist for $COMPONENT_PATTERN. The index template will not be loaded. Load failures: $load_failures"
       else

salt/firewall/containers.map.jinja

@@ -21,7 +21,7 @@
      'so-strelka-filestream'
 ] %}
 
-{% elif GLOBALS.role == 'so-manager' or GLOBALS.role == 'so-standalone' or GLOBALS.role == 'so-managersearch' %}
+{% elif GLOBALS.role in ['so-manager', 'so-standalone','so-managersearch', 'so-managerhype'] %}
 {% set NODE_CONTAINERS = [
      'so-dockerregistry',
      'so-elasticsearch',

salt/firewall/defaults.yaml

@@ -14,11 +14,13 @@ firewall:
     external_kafka: []
     fleet: []
     heavynode: []
+    hypervisor: []
     idh: []
     import: []
     localhost:
       - 127.0.0.1
     manager: []
+    managerhype: []
     managersearch: []
     receiver: []
     searchnode: []
@@ -489,6 +491,15 @@ firewall:
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
+            hypervisor:
+              portgroups:
+                - yum
+                - docker_registry
+                - influxdb
+                - elastic_agent_control
+                - elastic_agent_data
+                - elastic_agent_update
+                - sensoroni
             customhostgroup0:
               portgroups: []
             customhostgroup1:
@@ -541,6 +552,218 @@ firewall:
             desktop:
               portgroups:
                 - salt_manager
+            hypervisor:
+              portgroups:
+                - salt_manager
+            self:
+              portgroups:
+                - syslog
+            syslog:
+              portgroups:
+                - syslog
+            customhostgroup0:
+              portgroups: []
+            customhostgroup1:
+              portgroups: []
+            customhostgroup2:
+              portgroups: []
+            customhostgroup3:
+              portgroups: []
+            customhostgroup4:
+              portgroups: []
+            customhostgroup5:
+              portgroups: []
+            customhostgroup6:
+              portgroups: []
+            customhostgroup7:
+              portgroups: []
+            customhostgroup8:
+              portgroups: []
+            customhostgroup9:
+              portgroups: []
+    managerhype:
+      chain:
+        DOCKER-USER:
+          hostgroups:
+            managerhype:
+              portgroups:
+                - kibana
+                - redis
+                - influxdb
+                - elasticsearch_rest
+                - elasticsearch_node
+                - docker_registry
+                - elastic_agent_control
+                - elastic_agent_data
+                - elastic_agent_update
+                - localrules
+                - sensoroni
+            fleet:
+              portgroups:
+                - elasticsearch_rest
+                - docker_registry
+                - influxdb
+                - sensoroni
+                - yum
+                - beats_5044
+                - beats_5644
+                - beats_5056
+                - elastic_agent_control
+                - elastic_agent_data
+                - elastic_agent_update
+            idh:
+              portgroups:
+                - docker_registry
+                - influxdb
+                - sensoroni
+                - yum
+                - beats_5044
+                - beats_5644
+                - elastic_agent_control
+                - elastic_agent_data
+                - elastic_agent_update
+            sensor:
+              portgroups:
+                - beats_5044
+                - beats_5644
+                - elastic_agent_control
+                - elastic_agent_data
+                - elastic_agent_update
+                - yum
+                - docker_registry
+                - influxdb
+                - sensoroni
+            searchnode:
+              portgroups:
+                - redis
+                - elasticsearch_rest
+                - elasticsearch_node
+                - beats_5644
+                - yum
+                - docker_registry
+                - influxdb
+                - elastic_agent_control
+                - elastic_agent_data
+                - elastic_agent_update
+                - sensoroni
+            heavynode:
+              portgroups:
+                - redis
+                - elasticsearch_rest
+                - elasticsearch_node
+                - beats_5644
+                - yum
+                - docker_registry
+                - influxdb
+                - elastic_agent_control
+                - elastic_agent_data
+                - elastic_agent_update
+                - sensoroni
+            receiver:
+              portgroups:
+                - yum
+                - docker_registry
+                - influxdb
+                - elastic_agent_control
+                - elastic_agent_data
+                - elastic_agent_update
+                - sensoroni
+            analyst:
+              portgroups:
+                - nginx
+            beats_endpoint:
+              portgroups:
+                - beats_5044
+            beats_endpoint_ssl:
+              portgroups:
+                - beats_5644
+            elasticsearch_rest:
+              portgroups:
+                - elasticsearch_rest
+            elastic_agent_endpoint:
+              portgroups:
+                - elastic_agent_control
+                - elastic_agent_data
+                - elastic_agent_update
+            endgame:
+              portgroups:
+                - endgame
+            external_suricata:
+              portgroups:
+                - external_suricata
+            desktop:
+              portgroups:
+                - docker_registry
+                - influxdb
+                - sensoroni
+                - yum
+                - elastic_agent_control
+                - elastic_agent_data
+                - elastic_agent_update
+            hypervisor:
+              portgroups:
+                - yum
+                - docker_registry
+                - influxdb
+                - elastic_agent_control
+                - elastic_agent_data
+                - elastic_agent_update
+                - sensoroni
+            customhostgroup0:
+              portgroups: []
+            customhostgroup1:
+              portgroups: []
+            customhostgroup2:
+              portgroups: []
+            customhostgroup3:
+              portgroups: []
+            customhostgroup4:
+              portgroups: []
+            customhostgroup5:
+              portgroups: []
+            customhostgroup6:
+              portgroups: []
+            customhostgroup7:
+              portgroups: []
+            customhostgroup8:
+              portgroups: []
+            customhostgroup9:
+              portgroups: []
+        INPUT:
+          hostgroups:
+            anywhere:
+              portgroups:
+                - ssh
+            dockernet:
+              portgroups:
+                - all
+            fleet:
+              portgroups:
+                - salt_manager
+            idh:
+              portgroups:
+                - salt_manager
+            localhost:
+              portgroups:
+                - all
+            sensor:
+              portgroups:
+                - salt_manager
+            searchnode:
+              portgroups:
+                - salt_manager
+            heavynode:
+              portgroups:
+                - salt_manager
+            receiver:
+              portgroups:
+                - salt_manager
+            desktop:
+              portgroups:
+                - salt_manager
+            hypervisor:
+              portgroups:
+                - salt_manager
             self:
               portgroups:
                 - syslog
@@ -1472,3 +1695,64 @@ firewall:
               portgroups: []
             customhostgroup9:
               portgroups: []
+    hypervisor:
+      chain:
+        DOCKER-USER:
+          hostgroups:
+            customhostgroup0:
+              portgroups: []
+            customhostgroup1:
+              portgroups: []
+            customhostgroup2:
+              portgroups: []
+            customhostgroup3:
+              portgroups: []
+            customhostgroup4:
+              portgroups: []
+            customhostgroup5:
+              portgroups: []
+            customhostgroup6:
+              portgroups: []
+            customhostgroup7:
+              portgroups: []
+            customhostgroup8:
+              portgroups: []
+            customhostgroup9:
+              portgroups: []
+        INPUT:
+          hostgroups:
+            anywhere:
+              portgroups:
+                - ssh
+            dockernet:
+              portgroups:
+                - all
+            localhost:
+              portgroups:
+                - all
+            manager:
+              portgroups: []
+            managersearch:
+              portgroups: []
+            standalone:
+              portgroups: []
+            customhostgroup0:
+              portgroups: []
+            customhostgroup1:
+              portgroups: []
+            customhostgroup2:
+              portgroups: []
+            customhostgroup3:
+              portgroups: []
+            customhostgroup4:
+              portgroups: []
+            customhostgroup5:
+              portgroups: []
+            customhostgroup6:
+              portgroups: []
+            customhostgroup7:
+              portgroups: []
+            customhostgroup8:
+              portgroups: []
+            customhostgroup9:
+              portgroups: []

salt/firewall/iptables.jinja

@@ -91,6 +91,10 @@ COMMIT
 -A INPUT -m conntrack --ctstate INVALID -j DROP
 -A INPUT -p icmp -j ACCEPT
 -A INPUT -j LOGGING
+{% if GLOBALS.role in ['so-hypervisor', 'so-managerhyper'] -%}
+-A FORWARD -m state --state RELATED,ESTABLISHED -j ACCEPT
+-A FORWARD -i br0 -o br0 -j ACCEPT
+{%- endif %}
 -A FORWARD -j DOCKER-USER
 -A FORWARD -j DOCKER-ISOLATION-STAGE-1
 -A FORWARD -o sobridge -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT

salt/firewall/soc_firewall.yaml

@@ -36,6 +36,7 @@ firewall:
     external_kafka: *hostgroupsettings
     fleet: *hostgroupsettings
     heavynode: *hostgroupsettings
+    hypervisor: *hostgroupsettings
     idh: *hostgroupsettings
     import: *hostgroupsettings
     localhost: *ROhostgroupsettingsadv

salt/hypervisor/defaults.yaml

@@ -0,0 +1,125 @@
+hypervisor:
+  model:
+    testModel:
+      hardware:
+        cpu: 128
+        memory: 128
+        disk:
+          1: pci_0000_c7_00_0
+          2: pci_0000_c8_00_0
+        copper:
+          1: pci_0000_c4_00_0
+          2: pci_0000_c4_00_1
+          3: pci_0000_c4_00_2
+          4: pci_0000_c4_00_3
+        sfp:
+          5: pci_0000_02_00_0
+          6: pci_0000_02_00_1
+          7: pci_0000_41_00_0
+          8: pci_0000_41_00_1
+    SOSSNNV:
+      hardware:
+        cpu: 128
+        memory: 256
+        disk:
+          1: pci_0000_42_00_0
+          2: pci_0000_43_00_0
+          3: pci_0000_44_00_0
+          4: pci_0000_45_00_0
+        copper:
+        sfp:
+          1: pci_0000_02_00_0
+          2: pci_0000_02_00_1
+          3: pci_0000_41_00_0
+          4: pci_0000_41_00_1
+    SOSSNNV-DE02:
+      hardware:
+        cpu: 128
+        memory: 384
+        disk:
+          1: pci_0000_41_00_0
+          2: pci_0000_42_00_0
+          3: pci_0000_81_00_0
+          4: pci_0000_82_00_0
+          5: pci_0000_83_00_0
+          6: pci_0000_84_00_0
+        copper:
+          1: pci_0000_85_00_0
+          2: pci_0000_85_00_1
+          3: pci_0000_85_00_2
+          4: pci_0000_85_00_3
+        sfp:
+          5: pci_0000_c4_00_0
+          6: pci_0000_c4_00_1
+          7: pci_0000_c5_00_0
+          8: pci_0000_c5_00_1
+          9: pci_0000_c5_00_2
+          10: pci_0000_c5_00_3
+    SOSSN7200:
+      hardware:
+        cpu: 128
+        memory: 256
+        copper:
+          1: pci_0000_03_00_0
+          2: pci_0000_03_00_1
+          3: pci_0000_03_00_2
+          4: pci_0000_03_00_3
+        sfp:
+          5: pci_0000_02_00_0
+          6: pci_0000_02_00_1
+          7: pci_0000_81_00_0
+          8: pci_0000_81_00_1
+          9: pci_0000_81_00_2
+          10: pci_0000_81_00_3
+    SOSSN7200-DE02:
+      hardware:
+        cpu: 128
+        memory: 384
+        copper:
+          1: pci_0000_82_00_0
+          2: pci_0000_82_00_1
+          3: pci_0000_82_00_2
+          4: pci_0000_82_00_3
+        sfp:
+          5: pci_0000_c4_00_0
+          6: pci_0000_c4_00_1
+          7: pci_0000_c5_00_0
+          8: pci_0000_c5_00_1
+          9: pci_0000_c6_00_0
+          10: pci_0000_c6_00_1
+          11: pci_0000_c6_00_2
+          12: pci_0000_c6_00_3
+    SOS4000:
+      hardware:
+        cpu: 128
+        memory: 256
+        copper:
+          1: pci_0000_03_00_0
+          2: pci_0000_03_00_1
+          3: pci_0000_03_00_2
+          4: pci_0000_03_00_3
+        sfp:
+          5: pci_0000_02_00_0
+          6: pci_0000_02_00_1
+          7: pci_0000_81_00_0
+          8: pci_0000_81_00_1
+          9: pci_0000_81_00_2
+          10: pci_0000_81_00_3
+    SOS5000-DE02:
+      hardware:
+        cpu: 128
+        memory: 384
+        copper:
+          1: pci_0000_82_00_0
+          2: pci_0000_82_00_1
+          3: pci_0000_82_00_2
+          4: pci_0000_82_00_3
+        sfp:
+          5: pci_0000_c4_00_0
+          6: pci_0000_c4_00_1
+          7: pci_0000_c5_00_0
+          8: pci_0000_c5_00_1
+          9: pci_0000_c6_00_0
+          10: pci_0000_c6_00_1
+          11: pci_0000_c6_00_2
+          12: pci_0000_c6_00_3

salt/hypervisor/hosts/README

@@ -0,0 +1 @@
+This directory will contain hypervisor hosts. We need this README in place to ensure /opt/so/saltstack/local/salt/hypervisor/hosts directory gets created during setup.

salt/hypervisor/init.sls

@@ -0,0 +1,49 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+#
+# Note: Per the Elastic License 2.0, the second limitation states:
+#
+#   "You may not move, change, disable, or circumvent the license key functionality
+#    in the software, and you may not remove or obscure any functionality in the
+#    software that is protected by the license key."
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls in allowed_states %}
+{%   if 'vrt' in salt['pillar.get']('features', []) %}
+
+hypervisor_log_dir:
+  file.directory:
+    - name: /opt/so/log/hypervisor
+
+hypervisor_sbin:
+  file.recurse:
+    - name: /usr/sbin
+    - source: salt://hypervisor/tools/sbin
+    - file_mode: 744
+
+hypervisor_sbin_jinja:
+  file.recurse:
+    - name: /usr/sbin
+    - source: salt://hypervisor/tools/sbin_jinja
+    - template: jinja
+    - file_mode: 744
+
+{%   else %}
+{{sls}}_no_license_detected:
+  test.fail_without_changes:
+    - name: {{sls}}_no_license_detected
+    - comment:
+      - "Hypervisor nodes are a feature supported only for customers with a valid license.
+      Contact Security Onion Solutions, LLC via our website at https://securityonionsolutions.com
+      for more information about purchasing a license to enable this feature."
+{%   endif %}
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/hypervisor/map.jinja

@@ -0,0 +1,164 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. 
+   
+   Note: Per the Elastic License 2.0, the second limitation states:
+
+    "You may not move, change, disable, or circumvent the license key functionality
+     in the software, and you may not remove or obscure any functionality in the
+     software that is protected by the license key." #}
+
+{% if 'vrt' in salt['pillar.get']('features', []) %}
+
+{# Import defaults.yaml for model hardware capabilities #}
+{% import_yaml 'hypervisor/defaults.yaml' as DEFAULTS %}
+
+{# Get hypervisor nodes from pillar #}
+{% set NODES = salt['pillar.get']('hypervisor:nodes', {}) %}
+
+{# Build enhanced HYPERVISORS structure #}
+{% set HYPERVISORS = {} %}
+{% do salt.log.debug('salt/hypervisor/map.jinja: NODES content: ' ~ NODES | tojson) %}
+{% for role, hypervisors in NODES.items() %}
+  {% do salt.log.debug('salt/hypervisor/map.jinja: Processing role: ' ~ role) %}
+  {% do HYPERVISORS.update({role: {}}) %}
+  {% for hypervisor, config in hypervisors.items() %}
+    {% do salt.log.debug('salt/hypervisor/map.jinja: Processing hypervisor: ' ~ hypervisor ~ ' with config: ' ~ config | tojson) %}
+    {# Get model from cached grains using Salt runner #}
+    {% set grains = salt.saltutil.runner('cache.grains', tgt=hypervisor ~ '_*', tgt_type='glob') %}
+    {% set model = '' %}
+    {% if grains %}
+      {% set minion_id = grains.keys() | first %}
+      {% set model = grains[minion_id].get('sosmodel', '') %}
+    {% endif %}
+    {% set model_config = DEFAULTS.hypervisor.model.get(model, {}) %}
+    
+    {# Get VM list from VMs file #}
+    {% set vms = {} %}
+    {% set vm_list = [] %}
+    {% set vm_list_file = 'hypervisor/hosts/' ~ hypervisor ~ 'VMs' %}
+    {% do salt.log.debug('salt/hypervisor/map.jinja: VM list file: ' ~ vm_list_file) %}
+    {% if salt['file.file_exists']('/opt/so/saltstack/local/salt/' ~ vm_list_file) %}
+    {%   import_json vm_list_file as vm_list %}
+    {% endif %}
+    {% if vm_list %}
+      {% do salt.log.debug('salt/hypervisor/map.jinja: VM list content: ' ~ vm_list | tojson) %}
+    {% else %}
+      {# we won't get here if the vm_list_file doesn't exist because we will get TemplateNotFound on the import_json #}
+      {% do salt.log.debug('salt/hypervisor/map.jinja: VM list empty: ' ~ vm_list_file) %}
+    {% endif %}
+    
+    {# Load status and configuration for each VM #}
+    {% for vm in vm_list %}
+      {# Get VM details from list entry #}
+      {% set hostname = vm.get('hostname', '') %}
+      {% set role = vm.get('role', '') %}
+      {% do salt.log.debug('salt/hypervisor/map.jinja: Processing VM - hostname: ' ~ hostname ~ ', role: ' ~ role) %}
+      
+      {# Load VM configuration from config file #}
+      {% set vm_file = 'hypervisor/hosts/' ~ hypervisor ~ '/' ~ hostname ~ '_' ~ role %}
+      {% do salt.log.debug('salt/hypervisor/map.jinja: VM config file: ' ~ vm_file) %}
+      {% import_json vm_file as vm_state %}
+      {% if vm_state %}
+        {% do salt.log.debug('salt/hypervisor/map.jinja: VM config content: ' ~ vm_state | tojson) %}
+        {% set vm_data = {'config': vm_state.config} %}
+        
+        {# Load VM status from status file #}
+        {% set status_file = vm_file ~ '.status' %}
+        {% do salt.log.debug('salt/hypervisor/map.jinja: VM status file: ' ~ status_file) %}
+        {% import_json status_file as status_data %}
+        {% if status_data %}
+          {% do salt.log.debug('salt/hypervisor/map.jinja: VM status content: ' ~ status_data | tojson) %}
+          {% do vm_data.update({'status': status_data}) %}
+        {% else %}
+          {% do salt.log.debug('salt/hypervisor/map.jinja: Status file empty: ' ~ status_file) %}
+          {% do vm_data.update({
+            'status': {
+              'status': '',
+              'details': null,
+              'timestamp': ''
+            }
+          }) %}
+        {% endif %}
+        {% do vms.update({hostname ~ '_' ~ role: vm_data}) %}
+      {% else %}
+        {% do salt.log.debug('salt/hypervisor/map.jinja: Config file empty: ' ~ vm_file) %}
+      {% endif %}
+    {% endfor %}
+    
+    {# Find and add destroyed VMs from status files #}
+    {% set processed_vms = [] %}
+    {% for vm_full_name, vm_data in vms.items() %}
+      {% do processed_vms.append(vm_full_name) %}
+    {% endfor %}
+    
+    {# Find all status files for this hypervisor #}
+    {% set relative_path = 'hypervisor/hosts/' ~ hypervisor %}
+    {% set absolute_path = '/opt/so/saltstack/local/salt/' ~ relative_path %}
+    {% do salt.log.debug('salt/hypervisor/map.jinja: Scanning for status files in: ' ~ absolute_path) %}
+    
+    {# Try to find status files using file.find with absolute path #}
+    {% set status_files = salt['file.find'](absolute_path, name='*_*.status', type='f') %}
+    {% do salt.log.debug('salt/hypervisor/map.jinja: Found status files: ' ~ status_files | tojson) %}
+    
+    {# Convert absolute paths back to relative paths for processing #}
+    {% set relative_status_files = [] %}
+    {% for status_file in status_files %}
+      {% set relative_file = status_file | replace('/opt/so/saltstack/local/salt/', '') %}
+      {% do relative_status_files.append(relative_file) %}
+    {% endfor %}
+    {% set status_files = relative_status_files %}
+    
+    {% do salt.log.debug('salt/hypervisor/map.jinja: Converted to relative paths: ' ~ status_files | tojson) %}
+    
+    {% for status_file in status_files %}
+      {# Extract the VM name from the filename #}
+      {% set basename = status_file.split('/')[-1] %}
+      {% set vm_name = basename.replace('.status', '') %}
+      {% set hostname = vm_name.split('_')[0] %}
+      
+      {# Skip already processed VMs #}
+      {% if vm_name in processed_vms %}
+        {% continue %}
+      {% endif %}
+      
+      {# Read the status file #}
+      {% do salt.log.debug('salt/hypervisor/map.jinja: Processing potential destroyed VM status file: ' ~ status_file) %}
+      {% import_json status_file as status_data %}
+      
+      {# Only process files with "Destroyed Instance" status #}
+      {% if status_data and status_data.status == 'Destroyed Instance' %}
+        {% do salt.log.debug('salt/hypervisor/map.jinja: Found VM with Destroyed Instance status: ' ~ vm_name) %}
+        
+        {# Add to vms with minimal config #}
+        {% do vms.update({
+          vm_name: {
+            'status': status_data,
+            'config': {}
+          }
+        }) %}
+      {% endif %}
+    {% endfor %}
+    
+    {# Merge node config with model capabilities and VM states #}
+    {% do HYPERVISORS[role].update({
+      hypervisor: {
+        'config': config,
+        'model': model,
+        'hardware': model_config.get('hardware', {}),
+        'vms': vms
+      }
+    }) %}
+  {% endfor %}
+{% endfor %}
+
+{% else %}
+
+{% do salt.log.error(
+  'Hypervisor nodes are a feature supported only for customers with a valid license.'
+  'Contact Security Onion Solutions, LLC via our website at https://securityonionsolutions.com'
+  'for more information about purchasing a license to enable this feature.'
+) %}
+
+{% endif %}

salt/hypervisor/tools/sbin/so-nvme-raid1.sh

@@ -0,0 +1,335 @@
+#!/bin/bash
+
+#################################################################
+# RAID-1 Setup Script for NVMe Drives
+#################################################################
+# 
+# DESCRIPTION:
+# This script automatically sets up a RAID-1 (mirrored) array using two NVMe drives
+# (/dev/nvme0n1 and /dev/nvme1n1) and mounts it at /nsm with XFS filesystem.
+#
+# FUNCTIONALITY:
+# - Detects and reports existing RAID configurations
+# - Thoroughly cleans target drives of any existing data/configurations
+# - Creates GPT partition tables with RAID-type partitions
+# - Establishes RAID-1 array (${RAID_DEVICE}) for data redundancy
+# - Formats the array with XFS filesystem for performance
+# - Automatically mounts at /nsm and configures for boot persistence
+# - Provides monitoring information for resync operations
+#
+# SAFETY FEATURES:
+# - Requires root privileges
+# - Exits gracefully if RAID already exists and is mounted
+# - Performs comprehensive cleanup to avoid conflicts
+# - Forces partition table updates and waits for system recognition
+#
+# PREREQUISITES:
+# - Two NVMe drives: /dev/nvme0n1 and /dev/nvme1n1
+# - Root access
+# - mdadm, sgdisk, and standard Linux utilities
+#
+# WARNING: This script will DESTROY all data on the target drives!
+#
+# USAGE: sudo ./so-nvme-raid1.sh
+#
+#################################################################
+
+# Exit on any error
+set -e
+
+# Configuration variables
+RAID_ARRAY_NAME="md0"
+RAID_DEVICE="/dev/${RAID_ARRAY_NAME}"
+MOUNT_POINT="/nsm"
+
+# Function to log messages
+log() {
+    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1"
+}
+
+# Function to check if running as root
+check_root() {
+    if [ "$EUID" -ne 0 ]; then
+        log "Error: Please run as root"
+        exit 1
+    fi
+}
+
+# Function to find MD arrays using specific devices
+find_md_arrays_using_devices() {
+    local target_devices=("$@")
+    local found_arrays=()
+    
+    # Parse /proc/mdstat to find arrays using our target devices
+    if [ -f "/proc/mdstat" ]; then
+        while IFS= read -r line; do
+            if [[ $line =~ ^(md[0-9]+) ]]; then
+                local array_name="${BASH_REMATCH[1]}"
+                local array_path="/dev/$array_name"
+                
+                # Check if this array uses any of our target devices
+                for device in "${target_devices[@]}"; do
+                    if echo "$line" | grep -q "${device##*/}"; then
+                        found_arrays+=("$array_path")
+                        break
+                    fi
+                done
+            fi
+        done < /proc/mdstat
+    fi
+    
+    printf '%s\n' "${found_arrays[@]}"
+}
+
+# Function to check if RAID is already set up
+check_existing_raid() {
+    local target_devices=("/dev/nvme0n1p1" "/dev/nvme1n1p1")
+    local found_arrays=($(find_md_arrays_using_devices "${target_devices[@]}"))
+    
+    # Check if we found any arrays using our target devices
+    if [ ${#found_arrays[@]} -gt 0 ]; then
+        for array_path in "${found_arrays[@]}"; do
+            if mdadm --detail "$array_path" &>/dev/null; then
+                local raid_state=$(mdadm --detail "$array_path" | grep "State" | awk '{print $3}')
+                local mount_point="/nsm"
+                
+                log "Found existing RAID array $array_path (State: $raid_state)"
+                
+                # Check what's currently mounted at /nsm
+                local current_mount=$(findmnt -n -o SOURCE "$mount_point" 2>/dev/null || echo "")
+                
+                if [ -n "$current_mount" ]; then
+                    if [ "$current_mount" = "$array_path" ]; then
+                        log "RAID array $array_path is already correctly mounted at $mount_point"
+                        log "Current RAID details:"
+                        mdadm --detail "$array_path"
+                        
+                        # Check if resyncing
+                        if grep -q "resync" /proc/mdstat; then
+                            log "RAID is currently resyncing:"
+                            grep resync /proc/mdstat
+                            log "You can monitor progress with: watch -n 60 cat /proc/mdstat"
+                        else
+                            log "RAID is fully synced and operational"
+                        fi
+                        
+                        # Show disk usage
+                        log "Current disk usage:"
+                        df -h "$mount_point"
+                        
+                        exit 0
+                    else
+                        log "Found $mount_point mounted on $current_mount, but RAID array $array_path exists"
+                        log "Will unmount current filesystem and remount on RAID array"
+                        
+                        # Unmount current filesystem
+                        log "Unmounting $mount_point"
+                        umount "$mount_point"
+                        
+                        # Remove old fstab entry
+                        log "Removing old fstab entry for $current_mount"
+                        sed -i "\|$current_mount|d" /etc/fstab
+                        
+                        # Mount the RAID array
+                        log "Mounting RAID array $array_path at $mount_point"
+                        mount "$array_path" "$mount_point"
+                        
+                        # Update fstab
+                        log "Updating fstab for RAID array"
+                        sed -i "\|${array_path}|d" /etc/fstab
+                        echo "${array_path}  ${mount_point}  xfs  defaults,nofail  0  0" >> /etc/fstab
+                        
+                        log "RAID array is now mounted at $mount_point"
+                        log "Current RAID details:"
+                        mdadm --detail "$array_path"
+                        
+                        # Check if resyncing
+                        if grep -q "resync" /proc/mdstat; then
+                            log "RAID is currently resyncing:"
+                            grep resync /proc/mdstat
+                            log "You can monitor progress with: watch -n 60 cat /proc/mdstat"
+                        else
+                            log "RAID is fully synced and operational"
+                        fi
+                        
+                        # Show disk usage
+                        log "Current disk usage:"
+                        df -h "$mount_point"
+                        
+                        exit 0
+                    fi
+                else
+                    # /nsm not mounted, mount the RAID array
+                    log "Mounting RAID array $array_path at $mount_point"
+                    mount "$array_path" "$mount_point"
+                    
+                    # Update fstab
+                    log "Updating fstab for RAID array"
+                    sed -i "\|${array_path}|d" /etc/fstab
+                    echo "${array_path}  ${mount_point}  xfs  defaults,nofail  0  0" >> /etc/fstab
+                    
+                    log "RAID array is now mounted at $mount_point"
+                    log "Current RAID details:"
+                    mdadm --detail "$array_path"
+                    
+                    # Show disk usage
+                    log "Current disk usage:"
+                    df -h "$mount_point"
+                    
+                    exit 0
+                fi
+            fi
+        done
+    fi
+    
+    # Check if any of the target devices are in use
+    for device in "/dev/nvme0n1" "/dev/nvme1n1"; do        
+        if mdadm --examine "$device" &>/dev/null || mdadm --examine "${device}p1" &>/dev/null; then
+            # Find the actual array name for this device
+            local device_arrays=($(find_md_arrays_using_devices "${device}p1"))
+            local array_name=""
+            
+            if [ ${#device_arrays[@]} -gt 0 ]; then
+                array_name="${device_arrays[0]}"
+            else
+                # Fallback: try to find array name from /proc/mdstat
+                local partition_name="${device##*/}p1"
+                array_name=$(grep -l "$partition_name" /proc/mdstat 2>/dev/null | head -1)
+                if [ -n "$array_name" ]; then
+                    array_name=$(grep "^md[0-9]" /proc/mdstat | grep "$partition_name" | awk '{print "/dev/" $1}' | head -1)
+                fi
+                # Final fallback
+                if [ -z "$array_name" ]; then
+                    array_name="$RAID_DEVICE"
+                fi
+            fi
+            
+            log "Error: $device appears to be part of an existing RAID array"
+            log "To reuse this device, you must first:"
+            log "1. Unmount any filesystems"
+            log "2. Stop the RAID array: mdadm --stop $array_name"
+            log "3. Zero the superblock: mdadm --zero-superblock ${device}p1"
+            exit 1
+        fi
+    done
+}
+
+# Function to ensure devices are not in use
+ensure_devices_free() {
+    local device=$1
+    
+    log "Cleaning up device $device"
+    
+    # Kill any processes using the device
+    fuser -k "${device}"* 2>/dev/null || true
+    
+    # Force unmount any partitions
+    for part in "${device}"*; do
+        if mount | grep -q "$part"; then
+            umount -f "$part" 2>/dev/null || true
+        fi
+    done
+    
+    # Stop any MD arrays using this device
+    for md in $(ls /dev/md* 2>/dev/null || true); do
+        if mdadm --detail "$md" 2>/dev/null | grep -q "$device"; then
+            mdadm --stop "$md" 2>/dev/null || true
+        fi
+    done
+    
+    # Clear MD superblock
+    mdadm --zero-superblock "${device}"* 2>/dev/null || true
+    
+    # Remove LVM PV if exists
+    pvremove -ff -y "$device" 2>/dev/null || true
+    
+    # Clear all signatures
+    wipefs -af "$device" 2>/dev/null || true
+    
+    # Delete partition table
+    dd if=/dev/zero of="$device" bs=512 count=2048 2>/dev/null || true
+    dd if=/dev/zero of="$device" bs=512 seek=$(( $(blockdev --getsz "$device") - 2048 )) count=2048 2>/dev/null || true
+    
+    # Force kernel to reread
+    blockdev --rereadpt "$device" 2>/dev/null || true
+    partprobe -s "$device" 2>/dev/null || true
+    sleep 2
+}
+
+# Main script
+main() {
+    log "Starting RAID setup script"
+    
+    # Check if running as root
+    check_root
+    
+    # Check for existing RAID setup
+    check_existing_raid
+    
+    # Clean up any existing MD arrays
+    log "Cleaning up existing MD arrays"
+    mdadm --stop --scan 2>/dev/null || true
+    
+    # Clear mdadm configuration
+    log "Clearing mdadm configuration"
+    echo "DEVICE partitions" > /etc/mdadm.conf
+    
+    # Clean and prepare devices
+    for device in "/dev/nvme0n1" "/dev/nvme1n1"; do
+        ensure_devices_free "$device"
+        
+        log "Creating new partition table on $device"
+        sgdisk -Z "$device"
+        sgdisk -o "$device"
+        
+        log "Creating RAID partition"
+        sgdisk -n 1:0:0 -t 1:fd00 "$device"
+        
+        partprobe "$device"
+        udevadm settle
+        sleep 5
+    done
+    
+    log "Final verification of partition availability"
+    if ! [ -b "/dev/nvme0n1p1" ] || ! [ -b "/dev/nvme1n1p1" ]; then
+        log "Error: Partitions not available after creation"
+        exit 1
+    fi
+    
+    log "Creating RAID array"
+    mdadm --create "$RAID_DEVICE" --level=1 --raid-devices=2 \
+          --metadata=1.2 \
+          /dev/nvme0n1p1 /dev/nvme1n1p1 \
+          --force --run
+    
+    log "Creating XFS filesystem"
+    mkfs.xfs -f "$RAID_DEVICE"
+    
+    log "Creating mount point"
+    mkdir -p /nsm
+    
+    log "Updating fstab"
+    sed -i "\|${RAID_DEVICE}|d" /etc/fstab
+    echo "${RAID_DEVICE}  ${MOUNT_POINT}  xfs  defaults,nofail  0  0" >> /etc/fstab
+    
+    log "Reloading systemd daemon"
+    systemctl daemon-reload
+    
+    log "Mounting filesystem"
+    mount -a
+    
+    log "Saving RAID configuration"
+    mdadm --detail --scan > /etc/mdadm.conf
+    
+    log "RAID setup complete"
+    log "RAID array details:"
+    mdadm --detail "$RAID_DEVICE"
+    
+    if grep -q "resync" /proc/mdstat; then
+        log "RAID is currently resyncing. You can monitor progress with:"
+        log "watch -n 60 cat /proc/mdstat"
+    fi
+}
+
+# Run main function
+main "$@"

salt/hypervisor/tools/sbin/so-qcow2-network-predictable

@@ -0,0 +1,424 @@
+#!/usr/bin/python3
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+"""
+Script for configuring network interface predictability in Security Onion VMs.
+This script modifies the necessary files to ensure consistent network interface naming.
+
+The script performs the following operations:
+1. Modifies the BLS entry to set net.ifnames=1
+2. Removes any existing persistent network rules
+3. Updates GRUB configuration
+
+**Usage:**
+    so-qcow2-network-predictable -n <domain_name> [-I <qcow2_image_path>]
+
+**Options:**
+    -n, --name          Domain name of the VM to configure
+    -I, --image         (Optional) Path to the QCOW2 image. If not provided,
+                        defaults to /nsm/libvirt/images/<domain_name>/<domain_name>.qcow2
+
+**Examples:**
+
+1. **Configure using domain name:**
+    ```bash
+    so-qcow2-network-predictable -n sool9
+    ```
+    This command will:
+    - Use default image path: /nsm/libvirt/images/sool9/sool9.qcow2
+    - Configure network interface predictability
+
+2. **Configure using custom image path:**
+    ```bash
+    so-qcow2-network-predictable -n sool9 -I /path/to/custom/image.qcow2
+    ```
+    This command will:
+    - Use the specified image path
+    - Configure network interface predictability
+
+**Notes:**
+- The VM must not be running when executing this script
+- Requires root privileges
+- Will automatically find and modify the appropriate BLS entry
+- Removes /etc/udev/rules.d/70-persistent-net.rules if it exists
+- Updates GRUB configuration after changes
+
+**Exit Codes:**
+- 0: Success
+- 1: General error (invalid arguments, file operations, etc.)
+- 2: VM is running
+- 3: Required files not found
+- 4: Permission denied
+
+**Logging:**
+- Logs are written to /opt/so/log/hypervisor/so-qcow2-network-predictable.log
+- Both file and console logging are enabled
+- Log entries include:
+  - Timestamps
+  - Operation details
+  - Error messages
+  - Configuration changes
+"""
+
+import argparse
+import guestfs
+import glob
+import libvirt
+import logging
+import os
+import re
+import sys
+from so_logging_utils import setup_logging
+
+# Set up logging
+logger = setup_logging(
+    logger_name='so-qcow2-network-predictable',
+    log_file_path='/opt/so/log/hypervisor/so-qcow2-network-predictable.log',
+    log_level=logging.INFO,
+    format_str='%(asctime)s - %(levelname)s - %(message)s'
+)
+
+def check_domain_status(domain_name):
+    """
+    Check if the specified domain exists and is not running.
+    
+    Args:
+        domain_name (str): Name of the libvirt domain to check
+        
+    Returns:
+        bool: True if domain exists and is not running, False otherwise
+        
+    Raises:
+        RuntimeError: If domain is running or connection to libvirt fails
+    """
+    try:
+        conn = libvirt.open('qemu:///system')
+        try:
+            dom = conn.lookupByName(domain_name)
+            is_running = dom.isActive()
+            if is_running:
+                logger.error(f"Domain '{domain_name}' is running - cannot modify configuration")
+                raise RuntimeError(f"Domain '{domain_name}' must not be running")
+            logger.info(f"Domain '{domain_name}' exists and is not running")
+            return True
+        except libvirt.libvirtError as e:
+            if "no domain with matching name" in str(e):
+                logger.error(f"Domain '{domain_name}' not found")
+                raise RuntimeError(f"Domain '{domain_name}' not found")
+            raise
+        finally:
+            conn.close()
+    except libvirt.libvirtError as e:
+        logger.error(f"Failed to connect to libvirt: {e}")
+        raise RuntimeError(f"Failed to connect to libvirt: {e}")
+
+def modify_bls_entry(g):
+    """
+    Find and modify the BLS entry to set net.ifnames=1.
+    
+    Args:
+        g: Mounted guestfs handle
+        
+    Returns:
+        bool: True if successful, False if no changes needed
+        
+    Raises:
+        RuntimeError: If BLS entry cannot be found or modified
+    """
+    bls_dir = "/boot/loader/entries"
+    logger.info(f"Checking BLS directory: {bls_dir}")
+    if g.is_dir(bls_dir):
+        logger.info("BLS directory exists")
+    else:
+        logger.info("Listing /boot contents:")
+        try:
+            boot_contents = g.ls("/boot")
+            logger.info(f"/boot contains: {boot_contents}")
+            if g.is_dir("/boot/loader"):
+                logger.info("Listing /boot/loader contents:")
+                loader_contents = g.ls("/boot/loader")
+                logger.info(f"/boot/loader contains: {loader_contents}")
+        except Exception as e:
+            logger.error(f"Error listing /boot contents: {e}")
+        raise RuntimeError(f"BLS directory not found: {bls_dir}")
+
+    # Find BLS entry file
+    entries = g.glob_expand(f"{bls_dir}/*.conf")
+    logger.info(f"Found BLS entries: {entries}")
+    if not entries:
+        logger.error("No BLS entry files found")
+        raise RuntimeError("No BLS entry files found")
+
+    # Use the first entry found
+    bls_file = entries[0]
+    logger.info(f"Found BLS entry file: {bls_file}")
+
+    try:
+        logger.info(f"Reading BLS file contents from: {bls_file}")
+        content = g.read_file(bls_file).decode('utf-8')
+        logger.info("Current BLS file content:")
+        logger.info("---BEGIN BLS CONTENT---")
+        logger.info(content)
+        logger.info("---END BLS CONTENT---")
+
+        lines = content.splitlines()
+        modified = False
+
+        for i, line in enumerate(lines):
+            if line.startswith('options '):
+                logger.info(f"Found options line: {line}")
+                
+                # First remove any existing net.ifnames parameters (both =0 and =1)
+                new_line = re.sub(r'\s*net\.ifnames=[01]\s*', ' ', line)
+                # Also remove any quoted versions
+                new_line = re.sub(r'\s*"net\.ifnames=[01]"\s*', ' ', new_line)
+                # Clean up multiple spaces
+                new_line = re.sub(r'\s+', ' ', new_line).strip()
+                
+                # Now add net.ifnames=1 at the end
+                new_line = f"{new_line} net.ifnames=1"
+                
+                if new_line != line:
+                    lines[i] = new_line
+                    modified = True
+                    logger.info(f"Updated options line. New line: {new_line}")
+                break
+
+        if modified:
+            new_content = '\n'.join(lines) + '\n'
+            logger.info("New BLS file content:")
+            logger.info("---BEGIN NEW BLS CONTENT---")
+            logger.info(new_content)
+            logger.info("---END NEW BLS CONTENT---")
+            g.write(bls_file, new_content.encode('utf-8'))
+            logger.info("Successfully updated BLS entry")
+            return True
+        
+        logger.info("No changes needed for BLS entry")
+        return False
+
+    except Exception as e:
+        logger.error(f"Failed to modify BLS entry: {e}")
+        raise RuntimeError(f"Failed to modify BLS entry: {e}")
+
+def remove_persistent_net_rules(g):
+    """
+    Remove the persistent network rules file if it exists.
+    
+    Args:
+        g: Mounted guestfs handle
+        
+    Returns:
+        bool: True if file was removed, False if it didn't exist
+    """
+    rules_file = "/etc/udev/rules.d/70-persistent-net.rules"
+    logger.info(f"Checking for persistent network rules file: {rules_file}")
+    try:
+        if g.is_file(rules_file):
+            logger.info("Found persistent network rules file, removing...")
+            g.rm(rules_file)
+            logger.info(f"Successfully removed persistent network rules file: {rules_file}")
+            return True
+        logger.info("No persistent network rules file found")
+        return False
+    except Exception as e:
+        logger.error(f"Failed to remove persistent network rules: {e}")
+        raise RuntimeError(f"Failed to remove persistent network rules: {e}")
+
+def update_grub_config(g):
+    """
+    Update GRUB configuration.
+    
+    Args:
+        g: Mounted guestfs handle
+        
+    Raises:
+        RuntimeError: If GRUB update fails
+    """
+    try:
+        # First, read the current grubenv to get the existing kernelopts
+        logger.info("Reading current grubenv...")
+        grubenv_content = g.read_file('/boot/grub2/grubenv').decode('utf-8')
+        logger.info("Current grubenv content:")
+        logger.info(grubenv_content)
+        
+        # Extract current kernelopts
+        kernelopts_match = re.search(r'^kernelopts="([^"]+)"', grubenv_content, re.MULTILINE)
+        if kernelopts_match:
+            current_kernelopts = kernelopts_match.group(1)
+            logger.info(f"Current kernelopts: {current_kernelopts}")
+            
+            # Remove any existing net.ifnames parameters
+            new_kernelopts = re.sub(r'\s*net\.ifnames=[01]\s*', ' ', current_kernelopts)
+            # Clean up multiple spaces
+            new_kernelopts = re.sub(r'\s+', ' ', new_kernelopts).strip()
+            # Add net.ifnames=1
+            new_kernelopts = f"{new_kernelopts} net.ifnames=1"
+            
+            logger.info(f"New kernelopts: {new_kernelopts}")
+            
+            # Update grubenv with the new kernelopts
+            logger.info("Setting kernelopts with net.ifnames=1...")
+            output_editenv = g.command(['grub2-editenv', '-', 'set', f'kernelopts={new_kernelopts}'])
+            logger.info("grub2-editenv output:")
+            logger.info(output_editenv)
+        else:
+            # If we can't find existing kernelopts, use the default
+            logger.warning("Could not find existing kernelopts, using default")
+            output_editenv = g.command(['grub2-editenv', '-', 'set', 'kernelopts=console=tty0 no_timer_check biosdevname=0 resume=/dev/mapper/vg_main-lv_swap rd.lvm.lv=vg_main/lv_root rd.lvm.lv=vg_main/lv_swap net.ifnames=1 crashkernel=1G-64G:448M,64G-:512M'])
+            logger.info("grub2-editenv output:")
+            logger.info(output_editenv)
+
+        logger.info("Updating grubby with net.ifnames=1...")
+        # First remove any existing net.ifnames arguments
+        output_grubby_remove = g.command(['grubby', '--update-kernel=ALL', '--remove-args=net.ifnames=0 net.ifnames=1'])
+        logger.info("grubby remove output:")
+        logger.info(output_grubby_remove)
+        
+        # Then add net.ifnames=1
+        output_grubby_add = g.command(['grubby', '--update-kernel=ALL', '--args=net.ifnames=1'])
+        logger.info("grubby add output:")
+        logger.info(output_grubby_add)
+
+        logger.info("Updating GRUB configuration...")
+        output_mkconfig = g.command(['grub2-mkconfig', '-o', '/boot/grub2/grub.cfg'])
+        logger.info("GRUB update output:")
+        logger.info(output_mkconfig)
+        logger.info("Successfully updated GRUB configuration")
+    except Exception as e:
+        logger.error(f"Failed to update GRUB configuration: {e}")
+        raise RuntimeError(f"Failed to update GRUB configuration: {e}")
+
+def configure_network_predictability(domain_name, image_path=None):
+    """
+    Configure network interface predictability for a VM.
+    
+    Args:
+        domain_name (str): Name of the domain to configure
+        image_path (str, optional): Path to the QCOW2 image
+        
+    Raises:
+        RuntimeError: If configuration fails
+    """
+    # Check domain status
+    check_domain_status(domain_name)
+
+    # Use default image path if none provided
+    if not image_path:
+        image_path = f"/nsm/libvirt/images/{domain_name}/{domain_name}.qcow2"
+
+    if not os.path.exists(image_path):
+        logger.error(f"Image file not found: {image_path}")
+        raise RuntimeError(f"Image file not found: {image_path}")
+
+    if not os.access(image_path, os.R_OK | os.W_OK):
+        logger.error(f"Permission denied: Cannot access image file {image_path}")
+        raise RuntimeError(f"Permission denied: Cannot access image file {image_path}")
+
+    logger.info(f"Configuring network predictability for domain: {domain_name}")
+    logger.info(f"Using image: {image_path}")
+
+    g = guestfs.GuestFS(python_return_dict=True)
+    try:
+        logger.info("Initializing guestfs...")
+        g.set_network(False)
+        g.selinux = False
+        g.add_drive_opts(image_path, format="qcow2")
+        g.launch()
+
+        logger.info("Inspecting operating system...")
+        roots = g.inspect_os()
+        if not roots:
+            raise RuntimeError("No operating system found in image")
+
+        root = roots[0]
+        logger.info(f"Found root filesystem: {root}")
+        logger.info(f"Operating system type: {g.inspect_get_type(root)}")
+        logger.info(f"Operating system distro: {g.inspect_get_distro(root)}")
+        logger.info(f"Operating system major version: {g.inspect_get_major_version(root)}")
+        logger.info(f"Operating system minor version: {g.inspect_get_minor_version(root)}")
+
+        logger.info("Getting mount points...")
+        mountpoints = g.inspect_get_mountpoints(root)
+        logger.info(f"Found mount points: {mountpoints}")
+        logger.info("Converting mount points to sortable list...")
+        # Convert dictionary to list of tuples
+        mountpoints = list(mountpoints.items())
+        logger.info(f"Converted mount points: {mountpoints}")
+        logger.info("Sorting mount points by path length for proper mount order...")
+        mountpoints.sort(key=lambda m: len(m[0]))
+        logger.info(f"Mount order will be: {[mp[0] for mp in mountpoints]}")
+
+        for mp_path, mp_device in mountpoints:
+            try:
+                logger.info(f"Attempting to mount {mp_device} at {mp_path}")
+                g.mount(mp_device, mp_path)
+                logger.info(f"Successfully mounted {mp_device} at {mp_path}")
+            except Exception as e:
+                logger.warning(f"Could not mount {mp_device} at {mp_path}: {str(e)}")
+                # Continue with other mounts
+
+        # Perform configuration steps
+        bls_modified = modify_bls_entry(g)
+        rules_removed = remove_persistent_net_rules(g)
+        
+        if bls_modified or rules_removed:
+            update_grub_config(g)
+            logger.info("Network predictability configuration completed successfully")
+        else:
+            logger.info("No changes were necessary")
+
+    except Exception as e:
+        raise RuntimeError(f"Failed to configure network predictability: {e}")
+    finally:
+        try:
+            logger.info("Unmounting all filesystems...")
+            g.umount_all()
+            logger.info("Successfully unmounted all filesystems")
+        except Exception as e:
+            logger.warning(f"Error unmounting filesystems: {e}")
+        g.close()
+
+def parse_arguments():
+    """Parse command line arguments."""
+    parser = argparse.ArgumentParser(
+        description="Configure network interface predictability for Security Onion VMs"
+    )
+    parser.add_argument("-n", "--name", required=True,
+                      help="Domain name of the VM to configure")
+    parser.add_argument("-I", "--image",
+                      help="Path to the QCOW2 image (optional)")
+    return parser.parse_args()
+
+def main():
+    """Main entry point for the script."""
+    try:
+        args = parse_arguments()
+        configure_network_predictability(args.name, args.image)
+        sys.exit(0)
+    except RuntimeError as e:
+        if "must not be running" in str(e):
+            logger.error(str(e))
+            sys.exit(2)
+        elif "not found" in str(e):
+            logger.error(str(e))
+            sys.exit(3)
+        elif "Permission denied" in str(e):
+            logger.error(str(e))
+            sys.exit(4)
+        else:
+            logger.error(str(e))
+            sys.exit(1)
+    except KeyboardInterrupt:
+        logger.error("Operation cancelled by user")
+        sys.exit(1)
+    except Exception as e:
+        logger.error(f"Unexpected error: {e}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

salt/hypervisor/tools/sbin/so-wait-cloud-init

@@ -0,0 +1,206 @@
+#!/usr/bin/python3
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+"""
+Script for waiting for cloud-init to complete on a Security Onion VM.
+Monitors VM state to ensure proper cloud-init initialization and shutdown.
+
+**Usage:**
+    so-wait-cloud-init -n <domain_name>
+
+**Options:**
+    -n, --name          Domain name of the VM to monitor
+
+**Exit Codes:**
+- 0: Success (cloud-init completed and VM shutdown)
+- 1: General error
+- 2: VM never started
+- 3: VM stopped too quickly
+- 4: VM failed to shutdown
+
+**Description:**
+This script monitors a VM's state to ensure proper cloud-init initialization and completion:
+1. Waits for VM to start running
+2. Verifies VM remains running (not an immediate crash)
+3. Waits for VM to shutdown (indicating cloud-init completion)
+4. Verifies VM remains shutdown
+
+The script is typically used in the libvirt.images state after creating a new VM
+to ensure cloud-init completes its initialization before proceeding with further
+configuration.
+
+**Logging:**
+- Logs are written to /opt/so/log/hypervisor/so-wait-cloud-init.log
+- Both file and console logging are enabled
+- Log entries include:
+  - Timestamps
+  - State changes
+  - Error conditions
+  - Verification steps
+"""
+
+import argparse
+import logging
+import subprocess
+import sys
+import time
+from so_logging_utils import setup_logging
+
+# Set up logging
+logger = setup_logging(
+    logger_name='so-wait-cloud-init',
+    log_file_path='/opt/so/log/hypervisor/so-wait-cloud-init.log',
+    log_level=logging.INFO,
+    format_str='%(asctime)s - %(levelname)s - %(message)s'
+)
+
+def check_vm_running(domain_name):
+    """
+    Check if VM is in running state.
+    
+    Args:
+        domain_name (str): Name of the domain to check
+        
+    Returns:
+        bool: True if VM is running, False otherwise
+    """
+    try:
+        result = subprocess.run(['virsh', 'list', '--state-running', '--name'],
+                              capture_output=True, text=True, check=True)
+        return domain_name in result.stdout.splitlines()
+    except subprocess.CalledProcessError as e:
+        logger.error(f"Failed to check VM state: {e}")
+        return False
+
+def wait_for_vm_start(domain_name, timeout=300):
+    """
+    Wait for VM to start running.
+    
+    Args:
+        domain_name (str): Name of the domain to monitor
+        timeout (int): Maximum time to wait in seconds
+        
+    Returns:
+        bool: True if VM started, False if timeout occurred
+    """
+    logger.info(f"Waiting for VM {domain_name} to start...")
+    start_time = time.time()
+    
+    while time.time() - start_time < timeout:
+        if check_vm_running(domain_name):
+            logger.info("VM is running")
+            return True
+        time.sleep(1)
+    
+    logger.error(f"Timeout waiting for VM {domain_name} to start")
+    return False
+
+def verify_vm_running(domain_name):
+    """
+    Verify VM remains running after initial start.
+    
+    Args:
+        domain_name (str): Name of the domain to verify
+        
+    Returns:
+        bool: True if VM is still running after verification period
+    """
+    logger.info("Verifying VM remains running...")
+    time.sleep(5)  # Wait to ensure VM is stable
+    
+    if not check_vm_running(domain_name):
+        logger.error("VM stopped too quickly after starting")
+        return False
+    
+    logger.info("VM verified running")
+    return True
+
+def wait_for_vm_shutdown(domain_name, timeout=600):
+    """
+    Wait for VM to shutdown.
+    
+    Args:
+        domain_name (str): Name of the domain to monitor
+        timeout (int): Maximum time to wait in seconds
+        
+    Returns:
+        bool: True if VM shutdown, False if timeout occurred
+    """
+    logger.info("Waiting for cloud-init to complete and VM to shutdown...")
+    start_time = time.time()
+    check_count = 0
+    
+    while time.time() - start_time < timeout:
+        if not check_vm_running(domain_name):
+            logger.info("VM has shutdown")
+            return True
+            
+        # Log status every minute (after 12 checks at 5 second intervals)
+        check_count += 1
+        if check_count % 12 == 0:
+            elapsed = int(time.time() - start_time)
+            logger.info(f"Still waiting for cloud-init... ({elapsed} seconds elapsed)")
+        
+        time.sleep(5)
+    
+    logger.error(f"Timeout waiting for VM {domain_name} to shutdown")
+    return False
+
+def verify_vm_shutdown(domain_name):
+    """
+    Verify VM remains shutdown.
+    
+    Args:
+        domain_name (str): Name of the domain to verify
+        
+    Returns:
+        bool: True if VM remains shutdown after verification period
+    """
+    logger.info("Verifying VM remains shutdown...")
+    time.sleep(5)  # Wait to ensure VM state is stable
+    
+    if check_vm_running(domain_name):
+        logger.error("VM is still running after shutdown check")
+        return False
+    
+    logger.info("VM verified shutdown")
+    return True
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Wait for cloud-init to complete on a Security Onion VM"
+    )
+    parser.add_argument("-n", "--name", required=True,
+                      help="Domain name of the VM to monitor")
+    args = parser.parse_args()
+
+    try:
+        # Wait for VM to start
+        if not wait_for_vm_start(args.name):
+            sys.exit(2)  # VM never started
+        
+        # Verify VM remains running
+        if not verify_vm_running(args.name):
+            sys.exit(3)  # VM stopped too quickly
+        
+        # Wait for VM to shutdown
+        if not wait_for_vm_shutdown(args.name):
+            sys.exit(4)  # VM failed to shutdown
+        
+        # Verify VM remains shutdown
+        if not verify_vm_shutdown(args.name):
+            sys.exit(4)  # VM failed to stay shutdown
+        
+        logger.info("Cloud-init completed successfully")
+        sys.exit(0)
+        
+    except Exception as e:
+        logger.error(f"Unexpected error: {e}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

salt/hypervisor/tools/sbin/so_vm_utils.py

@@ -0,0 +1,60 @@
+#!/usr/bin/python3
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+import sys
+import time
+import libvirt
+import logging
+
+def stop_vm(conn, vm_name, logger):
+    """
+    Stops the specified virtual machine if it is running.
+
+    Parameters:
+        conn (libvirt.virConnect): The libvirt connection object.
+        vm_name (str): The name of the virtual machine.
+        logger (logging.Logger): The logger object.
+
+    Returns:
+        libvirt.virDomain: The domain object of the VM.
+
+    Raises:
+        SystemExit: If the VM cannot be found or an error occurs.
+    """
+    try:
+        dom = conn.lookupByName(vm_name)
+        if dom.isActive():
+            logger.info(f"Shutting down VM '{vm_name}'...")
+            dom.shutdown()
+            # Wait for the VM to shut down
+            while dom.isActive():
+                time.sleep(1)
+            logger.info(f"VM '{vm_name}' has been stopped.")
+        else:
+            logger.info(f"VM '{vm_name}' is already stopped.")
+        return dom
+    except libvirt.libvirtError as e:
+        logger.error(f"Failed to stop VM '{vm_name}': {e}")
+        sys.exit(1)
+
+def start_vm(dom, logger):
+    """
+    Starts the specified virtual machine.
+
+    Parameters:
+        dom (libvirt.virDomain): The domain object of the VM.
+        logger (logging.Logger): The logger object.
+
+    Raises:
+        SystemExit: If the VM cannot be started.
+    """
+    try:
+        dom.create()
+        logger.info(f"VM '{dom.name()}' started successfully.")
+    except libvirt.libvirtError as e:
+        logger.error(f"Failed to start VM '{dom.name()}': {e}")
+        sys.exit(1)

salt/hypervisor/tools/sbin_jinja/so-kvm-modify-hardware

@@ -0,0 +1,365 @@
+#!/usr/bin/python3
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+#
+# Note: Per the Elastic License 2.0, the second limitation states:
+#
+#   "You may not move, change, disable, or circumvent the license key functionality
+#    in the software, and you may not remove or obscure any functionality in the
+#    software that is protected by the license key."
+
+{% if 'vrt' in salt['pillar.get']('features', []) %}
+
+"""
+Script for managing hardware configurations of KVM virtual machines. This script provides
+functionality to modify CPU, memory, and PCI device settings without manual XML editing
+or direct libvirt interaction.
+
+The script offers three main configuration capabilities:
+1. CPU Management: Adjust virtual CPU count
+2. Memory Management: Modify memory allocation
+3. PCI Passthrough: Configure PCI device passthrough for direct hardware access
+
+This script is designed to work with Security Onion's virtualization infrastructure and is typically
+used during VM provisioning and hardware reconfiguration tasks.
+
+**Usage:**
+    so-kvm-modify-hardware -v <vm_name> [-c <cpu_count>] [-m <memory_amount>] [-p <pci_id>] [-p <pci_id> ...] [-s]
+
+**Options:**
+    -v, --vm          Name of the virtual machine to modify.
+    -c, --cpu         Number of virtual CPUs to assign.
+    -m, --memory      Amount of memory to assign in MiB.
+    -p, --pci         PCI hardware ID(s) to passthrough to the VM (e.g., 0000:00:1f.2). Can be specified multiple times.
+                      Format: domain:bus:device.function
+    -s, --start       Start the VM after modification.
+
+**Examples:**
+
+1. **Modify CPU and Memory with Multiple PCI Devices:**
+
+    ```bash
+    so-kvm-modify-hardware -v vm1_sensor -c 4 -m 8192 -p 0000:c7:00.0 -p 0000:c8:00.0 -s
+    ```
+
+    This command modifies a VM with the following settings:
+    - VM Name: `vm1_sensor`
+    - Hardware Configuration:
+        - CPUs: `4`
+        - Memory: `8192` MiB
+        - PCI Device Passthrough: `0000:c7:00.0`, `0000:c8:00.0`
+    - The VM is started after modification due to the `-s` flag
+
+2. **Add PCI Device Without Other Changes:**
+
+    ```bash
+    so-kvm-modify-hardware -v vm2_master -p 0000:c7:00.0
+    ```
+
+    This command adds a single PCI device passthrough to the VM:
+    - VM Name: `vm2_master`
+    - PCI Device: `0000:c7:00.0`
+    - Existing CPU and memory settings are preserved
+
+3. **Update Resource Allocation:**
+
+    ```bash
+    so-kvm-modify-hardware -v vm3_search -c 2 -m 4096
+    ```
+
+    This command updates only compute resources:
+    - VM Name: `vm3_search`
+    - CPUs: `2`
+    - Memory: `4096` MiB
+    - VM remains stopped after modification
+
+4. **Add Multiple PCI Devices:**
+
+    ```bash
+    so-kvm-modify-hardware -v vm4_node -p 0000:c7:00.0 -p 0000:c4:00.0 -p 0000:c4:00.1 -s
+    ```
+
+    This command adds multiple PCI devices and starts the VM:
+    - VM Name: `vm4_node`
+    - PCI Devices: `0000:c7:00.0`, `0000:c4:00.0`, `0000:c4:00.1`
+    - VM is started after modification
+
+**Notes:**
+
+- The script automatically stops the VM if it's running before making modifications.
+- At least one modification option (-c, -m, or -p) should be provided.
+- The PCI hardware IDs must be in the format `domain:bus:device.function` (e.g., `0000:c7:00.0`).
+- Multiple PCI devices can be added by using the `-p` option multiple times.
+- Without the `-s` flag, the VM remains stopped after modification.
+- Existing hardware configurations are preserved if not explicitly modified.
+
+**Description:**
+
+The `so-kvm-modify-hardware` script modifies hardware parameters of KVM virtual machines using the following process:
+
+1. **VM State Management:**
+   - Connects to the local libvirt daemon
+   - Stops the VM if it's currently running
+   - Retrieves current VM configuration
+
+2. **Hardware Configuration:**
+   - Modifies CPU count if specified
+   - Updates memory allocation if specified
+   - Adds PCI device passthrough configurations if specified
+   - All changes are made through libvirt XML configuration
+
+3. **VM Redefinition:**
+   - Applies the new configuration by redefining the VM
+   - Optionally starts the VM if requested
+   - Ensures clean shutdown and startup during modifications
+
+4. **Error Handling:**
+   - Validates all input parameters
+   - Ensures proper XML structure
+   - Provides detailed error messages for troubleshooting
+
+**Exit Codes:**
+
+- `0`: Success
+- `1`: An error occurred during execution
+
+**Logging:**
+
+- Logs are written to `/opt/so/log/hypervisor/so-kvm-modify-hardware.log`
+- Both file and console logging are enabled for real-time monitoring
+- Log entries include timestamps and severity levels
+- Detailed error messages are logged for troubleshooting
+"""
+
+import argparse
+import sys
+import libvirt
+import logging
+import socket
+import xml.etree.ElementTree as ET
+from io import StringIO
+from so_vm_utils import start_vm, stop_vm
+from so_logging_utils import setup_logging
+import subprocess
+
+# Get hypervisor name from local hostname
+HYPERVISOR = socket.gethostname()
+
+# Custom log handler to capture output
+class StringIOHandler(logging.Handler):
+    def __init__(self):
+        super().__init__()
+        self.strio = StringIO()
+
+    def emit(self, record):
+        msg = self.format(record)
+        self.strio.write(msg + '\n')
+
+    def get_value(self):
+        return self.strio.getvalue()
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description='Modify hardware parameters of a KVM virtual machine.')
+    parser.add_argument('-v', '--vm', required=True, help='Name of the virtual machine to modify.')
+    parser.add_argument('-c', '--cpu', type=int, help='Number of virtual CPUs to assign.')
+    parser.add_argument('-m', '--memory', type=int, help='Amount of memory to assign in MiB.')
+    parser.add_argument('-p', '--pci', action='append', help='PCI hardware ID(s) to passthrough to the VM (e.g., 0000:00:1f.2). Can be specified multiple times.')
+    parser.add_argument('-s', '--start', action='store_true', help='Start the VM after modification.')
+    args = parser.parse_args()
+    return args
+
+def modify_vm(dom, cpu_count, memory_amount, pci_ids, logger):
+    try:
+        # Get the XML description of the VM
+        xml_desc = dom.XMLDesc()
+        root = ET.fromstring(xml_desc)
+
+        # Modify CPU count
+        if cpu_count is not None:
+            vcpu_elem = root.find('./vcpu')
+            if vcpu_elem is not None:
+                vcpu_elem.text = str(cpu_count)
+                logger.info(f"Set CPU count to {cpu_count}.")
+            else:
+                logger.error("Could not find <vcpu> element in XML.")
+                sys.exit(1)
+
+        # Modify memory amount
+        if memory_amount is not None:
+            memory_elem = root.find('./memory')
+            current_memory_elem = root.find('./currentMemory')
+            if memory_elem is not None and current_memory_elem is not None:
+                memory_elem.text = str(memory_amount * 1024)  # Convert MiB to KiB
+                current_memory_elem.text = str(memory_amount * 1024)
+                logger.info(f"Set memory to {memory_amount} MiB.")
+            else:
+                logger.error("Could not find <memory> elements in XML.")
+                sys.exit(1)
+
+        # Add PCI device passthrough(s)
+        if pci_ids:
+            devices_elem = root.find('./devices')
+            if devices_elem is not None:
+                for pci_id in pci_ids:
+                    hostdev_elem = ET.SubElement(devices_elem, 'hostdev', attrib={
+                        'mode': 'subsystem',
+                        'type': 'pci',
+                        'managed': 'yes'
+                    })
+                    source_elem = ET.SubElement(hostdev_elem, 'source')
+                    # Split PCI ID into components (domain:bus:slot.function)
+                    parts = pci_id.split(':')
+                    if len(parts) != 3:
+                        logger.error(f"Invalid PCI ID format: {pci_id}. Expected format: domain:bus:slot.function")
+                        sys.exit(1)
+                    domain_id = parts[0]
+                    bus = parts[1]
+                    slot_func = parts[2].split('.')
+                    if len(slot_func) != 2:
+                        logger.error(f"Invalid PCI ID format: {pci_id}. Expected format: domain:bus:slot.function")
+                        sys.exit(1)
+                    slot = slot_func[0]
+                    function = slot_func[1]
+                    address_attrs = {
+                        'domain': f'0x{domain_id}',
+                        'bus': f'0x{bus}',
+                        'slot': f'0x{slot}',
+                        'function': f'0x{function}'
+                    }
+                    ET.SubElement(source_elem, 'address', attrib=address_attrs)
+                    logger.info(f"Added PCI device passthrough for {pci_id}.")
+            else:
+                logger.error("Could not find <devices> element in XML.")
+                sys.exit(1)
+
+        # Convert XML back to string
+        new_xml_desc = ET.tostring(root, encoding='unicode')
+        return new_xml_desc
+    except Exception as e:
+        logger.error(f"Failed to modify VM XML: {e}")
+        sys.exit(1)
+
+def redefine_vm(conn, new_xml_desc, logger):
+    try:
+        conn.defineXML(new_xml_desc)
+        logger.info("VM redefined with new hardware parameters.")
+    except libvirt.libvirtError as e:
+        logger.error(f"Failed to redefine VM: {e}")
+        sys.exit(1)
+
+def main():
+    # Set up logging using the so_logging_utils library
+    string_handler = StringIOHandler()
+    string_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
+    logger = setup_logging(
+        logger_name='so-kvm-modify-hardware',
+        log_file_path='/opt/so/log/hypervisor/so-kvm-modify-hardware.log',
+        log_level=logging.INFO,
+        format_str='%(asctime)s - %(levelname)s - %(message)s'
+    )
+    logger.addHandler(string_handler)
+
+    try:
+        args = parse_arguments()
+
+        vm_name = args.vm
+        cpu_count = args.cpu
+        memory_amount = args.memory
+        pci_ids = args.pci  # This will be a list or None
+        start_vm_flag = args.start
+
+        # Connect to libvirt
+        try:
+            conn = libvirt.open(None)
+        except libvirt.libvirtError as e:
+            logger.error(f"Failed to open connection to libvirt: {e}")
+            try:
+                subprocess.run([
+                    'so-salt-emit-vm-deployment-status-event',
+                    '-v', vm_name,
+                    '-H', HYPERVISOR,
+                    '-s', 'Hardware Configuration Failed'
+                ], check=True)
+            except subprocess.CalledProcessError as e:
+                logger.error(f"Failed to emit failure status event: {e}")
+            sys.exit(1)
+
+        # Stop VM if running
+        dom = stop_vm(conn, vm_name, logger)
+
+        # Modify VM XML
+        new_xml_desc = modify_vm(dom, cpu_count, memory_amount, pci_ids, logger)
+
+        # Redefine VM
+        redefine_vm(conn, new_xml_desc, logger)
+
+        # Start VM if -s or --start argument is provided
+        if start_vm_flag:
+            dom = conn.lookupByName(vm_name)
+            start_vm(dom, logger)
+            logger.info(f"VM '{vm_name}' started successfully.")
+        else:
+            logger.info("VM start flag not provided; VM will remain stopped.")
+
+        # Close connection
+        conn.close()
+
+        # Send success status event
+        try:
+            subprocess.run([
+                'so-salt-emit-vm-deployment-status-event',
+                '-v', vm_name,
+                '-H', HYPERVISOR,
+                '-s', 'Hardware Configuration'
+            ], check=True)
+        except subprocess.CalledProcessError as e:
+            logger.error(f"Failed to emit success status event: {e}")
+
+    except KeyboardInterrupt:
+        error_msg = "Operation cancelled by user"
+        logger.error(error_msg)
+        try:
+            subprocess.run([
+                'so-salt-emit-vm-deployment-status-event',
+                '-v', vm_name,
+                '-H', HYPERVISOR,
+                '-s', 'Hardware Configuration Failed'
+            ], check=True)
+        except subprocess.CalledProcessError as e:
+            logger.error(f"Failed to emit failure status event: {e}")
+        sys.exit(1)
+    except Exception as e:
+        error_msg = str(e)
+        if "Failed to open connection to libvirt" in error_msg:
+            error_msg = f"Failed to connect to libvirt: {error_msg}"
+        elif "Failed to redefine VM" in error_msg:
+            error_msg = f"Failed to apply hardware changes: {error_msg}"
+        elif "Failed to modify VM XML" in error_msg:
+            error_msg = f"Failed to update hardware configuration: {error_msg}"
+        else:
+            error_msg = f"An error occurred: {error_msg}"
+        logger.error(error_msg)
+        try:
+            subprocess.run([
+                'so-salt-emit-vm-deployment-status-event',
+                '-v', vm_name,
+                '-h', HYPERVISOR,
+                '-s', 'Hardware Configuration Failed'
+            ], check=True)
+        except subprocess.CalledProcessError as e:
+            logger.error(f"Failed to emit failure status event: {e}")
+        sys.exit(1)
+
+if __name__ == '__main__':
+    main()
+
+{%- else -%}
+
+echo "Hypervisor nodes are a feature supported only for customers with a valid license. \
+      Contact Security Onion Solutions, LLC via our website at https://securityonionsolutions.com \
+      for more information about purchasing a license to enable this feature."
+
+{% endif -%}

salt/hypervisor/tools/sbin_jinja/so-qcow2-modify-network

@@ -0,0 +1,531 @@
+#!/usr/bin/python3
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+#
+# Note: Per the Elastic License 2.0, the second limitation states:
+#
+#   "You may not move, change, disable, or circumvent the license key functionality
+#    in the software, and you may not remove or obscure any functionality in the
+#    software that is protected by the license key."
+
+{% if 'vrt' in salt['pillar.get']('features', []) -%}
+
+"""
+Script for modifying network configurations within QCOW2 virtual machine images. This script provides
+functionality to update NetworkManager settings, supporting both DHCP and static IP configurations
+without requiring the VM to be running.
+
+The script offers two main configuration modes:
+1. DHCP Configuration: Enable automatic IP address assignment
+2. Static IP Configuration: Set specific IP address, gateway, DNS servers, and search domains
+
+For both configuration modes, the script automatically sets the following NetworkManager connection properties:
+- connection.autoconnect: yes (ensures interface connects automatically)
+- connection.autoconnect-priority: 999 (sets connection priority)
+- connection.autoconnect-retries: -1 (unlimited connection retries)
+- connection.multi-connect: 0 (single connection mode)
+- connection.wait-device-timeout: -1 (wait indefinitely for device)
+
+This script is designed to work with Security Onion's virtualization infrastructure and is typically
+used during VM provisioning and network reconfiguration tasks.
+
+**Usage:**
+    so-qcow2-modify-network -I <qcow2_image_path> -i <interface> (--dhcp4 | --static4 --ip4 <ip_address> --gw4 <gateway>) 
+                           [--dns4 <dns_servers>] [--search4 <search_domain>]
+
+**Options:**
+    -I, --image          Path to the QCOW2 image.
+    -i, --interface      Network interface to modify (e.g., enp1s0).
+    --dhcp4              Configure interface for DHCP (IPv4).
+    --static4            Configure interface for static IPv4 settings.
+    --ip4                IPv4 address (e.g., 192.168.1.10/24). Required for static IPv4 configuration.
+    --gw4                IPv4 gateway (e.g., 192.168.1.1). Required for static IPv4 configuration.
+    --dns4               Comma-separated list of IPv4 DNS servers (e.g., 8.8.8.8,8.8.4.4).
+    --search4            DNS search domain for IPv4.
+
+**Examples:**
+
+1. **Static IP Configuration with DNS and Search Domain:**
+
+    ```bash
+    so-qcow2-modify-network -I /nsm/libvirt/images/sool9/sool9.qcow2 -i enp1s0 --static4 \
+        --ip4 192.168.1.10/24 --gw4 192.168.1.1 --dns4 192.168.1.1,192.168.1.2 --search4 example.local
+    ```
+
+    This command configures the network settings in the QCOW2 image with:
+    - Static IPv4 configuration:
+        - IP Address: `192.168.1.10/24`
+        - Gateway: `192.168.1.1`
+        - DNS Servers: `192.168.1.1`, `192.168.1.2`
+        - DNS Search Domain: `example.local`
+
+2. **DHCP Configuration:**
+
+    ```bash
+    so-qcow2-modify-network -I /nsm/libvirt/images/sool9/sool9.qcow2 -i enp1s0 --dhcp4
+    ```
+
+    This command configures the network interface to use DHCP for automatic IP address assignment.
+
+3. **Static IP Configuration without DNS Settings:**
+
+    ```bash
+    so-qcow2-modify-network -I /nsm/libvirt/images/sool9/sool9.qcow2 -i enp1s0 --static4 \
+        --ip4 192.168.1.20/24 --gw4 192.168.1.1
+    ```
+
+    This command sets only the basic static IP configuration:
+    - IP Address: `192.168.1.20/24`
+    - Gateway: `192.168.1.1`
+
+**Notes:**
+
+- When using `--static4`, both `--ip4` and `--gw4` options are required.
+- The script validates IP addresses, DNS servers, and interface names before making any changes.
+- DNS servers can be specified as a comma-separated list for multiple servers.
+- The script requires write permissions for the QCOW2 image file.
+- Interface names must contain only alphanumeric characters, underscores, and hyphens.
+
+**Description:**
+
+The `so-qcow2-modify-network` script modifies network configuration within a QCOW2 image using the following process:
+
+1. **Image Access:**
+   - Mounts the QCOW2 image using libguestfs
+   - Locates and accesses the NetworkManager configuration directory
+
+2. **Configuration Update:**
+   - Reads the existing network configuration for the specified interface
+   - Updates IPv4 settings based on provided parameters
+   - Supports both DHCP and static IP configurations
+   - Validates all input parameters before making changes
+
+3. **File Management:**
+   - Creates or updates the NetworkManager connection file
+   - Maintains proper file permissions and format
+   - Safely unmounts the image after changes
+
+**Exit Codes:**
+
+- `0`: Success
+- Non-zero: An error occurred during execution
+
+**Logging:**
+
+- Logs are written to `/opt/so/log/hypervisor/so-qcow2-modify-network.log`
+- Both file and console logging are enabled for real-time monitoring
+- Log entries include:
+  - Timestamps in ISO 8601 format
+  - Severity levels (INFO, WARNING, ERROR)
+  - Detailed error messages for troubleshooting
+- Critical operations logged:
+  - Network configuration changes
+  - Image mount/unmount operations
+  - Validation failures
+  - File access errors
+"""
+
+import argparse
+import guestfs
+import re
+import sys
+import logging
+import os
+import socket
+import ipaddress
+import configparser
+import uuid
+from io import StringIO
+import libvirt
+from so_logging_utils import setup_logging
+import subprocess
+
+# Get hypervisor name from local hostname
+HYPERVISOR = socket.gethostname()
+
+# Custom log handler to capture output
+class StringIOHandler(logging.Handler):
+    def __init__(self):
+        super().__init__()
+        self.strio = StringIO()
+
+    def emit(self, record):
+        msg = self.format(record)
+        self.strio.write(msg + '\n')
+
+    def get_value(self):
+        return self.strio.getvalue()
+
+# Set up logging using the so_logging_utils library
+string_handler = StringIOHandler()
+string_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
+logger = setup_logging(
+    logger_name='so-qcow2-modify-network',
+    log_file_path='/opt/so/log/hypervisor/so-qcow2-modify-network.log',
+    log_level=logging.INFO,
+    format_str='%(asctime)s - %(levelname)s - %(message)s'
+)
+logger.addHandler(string_handler)
+
+NETWORK_CONFIG_DIR = "/etc/NetworkManager/system-connections"
+
+def validate_ip_address(ip_str, description="IP address"):
+    try:
+        ipaddress.IPv4Interface(ip_str)
+    except ValueError:
+        try:
+            ipaddress.IPv4Address(ip_str)
+        except ValueError:
+            raise ValueError(f"Invalid {description}: {ip_str}")
+
+def validate_dns_addresses(dns_str):
+    dns_list = dns_str.split(',')
+    for dns in dns_list:
+        dns = dns.strip()
+        validate_ip_address(dns, description="DNS server address")
+
+def validate_interface_name(interface_name):
+    if not re.match(r'^[a-zA-Z0-9_\-]+$', interface_name):
+        raise ValueError(f"Invalid interface name: {interface_name}")
+
+def check_base_domain_status(image_path):
+    """
+    Check if the base domain corresponding to the image path is currently running.
+    Base domains should not be running when modifying their configuration.
+
+    Parameters:
+        image_path (str): Path to the QCOW2 image.
+
+    Returns:
+        bool: True if the base domain is running, False otherwise.
+    """
+    base_domain = os.path.basename(os.path.dirname(image_path))
+    logger.info(f"Verifying base domain status for image: {image_path}")
+    logger.info(f"Checking if base domain '{base_domain}' is running...")
+    
+    try:
+        conn = libvirt.open('qemu:///system')
+        try:
+            dom = conn.lookupByName(base_domain)
+            is_running = dom.isActive()
+            if is_running:
+                logger.error(f"Base domain '{base_domain}' is running - cannot modify configuration")
+            return is_running
+        except libvirt.libvirtError:
+            logger.info(f"Base domain '{base_domain}' not found or not running")
+            return False
+        finally:
+            conn.close()
+    except libvirt.libvirtError as e:
+        logger.error(f"Failed to connect to libvirt: {e}")
+        return False
+
+def update_network_config(content, mode, ip=None, gateway=None, dns=None, search_domain=None):
+    config = configparser.ConfigParser(strict=False)
+    config.optionxform = str
+    config.read_string(content)
+
+    # Ensure connection section exists and set required properties
+    if 'connection' not in config.sections():
+        logger.info("Creating new connection section in network configuration")
+        config.add_section('connection')
+    
+    # Set mandatory connection properties
+    config.set('connection', 'autoconnect', 'yes')
+    config.set('connection', 'autoconnect-priority', '999')
+    config.set('connection', 'autoconnect-retries', '-1')
+    config.set('connection', 'multi-connect', '0')
+    config.set('connection', 'wait-device-timeout', '-1')
+
+    # Ensure ipv4 section exists
+    if 'ipv4' not in config.sections():
+        logger.info("Creating new IPv4 section in network configuration")
+        config.add_section('ipv4')
+
+    if mode == "dhcp4":
+        logger.info("Configuring DHCP settings:")
+        logger.info("  method: auto (DHCP enabled)")
+        logger.info("  Removing any existing static configuration")
+        config.set('ipv4', 'method', 'auto')
+        config.remove_option('ipv4', 'address1')
+        config.remove_option('ipv4', 'addresses')
+        config.remove_option('ipv4', 'dns')
+        config.remove_option('ipv4', 'dns-search')
+    elif mode == "static4":
+        logger.info("Configuring static IP settings:")
+        logger.info("  method: manual (static configuration)")
+        config.set('ipv4', 'method', 'manual')
+        if ip and gateway:
+            logger.info(f"  Setting address: {ip}")
+            logger.info(f"  Setting gateway: {gateway}")
+            config.set('ipv4', 'address1', f"{ip},{gateway}")
+        else:
+            logger.error("Missing required IP address or gateway for static configuration")
+            raise ValueError("Both IP address and gateway are required for static configuration.")
+        if dns:
+            logger.info(f"  Setting DNS servers: {dns}")
+            config.set('ipv4', 'dns', f"{dns};")
+        else:
+            logger.info("  No DNS servers specified")
+            config.remove_option('ipv4', 'dns')
+        if search_domain:
+            logger.info(f"  Setting search domain: {search_domain}")
+            config.set('ipv4', 'dns-search', f"{search_domain};")
+        else:
+            logger.info("  No search domain specified")
+            config.remove_option('ipv4', 'dns-search')
+    else:
+        raise ValueError(f"Invalid mode '{mode}'. Expected 'dhcp4' or 'static4'.")
+
+    output = StringIO()
+    config.write(output, space_around_delimiters=False)
+    updated_content = output.getvalue()
+    output.close()
+
+    return updated_content
+
+def modify_network_config(image_path, interface, mode, ip=None, gateway=None, dns=None, search_domain=None):
+    """
+    Modifies network configuration in a QCOW2 image, ensuring specific connection settings are set.
+    
+    Handles both eth0 and predictable network interface names (e.g., enp1s0).
+    If the requested interface configuration is not found but eth0.nmconnection exists,
+    it will be renamed and updated with the proper interface configuration.
+    """
+    # Check if base domain is running
+    if check_base_domain_status(image_path):
+        raise RuntimeError("Cannot modify network configuration while base domain is running")
+
+    if not os.access(image_path, os.W_OK):
+        logger.error(f"Permission denied: Cannot write to image file {image_path}")
+        raise PermissionError(f"Write permission denied for image file: {image_path}")
+
+    logger.info(f"Configuring network for VM image: {image_path}")
+    logger.info(f"Network configuration details for interface {interface}:")
+    logger.info(f"  Mode: {mode.upper()}")
+    if mode == "static4":
+        logger.info(f"  IP Address: {ip}")
+        logger.info(f"  Gateway: {gateway}")
+        logger.info(f"  DNS Servers: {dns if dns else 'Not configured'}")
+        logger.info(f"  Search Domain: {search_domain if search_domain else 'Not configured'}")
+
+    g = guestfs.GuestFS(python_return_dict=True)
+    try:
+        logger.info("Initializing GuestFS and mounting image...")
+        g.set_network(False)
+        g.selinux = False
+        g.add_drive_opts(image_path, format="qcow2")
+        g.launch()
+    except RuntimeError as e:
+        logger.error(f"Failed to initialize GuestFS: {e}")
+        raise RuntimeError(f"Failed to initialize GuestFS or launch appliance: {e}")
+
+    try:
+        os_list = g.inspect_os()
+        if not os_list:
+            logger.error(f"No operating system found in image: {image_path}")
+            raise RuntimeError(f"Unable to find any OS in {image_path}.")
+
+        root_fs = os_list[0]
+        try:
+            g.mount(root_fs, "/")
+            logger.info("Successfully mounted VM image filesystem")
+        except RuntimeError as e:
+            logger.error(f"Failed to mount filesystem: {e}")
+            raise RuntimeError(f"Failed to mount the filesystem: {e}")
+
+        if not g.is_dir(NETWORK_CONFIG_DIR):
+            logger.error(f"NetworkManager configuration directory not found: {NETWORK_CONFIG_DIR}")
+            raise FileNotFoundError(f"NetworkManager configuration directory not found in the image at {NETWORK_CONFIG_DIR}.")
+
+        requested_config_path = f"{NETWORK_CONFIG_DIR}/{interface}.nmconnection"
+        eth0_config_path = f"{NETWORK_CONFIG_DIR}/eth0.nmconnection"
+        config_file_path = None
+        current_content = None
+
+        # Try to read the requested interface config first
+        try:
+            file_content = g.read_file(requested_config_path)
+            current_content = file_content.decode('utf-8')
+            config_file_path = requested_config_path
+            logger.info(f"Found existing network configuration for interface {interface}")
+        except RuntimeError:
+            # If not found, try eth0 config
+            try:
+                file_content = g.read_file(eth0_config_path)
+                current_content = file_content.decode('utf-8')
+                config_file_path = eth0_config_path
+                logger.info("Found eth0 network configuration, will update for new interface")
+            except RuntimeError:
+                logger.error(f"No network configuration found for either {interface} or eth0")
+                raise FileNotFoundError(f"No network configuration found at {requested_config_path} or {eth0_config_path}")
+        except UnicodeDecodeError:
+            logger.error(f"Failed to decode network configuration file")
+            raise ValueError(f"Failed to decode the configuration file")
+
+        # If using eth0 config, update interface-specific fields
+        if config_file_path == eth0_config_path:
+            config = configparser.ConfigParser(strict=False)
+            config.optionxform = str
+            config.read_string(current_content)
+            
+            if 'connection' not in config.sections():
+                config.add_section('connection')
+            
+            # Update interface-specific fields
+            config.set('connection', 'id', interface)
+            config.set('connection', 'interface-name', interface)
+            config.set('connection', 'uuid', str(uuid.uuid4()))
+            
+            # Write updated content back to string
+            output = StringIO()
+            config.write(output, space_around_delimiters=False)
+            current_content = output.getvalue()
+            output.close()
+            
+            # Update config file path to new interface name
+            config_file_path = requested_config_path
+
+        logger.info("Applying network configuration changes...")
+        updated_content = update_network_config(current_content, mode, ip, gateway, dns, search_domain)
+
+        try:
+            g.write(config_file_path, updated_content.encode('utf-8'))
+            # Set proper permissions (600) on the network configuration file
+            g.chmod(0o600, config_file_path)
+            logger.info("Successfully wrote updated network configuration with proper permissions (600)")
+
+            # If we renamed eth0 to the new interface, remove the old eth0 config
+            if config_file_path == requested_config_path and eth0_config_path != requested_config_path:
+                try:
+                    g.rm(eth0_config_path)
+                    logger.info("Removed old eth0 configuration file")
+                except RuntimeError:
+                    logger.warning("Could not remove old eth0 configuration file - it may have already been removed")
+
+        except RuntimeError as e:
+            logger.error(f"Failed to write network configuration: {e}")
+            raise IOError(f"Failed to write updated configuration to {config_file_path}: {e}")
+
+        logger.info(f"Successfully updated network configuration:")
+        logger.info(f"  Image: {image_path}")
+        logger.info(f"  Interface: {interface}")
+        logger.info(f"  Mode: {mode.upper()}")
+        if mode == "static4":
+            logger.info(f"  Settings applied:")
+            logger.info(f"    IP Address: {ip}")
+            logger.info(f"    Gateway: {gateway}")
+            logger.info(f"    DNS Servers: {dns if dns else 'Not configured'}")
+            logger.info(f"    Search Domain: {search_domain if search_domain else 'Not configured'}")
+
+    except Exception as e:
+        raise e
+    finally:
+        g.umount_all()
+        g.close()
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description="Modify IPv4 settings in a QCOW2 image for a specified network interface.")
+    parser.add_argument("-I", "--image", required=True, help="Path to the QCOW2 image.")
+    parser.add_argument("-i", "--interface", required=True, help="Network interface to modify (e.g., enp1s0).")
+    parser.add_argument("-n", "--vm-name", required=True, help="Full name of the VM (hostname_role).")
+    group = parser.add_mutually_exclusive_group(required=True)
+    group.add_argument("--dhcp4", action="store_true", help="Configure interface for DHCP (IPv4).")
+    group.add_argument("--static4", action="store_true", help="Configure interface for static IPv4 settings.")
+    parser.add_argument("--ip4", help="IPv4 address (e.g., 192.168.1.10/24). Required for static IPv4 configuration.")
+    parser.add_argument("--gw4", help="IPv4 gateway (e.g., 192.168.1.1). Required for static IPv4 configuration.")
+    parser.add_argument("--dns4", help="Comma-separated list of IPv4 DNS servers (e.g., 8.8.8.8,8.8.4.4).")
+    parser.add_argument("--search4", help="DNS search domain for IPv4.")
+
+    args = parser.parse_args()
+
+    if args.static4:
+        if not args.ip4 or not args.gw4:
+            parser.error("Both --ip4 and --gw4 are required for static IPv4 configuration.")
+    return args
+
+def main():
+    try:
+        logger.info("Starting network configuration update...")
+        args = parse_arguments()
+
+        logger.info("Validating interface name...")
+        validate_interface_name(args.interface)
+
+        if args.dhcp4:
+            mode = "dhcp4"
+            logger.info("Using DHCP configuration mode")
+        elif args.static4:
+            mode = "static4"
+            logger.info("Using static IP configuration mode")
+            if not args.ip4 or not args.gw4:
+                logger.error("Missing required parameters for static configuration")
+                raise ValueError("Both --ip4 and --gw4 are required for static IPv4 configuration.")
+            
+            logger.info("Validating IP addresses...")
+            validate_ip_address(args.ip4, description="IPv4 address")
+            validate_ip_address(args.gw4, description="IPv4 gateway")
+            if args.dns4:
+                validate_dns_addresses(args.dns4)
+        else:
+            logger.error("No configuration mode specified")
+            raise ValueError("Either --dhcp4 or --static4 must be specified.")
+
+        modify_network_config(args.image, args.interface, mode, args.ip4, args.gw4, args.dns4, args.search4)
+        logger.info("Network configuration update completed successfully")
+
+        # Send success status event
+        try:
+            subprocess.run([
+                'so-salt-emit-vm-deployment-status-event',
+                '-v', args.vm_name,
+                '-H', HYPERVISOR,
+                '-s', 'IP Configuration'
+            ], check=True)
+        except subprocess.CalledProcessError as e:
+            logger.error(f"Failed to emit success status event: {e}")
+
+    except KeyboardInterrupt:
+        error_msg = "Operation cancelled by user"
+        logger.error(error_msg)
+        try:
+            subprocess.run([
+                'so-salt-emit-vm-deployment-status-event',
+                '-v', args.vm_name,
+                '-H', HYPERVISOR,
+                '-s', 'IP Configuration Failed'
+            ], check=True)
+        except subprocess.CalledProcessError as e:
+            logger.error(f"Failed to emit failure status event: {e}")
+        sys.exit(1)
+    except Exception as e:
+        error_msg = str(e)
+        if "base domain is running" in error_msg:
+            logger.error("Cannot proceed: Base domain must not be running when modifying network configuration")
+            error_msg = "Base domain must not be running when modifying network configuration"
+        else:
+            logger.error(f"An error occurred: {e}")
+        try:
+            subprocess.run([
+                'so-salt-emit-vm-deployment-status-event',
+                '-v', args.vm_name,
+                '-H', HYPERVISOR,
+                '-s', 'IP Configuration Failed'
+            ], check=True)
+        except subprocess.CalledProcessError as e:
+            logger.error(f"Failed to emit failure status event: {e}")
+        sys.exit(1)
+
+if __name__ == '__main__':
+    main()
+
+{%- else -%}
+
+echo "Hypervisor nodes are a feature supported only for customers with a valid license. \
+      Contact Security Onion Solutions, LLC via our website at https://securityonionsolutions.com \
+      for more information about purchasing a license to enable this feature."
+
+{% endif -%}

salt/idh/config.sls

@@ -86,7 +86,7 @@ idh_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://idh/tools/sbin
-    - user: 934
+    - user: 939
     - group: 939
     - file_mode: 755
 

salt/idstools/config.sls

@@ -20,7 +20,7 @@ idstools_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://idstools/tools/sbin
-    - user: 934
+    - user: 939
     - group: 939
     - file_mode: 755
 
@@ -29,7 +29,7 @@ idstools_sbin:
 #  file.recurse:
 #    - name: /usr/sbin
 #    - source: salt://idstools/tools/sbin_jinja
-#    - user: 934
+#    - user: 939
 #    - group: 939 
 #    - file_mode: 755
 #    - template: jinja
@@ -38,7 +38,7 @@ idstools_so-rule-update:
   file.managed:
     - name: /usr/sbin/so-rule-update
     - source: salt://idstools/tools/sbin_jinja/so-rule-update
-    - user: 934
+    - user: 939
     - group: 939 
     - mode: 755
     - template: jinja

salt/idstools/rules/extraction.rules

@@ -1,26 +1,26 @@
 # Extract all PDF mime type
-alert http any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; sid:1100000; rev:1;)
-alert smtp any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; sid:1100001; rev:1;)
-alert nfs any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; sid:1100002; rev:1;)
-alert smb any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; sid:1100003; rev:1;)
+alert http any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100000; rev:1;)
+alert smtp any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100001; rev:1;)
+alert nfs any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100002; rev:1;)
+alert smb any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; noalert; sid:1100003; rev:1;)
 # Extract EXE/DLL file types
-alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:1100004; rev:1;)
-alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:1100005; rev:1;)
-alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:1100006; rev:1;)
-alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:1100007; rev:1;)
-alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:1100008; rev:1;)
-alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:1100009; rev:1;)
-alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:1100010; rev:1;)
-alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:1100011; rev:1;)
+alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100004; rev:1;)
+alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100005; rev:1;)
+alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100006; rev:1;)
+alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; noalert; sid:1100007; rev:1;)
+alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100008; rev:1;)
+alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100009; rev:1;)
+alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100010; rev:1;)
+alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; noalert; sid:1100011; rev:1;)
 
 # Extract all Zip files
-alert http any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; sid:1100012; rev:1;)
-alert smtp any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; sid:1100013; rev:1;)
-alert nfs any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; sid:1100014; rev:1;)
-alert smb any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; sid:1100015; rev:1;)
+alert http any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100012; rev:1;)
+alert smtp any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100013; rev:1;)
+alert nfs any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100014; rev:1;)
+alert smb any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; noalert; sid:1100015; rev:1;)
 
 # Extract Word Docs
-alert http any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100016; rev:1;)
-alert smtp any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100017; rev:1;)
-alert nfs any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100018; rev:1;)
-alert smb any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100019; rev:1;)
+alert http any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100016; rev:1;)
+alert smtp any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100017; rev:1;)
+alert nfs any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100018; rev:1;)
+alert smb any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100019; rev:1;)

salt/kibana/defaults.yaml

@@ -22,7 +22,7 @@ kibana:
           - default
           - file
     migrations:
-      discardCorruptObjects: "8.17.3"
+      discardCorruptObjects: "8.18.4"
     telemetry:
       enabled: False
     security:

salt/kibana/files/config_saved_objects.ndjson.jinja

@@ -1,3 +1,3 @@
 {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS -%}
 
-{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n  \"from\": \"now-24h\",\n  \"to\": \"now\"\n}"},"coreMigrationVersion": "{{ ELASTICSEARCHDEFAULTS.elasticsearch.version }}","id": "{{ ELASTICSEARCHDEFAULTS.elasticsearch.version }}","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="}
+{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"savedObjects:listingLimit":1500,"theme:darkMode": true,"timepicker:timeDefaults": "{\n  \"from\": \"now-24h\",\n  \"to\": \"now\"\n}"},"coreMigrationVersion": "{{ ELASTICSEARCHDEFAULTS.elasticsearch.version }}","id": "{{ ELASTICSEARCHDEFAULTS.elasticsearch.version }}","references": [],"type": "config","version": "WzI5NzUsMl0="}

salt/kibana/tools/sbin_jinja/so-kibana-space-defaults

@@ -13,6 +13,6 @@ echo "Setting up default Space:"
 {% if HIGHLANDER %}
 curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["enterpriseSearch"]} ' >> /opt/so/log/kibana/misc.log
 {% else %}
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV2","inventory","dataQuality","actions"]} ' >> /opt/so/log/kibana/misc.log
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","entityManager"]} ' >> /opt/so/log/kibana/misc.log
 {% endif %}
 echo

salt/libvirt/64962/init.sls

@@ -0,0 +1,18 @@
+python3_lief:
+  pkg.installed:
+    - name: securityonion-python3-lief
+
+so-fix-salt-ldap_script:
+  file.managed:
+    - name: /usr/sbin/so-fix-salt-ldap.py
+    - source: salt://libvirt/64962/scripts/so-fix-salt-ldap.py
+    - mode: 744
+
+fix-salt-ldap:
+  cmd.run:
+    - name: /usr/sbin/so-fix-salt-ldap.py
+    - require:
+      - pkg: python3_lief
+      - file: so-fix-salt-ldap_script
+    - onchanges:
+      - file: so-fix-salt-ldap_script

salt/libvirt/64962/scripts/so-fix-salt-ldap.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+
+# this script comes from the user nf-brentsaner located here https://github.com/saltstack/salt/issues/64962
+
+import datetime
+import grp
+import os
+import pathlib
+import pwd
+import shutil
+##
+import dbus  # dnf -y install python3-dbus
+##
+import lief  # https://pypi.org/project/lief/
+
+salt_root = pathlib.Path('/opt/saltstack')
+src_lib = pathlib.Path('/lib64/libldap.so.2')
+dst_lib = salt_root.joinpath('salt', 'lib', 'libldap.so.2')
+
+uname = 'salt'
+gname = 'salt'
+
+lib = lief.parse(str(src_lib))
+
+sym = next((i for i in lib.imported_symbols if i.name == 'EVP_md2'), None)
+
+if sym:
+    # Get the Salt services from DBus.
+    sysbus = dbus.SystemBus()
+    sysd = sysbus.get_object('org.freedesktop.systemd1', '/org/freedesktop/systemd1')
+    mgr = dbus.Interface(sysd, 'org.freedesktop.systemd1.Manager')
+    svcs = []
+    for i in mgr.ListUnits():
+        # first element is unit name.
+        if not str(i[0]).startswith('salt-'):
+            continue
+        svc = sysbus.get_object('org.freedesktop.systemd1', object_path = mgr.GetUnit(str(i[0])))
+        props = dbus.Interface(svc, dbus_interface = 'org.freedesktop.DBus.Properties')
+        state = props.Get('org.freedesktop.systemd1.Unit', 'ActiveState')
+        if str(state) == 'active':
+            svcs.append(i[0])
+    # Get the user/group
+    u = pwd.getpwnam(uname)
+    g = grp.getgrnam(gname)
+    # Modify
+    print('Modifications necessary.')
+    if svcs:
+        # Stop the services first.
+        for sn in svcs:
+            mgr.StopUnit(sn, 'replace')
+    if dst_lib.exists():
+        # 3.10 deprecated .utcnow().
+        #dst_lib_bak = pathlib.Path(str(dst_lib) + '.bak_{0}'.format(datetime.datetime.now(datetime.UTC).timestamp()))
+        dst_lib_bak = pathlib.Path(str(dst_lib) + '.bak_{0}'.format(datetime.datetime.utcnow().timestamp()))
+        os.rename(dst_lib, dst_lib_bak)
+        print('Destination file {0} exists; backed up to {1}.'.format(dst_lib, dst_lib_bak))
+    lib.remove_dynamic_symbol(sym)
+    lib.write(str(dst_lib))
+    os.chown(dst_lib, u.pw_uid, g.gr_gid)
+    os.chmod(dst_lib, src_lib.stat().st_mode)
+    # Before we restart services, we also want to remove any python caches.
+    for root, dirs, files in os.walk(salt_root):
+        for f in files:
+            if f.lower().endswith('.pyc'):
+                fpath = os.path.join(root, f)
+                os.remove(fpath)
+                print('Removed file {0}'.format(fpath))
+        if '__pycache__' in dirs:
+            dpath = os.path.join(root, '__pycache__')
+            shutil.rmtree(dpath)
+            print('Removed directory {0}'.format(dpath))
+    # And then start the units that were started before.
+    if svcs:
+        for sn in svcs:
+            mgr.RestartUnit(sn, 'replace')
+else:
+    print('No EVP_md2 symbol found in the library. No modifications needed.')
+
+print('Done.')

salt/libvirt/bridge.sls

@@ -0,0 +1,49 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{%  from 'libvirt/map.jinja' import LIBVIRTMERGED %}
+{%  from 'salt/map.jinja' import SYSTEMD_UNIT_FILE %}
+
+down_original_mgmt_interface:
+  cmd.run:
+    - name: "nmcli con down {{ pillar.host.mainint }}"
+    - unless:
+      - nmcli -f GENERAL.CONNECTION dev show {{ pillar.host.mainint }} | grep bridge-slave-{{ pillar.host.mainint }}
+    - order: last
+
+wait_for_br0_ip:
+  cmd.run:
+    - name: |
+        counter=0
+        until ip addr show br0 | grep -q "inet "; do
+          sleep 1
+          counter=$((counter+1))
+          if [ $counter -ge 90 ]; then
+            echo "Timeout waiting for br0 to get an IP address"
+            exit 1
+          fi
+        done
+        echo "br0 has IP address: $(ip addr show br0 | grep 'inet ' | awk '{print $2}')"
+    - timeout: 95
+    - onchanges:
+      - cmd: down_original_mgmt_interface
+
+update_mine_functions:
+  file.managed:
+    - name: /etc/salt/minion.d/mine_functions.conf
+    - contents: |
+        mine_interval: 25
+        mine_functions:
+          network.ip_addrs:
+            - interface: br0
+    - onchanges:
+      - cmd: wait_for_br0_ip
+
+restart_salt_minion_service:
+  service.running:
+    - name: salt-minion
+    - enable: True
+    - listen:
+      - file: update_mine_functions

salt/libvirt/defaults.yaml

@@ -0,0 +1,53 @@
+libvirt:
+  config:
+    listen_tls: 1
+    listen_tcp: 0
+    tls_port: "16514"
+    tcp_port: "16509"
+    listen_addr: "0.0.0.0"
+    unix_sock_group: "root"
+    unix_sock_ro_perms: "0777"
+    unix_sock_rw_perms: "0770"
+    unix_sock_admin_perms: "0700"
+    unix_sock_dir: "/run/libvirt"
+    auth_unix_ro: "polkit"
+    auth_unix_rw: "polkit"
+    auth_tcp: "sasl"
+    auth_tls: "none"
+    tcp_min_ssf: 112
+    access_drivers: ["polkit"]
+    key_file: "/etc/pki/libvirt/private/serverkey.pem"
+    cert_file: "/etc/pki/libvirt/servercert.pem"
+    ca_file: "/etc/pki/CA/cacert.pem"
+    #crl_file: "/etc/pki/CA/crl.pem"
+    tls_no_sanity_certificate: 0
+    tls_no_verify_certificate: 0
+    tls_allowed_dn_list: ["DN1", "DN2"]
+    tls_priority: "NORMAL"
+    sasl_allowed_username_list: ["joe@EXAMPLE.COM", "fred@EXAMPLE.COM"]
+    max_clients: 5000
+    max_queued_clients: 1000
+    max_anonymous_clients: 20
+    min_workers: 5
+    max_workers: 20
+    prio_workers: 5
+    max_client_requests: 5
+    admin_min_workers: 1
+    admin_max_workers: 5
+    admin_max_clients: 5
+    admin_max_queued_clients: 5
+    admin_max_client_requests: 5
+    log_level: 3
+    log_filters: "1:qemu 1:libvirt 4:object 4:json 4:event 1:util"
+    log_outputs: "3:syslog:libvirtd"
+    audit_level: 2
+    audit_logging: 1
+    #host_uuid: "00000000-0000-0000-0000-000000000000"
+    host_uuid_source: "smbios"
+    keepalive_interval: 5
+    keepalive_count: 5
+    keepalive_required: 1
+    admin_keepalive_required: 1
+    admin_keepalive_interval: 5
+    admin_keepalive_count: 5
+    ovs_timeout: 5

salt/libvirt/etc/libvirtd.conf

@@ -0,0 +1,536 @@
+# Master libvirt daemon configuration file
+#
+
+#################################################################
+#
+# Network connectivity controls
+#
+
+# Flag listening for secure TLS connections on the public TCP/IP port.
+#
+# To enable listening sockets with the 'libvirtd' daemon it's also required to
+# pass the '--listen' flag on the commandline of the daemon.
+# This is not needed with 'virtproxyd'.
+#
+# This setting is not required or honoured if using systemd socket
+# activation.
+#
+# It is necessary to setup a CA and issue server certificates before
+# using this capability.
+#
+# This is enabled by default, uncomment this to disable it
+#listen_tls = 0
+
+# Listen for unencrypted TCP connections on the public TCP/IP port.
+#
+# To enable listening sockets with the 'libvirtd' daemon it's also required to
+# pass the '--listen' flag on the commandline of the daemon.
+# This is not needed with 'virtproxyd'.
+#
+# This setting is not required or honoured if using systemd socket
+# activation.
+#
+# Using the TCP socket requires SASL authentication by default. Only
+# SASL mechanisms which support data encryption are allowed. This is
+# DIGEST_MD5 and GSSAPI (Kerberos5)
+#
+# This is disabled by default, uncomment this to enable it.
+#listen_tcp = 1
+
+
+
+# Override the port for accepting secure TLS connections
+# This can be a port number, or service name
+#
+# This setting is not required or honoured if using systemd socket
+# activation.
+#
+#tls_port = "16514"
+
+# Override the port for accepting insecure TCP connections
+# This can be a port number, or service name
+#
+# This setting is not required or honoured if using systemd socket
+# activation.
+#
+#tcp_port = "16509"
+
+
+# Override the default configuration which binds to all network
+# interfaces. This can be a numeric IPv4/6 address, or hostname
+#
+# This setting is not required or honoured if using systemd socket
+# activation.
+#
+# If the libvirtd service is started in parallel with network
+# startup (e.g. with systemd), binding to addresses other than
+# the wildcards (0.0.0.0/::) might not be available yet.
+#
+#listen_addr = "192.168.0.1"
+
+
+#################################################################
+#
+# UNIX socket access controls
+#
+
+# Set the UNIX domain socket group ownership. This can be used to
+# allow a 'trusted' set of users access to management capabilities
+# without becoming root.
+#
+# This setting is not required or honoured if using systemd socket
+# activation.
+#
+# This is restricted to 'root' by default.
+#unix_sock_group = "libvirt"
+
+# Set the UNIX socket permissions for the R/O socket. This is used
+# for monitoring VM status only
+#
+# This setting is not required or honoured if using systemd socket
+# activation.
+#
+# Default allows any user. If setting group ownership, you may want to
+# restrict this too.
+#unix_sock_ro_perms = "0777"
+
+# Set the UNIX socket permissions for the R/W socket. This is used
+# for full management of VMs
+#
+# This setting is not required or honoured if using systemd socket
+# activation.
+#
+# Default allows only root. If PolicyKit is enabled on the socket,
+# the default will change to allow everyone (eg, 0777)
+#
+# If not using PolicyKit and setting group ownership for access
+# control, then you may want to relax this too.
+#unix_sock_rw_perms = "0770"
+
+# Set the UNIX socket permissions for the admin interface socket.
+#
+# This setting is not required or honoured if using systemd socket
+# activation.
+#
+# Default allows only owner (root), do not change it unless you are
+# sure to whom you are exposing the access to.
+#unix_sock_admin_perms = "0700"
+
+# Set the name of the directory in which sockets will be found/created.
+#
+# This setting is not required or honoured if using systemd socket
+# activation.
+#
+#unix_sock_dir = "/run/libvirt"
+
+
+
+#################################################################
+#
+# Authentication.
+#
+# There are the following choices available:
+#
+#  - none: do not perform auth checks. If you can connect to the
+#          socket you are allowed. This is suitable if there are
+#          restrictions on connecting to the socket (eg, UNIX
+#          socket permissions), or if there is a lower layer in
+#          the network providing auth (eg, TLS/x509 certificates)
+#
+#  - sasl: use SASL infrastructure. The actual auth scheme is then
+#          controlled from /etc/sasl2/libvirt.conf. For the TCP
+#          socket only GSSAPI & DIGEST-MD5 mechanisms will be used.
+#          For non-TCP or TLS sockets, any scheme is allowed.
+#
+#  - polkit: use PolicyKit to authenticate. This is only suitable
+#            for use on the UNIX sockets. The default policy will
+#            require a user to supply their own password to gain
+#            full read/write access (aka sudo like), while anyone
+#            is allowed read/only access.
+#
+
+# Set an authentication scheme for UNIX read-only sockets
+#
+# By default socket permissions allow anyone to connect
+#
+# If libvirt was compiled without support for 'polkit', then
+# no access control checks are done, but libvirt still only
+# allows execution of APIs which don't change state.
+#
+# If libvirt was compiled with support for 'polkit', then
+# the libvirt socket will perform a check with polkit after
+# connections. The default policy still allows any local
+# user access.
+#
+# To restrict monitoring of domains you may wish to either
+# enable 'sasl' here, or change the polkit policy definition.
+#auth_unix_ro = "polkit"
+
+# Set an authentication scheme for UNIX read-write sockets.
+#
+# If libvirt was compiled without support for 'polkit', then
+# the systemd .socket files will use SocketMode=0600 by default
+# thus only allowing root user to connect, and 'auth_unix_rw'
+# will default to 'none'.
+#
+# If libvirt was compiled with support for 'polkit', then
+# the systemd .socket files will use SocketMode=0666 which
+# allows any user to connect and 'auth_unix_rw' will default
+# to 'polkit'. If you disable use of 'polkit' here, then it
+# is essential to change the systemd SocketMode parameter
+# back to 0600, to avoid an insecure configuration.
+#
+#auth_unix_rw = "polkit"
+
+# Change the authentication scheme for TCP sockets.
+#
+# If you don't enable SASL, then all TCP traffic is cleartext.
+# Don't do this outside of a dev/test scenario. For real world
+# use, always enable SASL and use the GSSAPI or DIGEST-MD5
+# mechanism in /etc/sasl2/libvirt.conf
+#auth_tcp = "sasl"
+
+# Change the authentication scheme for TLS sockets.
+#
+# TLS sockets already have encryption provided by the TLS
+# layer, and limited authentication is done by certificates
+#
+# It is possible to make use of any SASL authentication
+# mechanism as well, by using 'sasl' for this option
+#auth_tls = "none"
+
+# Enforce a minimum SSF value for TCP sockets
+#
+# The default minimum is currently 56 (single-DES) which will
+# be raised to 112 in the future.
+#
+# This option can be used to set values higher than 112
+#tcp_min_ssf = 112
+
+
+# Change the API access control scheme
+#
+# By default an authenticated user is allowed access
+# to all APIs. Access drivers can place restrictions
+# on this. By default the 'nop' driver is enabled,
+# meaning no access control checks are done once a
+# client has authenticated with libvirtd
+#
+#access_drivers = [ "polkit" ]
+
+#################################################################
+#
+# TLS x509 certificate configuration
+#
+
+# Use of TLS requires that x509 certificates be issued. The default locations
+# for the certificate files is as follows:
+#
+#   /etc/pki/CA/cacert.pem - The CA master certificate
+#   /etc/pki/libvirt/servercert.pem - The server certificate signed by cacert.pem
+#   /etc/pki/libvirt/private/serverkey.pem - The server private key
+#
+# It is possible to override the default locations by altering the 'key_file',
+# 'cert_file', and 'ca_file' values and uncommenting them below.
+#
+# NB, overriding the default of one location requires uncommenting and
+# possibly additionally overriding the other settings.
+#
+
+# Override the default server key file path
+#
+#key_file = "/etc/pki/libvirt/private/serverkey.pem"
+
+# Override the default server certificate file path
+#
+#cert_file = "/etc/pki/libvirt/servercert.pem"
+
+# Override the default CA certificate path
+#
+#ca_file = "/etc/pki/CA/cacert.pem"
+
+# Specify a certificate revocation list.
+#
+# Defaults to not using a CRL, uncomment to enable it
+#crl_file = "/etc/pki/CA/crl.pem"
+
+
+
+#################################################################
+#
+# Authorization controls
+#
+
+
+# Flag to disable verification of our own server certificates
+#
+# When libvirtd starts it performs some sanity checks against
+# its own certificates.
+#
+# Default is to always run sanity checks. Uncommenting this
+# will disable sanity checks which is not a good idea
+#tls_no_sanity_certificate = 1
+
+# Flag to disable verification of client certificates
+#
+# Client certificate verification is the primary authentication mechanism.
+# Any client which does not present a certificate signed by the CA
+# will be rejected.
+#
+# Default is to always verify. Uncommenting this will disable
+# verification.
+#tls_no_verify_certificate = 1
+
+
+# An access control list of allowed x509 Distinguished Names
+# This list may contain wildcards such as
+#
+#    "C=GB,ST=London,L=London,O=Red Hat,CN=*"
+#
+# Any * matches any number of consecutive spaces, like a simplified glob(7).
+#
+# The format of the DN for a particular certificate can be queried
+# using:
+#
+#    virt-pki-query-dn clientcert.pem
+#
+# NB If this is an empty list, no client can connect, so comment out
+# entirely rather than using empty list to disable these checks
+#
+# By default, no DN's are checked
+#tls_allowed_dn_list = ["DN1", "DN2"]
+
+
+# Override the compile time default TLS priority string. The
+# default is usually "NORMAL" unless overridden at build time.
+# Only set this is it is desired for libvirt to deviate from
+# the global default settings.
+#
+#tls_priority="NORMAL"
+
+
+# An access control list of allowed SASL usernames. The format for username
+# depends on the SASL authentication mechanism. Kerberos usernames
+# look like username@REALM
+#
+# This list may contain wildcards such as
+#
+#    "*@EXAMPLE.COM"
+#
+# See the g_pattern_match function for the format of the wildcards.
+#
+# https://developer.gnome.org/glib/stable/glib-Glob-style-pattern-matching.html
+#
+# NB If this is an empty list, no client can connect, so comment out
+# entirely rather than using empty list to disable these checks
+#
+# By default, no Username's are checked
+#sasl_allowed_username_list = ["joe@EXAMPLE.COM", "fred@EXAMPLE.COM" ]
+
+
+#################################################################
+#
+# Processing controls
+#
+
+# The maximum number of concurrent client connections to allow
+# over all sockets combined.
+#max_clients = 5000
+
+# The maximum length of queue of connections waiting to be
+# accepted by the daemon. Note, that some protocols supporting
+# retransmission may obey this so that a later reattempt at
+# connection succeeds.
+#max_queued_clients = 1000
+
+# The maximum length of queue of accepted but not yet
+# authenticated clients. The default value is 20. Set this to
+# zero to turn this feature off.
+#max_anonymous_clients = 20
+
+# The minimum limit sets the number of workers to start up
+# initially. If the number of active clients exceeds this,
+# then more threads are spawned, up to max_workers limit.
+# Typically you'd want max_workers to equal maximum number
+# of clients allowed
+#min_workers = 5
+#max_workers = 20
+
+
+# The number of priority workers. If all workers from above
+# pool are stuck, some calls marked as high priority
+# (notably domainDestroy) can be executed in this pool.
+#prio_workers = 5
+
+# Limit on concurrent requests from a single client
+# connection. To avoid one client monopolizing the server
+# this should be a small fraction of the global max_workers
+# parameter.
+# Setting this too low may cause keepalive timeouts.
+#max_client_requests = 5
+
+# Same processing controls, but this time for the admin interface.
+# For description of each option, be so kind to scroll few lines
+# upwards.
+
+#admin_min_workers = 1
+#admin_max_workers = 5
+#admin_max_clients = 5
+#admin_max_queued_clients = 5
+#admin_max_client_requests = 5
+
+#################################################################
+#
+# Logging controls
+#
+
+# Logging level: 4 errors, 3 warnings, 2 information, 1 debug
+# basically 1 will log everything possible
+#
+# WARNING: USE OF THIS IS STRONGLY DISCOURAGED.
+#
+# WARNING: It outputs too much information to practically read.
+# WARNING: The "log_filters" setting is recommended instead.
+#
+# WARNING: Journald applies rate limiting of messages and so libvirt
+# WARNING: will limit "log_level" to only allow values 3 or 4 if
+# WARNING: journald is the current output.
+#
+# WARNING: USE OF THIS IS STRONGLY DISCOURAGED.
+#log_level = 3
+
+# Logging filters:
+# A filter allows to select a different logging level for a given category
+# of logs. The format for a filter is:
+#
+#    level:match
+#
+# where 'match' is a string which is matched against the category
+# given in the VIR_LOG_INIT() at the top of each libvirt source
+# file, e.g., "remote", "qemu", or "util.json". The 'match' in the
+# filter matches using shell wildcard syntax (see 'man glob(7)').
+# The 'match' is always treated as a substring match. IOW a match
+# string 'foo' is equivalent to '*foo*'.
+#
+# 'level' is the minimal level where matching messages should
+#  be logged:
+#
+#    1: DEBUG
+#    2: INFO
+#    3: WARNING
+#    4: ERROR
+#
+# Multiple filters can be defined in a single @log_filters, they just need
+# to be separated by spaces. Note that libvirt performs "first" match, i.e.
+# if there are concurrent filters, the first one that matches will be applied,
+# given the order in @log_filters.
+#
+# A typical need is to capture information from a hypervisor driver,
+# public API entrypoints and some of the utility code. Some utility
+# code is very verbose and is generally not desired. Taking the QEMU
+# hypervisor as an example, a suitable filter string for debugging
+# might be to turn off object, json & event logging, but enable the
+# rest of the util code:
+#
+#log_filters="1:qemu 1:libvirt 4:object 4:json 4:event 1:util"
+
+# Logging outputs:
+# An output is one of the places to save logging information
+# The format for an output can be:
+#    level:stderr
+#      output goes to stderr
+#    level:syslog:name
+#      use syslog for the output and use the given name as the ident
+#    level:file:file_path
+#      output to a file, with the given filepath
+#    level:journald
+#      output to journald logging system
+# In all cases 'level' is the minimal priority, acting as a filter
+#    1: DEBUG
+#    2: INFO
+#    3: WARNING
+#    4: ERROR
+#
+# Multiple outputs can be defined, they just need to be separated by spaces.
+# e.g. to log all warnings and errors to syslog under the libvirtd ident:
+#log_outputs="3:syslog:libvirtd"
+
+
+##################################################################
+#
+# Auditing
+#
+# This setting allows usage of the auditing subsystem to be altered:
+#
+#   audit_level == 0  -> disable all auditing
+#   audit_level == 1  -> enable auditing, only if enabled on host (default)
+#   audit_level == 2  -> enable auditing, and exit if disabled on host
+#
+#audit_level = 2
+#
+# If set to 1, then audit messages will also be sent
+# via libvirt logging infrastructure. Defaults to 0
+#
+#audit_logging = 1
+
+###################################################################
+# UUID of the host:
+# Host UUID is read from one of the sources specified in host_uuid_source.
+#
+# - 'smbios': fetch the UUID from 'dmidecode -s system-uuid'
+# - 'machine-id': fetch the UUID from /etc/machine-id
+#
+# The host_uuid_source default is 'smbios'. If 'dmidecode' does not provide
+# a valid UUID a temporary UUID will be generated.
+#
+# Another option is to specify host UUID in host_uuid.
+#
+# Keep the format of the example UUID below. UUID must not have all digits
+# be the same.
+
+# NB This default all-zeros UUID will not work. Replace
+# it with the output of the 'uuidgen' command and then
+# uncomment this entry
+#host_uuid = "00000000-0000-0000-0000-000000000000"
+#host_uuid_source = "smbios"
+
+###################################################################
+# Keepalive protocol:
+# This allows libvirtd to detect broken client connections or even
+# dead clients.  A keepalive message is sent to a client after
+# keepalive_interval seconds of inactivity to check if the client is
+# still responding; keepalive_count is a maximum number of keepalive
+# messages that are allowed to be sent to the client without getting
+# any response before the connection is considered broken.  In other
+# words, the connection is automatically closed approximately after
+# keepalive_interval * (keepalive_count + 1) seconds since the last
+# message received from the client.  If keepalive_interval is set to
+# -1, libvirtd will never send keepalive requests; however clients
+# can still send them and the daemon will send responses.  When
+# keepalive_count is set to 0, connections will be automatically
+# closed after keepalive_interval seconds of inactivity without
+# sending any keepalive messages.
+#
+#keepalive_interval = 5
+#keepalive_count = 5
+
+#
+# These configuration options are no longer used.  There is no way to
+# restrict such clients from connecting since they first need to
+# connect in order to ask for keepalive.
+#
+#keepalive_required = 1
+#admin_keepalive_required = 1
+
+# Keepalive settings for the admin interface
+#admin_keepalive_interval = 5
+#admin_keepalive_count = 5
+
+###################################################################
+# Open vSwitch:
+# This allows to specify a timeout for openvswitch calls made by
+# libvirt. The ovs-vsctl utility is used for the configuration and
+# its timeout option is set by default to 5 seconds to avoid
+# potential infinite waits blocking libvirt.
+#
+#ovs_timeout = 5

salt/libvirt/etc/libvirtd.conf.jinja

@@ -0,0 +1,8 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. -#}
+
+{%- for k, v in LIBVIRTMERGED.config.items() %}
+{{ k }} = {{ v | json }}
+{%- endfor %}

salt/libvirt/images/init.sls

@@ -0,0 +1,201 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+#
+# Note: Per the Elastic License 2.0, the second limitation states:
+#
+#   "You may not move, change, disable, or circumvent the license key functionality
+#    in the software, and you may not remove or obscure any functionality in the
+#    software that is protected by the license key."
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states or sls in allowed_states %}
+{%   if 'vrt' in salt['pillar.get']('features', []) %}
+
+include:
+  - hypervisor
+  - libvirt.packages
+
+nsm_libvirt_images:
+  file.directory:
+    - name: /nsm/libvirt/images/sool9
+    - dir_mode: 775
+    - file_mode: 640
+    - recurse:
+      - mode
+    - makedirs: True
+
+# Remove hash file if image isn't present. This will allow for the image to redownload and initialize.
+remove_sha256_sool9:
+  file.absent:
+    - name: /nsm/libvirt/images/sool9/sool9.sha256
+    - unless: test -f /nsm/libvirt/images/sool9/sool9.qcow2
+
+# Manage SHA256 hash file
+manage_sha256_sool9:
+  file.managed:
+    - name: /nsm/libvirt/images/sool9/sool9.sha256
+    - source: salt://libvirt/images/sool9/sool9.sha256
+
+# Manage cloud-init files
+manage_metadata_sool9:
+  file.managed:
+    - name: /nsm/libvirt/images/sool9/meta-data
+    - source: salt://libvirt/images/sool9/meta-data
+
+manage_userdata_sool9:
+  file.managed:
+    - name: /nsm/libvirt/images/sool9/user-data
+    - source: salt://libvirt/images/sool9/user-data
+    - show_changes: False
+
+# Manage qcow2 image
+manage_qcow2_sool9:
+  file.managed:
+    - name: /nsm/libvirt/images/sool9/sool9.qcow2
+    - source: salt://libvirt/images/sool9/sool9.qcow2
+    - onchanges:
+      - file: manage_sha256_sool9
+      - file: manage_metadata_sool9
+      - file: manage_userdata_sool9
+
+manage_cidata_sool9:
+  file.managed:
+    - name: /nsm/libvirt/images/sool9/sool9-cidata.iso
+    - source: salt://libvirt/images/sool9/sool9-cidata.iso
+    - onchanges:
+      - file: manage_qcow2_sool9
+
+# Define the storage pool
+define_storage_pool_sool9:
+  virt.pool_defined:
+    - name: sool9
+    - ptype: dir
+    - target: /nsm/libvirt/images/sool9
+    - require:
+      - file: manage_metadata_sool9
+      - file: manage_userdata_sool9
+      - file: manage_cidata_sool9
+      - cmd: libvirt_python_module
+    - unless:
+      - virsh pool-list --all | grep -q sool9
+
+# Set pool autostart
+set_pool_autostart_sool9:
+  cmd.run:
+    - name: virsh pool-autostart sool9
+    - require:
+      - virt: define_storage_pool_sool9
+    - unless:
+      - virsh pool-info sool9 | grep -q "Autostart.*yes"
+
+# Start the storage pool
+start_storage_pool_sool9:
+  cmd.run:
+    - name: virsh pool-start sool9
+    - require:
+      - virt: define_storage_pool_sool9
+      - cmd: libvirt_python_module
+    - unless:
+      - virsh pool-info sool9 | grep -q "State.*running"
+
+# Stop the VM if running and base image files change
+stop_vm_sool9:
+  module.run:
+    - virt.stop:
+      - name: sool9
+    - onchanges:
+      - file: manage_qcow2_sool9
+      - file: manage_metadata_sool9
+      - file: manage_userdata_sool9
+      - file: manage_cidata_sool9
+    - require_in:
+      - module: undefine_vm_sool9
+    - onlyif:
+      # Only try to stop if VM is actually running
+      - virsh list --state-running --name | grep -q sool9
+
+undefine_vm_sool9:
+  module.run:
+    - virt.undefine:
+      - vm_: sool9
+    - onchanges:
+      - file: manage_qcow2_sool9
+      - file: manage_metadata_sool9
+      - file: manage_userdata_sool9
+      - file: manage_cidata_sool9
+    # Note: When VM doesn't exist, you'll see "error: failed to get domain 'sool9'" - this is expected
+    # [ERROR   ] Command 'virsh' failed with return code: 1
+    # [ERROR   ] stdout: error: failed to get domain 'sool9'
+    - onlyif:
+      - virsh dominfo sool9
+
+# Create and start the VM, letting cloud-init run
+create_vm_sool9:
+  cmd.run:
+    - name: |
+        virt-install --name sool9 \
+          --memory 12288 --vcpus 8 --cpu host-model \
+          --disk /nsm/libvirt/images/sool9/sool9.qcow2,format=qcow2,bus=virtio \
+          --disk /nsm/libvirt/images/sool9/sool9-cidata.iso,device=cdrom \
+          --network bridge=br0,model=virtio \
+          --os-variant=ol9.5 \
+          --import \
+          --noautoconsole
+    - require:
+      - cmd: start_storage_pool_sool9
+      - pkg: install_virt-install
+    - onchanges:
+      - file: manage_qcow2_sool9
+      - file: manage_metadata_sool9
+      - file: manage_userdata_sool9
+      - file: manage_cidata_sool9
+
+# Wait for cloud-init to complete and VM to shutdown
+wait_for_cloud_init_sool9:
+  cmd.run:
+    - name: /usr/sbin/so-wait-cloud-init -n sool9
+    - require:
+      - cmd: create_vm_sool9
+    - onchanges:
+      - cmd: create_vm_sool9
+    - timeout: 600
+
+# Configure network predictability after cloud-init
+configure_network_predictable_sool9:
+  cmd.run:
+    - name: /usr/sbin/so-qcow2-network-predictable -n sool9
+    - require:
+      - cmd: wait_for_cloud_init_sool9
+    - onchanges:
+      - cmd: create_vm_sool9
+
+# Fire event here that causes soc.dyanno.hypervisor state to be applied
+base_domain_ready:
+  event.send:
+    - name: soc/dyanno/hypervisor/baseDomain
+    - data:
+        status: 'Initialized'
+    - require:
+      - cmd: configure_network_predictable_sool9
+    - onchanges:
+      - cmd: create_vm_sool9
+
+{%   else %}
+{{sls}}_no_license_detected:
+  test.fail_without_changes:
+    - name: {{sls}}_no_license_detected
+    - comment:
+      - "Hypervisor nodes are a feature supported only for customers with a valid license.
+      Contact Security Onion Solutions, LLC via our website at https://securityonionsolutions.com
+      for more information about purchasing a license to enable this feature."
+{%   endif %}
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/libvirt/images/sool9/README

@@ -0,0 +1 @@
+# The files in this directory (/opt/so/saltstack/local/salt/libvirt/images/sool9) are generated by the setup_hypervisor runner. They are then distributed to the hypervisors where a storage pool will be created then the image can be installed.

salt/libvirt/init.sls

@@ -0,0 +1,112 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+#
+# Note: Per the Elastic License 2.0, the second limitation states:
+#
+#   "You may not move, change, disable, or circumvent the license key functionality
+#    in the software, and you may not remove or obscure any functionality in the
+#    software that is protected by the license key."
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls in allowed_states %}
+{%   if 'vrt' in salt['pillar.get']('features', []) %}
+{%     from 'libvirt/map.jinja' import LIBVIRTMERGED %}
+{%     from 'salt/map.jinja' import SYSTEMD_UNIT_FILE %}
+
+include:
+  - libvirt.64962
+  - libvirt.packages
+  - libvirt.ssh.users
+
+install_libvirt:
+  pkg.installed:
+    - name: libvirt
+
+libvirt_conf_dir:
+  file.directory:
+    - name: /opt/so/conf/libvirt
+    - user: 939
+    - group: 939
+    - makedirs: True
+
+libvirt_config:
+  file.managed:
+    - name: /opt/so/conf/libvirt/libvirtd.conf
+    - source: salt://libvirt/etc/libvirtd.conf
+#    - source: salt://libvirt/etc/libvirtd.conf.jinja
+#    - template: jinja
+#    - defaults:
+#        LIBVIRTMERGED: {{ LIBVIRTMERGED }}
+
+# since the libvirtd service looks for the config at /etc/libvirt/libvirtd.conf, and we dont want to manage the service looking in a new location, create this symlink to the managed config 
+config_symlink:
+  file.symlink:
+    - name: /etc/libvirt/libvirtd.conf
+    - target: /opt/so/conf/libvirt/libvirtd.conf
+    - force: True
+    - user: qemu
+    - group: qemu
+
+libvirt_service:
+  service.running:
+    - name: libvirtd
+    - enable: True
+    - watch:
+      - file: libvirt_config
+
+# places cacert, clientcert, clientkey, servercert and serverkey
+# /etc/pki/CA/cacert.pem
+# /etc/pki/libvirt/clientcert.pem and /etc/pki/libvirt/servercert.pem
+# /etc/pki/libvirt/private/clientkey.pem and /etc/pki/libvirt/private/serverkey.pem
+libvirt_keys:
+  virt.keys:
+    - name: libvirt_keys
+
+install_qemu:
+  pkg.installed:
+    - name: qemu-kvm
+
+create_host_bridge:
+  virt.network_running:
+    - name: host-bridge
+    - bridge: br0
+    - forward: bridge
+    - autostart: True
+
+# Disable the default storage pool to avoid conflicts
+disable_default_pool:
+  cmd.run:
+    - name: virsh pool-destroy default && virsh pool-autostart default --disable
+    - onlyif: virsh pool-list | grep default
+    - require:
+      - pkg: install_libvirt-client
+      - service: libvirt_service
+
+disable_default_bridge:
+  cmd.run:
+    - name: virsh net-destroy default && virsh net-autostart default --disable
+    - require:
+      - pkg: install_libvirt-client
+      - service: libvirt_service
+    - onlyif:
+      - virsh net-list | grep default
+
+{%   else %}
+{{sls}}_no_license_detected:
+  test.fail_without_changes:
+    - name: {{sls}}_no_license_detected
+    - comment:
+      - "Hypervisor nodes are a feature supported only for customers with a valid license.
+      Contact Security Onion Solutions, LLC via our website at https://securityonionsolutions.com
+      for more information about purchasing a license to enable this feature."
+{%   endif %}
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/libvirt/map.jinja

@@ -0,0 +1,7 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+   
+{% import_yaml 'libvirt/defaults.yaml' as LIBVIRTDEFAULTS %}
+{% set LIBVIRTMERGED = salt['pillar.get']('libvirt', LIBVIRTDEFAULTS.libvirt, merge=True) %}

salt/libvirt/packages.sls

@@ -0,0 +1,84 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+#
+# Note: Per the Elastic License 2.0, the second limitation states:
+#
+#   "You may not move, change, disable, or circumvent the license key functionality
+#    in the software, and you may not remove or obscure any functionality in the
+#    software that is protected by the license key."
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states or sls in allowed_states %}
+{%   if 'vrt' in salt['pillar.get']('features', []) %}
+
+# allows for creating vm images
+# any node manipulating images needs this
+# used on manager for setup_hypervisor runner
+install_qemu-img:
+  pkg.installed:
+    - name: qemu-img
+
+# used on manager for setup_hypervisor runner
+install_xorriso:
+  pkg.installed:
+    - name: xorriso
+
+install_libvirt-libs:
+  pkg.installed:
+    - name: libvirt-libs
+
+libvirt_python_wheel:
+  file.recurse:
+    - name: /opt/so/conf/libvirt/source-packages/libvirt-python
+    - source: salt://libvirt/source-packages/libvirt-python
+    - makedirs: True
+    - clean: True
+
+libvirt_python_module:
+  cmd.run:
+    - name: /opt/saltstack/salt/bin/python3 -m pip install --no-index --find-links=/opt/so/conf/libvirt/source-packages/libvirt-python libvirt-python
+    - onchanges:
+      - file: libvirt_python_wheel
+
+{%     if 'hype' in grains.id.split('_') | last %}
+
+# provides virsh
+install_libvirt-client:
+  pkg.installed:
+    - name: libvirt-client
+
+install_guestfs-tools:
+  pkg.installed:
+    - name: guestfs-tools
+
+install_virt-install:
+  pkg.installed:
+    - name: virt-install
+
+# needed for for so-qcow2-modify-network - import guestfs
+install_python3-libguestfs:
+  pkg.installed:
+    - name: python3-libguestfs
+###
+
+{%     endif %}
+
+{%   else %}
+{{sls}}_no_license_detected:
+  test.fail_without_changes:
+    - name: {{sls}}_no_license_detected
+    - comment:
+      - "Hypervisor nodes are a feature supported only for customers with a valid license.
+      Contact Security Onion Solutions, LLC via our website at https://securityonionsolutions.com
+      for more information about purchasing a license to enable this feature."
+{%   endif %}
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/libvirt/ssh/files/config

@@ -0,0 +1,2 @@
+Match user soqemussh
+        IdentityFile /etc/ssh/auth_keys/soqemussh/id_ecdsa

salt/libvirt/ssh/users.sls

@@ -0,0 +1,69 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+#
+# Note: Per the Elastic License 2.0, the second limitation states:
+#
+#   "You may not move, change, disable, or circumvent the license key functionality
+#    in the software, and you may not remove or obscure any functionality in the
+#    software that is protected by the license key."
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states or sls in allowed_states %}
+{%   if 'vrt' in salt['pillar.get']('features', []) %}
+{%     from 'vars/globals.map.jinja' import GLOBALS %}
+
+{%     if GLOBALS.is_manager %}
+
+root_ssh_config:
+  file.touch:
+    - name: /root/.ssh/config
+
+qemu_ssh_client_config:
+  file.blockreplace:
+    - name: /root/.ssh/config
+    - marker_start: "# START of block managed by Salt - soqemussh config"
+    - marker_end: "# END of block managed by Salt - soqemussh config"
+    - source: salt://libvirt/ssh/files/config
+    - prepend_if_not_found: True
+
+{%     endif %}
+
+{%     if GLOBALS.role in ['so-hypervisor', 'so-managerhype'] %}
+
+# used for qemu+ssh connection between manager and hypervisors
+create_soqemussh_user:
+  user.present:
+    - name: soqemussh
+    - shell: /bin/bash
+    - home: /home/soqemussh
+    - groups:
+      - wheel
+      - qemu
+      - libvirt
+
+soqemussh_pub_key:
+  ssh_auth.present:
+    - user: soqemussh
+    - source: salt://libvirt/ssh/keys/id_ecdsa.pub
+
+{%     endif %}
+
+{%   else %}
+{{sls}}_no_license_detected:
+  test.fail_without_changes:
+    - name: {{sls}}_no_license_detected
+    - comment:
+      - "Hypervisor nodes are a feature supported only for customers with a valid license.
+      Contact Security Onion Solutions, LLC via our website at https://securityonionsolutions.com
+      for more information about purchasing a license to enable this feature."
+{%   endif %}
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/logstash/defaults.yaml

@@ -12,6 +12,8 @@ logstash:
         - search
       manager:
         - manager
+      managerhype:
+        - manager
       managersearch:
         - manager
         - search

salt/logstash/enabled.sls

@@ -16,7 +16,7 @@ include:
   - elasticsearch.ca
 {%   endif %}
 {# Kafka ca runs on nodes that can run logstash for Kafka input / output. Only when Kafka is global pipeline #}
-{%   if GLOBALS.role in ['so-searchnode', 'so-manager', 'so-managersearch', 'so-receiver', 'so-standalone'] and GLOBALS.pipeline == 'KAFKA' %}
+{%   if GLOBALS.role in ['so-searchnode', 'so-manager', 'so-managerhype', 'so-managersearch', 'so-receiver', 'so-standalone'] and GLOBALS.pipeline == 'KAFKA' %}
   - kafka.ca
   - kafka.ssl
 {%   endif %}
@@ -65,26 +65,26 @@ so-logstash:
       - /opt/so/log/logstash:/var/log/logstash:rw
       - /sys/fs/cgroup:/sys/fs/cgroup:ro
       - /opt/so/conf/logstash/etc/certs:/usr/share/logstash/certs:ro
-      {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-receiver'] %}
+      {% if GLOBALS.role in ['so-manager', 'so-managerhype', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-receiver'] %}
       - /etc/pki/filebeat.crt:/usr/share/logstash/filebeat.crt:ro
       - /etc/pki/filebeat.p8:/usr/share/logstash/filebeat.key:ro
       {% endif %}
-      {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import', 'so-eval','so-fleet', 'so-heavynode', 'so-receiver'] %}
+      {% if GLOBALS.is_manager or GLOBALS.role in ['so-fleet', 'so-heavynode', 'so-receiver'] %}
       - /etc/pki/elasticfleet-logstash.crt:/usr/share/logstash/elasticfleet-logstash.crt:ro
       - /etc/pki/elasticfleet-logstash.key:/usr/share/logstash/elasticfleet-logstash.key:ro
       - /etc/pki/elasticfleet-lumberjack.crt:/usr/share/logstash/elasticfleet-lumberjack.crt:ro
       - /etc/pki/elasticfleet-lumberjack.key:/usr/share/logstash/elasticfleet-lumberjack.key:ro
       {% endif %}
-      {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import'] %}
+      {% if GLOBALS.role in ['so-manager', 'so-managerhype', 'so-managersearch', 'so-standalone', 'so-import'] %}
       - /etc/pki/ca.crt:/usr/share/filebeat/ca.crt:ro
       {% else %}
       - /etc/pki/tls/certs/intca.crt:/usr/share/filebeat/ca.crt:ro
       {% endif %}
-      {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-searchnode' ] %}
+      {% if GLOBALS.role in ['so-manager', 'so-managerhype', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-searchnode' ] %}
       - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro
       - /opt/so/conf/ca/tls-ca-bundle.pem:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem:ro
       {% endif %}
-      {% if GLOBALS.pipeline == "KAFKA" and GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-searchnode'] %}
+      {% if GLOBALS.pipeline == "KAFKA" and GLOBALS.role in ['so-manager', 'so-managerhype', 'so-managersearch', 'so-standalone', 'so-searchnode'] %}
       - /etc/pki/kafka-logstash.p12:/usr/share/logstash/kafka-logstash.p12:ro
       - /opt/so/conf/kafka/kafka-truststore.jks:/etc/pki/kafka-truststore.jks:ro
       {% endif %}
@@ -100,7 +100,7 @@ so-logstash:
         {% endfor %}
       {% endif %}
     - watch:
-      {% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-import', 'so-fleet', 'so-receiver'] %}
+      {% if GLOBALS.is_manager or GLOBALS.role in ['so-fleet', 'so-receiver'] %}
       - x509: etc_elasticfleet_logstash_key
       - x509: etc_elasticfleet_logstash_crt
       {% endif %}
@@ -111,23 +111,23 @@ so-logstash:
       - file: ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}
         {% endfor %}
       {% endfor %}
-      {% if GLOBALS.pipeline == 'KAFKA' and GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-searchnode'] %}
+      {% if GLOBALS.pipeline == 'KAFKA' and GLOBALS.role in ['so-manager', 'so-managerhype', 'so-managersearch', 'so-standalone', 'so-searchnode'] %}
       - file: kafkacertz
       {% endif %}
     - require:
-      {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-receiver'] %}
+      {% if grains['role'] in ['so-manager', 'so-managerhype', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-receiver'] %}
       - x509: etc_filebeat_crt
       {% endif %}
-      {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import'] %}
+      {% if grains['role'] in ['so-manager', 'so-managerhype', 'so-managersearch', 'so-standalone', 'so-import'] %}
       - x509: pki_public_ca_crt
       {% else %}
       - x509: trusttheca
       {% endif %}
-      {% if grains.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import'] %}
+      {% if grains.role in ['so-manager', 'so-managerhype', 'so-managersearch', 'so-standalone', 'so-import'] %}
       - file: cacertz
       - file: capemz
       {% endif %}
-      {% if GLOBALS.pipeline == 'KAFKA' and GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-searchnode'] %}
+      {% if GLOBALS.pipeline == 'KAFKA' and GLOBALS.role in ['so-manager', 'so-managerhype', 'so-managersearch', 'so-standalone', 'so-searchnode'] %}
       - file: kafkacertz
       {% endif %}
 

salt/manager/hypervisor.sls

@@ -0,0 +1,64 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+#
+# Note: Per the Elastic License 2.0, the second limitation states:
+#
+#   "You may not move, change, disable, or circumvent the license key functionality
+#    in the software, and you may not remove or obscure any functionality in the
+#    software that is protected by the license key."
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+{%   if 'vrt' in salt['pillar.get']('features', []) %}
+{% set manager_hostname = grains.id.split('_')[0] %}
+
+# Check if hypervisor environment has been set up
+{% set ssh_user_exists = salt['user.info']('soqemussh') %}
+{% set ssh_keys_exist = salt['file.file_exists']('/etc/ssh/auth_keys/soqemussh/id_ecdsa') and 
+                        salt['file.file_exists']('/etc/ssh/auth_keys/soqemussh/id_ecdsa.pub') and
+                        salt['file.file_exists']('/opt/so/saltstack/local/salt/libvirt/ssh/keys/id_ecdsa.pub') %}
+{% set base_image_exists = salt['file.file_exists']('/nsm/libvirt/boot/OL9U5_x86_64-kvm-b253.qcow2') %}
+{% set vm_files_exist = salt['file.directory_exists']('/opt/so/saltstack/local/salt/libvirt/images/sool9') and
+                        salt['file.file_exists']('/opt/so/saltstack/local/salt/libvirt/images/sool9/sool9.qcow2') and
+                        salt['file.file_exists']('/opt/so/saltstack/local/salt/libvirt/images/sool9/sool9-cidata.iso') %}
+{% set hypervisor_host_dir_exists = salt['file.directory_exists']('/opt/so/saltstack/local/salt/hypervisor/hosts/' ~ manager_hostname) %}
+
+{% if ssh_user_exists and ssh_keys_exist and base_image_exists and vm_files_exist and hypervisor_host_dir_exists %}
+# Hypervisor environment is already set up, include the necessary states
+include:
+  - hypervisor
+  - libvirt
+  - libvirt.images
+
+hypervisor_setup_verified:
+  test.succeed_without_changes:
+    - name: Hypervisor environment is already set up
+    - comment: All required files and configurations for the hypervisor environment exist
+
+{% else %}
+# Hypervisor environment needs to be set up
+run_setup_hypervisor:
+  salt.runner:
+    - name: setup_hypervisor.setup_environment
+    - minion_id: {{ grains.id }}
+{% endif %}
+
+{%   else %}
+{{sls}}_no_license_detected:
+  test.fail_without_changes:
+    - name: {{sls}}_no_license_detected
+    - comment:
+      - "Hypervisor nodes are a feature supported only for customers with a valid license.
+      Contact Security Onion Solutions, LLC via our website at https://securityonionsolutions.com
+      for more information about purchasing a license to enable this feature."
+{%   endif %}
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/manager/init.sls

@@ -156,6 +156,13 @@ rules_dir:
     - group: socore
     - makedirs: True
 
+nsm_playbooks_dir:
+  file.directory:
+    - name: /nsm/airgap-resources/playbooks
+    - user: socore
+    - group: socore
+    - makedirs: True
+
 git_config_set_safe_dirs:
   git.config_set:
     - name: safe.directory
@@ -166,6 +173,8 @@ git_config_set_safe_dirs:
       - /nsm/rules/custom-local-repos/local-yara
       - /nsm/securityonion-resources
       - /opt/so/conf/soc/ai_summary_repos/securityonion-resources
+      - /nsm/airgap-resources/playbooks
+      - /opt/so/conf/soc/playbooks
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/manager/managed_soc_annotations.sls

@@ -5,7 +5,7 @@
 
 {# Managed elasticsearch/soc_elasticsearch.yaml file for adding integration configuration items to UI #}
 {% set managed_integrations = salt['pillar.get']('elasticsearch:managed_integrations', []) %}
-{% if managed_integrations %}
+{% if managed_integrations and salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') and salt['file.file_exists']('/opt/so/state/esfleet_component_templates.json') %}
 {%   from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %}
 {%   set addon_integration_keys = ADDON_INTEGRATION_DEFAULTS.keys() %}
 {%   set matched_integration_names = [] %}

salt/manager/tools/sbin/so-firewall

@@ -10,27 +10,44 @@ import subprocess
 import sys
 import time
 import yaml
+import logging
+
+# Configure logging to both file and console
+logger = logging.getLogger('so-firewall')
+logger.setLevel(logging.INFO)
+
+# File handler
+file_handler = logging.FileHandler('/opt/so/log/so-firewall.log')
+file_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
+logger.addHandler(file_handler)
+
+# Console handler
+console_handler = logging.StreamHandler()
+console_handler.setFormatter(logging.Formatter('%(levelname)s - %(message)s'))
+logger.addHandler(console_handler)
 
 lockFile = "/tmp/so-firewall.lock"
 hostgroupsFilename = "/opt/so/saltstack/local/pillar/firewall/soc_firewall.sls"
 defaultsFilename = "/opt/so/saltstack/default/salt/firewall/defaults.yaml"
 
 def showUsage(options, args):
-  print('Usage: {} [OPTIONS] <COMMAND> [ARGS...]'.format(sys.argv[0]))
-  print('  Options:')
-  print('   --apply         - After updating the firewall configuration files, apply the new firewall state')
-  print('')
-  print('  General commands:')
-  print('    help           - Prints this usage information.')
-  print('    apply          - Apply the firewall state.')
-  print('')
-  print('  Host commands:')
-  print('    includehost    - Includes the given IP in the given group. Args: <GROUP_NAME> <IP>')
-  print('    addhostgroup   - Adds a new, custom host group. Args: <GROUP_NAME>')
-  print('')
-  print('  Where:')
-  print('   GROUP_NAME    - The name of an alias group (Ex: analyst)')
-  print('   IP            - Either a single IP address (Ex: 8.8.8.8) or a CIDR block (Ex: 10.23.0.0/16).')
+  usage = f'''Usage: {sys.argv[0]} [OPTIONS] <COMMAND> [ARGS...]
+  Options:
+   --apply         - After updating the firewall configuration files, apply the new firewall state with queue=True
+
+  General commands:
+    help           - Prints this usage information.
+    apply          - Apply the firewall state.
+
+  Host commands:
+    includehost    - Includes the given IP in the given group. Args: <GROUP_NAME> <IP>
+    removehost     - Removes the given IP from all hostgroups. Args: <IP>
+    addhostgroup   - Adds a new, custom host group. Args: <GROUP_NAME>
+
+  Where:
+   GROUP_NAME    - The name of an alias group (Ex: analyst)
+   IP            - Either a single IP address (Ex: 8.8.8.8) or a CIDR block (Ex: 10.23.0.0/16).'''
+  logger.error(usage)
   sys.exit(1)
 
 def checkApplyOption(options):
@@ -61,7 +78,7 @@ def addIp(name, ip):
     else:
       hostgroup = content['firewall']['hostgroups'][name]
   else:
-    print('Host group not defined in salt/firewall/defaults.yaml or hostgroup name is unallowed.', file=sys.stderr)
+    logger.error(f"Host group {name} not defined in defaults or is unallowed")
     return 4
   ips = hostgroup
   if ips is None:
@@ -69,15 +86,16 @@ def addIp(name, ip):
     hostgroup = ips
   if ip not in ips:
     ips.append(ip)
+    writeYaml(hostgroupsFilename, content)
+    logger.info(f"Successfully added IP {ip} to hostgroup {name}")
   else:
-    print('Already exists', file=sys.stderr)
+    logger.warning(f"IP {ip} already exists in hostgroup {name}")
     return 3
-  writeYaml(hostgroupsFilename, content)
   return 0
 
 def includehost(options, args):
   if len(args) != 2:
-    print('Missing host group name or ip argument', file=sys.stderr)
+    logger.error('Missing host group name or ip argument')
     showUsage(options, args)
   result = addIp(args[0], args[1])
   code = result
@@ -86,9 +104,45 @@ def includehost(options, args):
   return code
 
 def apply(options, args):
-  proc = subprocess.run(['salt-call', 'state.apply', 'firewall', 'queue=True'])
+  logger.info("Applying firewall configuration changes")
+  salt_args = ['salt-call', 'state.apply', 'firewall', 'queue=True']
+  proc = subprocess.run(salt_args)
+  if proc.returncode != 0:
+    logger.error("Failed to apply firewall changes")
+  else:
+    logger.info("Successfully applied firewall changes")
   return proc.returncode
 
+def removehost(options, args):
+  """Remove an IP from all hostgroups and apply changes if requested"""
+  if len(args) != 1:
+    logger.error('Missing IP argument')
+    showUsage(options, args)
+  
+  ip = args[0]
+  content = loadYaml(hostgroupsFilename)
+  if not content or 'firewall' not in content or 'hostgroups' not in content['firewall']:
+    logger.error("Invalid firewall configuration structure")
+    return 4
+    
+  modified = False
+  removed_from = []
+  for group_name, ips in content['firewall']['hostgroups'].items():
+    if ips and ip in ips:
+      ips.remove(ip)
+      modified = True
+      removed_from.append(group_name)
+  
+  if modified:
+    writeYaml(hostgroupsFilename, content)
+    logger.info(f"Successfully removed IP {ip} from hostgroups: {', '.join(removed_from)}")
+    if "--apply" in options:
+      return apply(None, None)
+  else:
+    logger.error(f"IP {ip} not found in any hostgroups")
+  
+  return 0
+
 def main():
   options = []
   args = sys.argv[1:]
@@ -103,6 +157,7 @@ def main():
   commands = {
     "help": showUsage,
     "includehost": includehost,
+    "removehost": removehost,
     "apply": apply
   }
 
@@ -121,7 +176,7 @@ def main():
         time.sleep(2)
 
     if lockAttempts == maxAttempts:
-      print("Lock file (" + lockFile + ") could not be created; proceeding without lock.")
+      logger.error(f"Lock file ({lockFile}) could not be created - proceeding without lock")
 
     cmd = commands.get(args[0], showUsage)
     code = cmd(options, args[1:])
@@ -129,9 +184,9 @@ def main():
     try:
       os.remove(lockFile)
     except:
-      print("Lock file (" + lockFile + ") already removed")
+      logger.error(f"Lock file ({lockFile}) already removed")
 
   sys.exit(code)
 
 if __name__ == "__main__":
-  main()
+  main()

salt/manager/tools/sbin/so-firewall-minion

@@ -51,6 +51,10 @@ fi
         'MANAGER')
 			so-firewall includehost manager "$IP"
 			;;
+		'MANAGERHYPE')
+			so-firewall includehost manager "$IP"
+			so-firewall includehost hypervisor "$IP" --apply
+			;;
 		'MANAGERSEARCH')
 			so-firewall includehost manager "$IP"
 			so-firewall includehost searchnode "$IP" --apply
@@ -82,4 +86,7 @@ fi
 		'DESKTOP')
 			so-firewall includehost desktop "$IP" --apply
 			;;
+		'HYPERVISOR')
+			so-firewall includehost hypervisor "$IP" --apply
+			;;
     esac

salt/manager/tools/sbin/soup

@@ -417,6 +417,8 @@ preupgrade_changes() {
     [[ "$INSTALLEDVERSION" == 2.4.130 ]] && up_to_2.4.140
     [[ "$INSTALLEDVERSION" == 2.4.140 ]] && up_to_2.4.141
     [[ "$INSTALLEDVERSION" == 2.4.141 ]] && up_to_2.4.150
+    [[ "$INSTALLEDVERSION" == 2.4.150 ]] && up_to_2.4.160
+    [[ "$INSTALLEDVERSION" == 2.4.160 ]] && up_to_2.4.170
     true
 }
 
@@ -444,6 +446,8 @@ postupgrade_changes() {
     [[ "$POSTVERSION"  == 2.4.130 ]] && post_to_2.4.140
     [[ "$POSTVERSION"  == 2.4.140 ]] && post_to_2.4.141
     [[ "$POSTVERSION"  == 2.4.141 ]] && post_to_2.4.150
+    [[ "$POSTVERSION"  == 2.4.150 ]] && post_to_2.4.160
+    [[ "$POSTVERSION"  == 2.4.160 ]] && post_to_2.4.170
     true
 }
 
@@ -574,9 +578,25 @@ post_to_2.4.141() {
 }
 
 post_to_2.4.150() {
+  echo "Nothing to apply"
+  POSTVERSION=2.4.150
+}
+
+post_to_2.4.160() {
+  echo "Nothing to apply"
+  POSTVERSION=2.4.160
+}
+
+post_to_2.4.170() {
   echo "Regenerating Elastic Agent Installers"
   /sbin/so-elastic-agent-gen-installers
-  POSTVERSION=2.4.150
+
+  # Update kibana default space
+  salt-call state.apply kibana.config queue=True
+  echo "Updating Kibana default space"
+  /usr/sbin/so-kibana-space-defaults
+
+  POSTVERSION=2.4.170
 }
 
 repo_sync() {
@@ -788,9 +808,6 @@ up_to_2.4.130() {
   # Remove any old Elastic Defend config files
   rm -f /opt/so/conf/elastic-fleet/integrations/endpoints-initial/elastic-defend-endpoints.json
 
-  # Elastic Update for this release, so download Elastic Agent files
-  determine_elastic_agent_upgrade
-
   # Ensure override exists to allow nmcli access to other devices
   touch /etc/NetworkManager/conf.d/10-globally-managed-devices.conf
 
@@ -816,6 +833,29 @@ up_to_2.4.150() {
   INSTALLEDVERSION=2.4.150
 }
 
+up_to_2.4.160() {
+  echo "Nothing to do for 2.4.160"
+
+  INSTALLEDVERSION=2.4.160
+}
+
+up_to_2.4.170() {
+  echo "Creating pillar files for virtualization feature"
+
+  states=("hypervisor" "vm" "libvirt")
+  
+  # Create pillar files for each state
+  for state in "${states[@]}"; do
+    mkdir -p /opt/so/saltstack/local/pillar/$state
+    touch /opt/so/saltstack/local/pillar/$state/adv_$state.sls /opt/so/saltstack/local/pillar/$state/soc_$state.sls
+  done
+
+  # Elastic Update for this release, so download Elastic Agent files
+  determine_elastic_agent_upgrade
+
+  INSTALLEDVERSION=2.4.170
+}
+
 add_hydra_pillars() {
   mkdir -p /opt/so/saltstack/local/pillar/hydra
   touch /opt/so/saltstack/local/pillar/hydra/soc_hydra.sls
@@ -1405,6 +1445,8 @@ main() {
     if [[ $is_airgap -eq 0 ]]; then
       echo "Updating Rule Files to the Latest."
       update_airgap_rules
+      echo "Updating Playbooks to the Latest."
+      airgap_playbooks "$UPDATE_DIR"
     fi
 
     # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars

salt/manager/tools/sbin_jinja/so-salt-cloud

@@ -0,0 +1,665 @@
+#!/opt/saltstack/salt/bin/python3
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+#
+# Note: Per the Elastic License 2.0, the second limitation states:
+#
+#   "You may not move, change, disable, or circumvent the license key functionality
+#    in the software, and you may not remove or obscure any functionality in the
+#    software that is protected by the license key."
+
+{%   if 'vrt' in salt['pillar.get']('features', []) -%}
+
+"""
+Script for automated virtual machine provisioning and configuration in Security Onion's virtualization infrastructure. 
+This script integrates multiple components to provide a streamlined VM deployment process:
+
+1. Salt Cloud Integration:
+   - Works with libvirt salt-cloud provider for VM creation
+   - Manages VM lifecycle from provisioning through configuration
+   - Handles profile-based deployment for consistent VM setups
+
+2. Network Configuration Management:
+   - Supports both DHCP and static IPv4 networking
+   - Pre-configures network settings before VM deployment
+   - Integrates with qcow2.modify_network_config for image modification
+   - Ensures VMs boot with correct network configuration
+
+3. Hardware Resource Management:
+   - Flexible CPU and memory allocation
+   - Advanced PCI device passthrough capabilities
+   - Controlled VM startup sequence
+   - Uses qcow2.modify_hardware_config for hardware settings
+
+4. Security Integration:
+   - Automatic firewall rule configuration
+   - Directly integrates with so-firewall for consistent VM management
+   - Configures role-based firewall rules for new VMs
+   - Uses same firewall integration approach for both adding and removing VMs
+
+This script serves as the primary interface for VM deployment in Security Onion, coordinating 
+between salt-cloud, network configuration, hardware management, and security components to 
+ensure proper VM provisioning and configuration.
+
+Usage:
+    # Create a VM:
+    so-salt-cloud -p <profile> <vm_name> (--dhcp4 | --static4 --ip4 <ip_address> --gw4 <gateway>) 
+                            [-c <cpu_count>] [-m <memory_amount>] [-P <pci_id>] [-P <pci_id> ...] [--dns4 <dns_servers>] [--search4 <search_domain>]
+
+    # Delete a VM:
+    so-salt-cloud -p <profile> <vm_name> -d [-y]
+
+Options:
+    -p, --profile         The cloud profile to build the VM from.
+    <vm_name>             The name of the VM.
+    -d, --destroy         Delete the specified VM.
+    -y, --assume-yes      Default yes in answer to all confirmation questions.
+
+    Network Configuration (required for VM creation):
+    --dhcp4               Configure interface for DHCP (IPv4).
+    --static4             Configure interface for static IPv4 settings.
+    --ip4                 IPv4 address (e.g., 192.168.1.10/24). Required for static IPv4 configuration.
+    --gw4                 IPv4 gateway (e.g., 192.168.1.1). Required for static IPv4 configuration.
+    --dns4                Comma-separated list of IPv4 DNS servers (e.g., 8.8.8.8,8.8.4.4).
+    --search4             DNS search domain for IPv4.
+
+    Hardware Configuration (optional):
+    -c, --cpu             Number of virtual CPUs to assign.
+    -m, --memory          Amount of memory to assign in MiB.
+    -P, --pci             PCI hardware ID(s) to passthrough to the VM (e.g., 0000:c7:00.0). Can be specified multiple times.
+                          Format: domain:bus:device.function
+
+Examples:
+
+1. Static IP Configuration with Multiple PCI Devices:
+
+    Command:
+    so-salt-cloud -p sool9_hyper1 vm1_sensor --static4 --ip4 192.168.1.10/24 --gw4 192.168.1.1 \
+        --dns4 192.168.1.1,192.168.1.2 --search4 example.local -c 4 -m 8192 -P 0000:c7:00.0 -P 0000:c4:00.0
+
+    This command provisions a VM named vm1_sensor using the sool9_hyper1 profile with the following settings:
+
+    - Static IPv4 configuration:
+        - IP Address: 192.168.1.10/24
+        - Gateway: 192.168.1.1
+        - DNS Servers: 192.168.1.1, 192.168.1.2
+        - DNS Search Domain: example.local
+    - Hardware Configuration:
+        - CPUs: 4
+        - Memory: 8192 MiB
+        - PCI Device Passthrough: 0000:c7:00.0, 0000:c4:00.0
+
+2. DHCP Configuration with Default Hardware Settings:
+
+    Command:
+    so-salt-cloud -p sool9_hyper1 vm2_master --dhcp4
+
+    This command provisions a VM named vm2_master using the sool9_hyper1 profile with DHCP for network configuration and default hardware settings.
+
+3. Static IP Configuration without Hardware Specifications:
+
+    Command:
+    so-salt-cloud -p sool9_hyper1 vm3_search --static4 --ip4 192.168.1.20/24 --gw4 192.168.1.1
+
+    This command provisions a VM named vm3_search with a static IP configuration and default hardware settings.
+
+4. DHCP Configuration with Custom Hardware Specifications and Multiple PCI Devices:
+
+    Command:
+    so-salt-cloud -p sool9_hyper1 vm4_node --dhcp4 -c 8 -m 16384 -P 0000:c7:00.0 -P 0000:c4:00.0 -P 0000:c4:00.1
+
+    This command provisions a VM named vm4_node using DHCP for network configuration and custom hardware settings:
+
+    - CPUs: 8
+    - Memory: 16384 MiB
+    - PCI Device Passthrough: 0000:c7:00.0, 0000:c4:00.0, 0000:c4:00.1
+
+5. Static IP Configuration with DNS and Search Domain:
+
+    Command:
+    so-salt-cloud -p sool9_hyper1 vm1_sensor --static4 --ip4 192.168.1.10/24 --gw4 192.168.1.1 --dns4 192.168.1.1 --search4 example.local
+
+    This command provisions a VM named vm1_sensor using the sool9_hyper1 profile with static IPv4 configuration:
+
+    - Static IPv4 configuration:
+        - IP Address: 192.168.1.10/24
+        - Gateway: 192.168.1.1
+        - DNS Server: 192.168.1.1
+        - DNS Search Domain: example.local
+
+6. Delete a VM with Confirmation:
+
+    Command:
+    so-salt-cloud -p sool9_hyper1 vm1_sensor -d
+
+    This command deletes the VM named vm1_sensor and will prompt for confirmation before proceeding.
+
+7. Delete a VM without Confirmation:
+
+    Command:
+    so-salt-cloud -p sool9_hyper1 vm1_sensor -yd
+
+    This command deletes the VM named vm1_sensor without prompting for confirmation.
+
+Notes:
+
+- When using --static4, both --ip4 and --gw4 options are required.
+- The script assumes the cloud profile name follows the format basedomain-hypervisorname.
+- Hardware parameters (-c, -m, -P) are optional. If not provided, default values from the profile will be used.
+- The -P or --pci option can be specified multiple times to pass through multiple PCI devices to the VM.
+- The vm_name should include the role of the VM after an underscore (e.g., hostname_role), as the script uses this to determine the VM's role for firewall configuration.
+- PCI hardware IDs must be in the format domain:bus:device.function (e.g., 0000:c7:00.0).
+
+Description:
+
+The so-salt-cloud script automates the provisioning and configuration of virtual machines in Security Onion's infrastructure. It orchestrates multiple components to ensure proper VM setup and security configuration. The script executes in the following phases:
+
+1. Network Configuration Phase:
+   - Pre-deployment network setup using qcow2.modify_network_config
+   - Supports both DHCP and static IPv4 configurations
+   - Modifies the base QCOW2 image directly to ensure network settings persist
+   - Handles DNS and search domain configuration for proper name resolution
+   - Validates network parameters before modification
+   - Ensures network settings are in place before VM creation
+
+2. VM Provisioning Phase:
+   - Leverages salt-cloud for consistent VM deployment
+   - Uses predefined profiles for standardized configurations
+   - Manages the VM lifecycle through libvirt
+   - Prevents automatic VM start to allow hardware configuration
+   - Validates profile and VM name format
+   - Extracts role information from VM name for security configuration
+
+3. Hardware Configuration Phase:
+   - Configures VM hardware through qcow2.modify_hardware_config
+   - Manages CPU allocation based on host capabilities
+   - Handles memory assignment in MiB units
+   - Supports multiple PCI device passthrough for advanced networking
+   - Validates hardware parameters against host resources
+   - Controls VM startup sequence after configuration
+
+4. Security Integration Phase:
+   - Monitors salt-cloud output for VM IP address assignment
+   - Extracts role information from VM name
+   - Calls so-firewall directly to configure firewall rules
+   - Configures role-based firewall rules automatically
+   - Ensures security policies are in place for VM access
+   - Logs all security-related operations for audit purposes
+
+The script implements extensive error handling and logging throughout each phase:
+- Validates all input parameters before execution
+- Provides detailed error messages for troubleshooting
+- Logs operations to both file and console
+- Handles process interruption gracefully
+- Ensures atomic operations where possible
+- Maintains audit trail of all configuration changes
+
+Integration points:
+- Works with Security Onion's salt-cloud provider
+- Interfaces with qcow2 module for image and hardware management
+- Directly integrates with so-firewall for security configuration
+- Uses libvirt for VM management
+- Leverages SaltStack for distributed execution
+
+Exit Codes:
+
+- 0: Success
+- Non-zero: An error occurred during execution.
+
+Logging:
+
+- Logs are written to /opt/so/log/salt/so-salt-cloud.log.
+- Both file and console logging are enabled for real-time monitoring.
+
+"""
+
+import argparse
+import os
+import subprocess
+import re
+import sys
+import threading
+import salt.client
+import logging
+import yaml
+
+# Initialize Salt local client
+local = salt.client.LocalClient()
+
+# Set up logging
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+file_handler = logging.FileHandler('/opt/so/log/salt/so-salt-cloud.log')
+console_handler = logging.StreamHandler()
+
+formatter = logging.Formatter('%(asctime)s %(message)s')
+file_handler.setFormatter(formatter)
+console_handler.setFormatter(formatter)
+
+logger.addHandler(file_handler)
+logger.addHandler(console_handler)
+
+def add_host_to_firewall(ip, role):
+    """Configure firewall rules for a new VM.
+    
+    Args:
+        ip (str): The IP address of the VM to add to the firewall
+        role (str): The role of the VM (e.g., 'sensor', 'manager', etc.)
+        
+    This function calls so-firewall directly to configure firewall rules,
+    maintaining consistency with how firewall rules are managed during
+    VM deletion.
+    """
+    try:
+        # Call so-firewall directly with --apply
+        process = subprocess.Popen(
+            ['/usr/sbin/so-firewall', 'includehost', role.lower(), ip, '--apply'],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True
+        )
+
+        # Read and log the output
+        for line in iter(process.stdout.readline, ''):
+            if line:
+                logger.info(line.rstrip('\n'))
+
+        process.stdout.close()
+        process.wait()
+
+    except Exception as e:
+        logger.error(f"An error occurred while adding host to firewall: {e}")
+
+def get_vm_ip(vm_name):
+    """Get IP address of VM before deletion"""
+    try:
+        # Get IP from minion's pillar file
+        pillar_file = f"/opt/so/saltstack/local/pillar/minions/{vm_name}.sls"
+        with open(pillar_file, 'r') as f:
+            pillar_data = yaml.safe_load(f)
+            
+        if pillar_data and 'host' in pillar_data and 'mainip' in pillar_data['host']:
+            return pillar_data['host']['mainip']
+        raise Exception(f"Could not find mainip in pillar file {pillar_file}")
+    except FileNotFoundError:
+        raise Exception(f"Pillar file not found: {pillar_file}")
+    except Exception as e:
+        logger.error(f"Failed to get IP for VM {vm_name}: {e}")
+        raise
+
+def cleanup_deleted_vm(ip, role):
+    """Handle cleanup tasks when a VM is deleted"""
+    try:
+        # Remove IP from firewall
+        process = subprocess.Popen(
+            ['/usr/sbin/so-firewall', '--apply', 'removehost', ip],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True
+        )
+        
+        for line in iter(process.stdout.readline, ''):
+            if line:
+                logger.info(line.rstrip('\n'))
+                
+        process.stdout.close()
+        process.wait()
+        
+        if process.returncode == 0:
+            logger.info(f"Successfully removed IP {ip} from firewall configuration")
+        else:
+            logger.error(f"Failed to remove IP {ip} from firewall configuration")
+            
+    except Exception as e:
+        logger.error(f"Error during VM cleanup: {e}")
+
+def delete_vm(profile, vm_name, assume_yes=False):
+    """Delete a VM and perform cleanup tasks"""
+    try:
+        # Get VM's IP before deletion for cleanup
+        ip = get_vm_ip(vm_name)
+        role = vm_name.split("_")[1]
+        
+        # Run salt-cloud destroy command
+        cmd = ['salt-cloud', '-p', profile, vm_name, '-d']
+        if assume_yes:
+            cmd.append('-y')
+            
+        process = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True
+        )
+        
+        # Pattern to detect when no machines were found to be destroyed
+        no_machines_string = 'No machines were found to be destroyed'
+        no_machines_pattern = re.compile(re.escape(no_machines_string))
+        
+        # Track if we found any successful destruction
+        machines_destroyed = False
+        output_lines = []
+        
+        # Monitor output
+        for line in iter(process.stdout.readline, ''):
+            if line:
+                logger.info(line.rstrip('\n'))
+                output_lines.append(line.strip())
+                
+                # Check if no machines were found to be destroyed
+                if no_machines_pattern.search(line):
+                    machines_destroyed = False
+                    break
+                
+        process.stdout.close()
+        process.wait()
+        
+        # If we hit the "No machines were found" case, it's a failure
+        if no_machines_pattern.search('\n'.join(output_lines)):
+            logger.error(f"VM {vm_name} was not found to be destroyed. Verify that all configured hypervisors are online.")
+            sys.exit(1)
+        
+        # Check for successful destruction patterns in the output
+        # Look for the VM name appearing in libvirt section - this indicates successful processing
+        full_output = '\n'.join(output_lines)
+        if vm_name in full_output and 'libvirt:' in full_output:
+            # VM was processed by libvirt, which means destruction was attempted
+            # If we reach here and didn't hit the "No machines found" case, it's success
+            machines_destroyed = True
+        
+        # Check success criteria: returncode == 0 AND we found evidence of destruction
+        if process.returncode == 0 and machines_destroyed:
+            # Start cleanup tasks only when actual deletion occurred
+            cleanup_deleted_vm(ip, role)
+            logger.info(f"Successfully deleted VM {vm_name}")
+        elif process.returncode == 0:
+            # Command succeeded but we couldn't confirm destruction - this is the edge case we're fixing
+            # If salt-cloud returned 0 and we didn't hit the "No machines found" case,
+            # but we also don't see clear destruction evidence, we should still consider it success
+            # because salt-cloud returning 0 means it completed successfully
+            cleanup_deleted_vm(ip, role)
+            logger.info(f"Successfully deleted VM {vm_name} (salt-cloud completed successfully)")
+        else:
+            logger.error(f"Failed to delete VM {vm_name}")
+            sys.exit(1)
+            
+    except Exception as e:
+        logger.error(f"Failed to delete VM {vm_name}: {e}")
+        raise
+
+def _add_hypervisor_host_key(hostname):
+    """Add hypervisor host key to root's known_hosts file.
+    
+    Args:
+        hostname (str): The hostname or IP of the hypervisor
+        
+    Returns:
+        bool: True if key was added or already exists, False on error
+    """
+    try:
+        known_hosts = '/root/.ssh/known_hosts'
+        os.makedirs(os.path.dirname(known_hosts), exist_ok=True)
+        
+        # Check if key already exists using ssh-keygen
+        if os.path.exists(known_hosts):
+            check_result = subprocess.run(['ssh-keygen', '-F', hostname],
+                                        capture_output=True, text=True)
+            if check_result.returncode == 0 and check_result.stdout.strip():
+                logger.info("Host key for %s already in known_hosts", hostname)
+                return True
+            
+        # Get host key using ssh-keyscan
+        logger.info("Scanning host key for %s", hostname)
+        process = subprocess.run(['ssh-keyscan', '-H', hostname],
+                               capture_output=True, text=True)
+        
+        if process.returncode == 0 and process.stdout:
+            # Append new key
+            with open(known_hosts, 'a') as f:
+                f.write(process.stdout)
+            logger.info("Added host key for %s to known_hosts", hostname)
+            return True
+        else:
+            logger.error("Failed to get host key for %s: %s",
+                      hostname, process.stderr)
+            return False
+            
+    except Exception as e:
+        logger.error("Error adding host key for %s: %s", hostname, str(e))
+        return False
+
+def call_salt_cloud(profile, vm_name, destroy=False, assume_yes=False):
+    """Call salt-cloud to create or destroy a VM"""
+    try:
+        if destroy:
+            delete_vm(profile, vm_name, assume_yes)
+            return
+
+        # Extract hypervisor hostname from profile (e.g., sool9_hype1 -> hype1)
+        hypervisor = profile.split('_', 1)[1] if '_' in profile else None
+        if hypervisor:
+            logger.info("Ensuring host key exists for hypervisor %s", hypervisor)
+            if not _add_hypervisor_host_key(hypervisor):
+                logger.error("Failed to add host key for %s, cannot proceed with VM creation", hypervisor)
+                sys.exit(1)
+
+        # Start the salt-cloud command as a subprocess
+        process = subprocess.Popen(
+            ['salt-cloud', '-p', profile, vm_name, '-l', 'info'],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True
+        )
+
+        role = vm_name.split("_")[1]
+
+        ip_search_string = '[INFO    ] Address ='
+        ip_search_pattern = re.compile(re.escape(ip_search_string))
+
+        # Continuously read the output from salt-cloud
+        while True:
+            # Read stdout line by line
+            line = process.stdout.readline()
+            if line:
+                logger.info(line.rstrip('\n'))
+
+                if ip_search_pattern.search(line):
+                    parts = line.split("Address =")
+                    if len(parts) > 1:
+                        ip_address = parts[1].strip()
+                        logger.info(f"Extracted IP address: {ip_address}")
+                        # Create and start a thread to add host to firewall
+                        thread = threading.Thread(target=add_host_to_firewall, args=(ip_address, role))
+                        thread.start()
+                    else:
+                        logger.error("No IP address found.")
+            else:
+                # Check if salt-cloud has terminated
+                if process.poll() is not None:
+                    break
+
+        process.stdout.close()
+        process.wait()
+
+    except Exception as e:
+        logger.error(f"An error occurred while calling salt-cloud: {e}")
+
+def format_qcow2_output(operation, result):
+    """Format the output from qcow2 module operations for better readability.
+    
+    Args:
+        operation (str): The name of the operation (e.g., 'Network configuration', 'Hardware configuration')
+        result (dict): The result dictionary from the qcow2 module
+        
+    Returns:
+        None - logs the formatted output directly
+    """
+    for host, host_result in result.items():
+        if isinstance(host_result, dict):
+            # Extract and format stderr which contains the detailed log
+            if 'stderr' in host_result:
+                logger.info(f"{operation} on {host}:")
+                for line in host_result['stderr'].split('\n'):
+                    if line.strip():
+                        logger.info(f"  {line.strip()}")
+            if host_result.get('retcode', 0) != 0:
+                logger.error(f"{operation} failed on {host} with return code {host_result.get('retcode')}")
+        else:
+            logger.info(f"{operation} result from {host}: {host_result}")
+
+def run_qcow2_modify_hardware_config(profile, vm_name, cpu=None, memory=None, pci_list=None, start=False):
+    hv_name = profile.split('_')[1]
+    target = hv_name + "_*"
+
+    try:
+        args_list = [
+            'vm_name=' + vm_name,
+            'cpu=' + str(cpu) if cpu else '',
+            'memory=' + str(memory) if memory else '',
+            'start=' + str(start)
+        ]
+
+        # Add PCI devices if provided
+        if pci_list:
+            # Pass all PCI devices as a comma-separated list
+            args_list.append('pci=' + ','.join(pci_list))
+
+        result = local.cmd(target, 'qcow2.modify_hardware_config', args_list)
+        format_qcow2_output('Hardware configuration', result)
+    except Exception as e:
+        logger.error(f"An error occurred while running qcow2.modify_hardware_config: {e}")
+
+def run_qcow2_modify_network_config(profile, vm_name, mode, ip=None, gateway=None, dns=None, search_domain=None):
+    hv_name = profile.split('_')[1]
+    target = hv_name + "_*"
+    image = '/nsm/libvirt/images/sool9/sool9.qcow2'
+    interface = 'enp1s0'
+
+    try:
+        # Base arguments that are always included
+        args = [
+            'image=' + image,
+            'interface=' + interface,
+            'mode=' + mode,
+            'vm_name=' + vm_name
+        ]
+        
+        # Only include IP-related arguments if not using DHCP
+        if mode != "dhcp4":
+            if ip:
+                args.append('ip4=' + ip)
+            if gateway:
+                args.append('gw4=' + gateway)
+            if dns:
+                args.append('dns4=' + dns)
+            if search_domain:
+                args.append('search4=' + search_domain)
+                
+        result = local.cmd(target, 'qcow2.modify_network_config', args)
+        format_qcow2_output('Network configuration', result)
+    except Exception as e:
+        logger.error(f"An error occurred while running qcow2.modify_network_config: {e}")
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description="Call salt-cloud and pass the profile and VM name to it.")
+    parser.add_argument('-p', '--profile', type=str, required=True, help="The cloud profile to build the VM from.")
+    parser.add_argument('vm_name', type=str, help="The name of the VM.")
+    parser.add_argument('-d', '--destroy', action='store_true', help='Delete the specified VM')
+    parser.add_argument('-y', '--assume-yes', action='store_true', help='Default yes in answer to all confirmation questions')
+
+    # Create a group for network config arguments
+    network_group = parser.add_argument_group('Network Configuration')
+    # Make the group mutually exclusive but not required by default
+    mode_group = network_group.add_mutually_exclusive_group()
+    mode_group.add_argument("--dhcp4", action="store_true", help="Configure interface for DHCP (IPv4).")
+    mode_group.add_argument("--static4", action="store_true", help="Configure interface for static IPv4 settings.")
+
+    # Add other network and hardware arguments
+    network_group.add_argument("--ip4", help="IPv4 address (e.g., 192.168.1.10/24). Required for static IPv4 configuration.")
+    network_group.add_argument("--gw4", help="IPv4 gateway (e.g., 192.168.1.1). Required for static IPv4 configuration.")
+    network_group.add_argument("--dns4", help="Comma-separated list of IPv4 DNS servers (e.g., 8.8.8.8,8.8.4.4).")
+    network_group.add_argument("--search4", help="DNS search domain for IPv4.")
+    network_group.add_argument('-c', '--cpu', type=int, help='Number of virtual CPUs to assign.')
+    network_group.add_argument('-m', '--memory', type=int, help='Amount of memory to assign in MiB.')
+    network_group.add_argument('-P', '--pci', action='append', help='PCI hardware ID(s) to passthrough to the VM (e.g., 0000:c7:00.0). Can be specified multiple times.')
+
+    args = parser.parse_args()
+
+    # Only validate network config if not destroying
+    if not args.destroy:
+        if not args.dhcp4 and not args.static4:
+            parser.error("One of --dhcp4 or --static4 is required for VM creation")
+        if args.static4 and (not args.ip4 or not args.gw4):
+            parser.error("Both --ip4 and --gw4 are required for static IPv4 configuration")
+
+    return args
+
+def main():
+    try:
+        args = parse_arguments()
+        
+        # Log the initial request
+        if args.destroy:
+            logger.info(f"Received request to destroy VM '{args.vm_name}' using profile '{args.profile}'{' with --assume-yes' if args.assume_yes else ''}")
+        else:
+            # Build network config string
+            network_config = "using DHCP" if args.dhcp4 else f"with static IP {args.ip4}, gateway {args.gw4}"
+            if args.dns4:
+                network_config += f", DNS {args.dns4}"
+            if args.search4:
+                network_config += f", search domain {args.search4}"
+            
+            # Build hardware config string
+            hw_config = []
+            if args.cpu:
+                hw_config.append(f"{args.cpu} CPUs")
+            if args.memory:
+                hw_config.append(f"{args.memory}MB RAM")
+            if args.pci:
+                hw_config.append(f"PCI devices: {', '.join(args.pci)}")
+            hw_string = f" and hardware config: {', '.join(hw_config)}" if hw_config else ""
+            
+            logger.info(f"Received request to create VM '{args.vm_name}' using profile '{args.profile}' {network_config}{hw_string}")
+
+        if args.destroy:
+            # Handle VM deletion
+            call_salt_cloud(args.profile, args.vm_name, destroy=True, assume_yes=args.assume_yes)
+        else:
+            # Handle VM creation
+            if args.dhcp4:
+                mode = "dhcp4"
+            elif args.static4:
+                mode = "static4"
+            else:
+                mode = "dhcp4"  # Default to DHCP if not specified
+
+            # Step 1: Modify network configuration
+            run_qcow2_modify_network_config(args.profile, args.vm_name, mode, args.ip4, args.gw4, args.dns4, args.search4)
+
+            # Step 2: Provision the VM (without starting it)
+            call_salt_cloud(args.profile, args.vm_name)
+
+            # Step 3: Modify hardware configuration
+            run_qcow2_modify_hardware_config(args.profile, args.vm_name, cpu=args.cpu, memory=args.memory, pci_list=args.pci, start=True)
+
+    except KeyboardInterrupt:
+        logger.error("so-salt-cloud: Operation cancelled by user.")
+        sys.exit(1)
+    except Exception as e:
+        logger.error(f"so-salt-cloud: An error occurred: {e}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()
+
+{%- else -%}
+
+echo "Hypervisor nodes are a feature supported only for customers with a valid license. \
+      Contact Security Onion Solutions, LLC via our website at https://securityonionsolutions.com \
+      for more information about purchasing a license to enable this feature."
+
+{% endif -%}