adding beacon

salt/_beacons/add_virtual_node_beacon.py

@@ -0,0 +1,113 @@
+'''
+Add Virtual Node Beacon
+
+This beacon monitors for creation or modification of files matching a specific pattern
+and sends the contents of the files up to the Salt Master's event bus, including
+the hypervisor and nodetype extracted from the file path.
+
+Configuration:
+
+    beacons:
+      add_virtual_node_beacon:
+        - base_path: /path/to/files/*
+
+If base_path is not specified, it defaults to '/opt/so/saltstack/local/salt/hypervisor/hosts/*/add_*'
+'''
+
+import os
+import glob
+import logging
+import re
+
+log = logging.getLogger(__name__)
+
+__virtualname__ = 'add_virtual_node_beacon'
+DEFAULT_BASE_PATH = '/opt/so/saltstack/local/salt/hypervisor/hosts/*/add_*'
+
+def __virtual__():
+    '''
+    Return the virtual name of the beacon.
+    '''
+    return __virtualname__
+
+def validate(config):
+    '''
+    Validate the beacon configuration.
+
+    Args:
+        config (list): Configuration of the beacon.
+
+    Returns:
+        tuple: A tuple of (bool, str) indicating success and message.
+    '''
+    if not isinstance(config, list):
+        return False, 'Configuration for add_virtual_node_beacon must be a list of dictionaries'
+    for item in config:
+        if not isinstance(item, dict):
+            return False, 'Each item in configuration must be a dictionary'
+        if 'base_path' in item and not isinstance(item['base_path'], str):
+            return False, 'base_path must be a string'
+    return True, 'Valid beacon configuration'
+
+def beacon(config):
+    '''
+    Monitor for creation or modification of files and send events.
+
+    Args:
+        config (list): Configuration of the beacon.
+
+    Returns:
+        list: A list of events to send to the Salt Master.
+    '''
+    if 'add_virtual_node_beacon' not in __context__:
+        __context__['add_virtual_node_beacon'] = {}
+
+    ret = []
+
+    for item in config:
+        base_path = item.get('base_path', DEFAULT_BASE_PATH)
+        file_list = glob.glob(base_path)
+
+        log.debug('Starting add_virtual_node_beacon. Found %d files matching pattern %s', len(file_list), base_path)
+
+        for file_path in file_list:
+            try:
+                mtime = os.path.getmtime(file_path)
+                prev_mtime = __context__['add_virtual_node_beacon'].get(file_path, 0)
+                if mtime > prev_mtime:
+                    log.info('File %s is new or modified', file_path)
+                    with open(file_path, 'r') as f:
+                        contents = f.read()
+
+                    data = {}
+                    # Parse the contents of the file
+                    for line in contents.splitlines():
+                        if ':' in line:
+                            key, value = line.split(':', 1)
+                            data[key.strip()] = value.strip()
+                        else:
+                            log.warning('Line in file %s does not contain colon: %s', file_path, line)
+
+                    # Extract hypervisor and nodetype from the file path
+                    match = re.match(r'^.*/hosts/(?P<hypervisor>[^/]+)/add_(?P<nodetype>[^/]+)$', file_path)
+                    if match:
+                        data['hypervisor'] = match.group('hypervisor')
+                        data['nodetype'] = match.group('nodetype')
+                    else:
+                        log.warning('Unable to extract hypervisor and nodetype from file path: %s', file_path)
+                        data['hypervisor'] = None
+                        data['nodetype'] = None
+
+                    event = {'tag': f'add_virtual_node/{os.path.basename(file_path)}', 'data': data}
+                    ret.append(event)
+                    __context__['add_virtual_node_beacon'][file_path] = mtime
+                else:
+                    log.debug('File %s has not been modified since last check', file_path)
+            except FileNotFoundError:
+                log.warning('File not found: %s', file_path)
+            except PermissionError:
+                log.error('Permission denied when accessing file: %s', file_path)
+            except Exception as e:
+                log.error('Error processing file %s: %s', file_path, str(e))
+
+    return ret

salt/_modules/qcow2.py

@@ -1,10 +1,5 @@
 #!py
 
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
 import logging
 import subprocess
 import shlex
@@ -71,3 +66,58 @@ def modify_network_config(image, interface, mode, ip4=None, gw4=None, dns4=None,
     except Exception as e:
         log.error('qcow2 module: An error occurred while executing the script: {}'.format(e))
         raise
+
+def modify_hardware_config(vm_name, cpu=None, memory=None, pci=None, start=False):
+    '''
+    Wrapper function to call so-kvm-modify-hardware
+
+    :param vm_name: Name of the virtual machine to modify.
+    :param cpu: Number of virtual CPUs to assign.
+    :param memory: Amount of memory to assign in MiB.
+    :param pci: PCI hardware ID to passthrough to the VM (e.g., '0000:00:1f.2').
+    :param start: Boolean flag to start the VM after modification.
+
+    :return: A dictionary with the result of the script execution.
+
+    CLI Example:
+
+    .. code-block:: bash
+
+        salt '*' qcow2.modify_hardware_config vm_name='my_vm' cpu=4 memory=8192 pci='0000:00:1f.2' start=True
+
+    '''
+
+    cmd = ['/usr/sbin/so-kvm-modify-hardware', '-v', vm_name]
+
+    if cpu is not None:
+        if isinstance(cpu, int) and cpu > 0:
+            cmd.extend(['-c', str(cpu)])
+        else:
+            raise ValueError('cpu must be a positive integer.')
+    if memory is not None:
+        if isinstance(memory, int) and memory > 0:
+            cmd.extend(['-m', str(memory)])
+        else:
+            raise ValueError('memory must be a positive integer.')
+    if pci:
+        cmd.extend(['-p', pci])
+    if start:
+        cmd.append('-s')
+
+    log.info('qcow2 module: Executing command: {}'.format(' '.join(shlex.quote(arg) for arg in cmd)))
+
+    try:
+        result = subprocess.run(cmd, capture_output=True, text=True, check=False)
+        ret = {
+            'retcode': result.returncode,
+            'stdout': result.stdout,
+            'stderr': result.stderr
+        }
+        if result.returncode != 0:
+            log.error('qcow2 module: Script execution failed with return code {}: {}'.format(result.returncode, result.stderr))
+        else:
+            log.info('qcow2 module: Script executed successfully.')
+        return ret
+    except Exception as e:
+        log.error('qcow2 module: An error occurred while executing the script: {}'.format(e))
+        raise

salt/hypervisor/tools/sbin/so-kvm-modify-hardware

@@ -8,11 +8,48 @@
 """
 Script to modify hardware parameters of a KVM virtual machine.
 
-Usage:
+**Usage:**
-    python so-kvm-modify-hardware.py -v <vm_name> [-c <cpu_count>] [-m <memory_amount>] [-p <pci_id>] [-s]
+    python so-kvm-modify-hardware.py -v <vm_name> [-c <cpu_count>] [-m <memory_amount>] [-p <pci_id>] [-p <pci_id> ...] [-s]
+
+**Options:**
+    -v, --vm          Name of the virtual machine to modify.
+    -c, --cpu         Number of virtual CPUs to assign.
+    -m, --memory      Amount of memory to assign in MiB.
+    -p, --pci         PCI hardware ID(s) to passthrough to the VM (e.g., 0000:00:1f.2). Can be specified multiple times.
+    -s, --start       Start the VM after modification.
+
+**Examples:**
+
+1. **Modify VM with Multiple PCI Devices:**
+
+    ```bash
+    python so-kvm-modify-hardware.py -v my_vm -c 4 -m 8192 -p 0000:00:1f.2 -p 0000:00:1f.3 -s
+    ```
+
+    This command modifies the VM named `my_vm`, setting the CPU count to 4, memory to 8192 MiB, and adds two PCI devices for passthrough (`0000:00:1f.2` and `0000:00:1f.3`). The VM is then started after modification due to the `-s` flag.
+
+2. **Modify VM with Single PCI Device:**
+
+    ```bash
+    python so-kvm-modify-hardware.py -v my_vm -p 0000:00:1f.2
+    ```
+
+    This command adds a single PCI device passthrough to the VM named `my_vm`.
+
+3. **Modify VM Without Starting It:**
+
+    ```bash
+    python so-kvm-modify-hardware.py -v my_vm -c 2 -m 4096
+    ```
+
+    This command sets the CPU count and memory for `my_vm` but does not start it afterward.
+
+**Notes:**
+
+- The `-p` or `--pci` option can be specified multiple times to pass through multiple PCI devices to the VM.
+- The PCI hardware IDs should be in the format `0000:00:1f.2`.
+- If the `-s` or `--start` flag is not provided, the VM will remain stopped after modification.
 
-Example:
-    python so-kvm-modify-hardware.py -v my_vm -c 4 -m 8192 -p 0000:00:1f.2 -s
 """
 
 import argparse
@@ -28,12 +65,12 @@ def parse_arguments():
     parser.add_argument('-v', '--vm', required=True, help='Name of the virtual machine to modify.')
     parser.add_argument('-c', '--cpu', type=int, help='Number of virtual CPUs to assign.')
     parser.add_argument('-m', '--memory', type=int, help='Amount of memory to assign in MiB.')
-    parser.add_argument('-p', '--pci', help='PCI hardware ID to passthrough to the VM (e.g., 0000:00:1f.2).')
+    parser.add_argument('-p', '--pci', action='append', help='PCI hardware ID(s) to passthrough to the VM (e.g., 0000:00:1f.2). Can be specified multiple times.')
     parser.add_argument('-s', '--start', action='store_true', help='Start the VM after modification.')
     args = parser.parse_args()
     return args
 
-def modify_vm(dom, cpu_count, memory_amount, pci_id, logger):
+def modify_vm(dom, cpu_count, memory_amount, pci_ids, logger):
     try:
         # Get the XML description of the VM
         xml_desc = dom.XMLDesc()
@@ -61,22 +98,24 @@ def modify_vm(dom, cpu_count, memory_amount, pci_id, logger):
                 logger.error("Could not find <memory> elements in XML.")
                 sys.exit(1)
 
-        # Add PCI device passthrough
+        # Add PCI device passthrough(s)
-        if pci_id is not None:
+        if pci_ids:
             devices_elem = root.find('./devices')
             if devices_elem is not None:
+                for pci_id in pci_ids:
                     hostdev_elem = ET.SubElement(devices_elem, 'hostdev', attrib={
                         'mode': 'subsystem',
                         'type': 'pci',
                         'managed': 'yes'
                     })
                     source_elem = ET.SubElement(hostdev_elem, 'source')
-                domain_id, bus_slot_func = pci_id.split(':')
+                    domain_id, bus_slot_func = pci_id.split(':', 1)
                     bus_slot, function = bus_slot_func.split('.')
+                    bus, slot = bus_slot[:2], bus_slot[2:]
                     address_attrs = {
                         'domain': f'0x{domain_id}',
-                    'bus': f'0x{bus_slot}',
+                        'bus': f'0x{bus}',
-                    'slot': f'0x{bus_slot}',
+                        'slot': f'0x{slot}',
                         'function': f'0x{function}'
                     }
                     ET.SubElement(source_elem, 'address', attrib=address_attrs)
@@ -115,7 +154,7 @@ def main():
         vm_name = args.vm
         cpu_count = args.cpu
         memory_amount = args.memory
-        pci_id = args.pci
+        pci_ids = args.pci  # This will be a list or None
         start_vm_flag = args.start
 
         # Connect to libvirt
@@ -129,7 +168,7 @@ def main():
         dom = stop_vm(conn, vm_name, logger)
 
         # Modify VM XML
-        new_xml_desc = modify_vm(dom, cpu_count, memory_amount, pci_id, logger)
+        new_xml_desc = modify_vm(dom, cpu_count, memory_amount, pci_ids, logger)
 
         # Redefine VM
         redefine_vm(conn, new_xml_desc, logger)

salt/manager/tools/sbin/so-salt-cloud

@@ -6,15 +6,119 @@
 # Elastic License 2.0.
 
 """
-Script to assist with salt-cloud VM provisioning. This is only intended to work with a libvirt salt-cloud provider.
+Script to assist with salt-cloud VM provisioning. This is intended to work with a libvirt salt-cloud provider.
 
-Usage:
+**Usage:**
-    python so-salt-cloud -p <profile> <vm_name> (--dhcp4 | --static4 --ip4 <ip_address> --gw4 <gateway>) [--dns4 <dns_servers>] [--search4 <search_domain>]
+    python so-salt-cloud -p <profile> <vm_name> (--dhcp4 | --static4 --ip4 <ip_address> --gw4 <gateway>) 
+                            [-c <cpu_count>] [-m <memory_amount>] [-P <pci_id>] [-P <pci_id> ...] [--dns4 <dns_servers>] [--search4 <search_domain>]
 
-Examples:
+**Options:**
-    python so-salt-cloud -p core-hype1 hostname_nodetype --static4 --ip4 192.168.1.10/24 --gw4 192.168.1.1 --dns4 192.168.1.1,192.168.1.2 --search4 example.local
+    -p, --profile         The cloud profile to build the VM from.
+    <vm_name>             The name of the VM.
+    --dhcp4               Configure interface for DHCP (IPv4).
+    --static4             Configure interface for static IPv4 settings.
+    --ip4                 IPv4 address (e.g., 192.168.1.10/24). Required for static IPv4 configuration.
+    --gw4                 IPv4 gateway (e.g., 192.168.1.1). Required for static IPv4 configuration.
+    --dns4                Comma-separated list of IPv4 DNS servers (e.g., 8.8.8.8,8.8.4.4).
+    --search4             DNS search domain for IPv4.
+    -c, --cpu             Number of virtual CPUs to assign.
+    -m, --memory          Amount of memory to assign in MiB.
+    -P, --pci             PCI hardware ID(s) to passthrough to the VM (e.g., 0000:00:1f.2). Can be specified multiple times.
+
+**Examples:**
+
+1. **Static IP Configuration with Multiple PCI Devices:**
+
+    ```bash
+    python so-salt-cloud -p core-hype1 vm1_sensor --static4 --ip4 192.168.1.10/24 --gw4 192.168.1.1 \
+        --dns4 192.168.1.1,192.168.1.2 --search4 example.local -c 4 -m 8192 -P 0000:00:1f.2 -P 0000:00:1f.3
+    ```
+
+    This command provisions a VM named `vm1_sensor` using the `core-hype1` profile with the following settings:
+
+    - Static IPv4 configuration:
+        - IP Address: `192.168.1.10/24`
+        - Gateway: `192.168.1.1`
+        - DNS Servers: `192.168.1.1`, `192.168.1.2`
+        - DNS Search Domain: `example.local`
+    - Hardware Configuration:
+        - CPUs: `4`
+        - Memory: `8192` MiB
+        - PCI Device Passthrough: `0000:00:1f.2`, `0000:00:1f.3`
+
+2. **DHCP Configuration with Default Hardware Settings:**
+
+    ```bash
+    python so-salt-cloud -p core-hype1 vm2_master --dhcp4
+    ```
+
+    This command provisions a VM named `vm2_master` using the `core-hype1` profile with DHCP for network configuration and default hardware settings.
+
+3. **Static IP Configuration without Hardware Specifications:**
+
+    ```bash
+    python so-salt-cloud -p core-hype1 vm3_search --static4 --ip4 192.168.1.20/24 --gw4 192.168.1.1
+    ```
+
+    This command provisions a VM named `vm3_search` with a static IP configuration and default hardware settings.
+
+4. **DHCP Configuration with Custom Hardware Specifications and Multiple PCI Devices:**
+
+    ```bash
+    python so-salt-cloud -p core-hype1 vm4_node --dhcp4 -c 8 -m 16384 -P 0000:00:1f.4 -P 0000:00:1f.5
+    ```
+
+    This command provisions a VM named `vm4_node` using DHCP for network configuration and custom hardware settings:
+
+    - CPUs: `8`
+    - Memory: `16384` MiB
+    - PCI Device Passthrough: `0000:00:1f.4`, `0000:00:1f.5`
+
+**Notes:**
+
+- When using `--static4`, both `--ip4` and `--gw4` options are required.
+- The script assumes the cloud profile name follows the format `basedomain-hypervisorname`.
+- Hardware parameters (`-c`, `-m`, `-P`) are optional. If not provided, default values from the profile will be used.
+- The `-P` or `--pci` option can be specified multiple times to pass through multiple PCI devices to the VM.
+- The `vm_name` should include the role of the VM after an underscore (e.g., `hostname_role`), as the script uses this to determine the VM's role for firewall configuration.
+
+**Description:**
+
+The `so-salt-cloud` script automates the provisioning of virtual machines using SaltStack's `salt-cloud` utility. It performs the following steps:
+
+1. **Network Configuration:**
+
+   - Modifies the network settings of the base QCOW2 image before provisioning.
+   - Supports both DHCP and static IPv4 configurations.
+   - Uses the `qcow2.modify_network_config` module via SaltStack to apply these settings on the target hypervisor.
+
+2. **VM Provisioning:**
+
+   - Calls `salt-cloud` to provision the VM using the specified profile and VM name.
+   - The VM is provisioned but not started immediately to allow for hardware configuration.
+
+3. **Hardware Configuration:**
+
+   - Modifies the hardware settings of the newly defined VM.
+   - Supports specifying multiple PCI devices for passthrough.
+   - Uses the `qcow2.modify_hardware_config` module via SaltStack to adjust CPU count, memory allocation, and PCI device passthrough.
+   - Starts the VM after hardware modifications.
+
+4. **Firewall Configuration:**
+
+   - Monitors the output of `salt-cloud` to extract the VM's IP address.
+   - Calls the `so-firewall-minion` script to apply firewall rules based on the VM's role.
+
+**Exit Codes:**
+
+- `0`: Success
+- Non-zero: An error occurred during execution.
+
+**Logging:**
+
+- Logs are written to `/opt/so/log/salt/so-salt-cloud.log`.
+- Both file and console logging are enabled for real-time monitoring.
 
-    python so-salt-cloud -p core-hype1 hostname_nodetype --dhcp4
 """
 
 import argparse
@@ -23,11 +127,12 @@ import re
 import sys
 import threading
 import salt.client
-
-local = salt.client.LocalClient()
-
 import logging
 
+# Initialize Salt local client
+local = salt.client.LocalClient()
+
+# Set up logging
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
 
@@ -77,7 +182,7 @@ def call_salt_cloud(profile, vm_name):
         ip_search_string = '[INFO    ] Address ='
         ip_search_pattern = re.compile(re.escape(ip_search_string))
 
-        # continuously read the output from salt-cloud 
+        # Continuously read the output from salt-cloud
         while True:
             # Read stdout line by line
             line = process.stdout.readline()
@@ -95,7 +200,7 @@ def call_salt_cloud(profile, vm_name):
                     else:
                         logger.error("No IP address found.")
             else:
-                # check if salt-cloud has terminated
+                # Check if salt-cloud has terminated
                 if process.poll() is not None:
                     break
 
@@ -105,7 +210,29 @@ def call_salt_cloud(profile, vm_name):
     except Exception as e:
         logger.error(f"An error occurred while calling salt-cloud: {e}")
 
-# This function requires the cloud profile to be in the form: basedomain_hypervisorhostname. The profile name will be used to target the hypervisor.
+def run_qcow2_modify_hardware_config(profile, vm_name, cpu=None, memory=None, pci_list=None, start=False):
+    hv_name = profile.split('-')[1]
+    target = hv_name + "_*"
+
+    try:
+        args_list = [
+            'vm_name=' + vm_name,
+            'cpu=' + str(cpu) if cpu else '',
+            'memory=' + str(memory) if memory else '',
+            'start=' + str(start)
+        ]
+
+        # Add PCI devices if provided
+        if pci_list:
+            # Join the list of PCI IDs into a comma-separated string
+            pci_devices = ','.join(pci_list)
+            args_list.append('pci=' + pci_devices)
+
+        r = local.cmd(target, 'qcow2.modify_hardware_config', args_list)
+        logger.info(f'qcow2.modify_hardware_config: {r}')
+    except Exception as e:
+        logger.error(f"An error occurred while running qcow2.modify_hardware_config: {e}")
+
 def run_qcow2_modify_network_config(profile, mode, ip=None, gateway=None, dns=None, search_domain=None):
     hv_name = profile.split('-')[1]
     target = hv_name + "_*"
@@ -130,13 +257,18 @@ def parse_arguments():
     parser = argparse.ArgumentParser(description="Call salt-cloud and pass the profile and VM name to it.")
     parser.add_argument('-p', '--profile', type=str, required=True, help="The cloud profile to build the VM from.")
     parser.add_argument('vm_name', type=str, help="The name of the VM.")
+
     group = parser.add_mutually_exclusive_group(required=True)
     group.add_argument("--dhcp4", action="store_true", help="Configure interface for DHCP (IPv4).")
     group.add_argument("--static4", action="store_true", help="Configure interface for static IPv4 settings.")
+
     parser.add_argument("--ip4", help="IPv4 address (e.g., 192.168.1.10/24). Required for static IPv4 configuration.")
     parser.add_argument("--gw4", help="IPv4 gateway (e.g., 192.168.1.1). Required for static IPv4 configuration.")
     parser.add_argument("--dns4", help="Comma-separated list of IPv4 DNS servers (e.g., 8.8.8.8,8.8.4.4).")
     parser.add_argument("--search4", help="DNS search domain for IPv4.")
+    parser.add_argument('-c', '--cpu', type=int, help='Number of virtual CPUs to assign.')
+    parser.add_argument('-m', '--memory', type=int, help='Amount of memory to assign in MiB.')
+    parser.add_argument('-P', '--pci', action='append', help='PCI hardware ID(s) to passthrough to the VM (e.g., 0000:00:1f.2). Can be specified multiple times.')
 
     args = parser.parse_args()
 
@@ -156,9 +288,15 @@ def main():
         else:
             mode = "dhcp4"  # Default to DHCP if not specified
 
+        # Step 1: Modify network configuration
         run_qcow2_modify_network_config(args.profile, mode, args.ip4, args.gw4, args.dns4, args.search4)
+
+        # Step 2: Provision the VM (without starting it)
         call_salt_cloud(args.profile, args.vm_name)
 
+        # Step 3: Modify hardware configuration
+        run_qcow2_modify_hardware_config(args.profile, args.vm_name, cpu=args.cpu, memory=args.memory, pci_list=args.pci, start=True)
+
     except KeyboardInterrupt:
         logger.error("so-salt-cloud: Operation cancelled by user.")
         sys.exit(1)

salt/salt/minion.sls

@@ -70,6 +70,11 @@ enable_startup_states:
     - regex: '^startup_states: highstate$'
     - unless: pgrep so-setup
 
+# manager with hypervisors with need this beacon added to the minion config
+#beacons:
+#  add_virtual_node_beacon:
+#    - base_path: /opt/so/saltstack/local/salt/hypervisor/hosts/*/add_*
+
 # prior to 2.4.30 this managed file would restart the salt-minion service when updated
 # since this file is currently only adding a sleep timer on service start
 # it is not required to restart the service

salt/soc/dynamic_annotations/hypervisor/add_node

@@ -3,7 +3,7 @@ network_mode:
 ip4:
 gw4:
 dns4:
-sarch4:
+search4:
 cpu:
 memory:
 disk: