From c8a1c8377a0e230c9df6e3fdae24a4e5896fac9b Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 27 Feb 2025 16:04:44 -0500 Subject: [PATCH] vm power operations --- .../engines/master/virtual_power_manager.py | 330 ++++++++++++++++++ salt/salt/files/hvn_engine.conf | 7 + .../files/virtual_node_manager_engine.conf | 4 - salt/salt/master.sls | 17 +- salt/soc/dyanno/hypervisor/hypervisor.yaml | 8 + 5 files changed, 357 insertions(+), 9 deletions(-) create mode 100644 salt/salt/engines/master/virtual_power_manager.py create mode 100644 salt/salt/files/hvn_engine.conf delete mode 100644 salt/salt/files/virtual_node_manager_engine.conf diff --git a/salt/salt/engines/master/virtual_power_manager.py b/salt/salt/engines/master/virtual_power_manager.py new file mode 100644 index 000000000..8cca82606 --- /dev/null +++ b/salt/salt/engines/master/virtual_power_manager.py @@ -0,0 +1,330 @@ +#!/opt/saltstack/salt/bin/python3 + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. +# +# Note: Per the Elastic License 2.0, the second limitation states: +# +# "You may not move, change, disable, or circumvent the license key functionality +# in the software, and you may not remove or obscure any functionality in the +# software that is protected by the license key." + +""" +Salt Engine for Virtual Machine Power Management + +This engine manages power control actions for virtual machines in Security Onion's +virtualization infrastructure. It monitors VM configurations for power control requests +and executes the appropriate virt module actions. + +Usage: + engines: + - virtual_power_manager: + interval: 60 + base_path: /opt/so/saltstack/local/salt/hypervisor/hosts + +Options: + interval: Time in seconds between engine runs (managed by salt-master, default: 60) + base_path: Base directory containing hypervisor configurations (default: /opt/so/saltstack/local/salt/hypervisor/hosts) + +Configuration Files: + VMs: JSON file containing VM configurations + - Located at /VMs + - Contains array of VM configurations + - Power control requests are specified with the "powercontrol" key + - Valid values for "powercontrol": "Reboot", "Reset", "Shutdown", "Start", "Stop" + +Examples: + 1. Basic Configuration: + engines: + - virtual_power_manager: {} + + Uses default settings to process power control requests every 60 seconds. + + 2. Custom Interval: + engines: + - virtual_power_manager: + interval: 120 + + Processes power control requests every 120 seconds. + +Power Control Actions: + - Reboot: Gracefully reboot the VM (virt.reboot) + - Reset: Force reset the VM (virt.reset) + - Shutdown: Gracefully shut down the VM (virt.shutdown) + - Start: Start the VM (virt.start) + - Stop: Force stop the VM (virt.stop) + +Notes: + - File locking is used to prevent race conditions when multiple processes access the VMs file + - The "powercontrol" key is removed from the VM configuration after successful execution + - Comprehensive logging for troubleshooting + - No continuous loop (salt-master handles scheduling) + - File locking is only applied when a powercontrol key is detected, not on every run + +Description: + The engine operates in the following phases: + + 1. Configuration Processing + - Reads VMs file for each hypervisor without locking + - Identifies VMs with "powercontrol" key + - If powercontrol key is found, acquires lock and reads file again + + 2. Power Control Execution + - Maps "powercontrol" value to virt module function + - Executes appropriate virt module command + - Removes "powercontrol" key after successful execution + + 3. File Locking + - Acquires lock only when a powercontrol key is detected + - Releases lock after modifications + - Handles lock acquisition failures + +Logging: + Log files are written to /opt/so/log/salt/master + Comprehensive logging includes: + - Power control action details + - Command execution results + - Error conditions with full context + - File locking operations +""" + +import os +import glob +import json +import logging +import fcntl +import salt.client +from typing import Dict, List, Optional, Any, Tuple + +# Configure logging +log = logging.getLogger(__name__) +log.setLevel(logging.DEBUG) + +# Constants +DEFAULT_INTERVAL = 60 +DEFAULT_BASE_PATH = '/opt/so/saltstack/local/salt/hypervisor/hosts' +VALID_POWER_ACTIONS = {'Reboot', 'Reset', 'Shutdown', 'Start', 'Stop'} + +class FileLock: + """ + Context manager for file locking. + + This class provides a context manager for file locking using fcntl. + It acquires an exclusive lock on the file when entering the context + and releases the lock when exiting. + + Example: + with FileLock(file_path): + # Read and modify file + # Lock is automatically released when exiting the context + """ + + def __init__(self, file_path: str): + self.file_path = file_path + self.lock_path = f"{file_path}.lock" + self.lock_file = None + + def __enter__(self): + try: + # Open the lock file + self.lock_file = open(self.lock_path, 'w') + + # Acquire exclusive lock + fcntl.flock(self.lock_file, fcntl.LOCK_EX) + log.debug("Acquired lock on %s", self.file_path) + + return self + + except Exception as e: + log.error("Failed to acquire lock on %s: %s", self.file_path, str(e)) + if self.lock_file: + self.lock_file.close() + raise + + def __exit__(self, exc_type, exc_val, exc_tb): + try: + # Release lock + if self.lock_file: + fcntl.flock(self.lock_file, fcntl.LOCK_UN) + self.lock_file.close() + log.debug("Released lock on %s", self.file_path) + + # Remove lock file + if os.path.exists(self.lock_path): + os.remove(self.lock_path) + + except Exception as e: + log.error("Error releasing lock on %s: %s", self.file_path, str(e)) + +def read_json_file(file_path: str) -> Any: + """ + Read and parse a JSON file. + Returns an empty array if the file is empty. + """ + try: + with open(file_path, 'r') as f: + content = f.read().strip() + if not content: + return [] + return json.loads(content) + except Exception as e: + log.error("Failed to read JSON file %s: %s", file_path, str(e)) + raise + +def write_json_file(file_path: str, data: Any) -> None: + """Write data to a JSON file.""" + try: + with open(file_path, 'w') as f: + json.dump(data, f, indent=2) + except Exception as e: + log.error("Failed to write JSON file %s: %s", file_path, str(e)) + raise + +def has_power_control_requests(nodes_config: List[Dict]) -> bool: + """ + Check if any VM in the configuration has a powercontrol key. + + Args: + nodes_config: List of VM configurations + + Returns: + True if at least one VM has a powercontrol key, False otherwise + """ + return any('powercontrol' in vm_config for vm_config in nodes_config) + +def process_power_control(hypervisor: str, vm_config: dict) -> bool: + """ + Process a power control request for a VM. + + Args: + hypervisor: Name of the hypervisor + vm_config: VM configuration dictionary + + Returns: + True if the power control action was successful, False otherwise + """ + try: + # Get VM name and power control action + vm_name = f"{vm_config['hostname']}_{vm_config['role']}" + power_action = vm_config['powercontrol'] + + # Validate power action + if power_action not in VALID_POWER_ACTIONS: + log.error("Invalid power control action: %s", power_action) + return False + + # Map power action to virt module function + virt_function = power_action.lower() + + # Execute power control action + log.info("Executing %s on VM %s", power_action, vm_name) + client = salt.client.LocalClient() + result = client.cmd( + f"{hypervisor}_*", + f"virt.{virt_function}", + [vm_name], + expr_form="glob" + ) + + # Check result + if result and any(success for success in result.values()): + log.info("Successfully executed %s on VM %s", power_action, vm_name) + return True + else: + log.error("Failed to execute %s on VM %s: %s", power_action, vm_name, result) + return False + + except Exception as e: + log.error("Error processing power control for VM %s: %s", vm_config.get('hostname', 'unknown'), str(e)) + return False + +def process_hypervisor_power_requests(hypervisor_path: str) -> None: + """ + Process power control requests for a single hypervisor. + + Args: + hypervisor_path: Path to the hypervisor directory + """ + try: + # Get hypervisor name from path + hypervisor = os.path.basename(hypervisor_path) + + # Read VMs file + vms_file = os.path.join(os.path.dirname(hypervisor_path), f"{hypervisor}VMs") + if not os.path.exists(vms_file): + log.debug("No VMs file found at %s", vms_file) + return + + # First, read the file without locking to check if any VM has a powercontrol key + nodes_config = read_json_file(vms_file) + if not nodes_config: + log.debug("Empty VMs configuration in %s", vms_file) + return + + # Check if any VM has a powercontrol key + if not has_power_control_requests(nodes_config): + log.debug("No power control requests found in %s", vms_file) + return + + # If we found powercontrol keys, lock the file and process the requests + with FileLock(vms_file): + # Read the VMs file again with the lock to ensure we have the latest data + nodes_config = read_json_file(vms_file) + if not nodes_config: + log.debug("Empty VMs configuration in %s (after lock)", vms_file) + return + + # Track if any changes were made + changes_made = False + + # Process each VM configuration + for i, vm_config in enumerate(nodes_config): + if 'powercontrol' in vm_config: + # Process power control request + log.info("Found power control request for VM %s_%s: %s", + vm_config.get('hostname', 'unknown'), + vm_config.get('role', 'unknown'), + vm_config['powercontrol']) + + success = process_power_control(hypervisor, vm_config) + if success: + # Remove powercontrol key + log.info("Power control action successful, removing powercontrol key") + del nodes_config[i]['powercontrol'] + changes_made = True + + # Write updated configuration if changes were made + if changes_made: + log.info("Writing updated VM configuration to %s", vms_file) + write_json_file(vms_file, nodes_config) + + except Exception as e: + log.error("Failed to process hypervisor %s: %s", hypervisor_path, str(e)) + raise + +def start(interval: int = DEFAULT_INTERVAL, + base_path: str = DEFAULT_BASE_PATH) -> None: + """ + Process virtual machine power control requests. + + This function processes power control requests for virtual machines + by monitoring the VMs files for the "powercontrol" key. + + Args: + interval: Time in seconds between engine runs (managed by salt-master) + base_path: Base path containing hypervisor configurations + """ + log.info("Starting virtual power manager engine") + + try: + # Process each hypervisor directory + for hypervisor_path in glob.glob(os.path.join(base_path, '*')): + if os.path.isdir(hypervisor_path): + process_hypervisor_power_requests(hypervisor_path) + + log.info("Virtual power manager completed successfully") + + except Exception as e: + log.error("Error in virtual power manager: %s", str(e)) diff --git a/salt/salt/files/hvn_engine.conf b/salt/salt/files/hvn_engine.conf new file mode 100644 index 000000000..9ec2bc998 --- /dev/null +++ b/salt/salt/files/hvn_engine.conf @@ -0,0 +1,7 @@ +engines: + - virtual_node_manager: + interval: 10 + base_path: /opt/so/saltstack/local/salt/hypervisor/hosts + - virtual_power_manager: + interval: 10 + base_path: /opt/so/saltstack/local/salt/hypervisor/hosts diff --git a/salt/salt/files/virtual_node_manager_engine.conf b/salt/salt/files/virtual_node_manager_engine.conf deleted file mode 100644 index 8374f551a..000000000 --- a/salt/salt/files/virtual_node_manager_engine.conf +++ /dev/null @@ -1,4 +0,0 @@ -engines: - - virtual_node_manager: - interval: 30 - base_path: /opt/so/saltstack/local/salt/hypervisor/hosts diff --git a/salt/salt/master.sls b/salt/salt/master.sls index ed8da3f73..56f529e89 100644 --- a/salt/salt/master.sls +++ b/salt/salt/master.sls @@ -49,6 +49,13 @@ pillarWatch_engine: - source: salt://salt/engines/master/pillarWatch.py {% if 'hvn' in salt['pillar.get']('features', []) %} +hvn_engine_config: + file.managed: + - name: /etc/salt/master.d/hvn_engine.conf + - source: salt://salt/files/hvn_engine.conf + - watch_in: + - service: salt_master_service + virtual_node_manager_engine: file.managed: - name: /etc/salt/engines/virtual_node_manager.py @@ -56,11 +63,12 @@ virtual_node_manager_engine: - watch_in: - service: salt_master_service -virtual_node_manager_engine_config: +virtual_power_manager_engine: file.managed: - - name: /etc/salt/master.d/virtual_node_manager_engine.conf - - source: salt://salt/files/virtual_node_manager_engine.conf - + - name: /etc/salt/engines/virtual_power_manager.py + - source: salt://salt/engines/master/virtual_power_manager.py + - watch_in: + - service: salt_master_service {% endif %} engines_config: @@ -88,7 +96,6 @@ reactor: - 'setup/so-minion': - /opt/so/saltstack/default/salt/reactor/sominion_setup.sls - 'salt/cloud/*/destroyed': - - /opt/so/saltstack/default/salt/reactor/virtReleaseHardware.sls - /opt/so/saltstack/default/salt/reactor/deleteKey.sls #} diff --git a/salt/soc/dyanno/hypervisor/hypervisor.yaml b/salt/soc/dyanno/hypervisor/hypervisor.yaml index 59e8da940..6b0458265 100644 --- a/salt/soc/dyanno/hypervisor/hypervisor.yaml +++ b/salt/soc/dyanno/hypervisor/hypervisor.yaml @@ -77,3 +77,11 @@ hypervisor: readonly: true options: [] forcedType: '[]int' + - field: powercontrol + label: "Execute VM power operations" + options: + - Start + - Reboot + - Shutdown + - Reset + - Stop