Compare commits

...

21 Commits

Author SHA1 Message Date
Josh Patterson
7b4d471d7e cert expire test 2025-12-01 12:02:55 -05:00
Josh Patterson
36a6a59d55 renew certs 7 days before expire 2025-12-01 11:54:10 -05:00
Mike Reeves
875de88cb4 Merge pull request #15271 from Security-Onion-Solutions/TOoSmOotH-patch-2
Add JA4D option to config.zeek.ja4
2025-12-01 10:03:12 -05:00
Mike Reeves
63bb44886e Add JA4D option to config.zeek.ja4 2025-12-01 10:00:42 -05:00
Josh Patterson
9c06713f32 Merge pull request #15251 from Security-Onion-Solutions/bravo
use timestamp in volume path to prevent duplicates
2025-11-21 14:54:30 -05:00
Josh Patterson
23da0d4ba0 use timestamp in filename to prevent duplicates 2025-11-21 14:49:03 -05:00
Josh Patterson
d5f2cfb354 Merge pull request #15248 from Security-Onion-Solutions/bravo
clarify hypervisor annotation
2025-11-20 17:28:32 -05:00
Josh Patterson
fb5ad4193d indicate base image download start 2025-11-20 17:13:36 -05:00
Josh Patterson
1f5f283c06 update hypervisor annotaion. preinit instead of initialized 2025-11-20 16:53:55 -05:00
Josh Patterson
cf048030c4 Merge pull request #15247 from Security-Onion-Solutions/bravo
Notify user of hypervisor environment setup failures
2025-11-20 16:04:49 -05:00
Josh Patterson
2d716b44a8 update comment 2025-11-20 15:52:21 -05:00
Jorge Reyes
d70d652310 Merge pull request #15244 from Security-Onion-Solutions/reyesj2/suricapfile
suricata capture file
2025-11-20 14:31:43 -06:00
reyesj2
c5db7c8752 suricata.capture_file keyword 2025-11-20 14:26:12 -06:00
reyesj2
6f42ff3442 suricata capture_file 2025-11-20 14:16:49 -06:00
reyesj2
433dab7376 format json 2025-11-20 14:16:10 -06:00
Josh Patterson
97c1a46013 update annotation for general failure 2025-11-20 15:08:04 -05:00
Josh Patterson
fbe97221bb set initialized status 2025-11-20 14:43:09 -05:00
Josh Patterson
841ce6b6ec update hypervisor annotation for image download or ssh key creation failure 2025-11-20 13:55:22 -05:00
Josh Patterson
dd0b4c3820 fix failed or hung qcow2 image download 2025-11-19 15:48:53 -05:00
Josh Patterson
b407c68d88 Merge remote-tracking branch 'origin/2.4/dev' into bravo 2025-11-19 10:23:11 -05:00
Josh Patterson
5b6a7035af need python_shell for pipes 2025-11-19 10:22:58 -05:00
12 changed files with 358 additions and 126 deletions

View File

@@ -172,7 +172,15 @@ MANAGER_HOSTNAME = socket.gethostname()
def _download_image(): def _download_image():
""" """
Download and validate the Oracle Linux KVM image. Download and validate the Oracle Linux KVM image with retry logic and progress monitoring.
Features:
- Detects stalled downloads (no progress for 30 seconds)
- Retries up to 3 times on failure
- Connection timeout of 30 seconds
- Read timeout of 60 seconds
- Cleans up partial downloads on failure
Returns: Returns:
bool: True if successful or file exists with valid checksum, False on error bool: True if successful or file exists with valid checksum, False on error
""" """
@@ -186,25 +194,54 @@ def _download_image():
log.info("Starting image download process") log.info("Starting image download process")
# Retry configuration
max_attempts = 3
retry_delay = 5 # seconds to wait between retry attempts
stall_timeout = 30 # seconds without progress before considering download stalled
connection_timeout = 30 # seconds to establish connection
read_timeout = 60 # seconds to wait for data chunks
for attempt in range(1, max_attempts + 1):
log.info("Download attempt %d of %d", attempt, max_attempts)
try: try:
# Download file # Download file with timeouts
log.info("Downloading Oracle Linux KVM image from %s to %s", IMAGE_URL, IMAGE_PATH) log.info("Downloading Oracle Linux KVM image from %s to %s", IMAGE_URL, IMAGE_PATH)
response = requests.get(IMAGE_URL, stream=True) response = requests.get(
IMAGE_URL,
stream=True,
timeout=(connection_timeout, read_timeout)
)
response.raise_for_status() response.raise_for_status()
# Get total file size for progress tracking # Get total file size for progress tracking
total_size = int(response.headers.get('content-length', 0)) total_size = int(response.headers.get('content-length', 0))
downloaded_size = 0 downloaded_size = 0
last_log_time = 0 last_log_time = 0
last_progress_time = time.time()
last_downloaded_size = 0
# Save file with progress logging # Save file with progress logging and stall detection
with salt.utils.files.fopen(IMAGE_PATH, 'wb') as f: with salt.utils.files.fopen(IMAGE_PATH, 'wb') as f:
for chunk in response.iter_content(chunk_size=8192): for chunk in response.iter_content(chunk_size=8192):
if chunk: # filter out keep-alive new chunks
f.write(chunk) f.write(chunk)
downloaded_size += len(chunk) downloaded_size += len(chunk)
current_time = time.time()
# Check for stalled download
if downloaded_size > last_downloaded_size:
# Progress made, reset stall timer
last_progress_time = current_time
last_downloaded_size = downloaded_size
elif current_time - last_progress_time > stall_timeout:
# No progress for stall_timeout seconds
raise Exception(
f"Download stalled: no progress for {stall_timeout} seconds "
f"at {downloaded_size}/{total_size} bytes"
)
# Log progress every second # Log progress every second
current_time = time.time()
if current_time - last_log_time >= 1: if current_time - last_log_time >= 1:
progress = (downloaded_size / total_size) * 100 if total_size > 0 else 0 progress = (downloaded_size / total_size) * 100 if total_size > 0 else 0
log.info("Progress - %.1f%% (%d/%d bytes)", log.info("Progress - %.1f%% (%d/%d bytes)",
@@ -212,17 +249,50 @@ def _download_image():
last_log_time = current_time last_log_time = current_time
# Validate downloaded file # Validate downloaded file
log.info("Download complete, validating checksum...")
if not _validate_image_checksum(IMAGE_PATH, IMAGE_SHA256): if not _validate_image_checksum(IMAGE_PATH, IMAGE_SHA256):
log.error("Checksum validation failed on attempt %d", attempt)
os.unlink(IMAGE_PATH) os.unlink(IMAGE_PATH)
if attempt < max_attempts:
log.info("Will retry download...")
continue
else:
log.error("All download attempts failed due to checksum mismatch")
return False return False
log.info("Successfully downloaded and validated Oracle Linux KVM image") log.info("Successfully downloaded and validated Oracle Linux KVM image")
return True return True
except Exception as e: except requests.exceptions.Timeout as e:
log.error("Error downloading hypervisor image: %s", str(e)) log.error("Download attempt %d failed: Timeout - %s", attempt, str(e))
if os.path.exists(IMAGE_PATH): if os.path.exists(IMAGE_PATH):
os.unlink(IMAGE_PATH) os.unlink(IMAGE_PATH)
if attempt < max_attempts:
log.info("Will retry download in %d seconds...", retry_delay)
time.sleep(retry_delay)
else:
log.error("All download attempts failed due to timeout")
except requests.exceptions.RequestException as e:
log.error("Download attempt %d failed: Network error - %s", attempt, str(e))
if os.path.exists(IMAGE_PATH):
os.unlink(IMAGE_PATH)
if attempt < max_attempts:
log.info("Will retry download in %d seconds...", retry_delay)
time.sleep(retry_delay)
else:
log.error("All download attempts failed due to network errors")
except Exception as e:
log.error("Download attempt %d failed: %s", attempt, str(e))
if os.path.exists(IMAGE_PATH):
os.unlink(IMAGE_PATH)
if attempt < max_attempts:
log.info("Will retry download in %d seconds...", retry_delay)
time.sleep(retry_delay)
else:
log.error("All download attempts failed")
return False return False
def _check_ssh_keys_exist(): def _check_ssh_keys_exist():
@@ -419,25 +489,28 @@ def _ensure_hypervisor_host_dir(minion_id: str = None):
log.error(f"Error creating hypervisor host directory: {str(e)}") log.error(f"Error creating hypervisor host directory: {str(e)}")
return False return False
def _apply_dyanno_hypervisor_state(): def _apply_dyanno_hypervisor_state(status):
""" """
Apply the soc.dyanno.hypervisor state on the salt master. Apply the soc.dyanno.hypervisor state on the salt master.
This function applies the soc.dyanno.hypervisor state on the salt master This function applies the soc.dyanno.hypervisor state on the salt master
to update the hypervisor annotation and ensure all hypervisor host directories exist. to update the hypervisor annotation and ensure all hypervisor host directories exist.
Args:
status: Status passed to the hypervisor annotation state
Returns: Returns:
bool: True if state was applied successfully, False otherwise bool: True if state was applied successfully, False otherwise
""" """
try: try:
log.info("Applying soc.dyanno.hypervisor state on salt master") log.info(f"Applying soc.dyanno.hypervisor state on salt master with status: {status}")
# Initialize the LocalClient # Initialize the LocalClient
local = salt.client.LocalClient() local = salt.client.LocalClient()
# Target the salt master to apply the soc.dyanno.hypervisor state # Target the salt master to apply the soc.dyanno.hypervisor state
target = MANAGER_HOSTNAME + '_*' target = MANAGER_HOSTNAME + '_*'
state_result = local.cmd(target, 'state.apply', ['soc.dyanno.hypervisor', "pillar={'baseDomain': {'status': 'PreInit'}}", 'concurrent=True'], tgt_type='glob') state_result = local.cmd(target, 'state.apply', ['soc.dyanno.hypervisor', f"pillar={{'baseDomain': {{'status': '{status}'}}}}", 'concurrent=True'], tgt_type='glob')
log.debug(f"state_result: {state_result}") log.debug(f"state_result: {state_result}")
# Check if state was applied successfully # Check if state was applied successfully
if state_result: if state_result:
@@ -454,17 +527,17 @@ def _apply_dyanno_hypervisor_state():
success = False success = False
if success: if success:
log.info("Successfully applied soc.dyanno.hypervisor state") log.info(f"Successfully applied soc.dyanno.hypervisor state with status: {status}")
return True return True
else: else:
log.error("Failed to apply soc.dyanno.hypervisor state") log.error(f"Failed to apply soc.dyanno.hypervisor state with status: {status}")
return False return False
else: else:
log.error("No response from salt master when applying soc.dyanno.hypervisor state") log.error(f"No response from salt master when applying soc.dyanno.hypervisor state with status: {status}")
return False return False
except Exception as e: except Exception as e:
log.error(f"Error applying soc.dyanno.hypervisor state: {str(e)}") log.error(f"Error applying soc.dyanno.hypervisor state with status: {status}: {str(e)}")
return False return False
def _apply_cloud_config_state(): def _apply_cloud_config_state():
@@ -598,11 +671,6 @@ def setup_environment(vm_name: str = 'sool9', disk_size: str = '220G', minion_id
log.warning("Failed to apply salt.cloud.config state, continuing with setup") log.warning("Failed to apply salt.cloud.config state, continuing with setup")
# We don't return an error here as we want to continue with the setup process # We don't return an error here as we want to continue with the setup process
# Apply the soc.dyanno.hypervisor state on the salt master
if not _apply_dyanno_hypervisor_state():
log.warning("Failed to apply soc.dyanno.hypervisor state, continuing with setup")
# We don't return an error here as we want to continue with the setup process
log.info("Starting setup_environment in setup_hypervisor runner") log.info("Starting setup_environment in setup_hypervisor runner")
# Check if environment is already set up # Check if environment is already set up
@@ -616,9 +684,12 @@ def setup_environment(vm_name: str = 'sool9', disk_size: str = '220G', minion_id
# Handle image setup if needed # Handle image setup if needed
if not image_valid: if not image_valid:
_apply_dyanno_hypervisor_state('ImageDownloadStart')
log.info("Starting image download/validation process") log.info("Starting image download/validation process")
if not _download_image(): if not _download_image():
log.error("Image download failed") log.error("Image download failed")
# Update hypervisor annotation with failure status
_apply_dyanno_hypervisor_state('ImageDownloadFailed')
return { return {
'success': False, 'success': False,
'error': 'Image download failed', 'error': 'Image download failed',
@@ -631,6 +702,8 @@ def setup_environment(vm_name: str = 'sool9', disk_size: str = '220G', minion_id
log.info("Setting up SSH keys") log.info("Setting up SSH keys")
if not _setup_ssh_keys(): if not _setup_ssh_keys():
log.error("SSH key setup failed") log.error("SSH key setup failed")
# Update hypervisor annotation with failure status
_apply_dyanno_hypervisor_state('SSHKeySetupFailed')
return { return {
'success': False, 'success': False,
'error': 'SSH key setup failed', 'error': 'SSH key setup failed',
@@ -655,6 +728,12 @@ def setup_environment(vm_name: str = 'sool9', disk_size: str = '220G', minion_id
success = vm_result.get('success', False) success = vm_result.get('success', False)
log.info("Setup environment completed with status: %s", "SUCCESS" if success else "FAILED") log.info("Setup environment completed with status: %s", "SUCCESS" if success else "FAILED")
# Update hypervisor annotation with success status
if success:
_apply_dyanno_hypervisor_state('PreInit')
else:
_apply_dyanno_hypervisor_state('SetupFailed')
# If setup was successful and we have a minion_id, run highstate # If setup was successful and we have a minion_id, run highstate
if success and minion_id: if success and minion_id:
log.info("Running highstate on hypervisor %s", minion_id) log.info("Running highstate on hypervisor %s", minion_id)

View File

@@ -10,7 +10,7 @@ x509_signing_policies:
- keyUsage: "digitalSignature, nonRepudiation" - keyUsage: "digitalSignature, nonRepudiation"
- subjectKeyIdentifier: hash - subjectKeyIdentifier: hash
- authorityKeyIdentifier: keyid,issuer:always - authorityKeyIdentifier: keyid,issuer:always
- days_valid: 820 - days_valid: 9
- copypath: /etc/pki/issued_certs/ - copypath: /etc/pki/issued_certs/
registry: registry:
- minions: '*' - minions: '*'
@@ -24,7 +24,7 @@ x509_signing_policies:
- subjectKeyIdentifier: hash - subjectKeyIdentifier: hash
- authorityKeyIdentifier: keyid,issuer:always - authorityKeyIdentifier: keyid,issuer:always
- extendedKeyUsage: serverAuth - extendedKeyUsage: serverAuth
- days_valid: 820 - days_valid: 9
- copypath: /etc/pki/issued_certs/ - copypath: /etc/pki/issued_certs/
managerssl: managerssl:
- minions: '*' - minions: '*'
@@ -38,7 +38,7 @@ x509_signing_policies:
- subjectKeyIdentifier: hash - subjectKeyIdentifier: hash
- authorityKeyIdentifier: keyid,issuer:always - authorityKeyIdentifier: keyid,issuer:always
- extendedKeyUsage: serverAuth - extendedKeyUsage: serverAuth
- days_valid: 820 - days_valid: 9
- copypath: /etc/pki/issued_certs/ - copypath: /etc/pki/issued_certs/
influxdb: influxdb:
- minions: '*' - minions: '*'
@@ -52,7 +52,7 @@ x509_signing_policies:
- subjectKeyIdentifier: hash - subjectKeyIdentifier: hash
- authorityKeyIdentifier: keyid,issuer:always - authorityKeyIdentifier: keyid,issuer:always
- extendedKeyUsage: serverAuth - extendedKeyUsage: serverAuth
- days_valid: 820 - days_valid: 9
- copypath: /etc/pki/issued_certs/ - copypath: /etc/pki/issued_certs/
elasticfleet: elasticfleet:
- minions: '*' - minions: '*'
@@ -65,7 +65,7 @@ x509_signing_policies:
- keyUsage: "digitalSignature, nonRepudiation" - keyUsage: "digitalSignature, nonRepudiation"
- subjectKeyIdentifier: hash - subjectKeyIdentifier: hash
- authorityKeyIdentifier: keyid,issuer:always - authorityKeyIdentifier: keyid,issuer:always
- days_valid: 820 - days_valid: 9
- copypath: /etc/pki/issued_certs/ - copypath: /etc/pki/issued_certs/
kafka: kafka:
- minions: '*' - minions: '*'
@@ -79,5 +79,5 @@ x509_signing_policies:
- subjectKeyIdentifier: hash - subjectKeyIdentifier: hash
- authorityKeyIdentifier: keyid,issuer:always - authorityKeyIdentifier: keyid,issuer:always
- extendedKeyUsage: "serverAuth, clientAuth" - extendedKeyUsage: "serverAuth, clientAuth"
- days_valid: 820 - days_valid: 9
- copypath: /etc/pki/issued_certs/ - copypath: /etc/pki/issued_certs/

View File

@@ -39,8 +39,8 @@ pki_public_ca_crt:
- extendedkeyUsage: "serverAuth, clientAuth" - extendedkeyUsage: "serverAuth, clientAuth"
- subjectKeyIdentifier: hash - subjectKeyIdentifier: hash
- authorityKeyIdentifier: keyid:always, issuer - authorityKeyIdentifier: keyid:always, issuer
- days_valid: 3650 - days_valid: 11
- days_remaining: 0 - days_remaining: 7
- backup: True - backup: True
- replace: False - replace: False
- require: - require:

View File

@@ -1,30 +1,155 @@
{ {
"description": "suricata.common", "description": "suricata.common",
"processors": [ "processors": [
{ "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, {
{ "rename": { "field": "message2.pkt_src", "target_field": "network.packet_source","ignore_failure": true } }, "json": {
{ "rename": { "field": "message2.proto", "target_field": "network.transport", "ignore_failure": true } }, "field": "message",
{ "rename": { "field": "message2.in_iface", "target_field": "observer.ingress.interface.name", "ignore_failure": true } }, "target_field": "message2",
{ "rename": { "field": "message2.flow_id", "target_field": "log.id.uid", "ignore_failure": true } }, "ignore_failure": true
{ "rename": { "field": "message2.src_ip", "target_field": "source.ip", "ignore_failure": true } }, }
{ "rename": { "field": "message2.src_port", "target_field": "source.port", "ignore_failure": true } }, },
{ "rename": { "field": "message2.dest_ip", "target_field": "destination.ip", "ignore_failure": true } }, {
{ "rename": { "field": "message2.dest_port", "target_field": "destination.port", "ignore_failure": true } }, "rename": {
{ "rename": { "field": "message2.vlan", "target_field": "network.vlan.id", "ignore_failure": true } }, "field": "message2.pkt_src",
{ "rename": { "field": "message2.community_id", "target_field": "network.community_id", "ignore_missing": true } }, "target_field": "network.packet_source",
{ "rename": { "field": "message2.xff", "target_field": "xff.ip", "ignore_missing": true } }, "ignore_failure": true
{ "set": { "field": "event.dataset", "value": "{{ message2.event_type }}" } }, }
{ "set": { "field": "observer.name", "value": "{{agent.name}}" } }, },
{ "set": { "field": "event.ingested", "value": "{{@timestamp}}" } }, {
{ "date": { "field": "message2.timestamp", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "timezone": "UTC", "ignore_failure": true } }, "rename": {
{ "remove":{ "field": "agent", "ignore_failure": true } }, "field": "message2.proto",
{"append":{"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"ignore_failure":true}}, "target_field": "network.transport",
"ignore_failure": true
}
},
{
"rename": {
"field": "message2.in_iface",
"target_field": "observer.ingress.interface.name",
"ignore_failure": true
}
},
{
"rename": {
"field": "message2.flow_id",
"target_field": "log.id.uid",
"ignore_failure": true
}
},
{
"rename": {
"field": "message2.src_ip",
"target_field": "source.ip",
"ignore_failure": true
}
},
{
"rename": {
"field": "message2.src_port",
"target_field": "source.port",
"ignore_failure": true
}
},
{
"rename": {
"field": "message2.dest_ip",
"target_field": "destination.ip",
"ignore_failure": true
}
},
{
"rename": {
"field": "message2.dest_port",
"target_field": "destination.port",
"ignore_failure": true
}
},
{
"rename": {
"field": "message2.vlan",
"target_field": "network.vlan.id",
"ignore_failure": true
}
},
{
"rename": {
"field": "message2.community_id",
"target_field": "network.community_id",
"ignore_missing": true
}
},
{
"rename": {
"field": "message2.xff",
"target_field": "xff.ip",
"ignore_missing": true
}
},
{
"set": {
"field": "event.dataset",
"value": "{{ message2.event_type }}"
}
},
{
"set": {
"field": "observer.name",
"value": "{{agent.name}}"
}
},
{
"set": {
"field": "event.ingested",
"value": "{{@timestamp}}"
}
},
{
"date": {
"field": "message2.timestamp",
"target_field": "@timestamp",
"formats": [
"ISO8601",
"UNIX"
],
"timezone": "UTC",
"ignore_failure": true
}
},
{
"remove": {
"field": "agent",
"ignore_failure": true
}
},
{
"append": {
"field": "related.ip",
"value": [
"{{source.ip}}",
"{{destination.ip}}"
],
"allow_duplicates": false,
"ignore_failure": true
}
},
{ {
"script": { "script": {
"source": "boolean isPrivate(def ip) { if (ip == null) return false; int dot1 = ip.indexOf('.'); if (dot1 == -1) return false; int dot2 = ip.indexOf('.', dot1 + 1); if (dot2 == -1) return false; int first = Integer.parseInt(ip.substring(0, dot1)); if (first == 10) return true; if (first == 192 && ip.startsWith('168.', dot1 + 1)) return true; if (first == 172) { int second = Integer.parseInt(ip.substring(dot1 + 1, dot2)); return second >= 16 && second <= 31; } return false; } String[] fields = new String[] {\"source\", \"destination\"}; for (int i = 0; i < fields.length; i++) { def field = fields[i]; def ip = ctx[field]?.ip; if (ip != null) { if (ctx.network == null) ctx.network = new HashMap(); if (isPrivate(ip)) { if (ctx.network.private_ip == null) ctx.network.private_ip = new ArrayList(); if (!ctx.network.private_ip.contains(ip)) ctx.network.private_ip.add(ip); } else { if (ctx.network.public_ip == null) ctx.network.public_ip = new ArrayList(); if (!ctx.network.public_ip.contains(ip)) ctx.network.public_ip.add(ip); } } }", "source": "boolean isPrivate(def ip) { if (ip == null) return false; int dot1 = ip.indexOf('.'); if (dot1 == -1) return false; int dot2 = ip.indexOf('.', dot1 + 1); if (dot2 == -1) return false; int first = Integer.parseInt(ip.substring(0, dot1)); if (first == 10) return true; if (first == 192 && ip.startsWith('168.', dot1 + 1)) return true; if (first == 172) { int second = Integer.parseInt(ip.substring(dot1 + 1, dot2)); return second >= 16 && second <= 31; } return false; } String[] fields = new String[] {\"source\", \"destination\"}; for (int i = 0; i < fields.length; i++) { def field = fields[i]; def ip = ctx[field]?.ip; if (ip != null) { if (ctx.network == null) ctx.network = new HashMap(); if (isPrivate(ip)) { if (ctx.network.private_ip == null) ctx.network.private_ip = new ArrayList(); if (!ctx.network.private_ip.contains(ip)) ctx.network.private_ip.add(ip); } else { if (ctx.network.public_ip == null) ctx.network.public_ip = new ArrayList(); if (!ctx.network.public_ip.contains(ip)) ctx.network.public_ip.add(ip); } } }",
"ignore_failure": false "ignore_failure": false
} }
}, },
{ "pipeline": { "if": "ctx?.event?.dataset != null", "name": "suricata.{{event.dataset}}" } } {
"rename": {
"field": "message2.capture_file",
"target_field": "suricata.capture_file",
"ignore_missing": true
}
},
{
"pipeline": {
"if": "ctx?.event?.dataset != null",
"name": "suricata.{{event.dataset}}"
}
}
] ]
} }

View File

@@ -841,6 +841,10 @@
"type": "long" "type": "long"
} }
} }
},
"capture_file": {
"type": "keyword",
"ignore_above": 1024
} }
} }
} }

View File

@@ -45,7 +45,7 @@ used during VM provisioning to add dedicated NSM storage volumes.
This command creates and attaches a volume with the following settings: This command creates and attaches a volume with the following settings:
- VM Name: `vm1_sensor` - VM Name: `vm1_sensor`
- Volume Size: `500` GB - Volume Size: `500` GB
- Volume Path: `/nsm/libvirt/volumes/vm1_sensor-nsm.img` - Volume Path: `/nsm/libvirt/volumes/vm1_sensor-nsm-<epoch_timestamp>.img`
- Device: `/dev/vdb` (virtio-blk) - Device: `/dev/vdb` (virtio-blk)
- VM remains stopped after attachment - VM remains stopped after attachment
@@ -75,7 +75,8 @@ used during VM provisioning to add dedicated NSM storage volumes.
- The script automatically stops the VM if it's running before creating and attaching the volume. - The script automatically stops the VM if it's running before creating and attaching the volume.
- Volumes are created with full pre-allocation for optimal performance. - Volumes are created with full pre-allocation for optimal performance.
- Volume files are stored in `/nsm/libvirt/volumes/` with naming pattern `<vm_name>-nsm.img`. - Volume files are stored in `/nsm/libvirt/volumes/` with naming pattern `<vm_name>-nsm-<epoch_timestamp>.img`.
- The epoch timestamp ensures unique volume names and prevents conflicts.
- Volumes are attached as `/dev/vdb` using virtio-blk for high performance. - Volumes are attached as `/dev/vdb` using virtio-blk for high performance.
- The script checks available disk space before creating the volume. - The script checks available disk space before creating the volume.
- Ownership is set to `qemu:qemu` with permissions `640`. - Ownership is set to `qemu:qemu` with permissions `640`.
@@ -142,6 +143,7 @@ import socket
import subprocess import subprocess
import pwd import pwd
import grp import grp
import time
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
from io import StringIO from io import StringIO
from so_vm_utils import start_vm, stop_vm from so_vm_utils import start_vm, stop_vm
@@ -242,10 +244,13 @@ def create_volume_file(vm_name, size_gb, logger):
Raises: Raises:
VolumeCreationError: If volume creation fails VolumeCreationError: If volume creation fails
""" """
# Define volume path (directory already created in main()) # Generate epoch timestamp for unique volume naming
volume_path = os.path.join(VOLUME_DIR, f"{vm_name}-nsm.img") epoch_timestamp = int(time.time())
# Check if volume already exists # Define volume path with epoch timestamp for uniqueness
volume_path = os.path.join(VOLUME_DIR, f"{vm_name}-nsm-{epoch_timestamp}.img")
# Check if volume already exists (shouldn't be possible with timestamp)
if os.path.exists(volume_path): if os.path.exists(volume_path):
logger.error(f"VOLUME: Volume already exists: {volume_path}") logger.error(f"VOLUME: Volume already exists: {volume_path}")
raise VolumeCreationError(f"Volume already exists: {volume_path}") raise VolumeCreationError(f"Volume already exists: {volume_path}")

View File

@@ -44,8 +44,8 @@ kafka_client_crt:
- signing_policy: kafka - signing_policy: kafka
- private_key: /etc/pki/kafka-client.key - private_key: /etc/pki/kafka-client.key
- CN: {{ GLOBALS.hostname }} - CN: {{ GLOBALS.hostname }}
- days_remaining: 0 - days_remaining: 7
- days_valid: 820 - days_valid: 9
- backup: True - backup: True
- timeout: 30 - timeout: 30
- retry: - retry:
@@ -92,8 +92,8 @@ kafka_crt:
- signing_policy: kafka - signing_policy: kafka
- private_key: /etc/pki/kafka.key - private_key: /etc/pki/kafka.key
- CN: {{ GLOBALS.hostname }} - CN: {{ GLOBALS.hostname }}
- days_remaining: 0 - days_remaining: 7
- days_valid: 820 - days_valid: 9
- backup: True - backup: True
- timeout: 30 - timeout: 30
- retry: - retry:
@@ -153,8 +153,8 @@ kafka_logstash_crt:
- signing_policy: kafka - signing_policy: kafka
- private_key: /etc/pki/kafka-logstash.key - private_key: /etc/pki/kafka-logstash.key
- CN: {{ GLOBALS.hostname }} - CN: {{ GLOBALS.hostname }}
- days_remaining: 0 - days_remaining: 7
- days_valid: 820 - days_valid: 9
- backup: True - backup: True
- timeout: 30 - timeout: 30
- retry: - retry:

View File

@@ -64,8 +64,8 @@ managerssl_crt:
- private_key: /etc/pki/managerssl.key - private_key: /etc/pki/managerssl.key
- CN: {{ GLOBALS.hostname }} - CN: {{ GLOBALS.hostname }}
- subjectAltName: "DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}, DNS:{{ GLOBALS.url_base }}" - subjectAltName: "DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}, DNS:{{ GLOBALS.url_base }}"
- days_remaining: 0 - days_remaining: 7
- days_valid: 820 - days_valid: 9
- backup: True - backup: True
- timeout: 30 - timeout: 30
- retry: - retry:

View File

@@ -727,7 +727,8 @@ def check_hypervisor_disk_space(hypervisor: str, size_gb: int) -> Tuple[bool, Op
result = local.cmd( result = local.cmd(
hypervisor_minion, hypervisor_minion,
'cmd.run', 'cmd.run',
["df -BG /nsm/libvirt/volumes | tail -1 | awk '{print $4}' | sed 's/G//'"] ["df -BG /nsm/libvirt/volumes | tail -1 | awk '{print $4}' | sed 's/G//'"],
kwarg={'python_shell': True}
) )
if not result or hypervisor_minion not in result: if not result or hypervisor_minion not in result:

View File

@@ -43,10 +43,26 @@
No Virtual Machines Found No Virtual Machines Found
{%- endif %} {%- endif %}
{%- else %} {%- elif baseDomainStatus == 'ImageDownloadStart' %}
#### INFO
Base domain image download started.
{%- elif baseDomainStatus == 'ImageDownloadFailed' %}
#### ERROR
Base domain image download failed. Please check the salt-master log for details and verify network connectivity.
{%- elif baseDomainStatus == 'SSHKeySetupFailed' %}
#### ERROR
SSH key setup failed. Please check the salt-master log for details.
{%- elif baseDomainStatus == 'SetupFailed' %}
#### WARNING #### WARNING
Base domain has not been initialized. Setup failed. Please check the salt-master log for details.
{%- elif baseDomainStatus == 'PreInit' %}
#### WARNING
Base domain has not been initialized. Waiting for hypervisor to highstate.
{%- endif %} {%- endif %}
{%- endmacro -%} {%- endmacro -%}

View File

@@ -84,8 +84,8 @@ influxdb_crt:
- private_key: /etc/pki/influxdb.key - private_key: /etc/pki/influxdb.key
- CN: {{ GLOBALS.hostname }} - CN: {{ GLOBALS.hostname }}
- subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
- days_remaining: 0 - days_remaining: 7
- days_valid: 820 - days_valid: 9
- backup: True - backup: True
- timeout: 30 - timeout: 30
- retry: - retry:
@@ -123,8 +123,8 @@ redis_crt:
- signing_policy: registry - signing_policy: registry
- private_key: /etc/pki/redis.key - private_key: /etc/pki/redis.key
- CN: {{ GLOBALS.hostname }} - CN: {{ GLOBALS.hostname }}
- days_remaining: 0 - days_remaining: 7
- days_valid: 820 - days_valid: 9
- backup: True - backup: True
- timeout: 30 - timeout: 30
- retry: - retry:
@@ -165,8 +165,8 @@ etc_elasticfleet_crt:
- private_key: /etc/pki/elasticfleet-server.key - private_key: /etc/pki/elasticfleet-server.key
- CN: {{ GLOBALS.hostname }} - CN: {{ GLOBALS.hostname }}
- subjectAltName: DNS:{{ GLOBALS.hostname }},DNS:{{ GLOBALS.url_base }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %} - subjectAltName: DNS:{{ GLOBALS.hostname }},DNS:{{ GLOBALS.url_base }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %}
- days_remaining: 0 - days_remaining: 7
- days_valid: 820 - days_valid: 9
- backup: True - backup: True
- timeout: 30 - timeout: 30
- retry: - retry:
@@ -222,8 +222,8 @@ etc_elasticfleet_logstash_crt:
- private_key: /etc/pki/elasticfleet-logstash.key - private_key: /etc/pki/elasticfleet-logstash.key
- CN: {{ GLOBALS.hostname }} - CN: {{ GLOBALS.hostname }}
- subjectAltName: DNS:{{ GLOBALS.hostname }},DNS:{{ GLOBALS.url_base }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %} - subjectAltName: DNS:{{ GLOBALS.hostname }},DNS:{{ GLOBALS.url_base }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %}
- days_remaining: 0 - days_remaining: 7
- days_valid: 820 - days_valid: 9
- backup: True - backup: True
- timeout: 30 - timeout: 30
- retry: - retry:
@@ -283,8 +283,8 @@ etc_elasticfleetlumberjack_crt:
- private_key: /etc/pki/elasticfleet-lumberjack.key - private_key: /etc/pki/elasticfleet-lumberjack.key
- CN: {{ GLOBALS.node_ip }} - CN: {{ GLOBALS.node_ip }}
- subjectAltName: DNS:{{ GLOBALS.hostname }} - subjectAltName: DNS:{{ GLOBALS.hostname }}
- days_remaining: 0 - days_remaining: 7
- days_valid: 820 - days_valid: 9
- backup: True - backup: True
- timeout: 30 - timeout: 30
- retry: - retry:
@@ -350,8 +350,8 @@ etc_elasticfleet_agent_crt:
- signing_policy: elasticfleet - signing_policy: elasticfleet
- private_key: /etc/pki/elasticfleet-agent.key - private_key: /etc/pki/elasticfleet-agent.key
- CN: {{ GLOBALS.hostname }} - CN: {{ GLOBALS.hostname }}
- days_remaining: 0 - days_remaining: 7
- days_valid: 820 - days_valid: 9
- backup: True - backup: True
- timeout: 30 - timeout: 30
- retry: - retry:
@@ -412,8 +412,8 @@ etc_filebeat_crt:
- private_key: /etc/pki/filebeat.key - private_key: /etc/pki/filebeat.key
- CN: {{ GLOBALS.hostname }} - CN: {{ GLOBALS.hostname }}
- subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
- days_remaining: 0 - days_remaining: 7
- days_valid: 820 - days_valid: 9
- backup: True - backup: True
- timeout: 30 - timeout: 30
- retry: - retry:
@@ -483,8 +483,8 @@ registry_crt:
- signing_policy: registry - signing_policy: registry
- private_key: /etc/pki/registry.key - private_key: /etc/pki/registry.key
- CN: {{ GLOBALS.manager }} - CN: {{ GLOBALS.manager }}
- days_remaining: 0 - days_remaining: 7
- days_valid: 820 - days_valid: 9
- backup: True - backup: True
- timeout: 30 - timeout: 30
- retry: - retry:
@@ -521,8 +521,8 @@ regkeyperms:
- private_key: /etc/pki/elasticsearch.key - private_key: /etc/pki/elasticsearch.key
- CN: {{ GLOBALS.hostname }} - CN: {{ GLOBALS.hostname }}
- subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
- days_remaining: 0 - days_remaining: 7
- days_valid: 820 - days_valid: 9
- backup: True - backup: True
- timeout: 30 - timeout: 30
- retry: - retry:
@@ -582,8 +582,8 @@ conf_filebeat_crt:
- private_key: /opt/so/conf/filebeat/etc/pki/filebeat.key - private_key: /opt/so/conf/filebeat/etc/pki/filebeat.key
- CN: {{ GLOBALS.hostname }} - CN: {{ GLOBALS.hostname }}
- subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
- days_remaining: 0 - days_remaining: 7
- days_valid: 820 - days_valid: 9
- backup: True - backup: True
- timeout: 30 - timeout: 30
- retry: - retry:
@@ -636,8 +636,8 @@ chownfilebeatp8:
- private_key: /etc/pki/elasticsearch.key - private_key: /etc/pki/elasticsearch.key
- CN: {{ GLOBALS.hostname }} - CN: {{ GLOBALS.hostname }}
- subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
- days_remaining: 0 - days_remaining: 7
- days_valid: 820 - days_valid: 9
- backup: True - backup: True
- timeout: 30 - timeout: 30
- retry: - retry:
@@ -686,8 +686,8 @@ elasticfleet_kafka_crt:
- private_key: /etc/pki/elasticfleet-kafka.key - private_key: /etc/pki/elasticfleet-kafka.key
- CN: {{ GLOBALS.hostname }} - CN: {{ GLOBALS.hostname }}
- subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
- days_remaining: 0 - days_remaining: 7
- days_valid: 820 - days_valid: 9
- backup: True - backup: True
- timeout: 30 - timeout: 30
- retry: - retry:

View File

@@ -11,6 +11,8 @@ export {
option JA4S_enabled: bool = F; option JA4S_enabled: bool = F;
option JA4S_raw: bool = F; option JA4S_raw: bool = F;
option JA4D_enabled: bool = F;
option JA4H_enabled: bool = F; option JA4H_enabled: bool = F;
option JA4H_raw: bool = F; option JA4H_raw: bool = F;