cert expire test

Add JA4D option to config.zeek.ja4

salt/_runners/setup_hypervisor.py

@@ -172,7 +172,15 @@ MANAGER_HOSTNAME = socket.gethostname()
 
 def _download_image():
     """
-    Download and validate the Oracle Linux KVM image.
+    Download and validate the Oracle Linux KVM image with retry logic and progress monitoring.
+    
+    Features:
+    - Detects stalled downloads (no progress for 30 seconds)
+    - Retries up to 3 times on failure
+    - Connection timeout of 30 seconds
+    - Read timeout of 60 seconds
+    - Cleans up partial downloads on failure
+    
     Returns:
         bool: True if successful or file exists with valid checksum, False on error
     """
@@ -185,45 +193,107 @@ def _download_image():
             os.unlink(IMAGE_PATH)
     
     log.info("Starting image download process")
+    
+    # Retry configuration
+    max_attempts = 3
+    retry_delay = 5  # seconds to wait between retry attempts
+    stall_timeout = 30  # seconds without progress before considering download stalled
+    connection_timeout = 30  # seconds to establish connection
+    read_timeout = 60  # seconds to wait for data chunks
+    
+    for attempt in range(1, max_attempts + 1):
+        log.info("Download attempt %d of %d", attempt, max_attempts)
+        
+        try:
+            # Download file with timeouts
+            log.info("Downloading Oracle Linux KVM image from %s to %s", IMAGE_URL, IMAGE_PATH)
+            response = requests.get(
+                IMAGE_URL,
+                stream=True,
+                timeout=(connection_timeout, read_timeout)
+            )
+            response.raise_for_status()
 
-    try:
+            # Get total file size for progress tracking
-        # Download file
+            total_size = int(response.headers.get('content-length', 0))
-        log.info("Downloading Oracle Linux KVM image from %s to %s", IMAGE_URL, IMAGE_PATH)
+            downloaded_size = 0
-        response = requests.get(IMAGE_URL, stream=True)
+            last_log_time = 0
-        response.raise_for_status()
+            last_progress_time = time.time()
+            last_downloaded_size = 0
 
-        # Get total file size for progress tracking
+            # Save file with progress logging and stall detection
-        total_size = int(response.headers.get('content-length', 0))
+            with salt.utils.files.fopen(IMAGE_PATH, 'wb') as f:
-        downloaded_size = 0
+                for chunk in response.iter_content(chunk_size=8192):
-        last_log_time = 0
+                    if chunk:  # filter out keep-alive new chunks
+                        f.write(chunk)
+                        downloaded_size += len(chunk)
+                        current_time = time.time()
+                        
+                        # Check for stalled download
+                        if downloaded_size > last_downloaded_size:
+                            # Progress made, reset stall timer
+                            last_progress_time = current_time
+                            last_downloaded_size = downloaded_size
+                        elif current_time - last_progress_time > stall_timeout:
+                            # No progress for stall_timeout seconds
+                            raise Exception(
+                                f"Download stalled: no progress for {stall_timeout} seconds "
+                                f"at {downloaded_size}/{total_size} bytes"
+                            )
+                        
+                        # Log progress every second
+                        if current_time - last_log_time >= 1:
+                            progress = (downloaded_size / total_size) * 100 if total_size > 0 else 0
+                            log.info("Progress - %.1f%% (%d/%d bytes)",
+                                    progress, downloaded_size, total_size)
+                            last_log_time = current_time
 
-        # Save file with progress logging
+            # Validate downloaded file
-        with salt.utils.files.fopen(IMAGE_PATH, 'wb') as f:
+            log.info("Download complete, validating checksum...")
-            for chunk in response.iter_content(chunk_size=8192):
+            if not _validate_image_checksum(IMAGE_PATH, IMAGE_SHA256):
-                f.write(chunk)
+                log.error("Checksum validation failed on attempt %d", attempt)
-                downloaded_size += len(chunk)
+                os.unlink(IMAGE_PATH)
+                if attempt < max_attempts:
+                    log.info("Will retry download...")
+                    continue
+                else:
+                    log.error("All download attempts failed due to checksum mismatch")
+                    return False
+
+            log.info("Successfully downloaded and validated Oracle Linux KVM image")
+            return True
+
+        except requests.exceptions.Timeout as e:
+            log.error("Download attempt %d failed: Timeout - %s", attempt, str(e))
+            if os.path.exists(IMAGE_PATH):
+                os.unlink(IMAGE_PATH)
+            if attempt < max_attempts:
+                log.info("Will retry download in %d seconds...", retry_delay)
+                time.sleep(retry_delay)
+            else:
+                log.error("All download attempts failed due to timeout")
                 
-                # Log progress every second
+        except requests.exceptions.RequestException as e:
-                current_time = time.time()
+            log.error("Download attempt %d failed: Network error - %s", attempt, str(e))
-                if current_time - last_log_time >= 1:
+            if os.path.exists(IMAGE_PATH):
-                    progress = (downloaded_size / total_size) * 100 if total_size > 0 else 0
+                os.unlink(IMAGE_PATH)
-                    log.info("Progress - %.1f%% (%d/%d bytes)", 
+            if attempt < max_attempts:
-                            progress, downloaded_size, total_size)
+                log.info("Will retry download in %d seconds...", retry_delay)
-                    last_log_time = current_time
+                time.sleep(retry_delay)
-
+            else:
-        # Validate downloaded file
+                log.error("All download attempts failed due to network errors")
-        if not _validate_image_checksum(IMAGE_PATH, IMAGE_SHA256):
+                
-            os.unlink(IMAGE_PATH)
+        except Exception as e:
-            return False
+            log.error("Download attempt %d failed: %s", attempt, str(e))
-
+            if os.path.exists(IMAGE_PATH):
-        log.info("Successfully downloaded and validated Oracle Linux KVM image")
+                os.unlink(IMAGE_PATH)
-        return True
+            if attempt < max_attempts:
-
+                log.info("Will retry download in %d seconds...", retry_delay)
-    except Exception as e:
+                time.sleep(retry_delay)
-        log.error("Error downloading hypervisor image: %s", str(e))
+            else:
-        if os.path.exists(IMAGE_PATH):
+                log.error("All download attempts failed")
-            os.unlink(IMAGE_PATH)
+    
-        return False
+    return False
 
 def _check_ssh_keys_exist():
     """
@@ -419,25 +489,28 @@ def _ensure_hypervisor_host_dir(minion_id: str = None):
         log.error(f"Error creating hypervisor host directory: {str(e)}")
         return False
 
-def _apply_dyanno_hypervisor_state():
+def _apply_dyanno_hypervisor_state(status):
     """
     Apply the soc.dyanno.hypervisor state on the salt master.
     
     This function applies the soc.dyanno.hypervisor state on the salt master
     to update the hypervisor annotation and ensure all hypervisor host directories exist.
     
+    Args:
+        status: Status passed to the hypervisor annotation state
+    
     Returns:
         bool: True if state was applied successfully, False otherwise
     """
     try:
-        log.info("Applying soc.dyanno.hypervisor state on salt master")
+        log.info(f"Applying soc.dyanno.hypervisor state on salt master with status: {status}")
         
         # Initialize the LocalClient
         local = salt.client.LocalClient()
         
         # Target the salt master to apply the soc.dyanno.hypervisor state
         target = MANAGER_HOSTNAME + '_*'
-        state_result = local.cmd(target, 'state.apply', ['soc.dyanno.hypervisor', "pillar={'baseDomain': {'status': 'PreInit'}}", 'concurrent=True'], tgt_type='glob')
+        state_result = local.cmd(target, 'state.apply', ['soc.dyanno.hypervisor', f"pillar={{'baseDomain': {{'status': '{status}'}}}}", 'concurrent=True'], tgt_type='glob')
         log.debug(f"state_result: {state_result}")
         # Check if state was applied successfully
         if state_result:
@@ -454,17 +527,17 @@ def _apply_dyanno_hypervisor_state():
                         success = False
             
             if success:
-                log.info("Successfully applied soc.dyanno.hypervisor state")
+                log.info(f"Successfully applied soc.dyanno.hypervisor state with status: {status}")
                 return True
             else:
-                log.error("Failed to apply soc.dyanno.hypervisor state")
+                log.error(f"Failed to apply soc.dyanno.hypervisor state with status: {status}")
                 return False
         else:
-            log.error("No response from salt master when applying soc.dyanno.hypervisor state")
+            log.error(f"No response from salt master when applying soc.dyanno.hypervisor state with status: {status}")
             return False
             
     except Exception as e:
-        log.error(f"Error applying soc.dyanno.hypervisor state: {str(e)}")
+        log.error(f"Error applying soc.dyanno.hypervisor state with status: {status}: {str(e)}")
         return False
 
 def _apply_cloud_config_state():
@@ -598,11 +671,6 @@ def setup_environment(vm_name: str = 'sool9', disk_size: str = '220G', minion_id
         log.warning("Failed to apply salt.cloud.config state, continuing with setup")
         # We don't return an error here as we want to continue with the setup process
 
-    # Apply the soc.dyanno.hypervisor state on the salt master
-    if not _apply_dyanno_hypervisor_state():
-        log.warning("Failed to apply soc.dyanno.hypervisor state, continuing with setup")
-        # We don't return an error here as we want to continue with the setup process
-
     log.info("Starting setup_environment in setup_hypervisor runner")
     
     # Check if environment is already set up
@@ -616,9 +684,12 @@ def setup_environment(vm_name: str = 'sool9', disk_size: str = '220G', minion_id
     
     # Handle image setup if needed
     if not image_valid:
+        _apply_dyanno_hypervisor_state('ImageDownloadStart')
         log.info("Starting image download/validation process")
         if not _download_image():
             log.error("Image download failed")
+            # Update hypervisor annotation with failure status
+            _apply_dyanno_hypervisor_state('ImageDownloadFailed')
             return {
                 'success': False,
                 'error': 'Image download failed',
@@ -631,6 +702,8 @@ def setup_environment(vm_name: str = 'sool9', disk_size: str = '220G', minion_id
         log.info("Setting up SSH keys")
         if not _setup_ssh_keys():
             log.error("SSH key setup failed")
+            # Update hypervisor annotation with failure status
+            _apply_dyanno_hypervisor_state('SSHKeySetupFailed')
             return {
                 'success': False,
                 'error': 'SSH key setup failed',
@@ -655,6 +728,12 @@ def setup_environment(vm_name: str = 'sool9', disk_size: str = '220G', minion_id
     success = vm_result.get('success', False)
     log.info("Setup environment completed with status: %s", "SUCCESS" if success else "FAILED")
     
+    # Update hypervisor annotation with success status
+    if success:
+        _apply_dyanno_hypervisor_state('PreInit')
+    else:
+        _apply_dyanno_hypervisor_state('SetupFailed')
+    
     # If setup was successful and we have a minion_id, run highstate
     if success and minion_id:
         log.info("Running highstate on hypervisor %s", minion_id)

salt/ca/files/signing_policies.conf

@@ -10,7 +10,7 @@ x509_signing_policies:
     - keyUsage: "digitalSignature, nonRepudiation"
     - subjectKeyIdentifier: hash
     - authorityKeyIdentifier: keyid,issuer:always
-    - days_valid: 820
+    - days_valid: 9
     - copypath: /etc/pki/issued_certs/
   registry:
     - minions: '*'
@@ -24,7 +24,7 @@ x509_signing_policies:
     - subjectKeyIdentifier: hash
     - authorityKeyIdentifier: keyid,issuer:always
     - extendedKeyUsage: serverAuth
-    - days_valid: 820
+    - days_valid: 9
     - copypath: /etc/pki/issued_certs/
   managerssl:
     - minions: '*'
@@ -38,7 +38,7 @@ x509_signing_policies:
     - subjectKeyIdentifier: hash
     - authorityKeyIdentifier: keyid,issuer:always
     - extendedKeyUsage: serverAuth
-    - days_valid: 820
+    - days_valid: 9
     - copypath: /etc/pki/issued_certs/
   influxdb:
     - minions: '*'
@@ -52,7 +52,7 @@ x509_signing_policies:
     - subjectKeyIdentifier: hash
     - authorityKeyIdentifier: keyid,issuer:always
     - extendedKeyUsage: serverAuth
-    - days_valid: 820
+    - days_valid: 9
     - copypath: /etc/pki/issued_certs/
   elasticfleet:
     - minions: '*'
@@ -65,7 +65,7 @@ x509_signing_policies:
     - keyUsage: "digitalSignature, nonRepudiation"
     - subjectKeyIdentifier: hash
     - authorityKeyIdentifier: keyid,issuer:always
-    - days_valid: 820
+    - days_valid: 9
     - copypath: /etc/pki/issued_certs/
   kafka:
     - minions: '*'
@@ -79,5 +79,5 @@ x509_signing_policies:
     - subjectKeyIdentifier: hash
     - authorityKeyIdentifier: keyid,issuer:always
     - extendedKeyUsage: "serverAuth, clientAuth"
-    - days_valid: 820
+    - days_valid: 9
     - copypath: /etc/pki/issued_certs/

salt/ca/init.sls

@@ -39,8 +39,8 @@ pki_public_ca_crt:
     - extendedkeyUsage: "serverAuth, clientAuth"
     - subjectKeyIdentifier: hash
     - authorityKeyIdentifier: keyid:always, issuer
-    - days_valid: 3650
+    - days_valid: 11
-    - days_remaining: 0
+    - days_remaining: 7
     - backup: True
     - replace: False
     - require:

salt/elasticfleet/config.map.jinja

@@ -1,34 +0,0 @@
-{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-   https://securityonion.net/license; you may not use this file except in compliance with the
-   Elastic License 2.0. #}
-
-{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
-
-{# advanced config_yaml options for elasticfleet logstash output #}
-{% set ADV_OUTPUT_LOGSTASH_RAW = ELASTICFLEETMERGED.config.outputs.logstash %}
-{% set ADV_OUTPUT_LOGSTASH = {} %}
-{% for k, v in ADV_OUTPUT_LOGSTASH_RAW.items() %}
-{%   if v != "" and v is not none %}
-{%     if k == 'queue_mem_events' %}
-{#       rename queue_mem_events queue.mem.events #}
-{%       do ADV_OUTPUT_LOGSTASH.update({'queue.mem.events':v}) %}
-{%     elif k == 'loadbalance' %}
-{%       if v %}
-{#         only include loadbalance config when its True #}
-{%         do ADV_OUTPUT_LOGSTASH.update({k:v}) %}
-{%       endif %}
-{%     else %}
-{%       do ADV_OUTPUT_LOGSTASH.update({k:v}) %}
-{%     endif %}
-{%   endif %}
-{% endfor %}
-
-{% set LOGSTASH_CONFIG_YAML_RAW = [] %}
-{% if ADV_OUTPUT_LOGSTASH %}
-{%   for k, v in ADV_OUTPUT_LOGSTASH.items() %}
-{%     do LOGSTASH_CONFIG_YAML_RAW.append(k ~ ': ' ~ v) %}
-{%   endfor %}
-{% endif %}
-
-{% set LOGSTASH_CONFIG_YAML = LOGSTASH_CONFIG_YAML_RAW | join('\\n') if LOGSTASH_CONFIG_YAML_RAW else '' %}

salt/elasticfleet/defaults.yaml

@@ -10,14 +10,6 @@ elasticfleet:
       grid_enrollment: ''
     defend_filters:
       enable_auto_configuration: False
-    outputs:
-      logstash:
-        bulk_max_size: ''
-        worker: ''
-        queue_mem_events: ''
-        timeout: ''
-        loadbalance: False
-        compression_level: ''
     subscription_integrations: False
     auto_upgrade_integrations: False
   logging:

salt/elasticfleet/integration-defaults.map.jinja

@@ -121,9 +121,6 @@
                  "phases": {
                    "cold": {
                      "actions": {
-                        "allocate":{
-                          "number_of_replicas": ""
-                        },
                        "set_priority": {"priority": 0}
                      },
                      "min_age": "60d"
@@ -140,31 +137,12 @@
                          "max_age": "30d",
                          "max_primary_shard_size": "50gb"
                        },
-                       "forcemerge":{
-                         "max_num_segments": ""
-                       },
-                       "shrink":{
-                         "max_primary_shard_size": "",
-                         "method": "COUNT",
-                         "number_of_shards": ""
-                       },
                        "set_priority": {"priority": 100}
                       },
                       "min_age": "0ms"
                    },
                    "warm": {
                      "actions": {
-                        "allocate": {
-                          "number_of_replicas": ""
-                        },
-                        "forcemerge": {
-                          "max_num_segments": ""
-                        },
-                        "shrink":{
-                          "max_primary_shard_size": "",
-                          "method": "COUNT",
-                          "number_of_shards": ""
-                        },
                        "set_priority": {"priority": 50}
                      },
                      "min_age": "30d"

salt/elasticfleet/soc_elasticfleet.yaml

@@ -50,46 +50,6 @@ elasticfleet:
       global: True
       forcedType: bool
       helpLink: elastic-fleet.html
-    outputs:
-      logstash:
-        bulk_max_size:
-          description: The maximum number of events to bulk in a single Logstash request.
-          global: True
-          forcedType: int
-          advanced: True
-          helpLink: elastic-fleet.html
-        worker:
-          description: The number of workers per configured host publishing events.
-          global: True
-          forcedType: int
-          advanced: true
-          helpLink: elastic-fleet.html
-        queue_mem_events:
-          title: queued events
-          description: The number of events the queue can store. This value should be evenly divisible by the smaller of 'bulk_max_size' to avoid sending partial batches to the output.
-          global: True
-          forcedType: int
-          advanced: True
-          helpLink: elastic-fleet.html
-        timeout:
-          description: The number of seconds to wait for responses from the Logstash server before timing out. Eg 30s
-          regex: ^[0-9]+s$
-          advanced: True
-          global: True
-          helpLink: elastic-fleet.html
-        loadbalance:
-          description: If true and multiple Logstash hosts are configured, the output plugin load balances published events onto all Logstash hosts. If false, the output plugin sends all events to one host (determined at random) and switches to another host if the selected one becomes unresponsive.
-          forcedType: bool
-          advanced: True
-          global: True
-          helpLink: elastic-fleet.html
-        compression:
-          description: The gzip compression level. The compression level must be in the range of 1 (best speed) to 9 (best compression).
-          regex: ^[1-9]$
-          forcedType: int
-          advanced: True
-          global: True
-          helpLink: elastic-fleet.html
     server:
       custom_fqdn:
         description: Custom FQDN for Agents to connect to. One per line.

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update

@@ -3,13 +3,11 @@
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
 # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
 # this file except in compliance with the Elastic License 2.0.
-{%- from 'vars/globals.map.jinja' import GLOBALS %}
+{% from 'vars/globals.map.jinja' import GLOBALS %}
-{%- from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
+{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
-{%- from 'elasticfleet/config.map.jinja' import LOGSTASH_CONFIG_YAML %}
 
 . /usr/sbin/so-common
 
-FORCE_UPDATE=false
 # Only run on Managers
 if ! is_manager_node; then
     printf "Not a Manager Node... Exiting"
@@ -24,7 +22,7 @@ function update_logstash_outputs() {
                 --arg UPDATEDLIST "$NEW_LIST_JSON" \
                 --argjson SECRETS "$SECRETS" \
                 --argjson SSL_CONFIG "$SSL_CONFIG" \
-                '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":"{{ LOGSTASH_CONFIG_YAML }}","ssl": $SSL_CONFIG,"secrets": $SECRETS}')
+                '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl": $SSL_CONFIG,"secrets": $SECRETS}')
         else
             JSON_STRING=$(jq -n \
                 --arg UPDATEDLIST "$NEW_LIST_JSON" \
@@ -99,18 +97,9 @@ function update_kafka_outputs() {
    exit 1
   fi
 
-  CURRENT_LOGSTASH_ADV_CONFIG=$(jq -r '.item.config_yaml // ""' <<< "$RAW_JSON")
-  CURRENT_LOGSTASH_ADV_CONFIG_HASH=$(sha256sum <<< "$CURRENT_LOGSTASH_ADV_CONFIG" | awk '{print $1}')
-  NEW_LOGSTASH_ADV_CONFIG=$'{{ LOGSTASH_CONFIG_YAML }}'
-  NEW_LOGSTASH_ADV_CONFIG_HASH=$(sha256sum <<< "$NEW_LOGSTASH_ADV_CONFIG" | awk '{print $1}')
-
-  if [ "$CURRENT_LOGSTASH_ADV_CONFIG_HASH" != "$NEW_LOGSTASH_ADV_CONFIG_HASH" ]; then
-    FORCE_UPDATE=true
-  fi
-
   # Get the current list of Logstash outputs & hash them
   CURRENT_LIST=$(jq -c -r '.item.hosts' <<<  "$RAW_JSON")
-  CURRENT_HASH=$(sha256sum <<< "$CURRENT_LIST" | awk '{print $1}')
+  CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}')
 
   declare -a NEW_LIST=()
 
@@ -159,10 +148,10 @@ function update_kafka_outputs() {
 
 # Sort & hash the new list of Logstash Outputs
 NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${NEW_LIST[@]}")
-NEW_HASH=$(sha256sum <<< "$NEW_LIST_JSON" | awk '{print $1}')
+NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}')
 
 # Compare the current & new list of outputs - if different, update the Logstash outputs
-if [[ "$NEW_HASH" = "$CURRENT_HASH" ]] && [[ "$FORCE_UPDATE" != "true" ]]; then
+if [ "$NEW_HASH" = "$CURRENT_HASH" ]; then
     printf "\nHashes match - no update needed.\n"
     printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n"
 

salt/elasticsearch/defaults.yaml

@@ -72,8 +72,6 @@ elasticsearch:
             actions:
               set_priority:
                 priority: 0
-              allocate:
-                number_of_replicas: ""
             min_age: 60d
           delete:
             actions:
@@ -86,25 +84,11 @@ elasticsearch:
                 max_primary_shard_size: 50gb
               set_priority:
                 priority: 100
-              forcemerge:
-                max_num_segments: ""
-              shrink:
-                max_primary_shard_size: ""
-                method: COUNT
-                number_of_shards: ""
             min_age: 0ms
           warm:
             actions:
               set_priority:
                 priority: 50
-              forcemerge:
-                max_num_segments: ""
-              shrink:
-                max_primary_shard_size: ""
-                method: COUNT
-                number_of_shards: ""
-              allocate:
-                number_of_replicas: ""
             min_age: 30d
     so-case:
       index_sorting: false
@@ -261,6 +245,7 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
+      warm: 7
     so-detection:
       index_sorting: false
       index_template:
@@ -599,6 +584,7 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
+      warm: 7
     so-import:
       index_sorting: false
       index_template:
@@ -946,6 +932,7 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
+      warm: 7
     so-hydra:
       close: 30
       delete: 365
@@ -1056,6 +1043,7 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
+      warm: 7
     so-lists:
       index_sorting: false
       index_template:
@@ -1139,8 +1127,6 @@ elasticsearch:
             actions:
               set_priority:
                 priority: 0
-              allocate:
-                number_of_replicas: ""
             min_age: 60d
           delete:
             actions:
@@ -1153,25 +1139,11 @@ elasticsearch:
                 max_primary_shard_size: 50gb
               set_priority:
                 priority: 100
-              forcemerge:
-                max_num_segments: ""
-              shrink:
-                max_primary_shard_size: ""
-                method: COUNT
-                number_of_shards: ""
             min_age: 0ms
           warm:
             actions:
               set_priority:
                 priority: 50
-              allocate:
-                number_of_replicas: ""
-              forcemerge:
-                max_num_segments: ""
-              shrink:
-                max_primary_shard_size: ""
-                method: COUNT
-                number_of_shards: ""
             min_age: 30d
     so-logs-detections_x_alerts:
       index_sorting: false
@@ -3151,6 +3123,7 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
+      warm: 7
     so-logs-system_x_application:
       index_sorting: false
       index_template:

salt/elasticsearch/files/ingest/suricata.common

@@ -1,30 +1,155 @@
 {
-  "description" : "suricata.common",
+    "description": "suricata.common",
-  "processors" : [
+    "processors": [
-    { "json":   { "field": "message",                   "target_field": "message2",             "ignore_failure": true  } },
+        {
-    { "rename": { "field": "message2.pkt_src",          "target_field": "network.packet_source","ignore_failure": true  } },
+            "json": {
-    { "rename": { "field": "message2.proto",            "target_field": "network.transport",    "ignore_failure": true  } },
+                "field": "message",
-    { "rename": { "field": "message2.in_iface",         "target_field": "observer.ingress.interface.name",    "ignore_failure": true  } },
+                "target_field": "message2",
-    { "rename": { "field": "message2.flow_id",          "target_field": "log.id.uid",           "ignore_failure": true  } },
+                "ignore_failure": true
-    { "rename": { "field": "message2.src_ip",           "target_field": "source.ip",            "ignore_failure": true  } },
+            }
-    { "rename": { "field": "message2.src_port",         "target_field": "source.port",          "ignore_failure": true  } },
+        },
-    { "rename": { "field": "message2.dest_ip",          "target_field": "destination.ip",       "ignore_failure": true  } },
+        {
-    { "rename": { "field": "message2.dest_port",        "target_field": "destination.port",     "ignore_failure": true  } },
+            "rename": {
-    { "rename": { "field": "message2.vlan",             "target_field": "network.vlan.id",      "ignore_failure": true  } },
+                "field": "message2.pkt_src",
-    { "rename": { "field": "message2.community_id",     "target_field": "network.community_id", "ignore_missing": true  } },
+                "target_field": "network.packet_source",
-    { "rename": { "field": "message2.xff",              "target_field": "xff.ip",               "ignore_missing": true  } },
+                "ignore_failure": true
-    { "set":    { "field": "event.dataset",             "value": "{{ message2.event_type }}"                            } },
+            }
-    { "set":    { "field": "observer.name",             "value": "{{agent.name}}"                                       } },
+        },
-    { "set":    { "field": "event.ingested",            "value": "{{@timestamp}}"                                       } },
+        {
-    { "date": { "field": "message2.timestamp", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "timezone": "UTC", "ignore_failure": true } },
+            "rename": {
-    { "remove":{ "field": "agent",                                                              "ignore_failure": true  } },
+                "field": "message2.proto",
-    {"append":{"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"ignore_failure":true}},
+                "target_field": "network.transport",
-    {
+                "ignore_failure": true
-      "script": {
+            }
-        "source": "boolean isPrivate(def ip) { if (ip == null) return false; int dot1 = ip.indexOf('.'); if (dot1 == -1) return false; int dot2 = ip.indexOf('.', dot1 + 1); if (dot2 == -1) return false; int first = Integer.parseInt(ip.substring(0, dot1)); if (first == 10) return true; if (first == 192 && ip.startsWith('168.', dot1 + 1)) return true; if (first == 172) { int second = Integer.parseInt(ip.substring(dot1 + 1, dot2)); return second >= 16 && second <= 31; } return false; } String[] fields = new String[] {\"source\", \"destination\"}; for (int i = 0; i < fields.length; i++) { def field = fields[i]; def ip = ctx[field]?.ip; if (ip != null) { if (ctx.network == null) ctx.network = new HashMap(); if (isPrivate(ip)) { if (ctx.network.private_ip == null) ctx.network.private_ip = new ArrayList(); if (!ctx.network.private_ip.contains(ip)) ctx.network.private_ip.add(ip); } else { if (ctx.network.public_ip == null) ctx.network.public_ip = new ArrayList(); if (!ctx.network.public_ip.contains(ip)) ctx.network.public_ip.add(ip); } } }",
+        },
-        "ignore_failure": false
+        {
-      }
+            "rename": {
-    },
+                "field": "message2.in_iface",
-    { "pipeline": { "if": "ctx?.event?.dataset != null", "name": "suricata.{{event.dataset}}" } }
+                "target_field": "observer.ingress.interface.name",
-  ]
+                "ignore_failure": true
-}
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.flow_id",
+                "target_field": "log.id.uid",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.src_ip",
+                "target_field": "source.ip",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.src_port",
+                "target_field": "source.port",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dest_ip",
+                "target_field": "destination.ip",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.dest_port",
+                "target_field": "destination.port",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.vlan",
+                "target_field": "network.vlan.id",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.community_id",
+                "target_field": "network.community_id",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.xff",
+                "target_field": "xff.ip",
+                "ignore_missing": true
+            }
+        },
+        {
+            "set": {
+                "field": "event.dataset",
+                "value": "{{ message2.event_type }}"
+            }
+        },
+        {
+            "set": {
+                "field": "observer.name",
+                "value": "{{agent.name}}"
+            }
+        },
+        {
+            "set": {
+                "field": "event.ingested",
+                "value": "{{@timestamp}}"
+            }
+        },
+        {
+            "date": {
+                "field": "message2.timestamp",
+                "target_field": "@timestamp",
+                "formats": [
+                    "ISO8601",
+                    "UNIX"
+                ],
+                "timezone": "UTC",
+                "ignore_failure": true
+            }
+        },
+        {
+            "remove": {
+                "field": "agent",
+                "ignore_failure": true
+            }
+        },
+        {
+            "append": {
+                "field": "related.ip",
+                "value": [
+                    "{{source.ip}}",
+                    "{{destination.ip}}"
+                ],
+                "allow_duplicates": false,
+                "ignore_failure": true
+            }
+        },
+        {
+            "script": {
+                "source": "boolean isPrivate(def ip) { if (ip == null) return false; int dot1 = ip.indexOf('.'); if (dot1 == -1) return false; int dot2 = ip.indexOf('.', dot1 + 1); if (dot2 == -1) return false; int first = Integer.parseInt(ip.substring(0, dot1)); if (first == 10) return true; if (first == 192 && ip.startsWith('168.', dot1 + 1)) return true; if (first == 172) { int second = Integer.parseInt(ip.substring(dot1 + 1, dot2)); return second >= 16 && second <= 31; } return false; } String[] fields = new String[] {\"source\", \"destination\"}; for (int i = 0; i < fields.length; i++) { def field = fields[i]; def ip = ctx[field]?.ip; if (ip != null) { if (ctx.network == null) ctx.network = new HashMap(); if (isPrivate(ip)) { if (ctx.network.private_ip == null) ctx.network.private_ip = new ArrayList(); if (!ctx.network.private_ip.contains(ip)) ctx.network.private_ip.add(ip); } else { if (ctx.network.public_ip == null) ctx.network.public_ip = new ArrayList(); if (!ctx.network.public_ip.contains(ip)) ctx.network.public_ip.add(ip); } } }",
+                "ignore_failure": false
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.capture_file",
+                "target_field": "suricata.capture_file",
+                "ignore_missing": true
+            }
+        },
+        {
+            "pipeline": {
+                "if": "ctx?.event?.dataset != null",
+                "name": "suricata.{{event.dataset}}"
+            }
+        }
+    ]
+}

salt/elasticsearch/soc_elasticsearch.yaml

@@ -131,47 +131,6 @@ elasticsearch:
                   description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index.
                   global: True
                   helpLink: elasticsearch.html
-              shrink:
-                method:
-                  description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
-                  options:
-                  - COUNT
-                  - SIZE
-                  global: True
-                  advanced: True
-                  forcedType: string
-                number_of_shards:
-                  title: shard count
-                  description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
-                  global: True
-                  forcedType: int
-                  advanced: True
-                max_primary_shard_size:
-                  title: max shard size
-                  description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
-                  regex: ^[0-9]+(?:gb|tb|pb)$
-                  global: True
-                  forcedType: string
-                  advanced: True
-                allow_write_after_shrink:
-                  description: Allow writes after shrink.
-                  global: True
-                  forcedType: bool
-                  default: False
-                  advanced: True
-              forcemerge:
-                max_num_segments:
-                  description: Reduce the number of segments in each index shard and clean up deleted documents.
-                  global: True
-                  forcedType: int
-                  advanced: True
-                index_codec:
-                  title: compression
-                  description: Use higher compression for stored fields at the cost of slower performance.
-                  forcedType: bool
-                  global: True
-                  default: False
-                  advanced: True
           cold:
             min_age:
               description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and cold min_age set to 60 then there will be 30 days from index creation to rollover and then an additional 60 days before moving to cold tier.
@@ -185,12 +144,6 @@ elasticsearch:
                   description: Used for index recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
                   global: True
                   helpLink: elasticsearch.html
-              allocate:
-                number_of_replicas:
-                  description: Set the number of replicas. Remains the same as the previous phase by default.
-                  forcedType: int
-                  global: True
-                  advanced: True
           warm:
             min_age: 
               description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier.
@@ -205,52 +158,6 @@ elasticsearch:
                   forcedType: int
                   global: True
                   helpLink: elasticsearch.html
-              shrink:
-                method:
-                  description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
-                  options:
-                  - COUNT
-                  - SIZE
-                  global: True
-                  advanced: True
-                number_of_shards:
-                  title: shard count
-                  description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
-                  global: True
-                  forcedType: int
-                  advanced: True
-                max_primary_shard_size:
-                  title: max shard size
-                  description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
-                  regex: ^[0-9]+(?:gb|tb|pb)$
-                  global: True
-                  forcedType: string
-                  advanced: True
-                allow_write_after_shrink:
-                  description: Allow writes after shrink.
-                  global: True
-                  forcedType: bool
-                  default: False
-                  advanced: True
-              forcemerge:
-                max_num_segments:
-                  description: Reduce the number of segments in each index shard and clean up deleted documents.
-                  global: True
-                  forcedType: int
-                  advanced: True
-                index_codec:
-                  title: compression
-                  description: Use higher compression for stored fields at the cost of slower performance.
-                  forcedType: bool
-                  global: True
-                  default: False
-                  advanced: True
-              allocate:
-                number_of_replicas:
-                  description: Set the number of replicas. Remains the same as the previous phase by default.
-                  forcedType: int
-                  global: True
-                  advanced: True
           delete:
             min_age:
               description: Minimum age of index. ex. 90d - This determines when the index should be deleted. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and delete min_age set to 90 then there will be 30 days from index creation to rollover and then an additional 90 days before deletion.
@@ -380,47 +287,6 @@ elasticsearch:
                   global: True
                   advanced: True
                   helpLink: elasticsearch.html
-              shrink:
-                method:
-                  description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
-                  options:
-                  - COUNT
-                  - SIZE
-                  global: True
-                  advanced: True
-                  forcedType: string
-                number_of_shards:
-                  title: shard count
-                  description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
-                  global: True
-                  forcedType: int
-                  advanced: True
-                max_primary_shard_size:
-                  title: max shard size
-                  description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
-                  regex: ^[0-9]+(?:gb|tb|pb)$
-                  global: True
-                  forcedType: string
-                  advanced: True
-                allow_write_after_shrink:
-                  description: Allow writes after shrink.
-                  global: True
-                  forcedType: bool
-                  default: False
-                  advanced: True
-              forcemerge:
-                max_num_segments:
-                  description: Reduce the number of segments in each index shard and clean up deleted documents.
-                  global: True
-                  forcedType: int
-                  advanced: True
-                index_codec:
-                  title: compression
-                  description: Use higher compression for stored fields at the cost of slower performance.
-                  forcedType: bool
-                  global: True
-                  default: False
-                  advanced: True
           warm:
             min_age:
               description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier.
@@ -448,52 +314,6 @@ elasticsearch:
                   global: True
                   advanced: True
                   helpLink: elasticsearch.html
-              shrink:
-                method:
-                  description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
-                  options:
-                  - COUNT
-                  - SIZE
-                  global: True
-                  advanced: True
-                number_of_shards:
-                  title: shard count
-                  description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
-                  global: True
-                  forcedType: int
-                  advanced: True
-                max_primary_shard_size:
-                  title: max shard size
-                  description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
-                  regex: ^[0-9]+(?:gb|tb|pb)$
-                  global: True
-                  forcedType: string
-                  advanced: True
-                allow_write_after_shrink:
-                  description: Allow writes after shrink.
-                  global: True
-                  forcedType: bool
-                  default: False
-                  advanced: True
-              forcemerge:
-                max_num_segments:
-                  description: Reduce the number of segments in each index shard and clean up deleted documents.
-                  global: True
-                  forcedType: int
-                  advanced: True
-                index_codec:
-                  title: compression
-                  description: Use higher compression for stored fields at the cost of slower performance.
-                  forcedType: bool
-                  global: True
-                  default: False
-                  advanced: True
-              allocate:
-                number_of_replicas:
-                  description: Set the number of replicas. Remains the same as the previous phase by default.
-                  forcedType: int
-                  global: True
-                  advanced: True
           cold:
             min_age:
               description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier.  While still searchable, this tier is typically optimized for lower storage costs rather than search speed. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and cold min_age set to 60 then there will be 30 days from index creation to rollover and then an additional 60 days before moving to cold tier.
@@ -510,12 +330,6 @@ elasticsearch:
                   global: True
                   advanced: True
                   helpLink: elasticsearch.html
-              allocate:
-                number_of_replicas:
-                  description: Set the number of replicas. Remains the same as the previous phase by default.
-                  forcedType: int
-                  global: True
-                  advanced: True
           delete:
             min_age:
               description: Minimum age of index. ex. 90d - This determines when the index should be deleted. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and delete min_age set to 90 then there will be 30 days from index creation to rollover and then an additional 90 days before deletion.

salt/elasticsearch/template.map.jinja

@@ -61,55 +61,5 @@
 {%       do settings.index_template.template.settings.index.pop('sort') %}
 {%     endif %}
 {%   endif %}
-
-{# advanced ilm actions #}
-{%   if settings.policy is defined and settings.policy.phases is defined %}
-{%     set PHASE_NAMES = ["hot", "warm", "cold"] %}
-{%     for P in PHASE_NAMES %}
-{%       if settings.policy.phases[P] is defined and settings.policy.phases[P].actions is defined %}
-{%         set PHASE = settings.policy.phases[P].actions %}
-{#         remove allocate action if number_of_replicas isn't configured #}
-{%         if PHASE.allocate is defined %}
-{%           if PHASE.allocate.number_of_replicas is not defined or PHASE.allocate.number_of_replicas == "" %}
-{%             do PHASE.pop('allocate', none) %}
-{%           endif %}
-{%         endif %}
-{#         start shrink action #}
-{%         if PHASE.shrink is defined %}
-{%           if PHASE.shrink.method is defined %}
-{%             if PHASE.shrink.method == 'COUNT' and PHASE.shrink.number_of_shards is defined and PHASE.shrink.number_of_shards %}
-{#               remove max_primary_shard_size value when doing shrink operation by count vs size #}
-{%               do PHASE.shrink.pop('max_primary_shard_size', none) %}
-{%             elif PHASE.shrink.method == 'SIZE' and PHASE.shrink.max_primary_shard_size is defined and PHASE.shrink.max_primary_shard_size %}
-{#               remove number_of_shards value when doing shrink operation by size vs count #}
-{%               do PHASE.shrink.pop('number_of_shards', none) %}
-{%             else %}
-{#               method isn't defined or missing a required config number_of_shards/max_primary_shard_size #}
-{%               do PHASE.pop('shrink', none) %}
-{%             endif %}
-{%           endif %}
-{%         endif %}
-{#         always remove shrink method since its only used for SOC config, not in the actual ilm policy #}
-{%         if PHASE.shrink is defined %}
-{%            do PHASE.shrink.pop('method', none) %}
-{%         endif %}
-{#         end shrink action #}
-{#         start force merge #}
-{%         if PHASE.forcemerge is defined %}
-{%           if PHASE.forcemerge.index_codec is defined and PHASE.forcemerge.index_codec %}
-{%             do PHASE.forcemerge.update({'index_codec': 'best_compression'}) %}
-{%           else %}
-{%             do PHASE.forcemerge.pop('index_codec', none) %}
-{%           endif %}
-{%           if PHASE.forcemerge.max_num_segments is not defined or not PHASE.forcemerge.max_num_segments %}
-{#             max_num_segments is empty, drop it #}
-{%             do PHASE.pop('forcemerge', none) %}
-{%           endif %}
-{%         endif %}
-{#         end force merge #}
-{%       endif %}
-{%     endfor %}
-{%   endif %}
-
 {%   do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index]}) %}
 {% endfor %}

salt/elasticsearch/templates/component/ecs/suricata.json

@@ -841,6 +841,10 @@
                   "type": "long"
                 }
               }
+            },
+            "capture_file": {
+                "type": "keyword",
+                "ignore_above": 1024
             }
           }
         }

salt/hypervisor/tools/sbin_jinja/so-kvm-create-volume

@@ -45,7 +45,7 @@ used during VM provisioning to add dedicated NSM storage volumes.
     This command creates and attaches a volume with the following settings:
     - VM Name: `vm1_sensor`
     - Volume Size: `500` GB
-    - Volume Path: `/nsm/libvirt/volumes/vm1_sensor-nsm.img`
+    - Volume Path: `/nsm/libvirt/volumes/vm1_sensor-nsm-<epoch_timestamp>.img`
     - Device: `/dev/vdb` (virtio-blk)
     - VM remains stopped after attachment
 
@@ -75,7 +75,8 @@ used during VM provisioning to add dedicated NSM storage volumes.
 
 - The script automatically stops the VM if it's running before creating and attaching the volume.
 - Volumes are created with full pre-allocation for optimal performance.
-- Volume files are stored in `/nsm/libvirt/volumes/` with naming pattern `<vm_name>-nsm.img`.
+- Volume files are stored in `/nsm/libvirt/volumes/` with naming pattern `<vm_name>-nsm-<epoch_timestamp>.img`.
+- The epoch timestamp ensures unique volume names and prevents conflicts.
 - Volumes are attached as `/dev/vdb` using virtio-blk for high performance.
 - The script checks available disk space before creating the volume.
 - Ownership is set to `qemu:qemu` with permissions `640`.
@@ -142,6 +143,7 @@ import socket
 import subprocess
 import pwd
 import grp
+import time
 import xml.etree.ElementTree as ET
 from io import StringIO
 from so_vm_utils import start_vm, stop_vm
@@ -242,10 +244,13 @@ def create_volume_file(vm_name, size_gb, logger):
     Raises:
         VolumeCreationError: If volume creation fails
     """
-    # Define volume path (directory already created in main())
+    # Generate epoch timestamp for unique volume naming
-    volume_path = os.path.join(VOLUME_DIR, f"{vm_name}-nsm.img")
+    epoch_timestamp = int(time.time())
     
-    # Check if volume already exists
+    # Define volume path with epoch timestamp for uniqueness
+    volume_path = os.path.join(VOLUME_DIR, f"{vm_name}-nsm-{epoch_timestamp}.img")
+    
+    # Check if volume already exists (shouldn't be possible with timestamp)
     if os.path.exists(volume_path):
         logger.error(f"VOLUME: Volume already exists: {volume_path}")
         raise VolumeCreationError(f"Volume already exists: {volume_path}")

salt/kafka/ssl.sls

@@ -44,8 +44,8 @@ kafka_client_crt:
     - signing_policy: kafka
     - private_key: /etc/pki/kafka-client.key
     - CN: {{ GLOBALS.hostname }}
-    - days_remaining: 0
+    - days_remaining: 7
-    - days_valid: 820
+    - days_valid: 9
     - backup: True
     - timeout: 30
     - retry:
@@ -92,8 +92,8 @@ kafka_crt:
     - signing_policy: kafka
     - private_key: /etc/pki/kafka.key
     - CN: {{ GLOBALS.hostname }}
-    - days_remaining: 0
+    - days_remaining: 7
-    - days_valid: 820
+    - days_valid: 9
     - backup: True
     - timeout: 30
     - retry:
@@ -153,8 +153,8 @@ kafka_logstash_crt:
     - signing_policy: kafka
     - private_key: /etc/pki/kafka-logstash.key
     - CN: {{ GLOBALS.hostname }}
-    - days_remaining: 0
+    - days_remaining: 7
-    - days_valid: 820
+    - days_valid: 9
     - backup: True
     - timeout: 30
     - retry:
@@ -198,4 +198,4 @@ kafka_logstash_pkcs12_perms:
   test.fail_without_changes:
     - name: {{sls}}_state_not_allowed
 
-{% endif %}
+{% endif %}

salt/manager/managed_soc_annotations.sls

@@ -25,11 +25,13 @@
            {% set index_settings = es.get('index_settings', {}) %}
            {% set input = index_settings.get('so-logs', {}) %}
            {% for k in matched_integration_names %}
-           {%   do index_settings.update({k: input}) %}
+           {%   if k not in index_settings %}
+           {%     set _ = index_settings.update({k: input}) %}
+           {%   endif %}
            {% endfor %}
            {% for k in addon_integration_keys %}
            {%   if k not in matched_integration_names and k in index_settings %}
-           {%     do index_settings.pop(k) %}
+           {%     set _ = index_settings.pop(k) %}
            {%   endif %}
            {% endfor %}
            {{ data }}
@@ -43,12 +45,14 @@
            {% set es = data.get('elasticsearch', {}) %}
            {% set index_settings = es.get('index_settings', {}) %}
            {% for k in matched_integration_names %}
-           {%   set input = ADDON_INTEGRATION_DEFAULTS[k] %}
+           {%   if k not in index_settings %}
-           {%     do index_settings.update({k: input})%}
+           {%     set input = ADDON_INTEGRATION_DEFAULTS[k] %}
+           {%     set _ = index_settings.update({k: input})%}
+           {%   endif %}
            {% endfor %}
            {% for k in addon_integration_keys %}
            {%   if k not in matched_integration_names and k in index_settings %}
-           {%     do index_settings.pop(k) %}
+           {%     set _ = index_settings.pop(k) %}
            {%   endif %}
            {% endfor %}
            {{ data }}

salt/nginx/enabled.sls

@@ -64,8 +64,8 @@ managerssl_crt:
     - private_key: /etc/pki/managerssl.key
     - CN: {{ GLOBALS.hostname }}
     - subjectAltName: "DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}, DNS:{{ GLOBALS.url_base }}" 
-    - days_remaining: 0
+    - days_remaining: 7
-    - days_valid: 820
+    - days_valid: 9
     - backup: True
     - timeout: 30
     - retry:

salt/salt/engines/master/virtual_node_manager.py

@@ -727,7 +727,8 @@ def check_hypervisor_disk_space(hypervisor: str, size_gb: int) -> Tuple[bool, Op
         result = local.cmd(
             hypervisor_minion,
             'cmd.run',
-            ["df -BG /nsm/libvirt/volumes | tail -1 | awk '{print $4}' | sed 's/G//'"]
+            ["df -BG /nsm/libvirt/volumes | tail -1 | awk '{print $4}' | sed 's/G//'"],
+            kwarg={'python_shell': True}
         )
         
         if not result or hypervisor_minion not in result:

salt/soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja

@@ -43,10 +43,26 @@
 
 No Virtual Machines Found
 {%-   endif %}
-{%- else %}
+{%- elif baseDomainStatus == 'ImageDownloadStart' %}
+#### INFO
+
+Base domain image download started.
+{%- elif baseDomainStatus == 'ImageDownloadFailed' %}
+#### ERROR
+
+Base domain image download failed. Please check the salt-master log for details and verify network connectivity.
+{%- elif baseDomainStatus == 'SSHKeySetupFailed' %}
+#### ERROR
+
+SSH key setup failed. Please check the salt-master log for details.
+{%- elif baseDomainStatus == 'SetupFailed' %}
 #### WARNING
 
-Base domain has not been initialized.
+Setup failed. Please check the salt-master log for details.
+{%- elif baseDomainStatus == 'PreInit' %}
+#### WARNING
+
+Base domain has not been initialized. Waiting for hypervisor to highstate.
 {%- endif %}
 {%- endmacro -%}
 

salt/ssl/init.sls

@@ -84,8 +84,8 @@ influxdb_crt:
     - private_key: /etc/pki/influxdb.key
     - CN: {{ GLOBALS.hostname }}
     - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} 
-    - days_remaining: 0
+    - days_remaining: 7
-    - days_valid: 820
+    - days_valid: 9
     - backup: True
     - timeout: 30
     - retry:
@@ -123,8 +123,8 @@ redis_crt:
     - signing_policy: registry
     - private_key: /etc/pki/redis.key
     - CN: {{ GLOBALS.hostname }}
-    - days_remaining: 0
+    - days_remaining: 7
-    - days_valid: 820
+    - days_valid: 9
     - backup: True
     - timeout: 30
     - retry:
@@ -165,8 +165,8 @@ etc_elasticfleet_crt:
     - private_key: /etc/pki/elasticfleet-server.key
     - CN: {{ GLOBALS.hostname }}
     - subjectAltName: DNS:{{ GLOBALS.hostname }},DNS:{{ GLOBALS.url_base }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %}
-    - days_remaining: 0
+    - days_remaining: 7
-    - days_valid: 820
+    - days_valid: 9
     - backup: True
     - timeout: 30
     - retry:
@@ -222,8 +222,8 @@ etc_elasticfleet_logstash_crt:
     - private_key: /etc/pki/elasticfleet-logstash.key
     - CN: {{ GLOBALS.hostname }}
     - subjectAltName: DNS:{{ GLOBALS.hostname }},DNS:{{ GLOBALS.url_base }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %}
-    - days_remaining: 0
+    - days_remaining: 7
-    - days_valid: 820
+    - days_valid: 9
     - backup: True
     - timeout: 30
     - retry:
@@ -283,8 +283,8 @@ etc_elasticfleetlumberjack_crt:
     - private_key: /etc/pki/elasticfleet-lumberjack.key
     - CN: {{ GLOBALS.node_ip }}
     - subjectAltName: DNS:{{ GLOBALS.hostname }}
-    - days_remaining: 0
+    - days_remaining: 7
-    - days_valid: 820
+    - days_valid: 9
     - backup: True
     - timeout: 30
     - retry:
@@ -350,8 +350,8 @@ etc_elasticfleet_agent_crt:
     - signing_policy: elasticfleet
     - private_key: /etc/pki/elasticfleet-agent.key
     - CN: {{ GLOBALS.hostname }}
-    - days_remaining: 0
+    - days_remaining: 7
-    - days_valid: 820
+    - days_valid: 9
     - backup: True
     - timeout: 30
     - retry:
@@ -412,8 +412,8 @@ etc_filebeat_crt:
     - private_key: /etc/pki/filebeat.key
     - CN: {{ GLOBALS.hostname }}
     - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
-    - days_remaining: 0
+    - days_remaining: 7
-    - days_valid: 820
+    - days_valid: 9
     - backup: True
     - timeout: 30
     - retry:
@@ -483,8 +483,8 @@ registry_crt:
     - signing_policy: registry
     - private_key: /etc/pki/registry.key
     - CN: {{ GLOBALS.manager }}
-    - days_remaining: 0
+    - days_remaining: 7
-    - days_valid: 820
+    - days_valid: 9
     - backup: True
     - timeout: 30
     - retry:
@@ -521,8 +521,8 @@ regkeyperms:
     - private_key: /etc/pki/elasticsearch.key
     - CN: {{ GLOBALS.hostname }}
     - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
-    - days_remaining: 0
+    - days_remaining: 7
-    - days_valid: 820
+    - days_valid: 9
     - backup: True
     - timeout: 30
     - retry:
@@ -582,8 +582,8 @@ conf_filebeat_crt:
     - private_key: /opt/so/conf/filebeat/etc/pki/filebeat.key
     - CN: {{ GLOBALS.hostname }}
     - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
-    - days_remaining: 0
+    - days_remaining: 7
-    - days_valid: 820
+    - days_valid: 9
     - backup: True
     - timeout: 30
     - retry:
@@ -636,8 +636,8 @@ chownfilebeatp8:
     - private_key: /etc/pki/elasticsearch.key
     - CN: {{ GLOBALS.hostname }}
     - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
-    - days_remaining: 0
+    - days_remaining: 7
-    - days_valid: 820
+    - days_valid: 9
     - backup: True
     - timeout: 30
     - retry:
@@ -686,8 +686,8 @@ elasticfleet_kafka_crt:
     - private_key: /etc/pki/elasticfleet-kafka.key
     - CN: {{ GLOBALS.hostname }}
     - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
-    - days_remaining: 0
+    - days_remaining: 7
-    - days_valid: 820
+    - days_valid: 9
     - backup: True
     - timeout: 30
     - retry:

salt/zeek/files/config.zeek.ja4

@@ -11,6 +11,8 @@ export {
   option JA4S_enabled:   bool = F;
   option JA4S_raw:   bool = F;
 
+  option JA4D_enabled:  bool = F;
+
   option JA4H_enabled:   bool = F;
   option JA4H_raw:   bool = F;