Update defaults.yaml

update so-elasticsearch-retention-estimate

.github/DISCUSSION_TEMPLATE/2-4.yml

@@ -32,6 +32,7 @@ body:
         - 2.4.170
         - 2.4.180
         - 2.4.190
+        - 2.4.200
         - Other (please provide detail below)
     validations:
       required: true

VERSION

@@ -1 +1 @@
-2.4.190
+2.4.200

salt/common/tools/sbin/so-common

@@ -220,12 +220,22 @@ compare_es_versions() {
 }
 
 copy_new_files() {
+  # Define files to exclude from deletion (relative to their respective base directories)
+  local EXCLUDE_FILES=(
+    "salt/hypervisor/soc_hypervisor.yaml"
+  )
+  
+  # Build rsync exclude arguments
+  local EXCLUDE_ARGS=()
+  for file in "${EXCLUDE_FILES[@]}"; do
+    EXCLUDE_ARGS+=(--exclude="$file")
+  done
+  
   # Copy new files over to the salt dir
   cd $UPDATE_DIR
-  rsync -a salt $DEFAULT_SALT_DIR/ --delete
-  rsync -a pillar $DEFAULT_SALT_DIR/ --delete
+  rsync -a salt $DEFAULT_SALT_DIR/ --delete "${EXCLUDE_ARGS[@]}"
+  rsync -a pillar $DEFAULT_SALT_DIR/ --delete "${EXCLUDE_ARGS[@]}"
   chown -R socore:socore $DEFAULT_SALT_DIR/
-  chmod 755 $DEFAULT_SALT_DIR/pillar/firewall/addfirewall.sh
   cd /tmp
 }
 
@@ -385,7 +395,7 @@ is_manager_node() {
 }
 
 is_sensor_node() {
-	# Check to see if this is a sensor (forward) node
+	# Check to see if this is a sensor node
 	is_single_node_grid && return 0
 	grep "role: so-" /etc/salt/grains | grep -E "sensor|heavynode" &> /dev/null
 }

salt/common/tools/sbin/so-image-common

@@ -62,8 +62,6 @@ container_list() {
       "so-soc"
       "so-steno"
       "so-strelka-backend"
-      "so-strelka-filestream"
-      "so-strelka-frontend"
       "so-strelka-manager"
       "so-suricata"
       "so-telegraf"

salt/elasticfleet/install_agent_grid.sls

@@ -2,26 +2,30 @@
 # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
 # this file except in compliance with the Elastic License 2.0.
 
-{%- set GRIDNODETOKENGENERAL = salt['pillar.get']('global:fleet_grid_enrollment_token_general') -%}
-{%- set GRIDNODETOKENHEAVY = salt['pillar.get']('global:fleet_grid_enrollment_token_heavy') -%}
+{% set GRIDNODETOKEN = salt['pillar.get']('global:fleet_grid_enrollment_token_general') -%}
+{% if grains.role == 'so-heavynode' %}
+{%   set GRIDNODETOKEN = salt['pillar.get']('global:fleet_grid_enrollment_token_heavy') -%}
+{% endif %}
 
 {% set AGENT_STATUS = salt['service.available']('elastic-agent') %}
 {% if not AGENT_STATUS  %}
 
-{% if grains.role not in ['so-heavynode'] %}
-run_installer:
-  cmd.script:
-    - name: salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux_amd64
-    - cwd: /opt/so
-    - args: -token={{ GRIDNODETOKENGENERAL }}
-    - retry: True
-{% else %} 
-run_installer:
-  cmd.script:
-    - name: salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux_amd64
-    - cwd: /opt/so
-    - args: -token={{ GRIDNODETOKENHEAVY }}
-    - retry: True
-{% endif %}  
+pull_agent_installer:
+  file.managed:
+    - name: /opt/so/so-elastic-agent_linux_amd64
+    - source: salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux_amd64
+    - mode: 755
+    - makedirs: True
 
+run_installer:
+  cmd.run:
+    - name: ./so-elastic-agent_linux_amd64 -token={{ GRIDNODETOKEN }}
+    - cwd: /opt/so
+    - retry:
+        attempts: 3
+        interval: 20
+
+cleanup_agent_installer:
+  file.absent:
+    - name: /opt/so/so-elastic-agent_linux_amd64
 {% endif %}

salt/libvirt/init.sls

@@ -31,6 +31,19 @@ libvirt_conf_dir:
     - group: 939
     - makedirs: True
 
+libvirt_volumes:
+  file.directory:
+    - name: /nsm/libvirt/volumes
+    - user: qemu
+    - group: qemu
+    - dir_mode: 755
+    - file_mode: 640
+    - recurse:
+      - user
+      - group
+      - mode
+    - makedirs: True
+
 libvirt_config:
   file.managed:
     - name: /opt/so/conf/libvirt/libvirtd.conf

salt/manager/tools/sbin/soup

@@ -21,6 +21,9 @@ whiptail_title='Security Onion UPdater'
 NOTIFYCUSTOMELASTICCONFIG=false
 TOPFILE=/opt/so/saltstack/default/salt/top.sls
 BACKUPTOPFILE=/opt/so/saltstack/default/salt/top.sls.backup
+SALTUPGRADED=false
+SALT_CLOUD_INSTALLED=false
+SALT_CLOUD_CONFIGURED=false
 # used to display messages to the user at the end of soup
 declare -a FINAL_MESSAGE_QUEUE=()
 
@@ -1260,24 +1263,43 @@ upgrade_check_salt() {
 }
 
 upgrade_salt() {
-  SALTUPGRADED=True
   echo "Performing upgrade of Salt from $INSTALLEDSALTVERSION to $NEWSALTVERSION."
   echo ""
   # If rhel family
   if [[ $is_rpm ]]; then
+    # Check if salt-cloud is installed
+    if rpm -q salt-cloud &>/dev/null; then
+      SALT_CLOUD_INSTALLED=true
+    fi
+    # Check if salt-cloud is configured
+    if [[ -f /etc/salt/cloud.profiles.d/socloud.conf ]]; then
+      SALT_CLOUD_CONFIGURED=true
+    fi
+    
     echo "Removing yum versionlock for Salt."
     echo ""
     yum versionlock delete "salt"
     yum versionlock delete "salt-minion"
     yum versionlock delete "salt-master"
+    # Remove salt-cloud versionlock if installed
+    if [[ $SALT_CLOUD_INSTALLED == true ]]; then
+      yum versionlock delete "salt-cloud"
+    fi
     echo "Updating Salt packages."
     echo ""
     set +e
     # if oracle run with -r to ignore repos set by bootstrap
     if [[ $OS == 'oracle' ]]; then
+      # Add -L flag only if salt-cloud is already installed
+      if [[ $SALT_CLOUD_INSTALLED == true ]]; then
+        run_check_net_err \
+        "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -X -r -L -F -M stable \"$NEWSALTVERSION\"" \
+        "Could not update salt, please check $SOUP_LOG for details."
+      else
         run_check_net_err \
         "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -X -r -F -M stable \"$NEWSALTVERSION\"" \
         "Could not update salt, please check $SOUP_LOG for details."
+      fi
     # if another rhel family variant we want to run without -r to allow the bootstrap script to manage repos
     else
       run_check_net_err \
@@ -1290,6 +1312,10 @@ upgrade_salt() {
     yum versionlock add "salt-0:$NEWSALTVERSION-0.*"
     yum versionlock add "salt-minion-0:$NEWSALTVERSION-0.*"
     yum versionlock add "salt-master-0:$NEWSALTVERSION-0.*"
+    # Add salt-cloud versionlock if installed
+    if [[ $SALT_CLOUD_INSTALLED == true ]]; then
+      yum versionlock add "salt-cloud-0:$NEWSALTVERSION-0.*"
+    fi
   # Else do Ubuntu things
   elif [[ $is_deb ]]; then
     echo "Removing apt hold for Salt."
@@ -1322,6 +1348,7 @@ upgrade_salt() {
     echo ""
     exit 1
   else
+    SALTUPGRADED=true
     echo "Salt upgrade success."
     echo ""
   fi
@@ -1565,6 +1592,11 @@ main() {
     # ensure the mine is updated and populated before highstates run, following the salt-master restart
     update_salt_mine
 
+    if [[ $SALT_CLOUD_CONFIGURED == true && $SALTUPGRADED == true ]]; then
+      echo "Updating salt-cloud config to use the new Salt version"
+      salt-call state.apply salt.cloud.config concurrent=True
+    fi
+
     enable_highstate
 
     echo ""
@@ -1647,7 +1679,7 @@ This appears to be a distributed deployment. Other nodes should update themselve
 
 Each minion is on a random 15 minute check-in period and things like network bandwidth can be a factor in how long the actual upgrade takes. If you have a heavy node on a slow link, it is going to take a while to get the containers to it. Depending on what changes happened between the versions, Elasticsearch might not be able to talk to said heavy node until the update is complete.
 
-If it looks like you’re missing data after the upgrade, please avoid restarting services and instead make sure at least one search node has completed its upgrade. The best way to do this is to run 'sudo salt-call state.highstate' from a search node and make sure there are no errors. Typically if it works on one node it will work on the rest. Forward nodes are less complex and will update as they check in so you can monitor those from the Grid section of SOC.
+If it looks like you’re missing data after the upgrade, please avoid restarting services and instead make sure at least one search node has completed its upgrade. The best way to do this is to run 'sudo salt-call state.highstate' from a search node and make sure there are no errors. Typically if it works on one node it will work on the rest. Sensor nodes are less complex and will update as they check in so you can monitor those from the Grid section of SOC.
 
 For more information, please see $DOC_BASE_URL/soup.html#distributed-deployments.
 

salt/pcap/soc_pcap.yaml

@@ -7,7 +7,7 @@ pcap:
       description: By default, Stenographer limits the number of files in the pcap directory to 30000 to avoid limitations with the ext3 filesystem. However, if you're using the ext4 or xfs filesystems, then it is safe to increase this value. So if you have a large amount of storage and find that you only have 3 weeks worth of PCAP on disk while still having plenty of free space, then you may want to increase this default setting.
       helpLink: stenographer.html
     diskfreepercentage:
-      description: Stenographer will purge old PCAP on a regular basis to keep the disk free percentage at this level. If you have a distributed deployment with dedicated forward nodes, then the default value of 10 should be reasonable since Stenographer should be the main consumer of disk space in the /nsm partition. However, if you have systems that run both Stenographer and Elasticsearch at the same time (like eval and standalone installations), then you’ll want to make sure that this value is no lower than 21 so that you avoid Elasticsearch hitting its watermark setting at 80% disk usage. If you have an older standalone installation, then you may need to manually change this value to 21.
+      description: Stenographer will purge old PCAP on a regular basis to keep the disk free percentage at this level. If you have a distributed deployment with dedicated Sensor nodes, then the default value of 10 should be reasonable since Stenographer should be the main consumer of disk space in the /nsm partition. However, if you have systems that run both Stenographer and Elasticsearch at the same time (like eval and standalone installations), then you’ll want to make sure that this value is no lower than 21 so that you avoid Elasticsearch hitting its watermark setting at 80% disk usage. If you have an older standalone installation, then you may need to manually change this value to 21.
       helpLink: stenographer.html
     blocks:
       description: The number of 1MB packet blocks used by Stenographer and AF_PACKET to store packets in memory, per thread. You shouldn't need to change this. 

salt/salt/cloud/cloud.profiles.d/socloud.conf.jinja

@@ -14,7 +14,7 @@ sool9_{{host}}:
   private_key: /etc/ssh/auth_keys/soqemussh/id_ecdsa
   sudo: True
   deploy_command: sh /tmp/.saltcloud-*/deploy.sh
-  script_args: -r -F -x python3 stable 3006.9
+  script_args: -r -F -x python3 stable {{ SALTVERSION }}
   minion:
     master: {{ grains.host }}
     master_port: 4506

salt/salt/cloud/config.sls

@@ -13,6 +13,7 @@
 {% if '.'.join(sls.split('.')[:2]) in allowed_states %}
 {%   if 'vrt' in salt['pillar.get']('features', []) %}
 {%     set HYPERVISORS = salt['pillar.get']('hypervisor:nodes', {} ) %}
+{%     from 'salt/map.jinja' import SALTVERSION %}
 
 {%     if HYPERVISORS %}
 cloud_providers:
@@ -20,7 +21,7 @@ cloud_providers:
     - name: /etc/salt/cloud.providers.d/libvirt.conf
     - source: salt://salt/cloud/cloud.providers.d/libvirt.conf.jinja
     - defaults:
-        HYPERVISORS: {{HYPERVISORS}}
+        HYPERVISORS: {{ HYPERVISORS }}
     - template: jinja
     - makedirs: True
 
@@ -29,11 +30,17 @@ cloud_profiles:
     - name: /etc/salt/cloud.profiles.d/socloud.conf
     - source: salt://salt/cloud/cloud.profiles.d/socloud.conf.jinja
     - defaults:
-        HYPERVISORS: {{HYPERVISORS}}
+        HYPERVISORS: {{ HYPERVISORS }}
         MANAGERHOSTNAME: {{ grains.host }}
         MANAGERIP: {{ pillar.host.mainip }}
+        SALTVERSION: {{ SALTVERSION }}
     - template: jinja
     - makedirs: True
+{%     else %}
+no_hypervisors_configured:
+  test.succeed_without_changes:
+    - name: no_hypervisors_configured
+    - comment: No hypervisors are configured
 {%     endif %}
 
 {%   else %}

salt/salt/master.defaults.yaml

@@ -1,4 +1,4 @@
 # version cannot be used elsewhere in this pillar as soup is grepping for it to determine if Salt needs to be patched
 salt:
   master:
-    version: '3006.9'
+    version: '3006.16'

salt/salt/minion.defaults.yaml

@@ -1,5 +1,5 @@
 # version cannot be used elsewhere in this pillar as soup is grepping for it to determine if Salt needs to be patched
 salt:
   minion:
-    version: '3006.9'
+    version: '3006.16'
     check_threshold: 3600 # in seconds, threshold used for so-salt-minion-check. any value less than 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default

salt/soc/defaults.yaml

@@ -2552,9 +2552,27 @@ soc:
         assistant:
           enabled: false
           investigationPrompt: Investigate Alert ID {socId}
-          contextLimitSmall: 200000
-          contextLimitLarge: 1000000
           thresholdColorRatioLow: 0.5
           thresholdColorRatioMed: 0.75
           thresholdColorRatioMax: 1
+          availableModels:
+            - id: sonnet-4
+              displayName: Claude Sonnet 4
+              contextLimitSmall: 200000
+              contextLimitLarge: 1000000
+              lowBalanceColorAlert: 500000
+            - id: sonnet-4.5
+              displayName: Claude Sonnet 4.5
+              contextLimitSmall: 200000
+              contextLimitLarge: 1000000
+              lowBalanceColorAlert: 500000
+            - id: gptoss-120b
+              displayName: GPT-OSS 120B
+              contextLimitSmall: 128000
+              contextLimitLarge: 128000
+              lowBalanceColorAlert: 500000
+            - id: qwen-235b
+              displayName: QWEN 235B
+              contextLimitSmall: 256000
+              contextLimitLarge: 256000
               lowBalanceColorAlert: 500000

salt/soc/soc_soc.yaml

@@ -606,14 +606,6 @@ soc:
           investigationPrompt: 
             description: Prompt given to Onion AI when beginning an investigation.
             global: True
-          contextLimitSmall:
-            description: Smaller context limit for Onion AI.
-            global: True
-            advanced: True
-          contextLimitLarge:
-            description: Larger context limit for Onion AI.
-            global: True
-            advanced: True
           thresholdColorRatioLow:
             description: Lower visual context color change threshold.
             global: True
@@ -630,6 +622,32 @@ soc:
             description: Onion AI credit amount at which balance turns red.
             global: True
             advanced: True
+          availableModels:
+            description: List of AI models available for use in SOC as well as model specific warning thresholds.
+            global: True
+            advanced: True
+            forcedType: "[]{}"
+            helpLink: assistant.html
+            syntax: json
+            uiElements:
+            - field: id
+              label: Model ID
+              required: True
+            - field: displayName
+              label: Display Name
+              required: True
+            - field: contextLimitSmall
+              label: Context Limit (Small)
+              forcedType: int
+              required: True
+            - field: contextLimitLarge
+              label: Context Limit (Large)
+              forcedType: int
+              required: True
+            - field: lowBalanceColorAlert
+              label: Low Balance Color Alert
+              forcedType: int
+              required: True
         apiTimeoutMs:
           description: Duration (in milliseconds) to wait for a response from the SOC server API before giving up and showing an error on the SOC UI.
           global: True

salt/strelka/filestream/enabled.sls

@@ -14,7 +14,7 @@ include:
 
 strelka_filestream:
   docker_container.running:
-    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-strelka-filestream:{{ GLOBALS.so_version }}
+    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-strelka-manager:{{ GLOBALS.so_version }}
     - binds:
       - /opt/so/conf/strelka/filestream/:/etc/strelka/:ro
       - /nsm/strelka:/nsm/strelka

salt/strelka/frontend/enabled.sls

@@ -14,7 +14,7 @@ include:
 
 strelka_frontend:
   docker_container.running:
-    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-strelka-frontend:{{ GLOBALS.so_version }}
+    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-strelka-manager:{{ GLOBALS.so_version }}
     - binds:
       - /opt/so/conf/strelka/frontend/:/etc/strelka/:ro
       - /nsm/strelka/log/:/var/log/strelka/:rw

salt/telegraf/etc/telegraf.conf

@@ -337,4 +337,5 @@
   ]
   data_format = "influx"
   interval = "1h"
+  timeout = "120s"
 {%- endif %}

setup/so-functions

@@ -1646,6 +1646,12 @@ reserve_ports() {
 	fi
 }
 
+clear_previous_setup_results() {
+	# Disregard previous setup outcomes.
+	rm -f /root/failure
+	rm -f /root/success
+}
+
 reinstall_init() {
 	info "Putting system in state to run setup again"
 
@@ -1657,10 +1663,6 @@ reinstall_init() {
 
 	local service_retry_count=20
 
-	# Disregard previous install outcomes
-	rm -f /root/failure
-	rm -f /root/success
-
 	{
 		# remove all of root's cronjobs
 		logCmd "crontab -r -u root"

setup/so-setup

@@ -132,6 +132,10 @@ if [[ -f /root/accept_changes ]]; then
 	reset_proxy
 fi
 
+# Previous setup attempts, even if setup doesn't actually start the installation,
+# can leave behind results that may interfere with the current setup attempt.
+clear_previous_setup_results
+
 title "Parsing Username for Install"
 parse_install_username
 

setup/so-whiptail

@@ -676,8 +676,8 @@ whiptail_install_type_dist_existing() {
 	EOM
 	
 	install_type=$(whiptail --title "$whiptail_title" --menu "$node_msg" 19 75 7 \
-		"SENSOR" "Create a forward only sensor " \
-		"SEARCHNODE" "Add a search node with parsing " \
+		"SENSOR" "Add a Sensor Node for monitoring network traffic " \
+		"SEARCHNODE" "Add a Search Node with parsing " \
 		"FLEET" "Dedicated Elastic Fleet Node " \
 		"HEAVYNODE" "Sensor + Search Node " \
 		"IDH" "Intrusion Detection Honeypot Node " \