Merge remote-tracking branch 'origin/2.4/dev' into idstools-refactor

2026-04-26 22:47:49 +02:00 · 2025-11-06 10:38:37 -05:00
parent ded520c2c1 202b03b32b
commit 2f6fb717c1
84 changed files with 3913 additions and 312 deletions
@@ -1493,6 +1493,9 @@ soc:
                folder: securityonion-normalized
        assistant:
          apiUrl: https://onionai.securityonion.net
+          healthTimeoutSeconds: 3
+          systemPromptAddendum: ""
+          systemPromptAddendumMaxLength: 50000
        salt:
          queueDir: /opt/sensoroni/queue
          timeoutMs: 45000
@@ -1695,6 +1698,9 @@ soc:
            - name: socExcludeToggle
              filter: 'NOT event.module:"soc"'
              enabled: true
+            - name: onionaiExcludeToggle
+              filter: 'NOT _index:"*:so-assistant-*"'
+              enabled: true
          queries:
            - name: Default Query
              description: Show all events grouped by the observer host
@@ -2605,10 +2611,23 @@ soc:
              level: 'high'  # info | low | medium | high | critical
        assistant:
          enabled: false
-          investigationPrompt: Investigate Alert ID {socid}
-          contextLimitSmall: 200000
-          contextLimitLarge: 1000000
+          investigationPrompt: Investigate Alert ID {socId}
          thresholdColorRatioLow: 0.5
          thresholdColorRatioMed: 0.75
          thresholdColorRatioMax: 1
-          lowBalanceColorAlert: 500000
+          availableModels:
+            - id: sonnet-4
+              displayName: Claude Sonnet 4
+              contextLimitSmall: 200000
+              contextLimitLarge: 1000000
+              lowBalanceColorAlert: 500000
+            - id: sonnet-4.5
+              displayName: Claude Sonnet 4.5
+              contextLimitSmall: 200000
+              contextLimitLarge: 1000000
+              lowBalanceColorAlert: 500000
+            - id: gptoss-120b
+              displayName: GPT-OSS 120B
+              contextLimitSmall: 128000
+              contextLimitLarge: 128000
+              lowBalanceColorAlert: 500000
@@ -63,18 +63,22 @@ hypervisor:
        required: true
        readonly: true
        forcedType: int
+      - field: nsm_size
+        label: "Size of virtual disk to create and use for /nsm, in GB. Only applicable if no pass-through disk."
+        forcedType: int
+        readonly: true
      - field: disk
-        label: "Disk(s) for passthrough. Free: FREE | Total: TOTAL"
+        label: "Disk(s) to pass through for /nsm. Free: FREE | Total: TOTAL"
        readonly: true
        options: []
        forcedType: '[]int'
      - field: copper
-        label: "Copper port(s) for passthrough. Free: FREE | Total: TOTAL"
+        label: "Copper port(s) to pass through. Free: FREE | Total: TOTAL"
        readonly: true
        options: []
        forcedType: '[]int'
      - field: sfp
-        label: "SFP port(s) for passthrough. Free: FREE | Total: TOTAL"
+        label: "SFP port(s) to pass through. Free: FREE | Total: TOTAL"
        readonly: true
        options: []
        forcedType: '[]int'
@@ -3,11 +3,14 @@
 {# Define the list of process steps in order (case-sensitive) #}
 {% set PROCESS_STEPS = [
    'Processing',
+    'Hypervisor NSM Disk Full',
    'IP Configuration',
    'Starting Create',
    'Executing Deploy Script',
    'Initialize Minion Pillars',
    'Created Instance',
+    'Volume Creation',
+    'Volume Configuration',
    'Hardware Configuration',
    'Highstate Initiated',
    'Destroyed Instance'
@@ -0,0 +1,51 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+#
+# Note: Per the Elastic License 2.0, the second limitation states:
+#
+#   "You may not move, change, disable, or circumvent the license key functionality
+#    in the software, and you may not remove or obscure any functionality in the
+#    software that is protected by the license key."
+
+{% if 'vrt' in salt['pillar.get']('features', []) %}
+
+{% do salt.log.info('soc/dyanno/hypervisor/remove_failed_vm: Running') %}
+{% set vm_name = pillar.get('vm_name') %}
+{% set hypervisor = pillar.get('hypervisor') %}
+
+{% if vm_name and hypervisor %}
+{%   set vm_parts = vm_name.split('_') %}
+{%   if vm_parts | length >= 2 %}
+{%     set vm_role = vm_parts[-1] %}
+{%     set vm_hostname = '_'.join(vm_parts[:-1]) %}
+{%     set vms_file = '/opt/so/saltstack/local/salt/hypervisor/hosts/' ~ hypervisor ~ 'VMs' %}
+
+{%     do salt.log.info('soc/dyanno/hypervisor/remove_failed_vm: Removing VM ' ~ vm_name ~ ' from ' ~ vms_file) %}
+
+remove_vm_{{ vm_name }}_from_vms_file:
+  module.run:
+    - name: hypervisor.remove_vm_from_vms_file
+    - vms_file_path: {{ vms_file }}
+    - vm_hostname: {{ vm_hostname }}
+    - vm_role: {{ vm_role }}
+
+{%   else %}
+{%     do salt.log.error('soc/dyanno/hypervisor/remove_failed_vm: Invalid vm_name format: ' ~ vm_name) %}
+{%   endif %}
+{% else %}
+{%   do salt.log.error('soc/dyanno/hypervisor/remove_failed_vm: Missing required pillar data (vm_name or hypervisor)') %}
+{% endif %}
+
+{% do salt.log.info('soc/dyanno/hypervisor/remove_failed_vm: Completed') %}
+
+{% else %}
+
+{% do salt.log.error(
+  'Hypervisor nodes are a feature supported only for customers with a valid license. '
+  'Contact Security Onion Solutions, LLC via our website at https://securityonionsolutions.com '
+  'for more information about purchasing a license to enable this feature.'
+) %}
+
+{% endif %}
@@ -13,7 +13,6 @@

 {%- import_yaml 'soc/dyanno/hypervisor/hypervisor.yaml' as ANNOTATION -%}
 {%- from 'hypervisor/map.jinja' import HYPERVISORS -%}
-{%- from 'soc/dyanno/hypervisor/map.jinja' import PROCESS_STEPS -%}

 {%- set TEMPLATE = ANNOTATION.hypervisor.hosts.pop('defaultHost') -%}

@@ -27,7 +26,6 @@
 {%- if baseDomainStatus == 'Initialized' %}
 {%-   if vm_list %}
 #### Virtual Machines
-Status values: {% for step in PROCESS_STEPS %}{{ step }}{% if not loop.last %}, {% endif %}{% endfor %}. "Last Updated" shows when status changed. After "Highstate Initiated", only "Destroyed Instance" updates the timestamp.

 | Name               | Status             | CPU Cores | Memory (GB)| Disk | Copper | SFP  | Last Updated        |
 |--------------------|--------------------|-----------|------------|------|--------|------|---------------------|
@@ -42,7 +40,6 @@ Status values: {% for step in PROCESS_STEPS %}{{ step }}{% if not loop.last %},
 {%-     endfor %}
 {%-   else %}
 #### Virtual Machines
-Status values: {% for step in PROCESS_STEPS %}{{ step }}{% if not loop.last %}, {% endif %}{% endfor %}. "Last Updated" shows when status changed. After "Highstate Initiated", only "Destroyed Instance" updates the timestamp.

 No Virtual Machines Found
 {%-   endif %}
@@ -96,9 +93,21 @@ Base domain has not been initialized.
 {%-       endif -%}
 {%-     endfor -%}

-{# Calculate available resources #}
-{%-     set cpu_free = hw_config.cpu - ns.used_cpu -%}
-{%-     set mem_free = hw_config.memory - ns.used_memory -%}
+{# Determine host OS overhead based on role #}
+{%-     if role == 'hypervisor' -%}
+{%-       set host_os_cpu = 8 -%}
+{%-       set host_os_memory = 16 -%}
+{%-     elif role == 'managerhype' -%}
+{%-       set host_os_cpu = 16 -%}
+{%-       set host_os_memory = 32 -%}
+{%-     else -%}
+{%-       set host_os_cpu = 0 -%}
+{%-       set host_os_memory = 0 -%}
+{%-     endif -%}
+
+{# Calculate available resources (subtract both VM usage and host OS overhead) #}
+{%-     set cpu_free = hw_config.cpu - ns.used_cpu - host_os_cpu -%}
+{%-     set mem_free = hw_config.memory - ns.used_memory - host_os_memory -%}

 {# Get used PCI indices #}
 {%-     set used_disk = [] -%}
@@ -237,10 +237,22 @@ function manage_salt() {

  case "$op" in
    state)
-      log "Performing '$op' for '$state' on minion '$minion'"
      state=$(echo "$request" | jq -r .state)
-      response=$(salt --async "$minion" state.apply "$state" queue=2)
+      async=$(echo "$request" | jq -r .async)
+      if [[ $async == "true" ]]; then
+        log "Performing async '$op' on minion $minion with state '$state'"
+        response=$(salt --async "$minion" state.apply "$state" queue=2)
+      else
+        log "Performing '$op' on minion $minion with state '$state'"
+        response=$(salt "$minion" state.apply "$state")
+      fi
+     
      exit_code=$?
+      if [[ $exit_code -ne 0 && "$response" =~ "is running as PID" ]]; then
+        log "Salt already running: $response ($exit_code)"
+        respond "$id" "ERROR_SALT_ALREADY_RUNNING"
+        return
+      fi
      ;;
    highstate)
      log "Performing '$op' on minion $minion"
@@ -259,7 +271,7 @@ function manage_salt() {
      ;;
  esac

-  if [[ exit_code -eq 0 ]]; then
+  if [[ $exit_code -eq 0 ]]; then
    log "Successful command execution: $response"
    respond "$id" "true"
  else
@@ -631,6 +631,19 @@ soc:
            description: The URL of the AI gateway.
            advanced: True
            global: True
+          healthTimeoutSeconds:
+            description: Timeout in seconds for the Onion AI health check.
+            global: True
+            advanced: True
+          systemPromptAddendum:
+            description: Additional context to provide to the AI assistant about this SOC deployment. This can include information about your environment, policies, or any other relevant details that can help the AI provide more accurate and tailored assistance. Long prompts may be shortened.
+            global: True
+            advanced: False
+            multiline: True
+          systemPromptAddendumMaxLength:
+            description: Maximum length of the system prompt addendum. Longer prompts will be truncated.
+            global: True
+            advanced: True
      client:
        assistant:
          enabled:
@@ -639,14 +652,6 @@ soc:
          investigationPrompt: 
            description: Prompt given to Onion AI when beginning an investigation.
            global: True
-          contextLimitSmall:
-            description: Smaller context limit for Onion AI.
-            global: True
-            advanced: True
-          contextLimitLarge:
-            description: Larger context limit for Onion AI.
-            global: True
-            advanced: True
          thresholdColorRatioLow:
            description: Lower visual context color change threshold.
            global: True
@@ -661,7 +666,34 @@ soc:
            advanced: True 
          lowBalanceColorAlert:
            description: Onion AI credit amount at which balance turns red.
+            global: True
            advanced: True
+          availableModels:
+            description: List of AI models available for use in SOC as well as model specific warning thresholds.
+            global: True
+            advanced: True
+            forcedType: "[]{}"
+            helpLink: assistant.html
+            syntax: json
+            uiElements:
+            - field: id
+              label: Model ID
+              required: True
+            - field: displayName
+              label: Display Name
+              required: True
+            - field: contextLimitSmall
+              label: Context Limit (Small)
+              forcedType: int
+              required: True
+            - field: contextLimitLarge
+              label: Context Limit (Large)
+              forcedType: int
+              required: True
+            - field: lowBalanceColorAlert
+              label: Low Balance Color Alert
+              forcedType: int
+              required: True
        apiTimeoutMs:
          description: Duration (in milliseconds) to wait for a response from the SOC server API before giving up and showing an error on the SOC UI.
          global: True