Update echo messages for PCAP engine clarity

Continue upgrade after pcapengine is changed to SURICATA
Instead of exiting and requiring the user to rerun the script after changing pcapengine to SURICATA, let the script continue to the version check and upgrade.
2026-02-28 09:55:32 +01:00 · 2026-02-24 10:04:26 -05:00 · 2026-02-23 11:35:32 -05:00 · 2026-02-23 11:16:30 -05:00 · 2026-02-23 11:05:30 -05:00 · 2026-02-23 10:49:54 -05:00
13 changed files with 294 additions and 291 deletions
--- a/2
+++ b/2
@@ -1 +1 @@
-2.4.0-foxtrot
+2.4.210
--- a/salt/common/tools/sbin/so-common
+++ b/salt/common/tools/sbin/so-common
@@ -404,25 +404,6 @@ is_single_node_grid() {
 	grep "role: so-" /etc/salt/grains | grep -E "eval|standalone|import" &> /dev/null
 }

-initialize_elasticsearch_indices() {
-    local index_names=$1
-    local default_entry=${2:-'{"@timestamp":"0"}'}
-
-    for idx in $index_names; do
-        if ! so-elasticsearch-query "$idx" --fail --retry 3 --retry-delay 30 >/dev/null 2>&1; then
-            echo "Index does not already exist. Initializing $idx index."
-
-            if retry 3 10 "so-elasticsearch-query "$idx/_doc" -d '$default_entry' -XPOST --fail 2>/dev/null" '"successful":1'; then
-                echo "Successfully initialized $idx index."
-            else
-                echo "Failed to initialize $idx index after 3 attempts."
-            fi
-        else
-            echo "Index $idx already exists. No action needed."
-        fi
-    done
-}
-
 lookup_bond_interfaces() {
 	cat /proc/net/bonding/bond0 | grep "Slave Interface:" | sed -e "s/Slave Interface: //g"
 }
--- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json
+++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json
@@ -22,7 +22,7 @@
            "data_stream.dataset": "import",
            "pipeline": "",
            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
-            "exclude_files": ["({%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%})(\\..+)?\\.log$"],
+            "exclude_files": ["({%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%}).log$"],
            "include_files": [],
            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/zeek/logs/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- script:\n      lang: javascript\n      source: >\n        function process(event) {\n          var pl = event.Get(\"import.file\").slice(0,-4);\n          event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n        }\n- add_fields:\n    target: event\n    fields:\n      category: network\n      module: zeek\n      imported: true\n- add_tags:\n    tags: \"ics\"\n    when:\n      regexp:\n        import.file: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"",
            "tags": [],
--- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json
+++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json
@@ -21,7 +21,7 @@
            ],
            "data_stream.dataset": "zeek",
            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
-            "exclude_files": ["({%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%})(\\..+)?\\.log$"],
+            "exclude_files": ["({%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%}).log$"],
            "include_files": [],
            "processors": "- dissect:\n    tokenizer: \"/nsm/zeek/logs/current/%{pipeline}.log\"\n    field: \"log.file.path\"\n    trim_chars: \".log\"\n    target_prefix: \"\"\n- script:\n      lang: javascript\n      source: >\n        function process(event) {\n          var pl = event.Get(\"pipeline\");\n          event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n        }\n- add_fields:\n    target: event\n    fields:\n      category: network\n      module: zeek\n- add_tags:\n    tags: \"ics\"\n    when:\n      regexp:\n        pipeline: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"",
            "tags": [],
--- a/salt/elasticsearch/defaults.yaml
+++ b/salt/elasticsearch/defaults.yaml
@@ -6,8 +6,6 @@ elasticsearch:
    action:
      destructive_requires_name: true
    cluster:
-      logsdb:
-        enabled: false
      routing:
        allocation:
          disk:
--- a/salt/elasticsearch/soc_elasticsearch.yaml
+++ b/salt/elasticsearch/soc_elasticsearch.yaml
@@ -27,13 +27,6 @@ elasticsearch:
        readonly: True
        global: True
        helpLink: elasticsearch.html
-      logsdb:
-        enabled:
-          description: Enables or disables the Elasticsearch logsdb index mode. When enabled, most logs-* datastreams will convert to logsdb from standard after rolling over.
-          forcedType: bool
-          global: True
-          advanced: True
-          helpLink: elasticsearch.html
      routing:
        allocation:
          disk:
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -93,10 +93,6 @@ check_err() {
      161)
        echo 'Required intermediate Elasticsearch upgrade not complete'
      ;;
-      170)
-        echo "Intermediate upgrade completed successfully to $next_step_so_version, but next soup to Security Onion $originally_requested_so_version could not be started automatically."
-        echo "Start soup again manually to continue the upgrade to Security Onion $originally_requested_so_version."
-      ;;
      *)
        echo 'Unhandled error'
        echo "$err_msg"
@@ -158,7 +154,7 @@ EOF
        echo "Ensure you verify the ISO that you downloaded."
        exit 0
      else
-        echo "Device has been mounted! $(cat /tmp/soagupdate/SecurityOnion/VERSION)"
+        echo "Device has been mounted!"
      fi
    else
      echo "Could not find Security Onion ISO content at ${ISOLOC}"
@@ -696,8 +692,6 @@ post_to_2.4.210() {

  disable_redis_heavynodes

-  initialize_elasticsearch_indices "so-case so-casehistory so-assistant-session so-assistant-chat"
-
  echo "Regenerating Elastic Agent Installers"
  /sbin/so-elastic-agent-gen-installers

@@ -1686,218 +1680,115 @@ verify_latest_update_script() {

 verify_es_version_compatibility() {

-    local es_required_version_statefile_base="/opt/so/state/so_es_required_upgrade_version"
-    local es_verification_script="/tmp/so_intermediate_upgrade_verification.sh"
-    local is_active_intermediate_upgrade=1
-    # supported upgrade paths for SO-ES versions
-    declare -A es_upgrade_map=(
-	    ["8.14.3"]="8.17.3 8.18.4 8.18.6 8.18.8"
-	    ["8.17.3"]="8.18.4 8.18.6 8.18.8"
-	    ["8.18.4"]="8.18.6 8.18.8 9.0.8"
-	    ["8.18.6"]="8.18.8 9.0.8"
-	    ["8.18.8"]="9.0.8"
-    )
+  local es_required_version_statefile="/opt/so/state/so_es_required_upgrade_version.txt"
+  local es_verification_script="/tmp/so_intermediate_upgrade_verification.sh"
+  # supported upgrade paths for SO-ES versions
+  declare -A es_upgrade_map=(
+    ["8.14.3"]="8.17.3 8.18.4 8.18.6 8.18.8"
+    ["8.17.3"]="8.18.4 8.18.6 8.18.8"
+    ["8.18.4"]="8.18.6 8.18.8 9.0.8"
+    ["8.18.6"]="8.18.8 9.0.8"
+    ["8.18.8"]="9.0.8"
+  )

-    # Elasticsearch MUST upgrade through these versions
-    declare -A es_to_so_version=(
-	    ["8.18.8"]="2.4.190-20251024"
-    )
+  # Elasticsearch MUST upgrade through these versions
+  declare -A es_to_so_version=(
+    ["8.18.8"]="2.4.190-20251024"
+  )

-    # Get current Elasticsearch version
-    if es_version_raw=$(so-elasticsearch-query / --fail --retry 5 --retry-delay 10); then
-        es_version=$(echo "$es_version_raw" | jq -r '.version.number' )
-    else
-        echo "Could not determine current Elasticsearch version to validate compatibility with post soup Elasticsearch version."
+  # Get current Elasticsearch version
+  if es_version_raw=$(so-elasticsearch-query / --fail --retry 5 --retry-delay 10); then
+    es_version=$(echo "$es_version_raw" | jq -r '.version.number' )
+  else
+    echo "Could not determine current Elasticsearch version to validate compatibility with post soup Elasticsearch version."
+    exit 160
+  fi

-        exit 160
+  if ! target_es_version=$(so-yaml.py get $UPDATE_DIR/salt/elasticsearch/defaults.yaml elasticsearch.version | sed -n '1p'); then
+    # so-yaml.py failed to get the ES version from upgrade versions elasticsearch/defaults.yaml file. Likely they are upgrading to an SO version older than 2.4.110 prior to the ES version pinning and should be OKAY to continue with the upgrade.
+
+    # if so-yaml.py failed to get the ES version AND the version we are upgrading to is newer than 2.4.110 then we should bail
+    if [[ $(cat $UPDATE_DIR/VERSION | cut -d'.' -f3) > 110 ]]; then
+      echo "Couldn't determine the target Elasticsearch version (post soup version) to ensure compatibility with current Elasticsearch version. Exiting"
+      exit 160
    fi

-    if ! target_es_version_raw=$(so-yaml.py get $UPDATE_DIR/salt/elasticsearch/defaults.yaml elasticsearch.version); then
-        # so-yaml.py failed to get the ES version from upgrade versions elasticsearch/defaults.yaml file. Likely they are upgrading to an SO version older than 2.4.110 prior to the ES version pinning and should be OKAY to continue with the upgrade.
+    # allow upgrade to version < 2.4.110 without checking ES version compatibility
+    return 0

-        # if so-yaml.py failed to get the ES version AND the version we are upgrading to is newer than 2.4.110 then we should bail
-        if [[ $(cat $UPDATE_DIR/VERSION | cut -d'.' -f3) > 110 ]]; then
-            echo "Couldn't determine the target Elasticsearch version (post soup version) to ensure compatibility with current Elasticsearch version. Exiting"
+  fi

-            exit 160
-        fi
-
-        # allow upgrade to version < 2.4.110 without checking ES version compatibility
-        return 0
-    else
-        target_es_version=$(sed -n '1p' <<< "$target_es_version_raw")
+  # if this statefile exists then we have done an intermediate upgrade and we need to ensure that ALL ES nodes have been upgraded to the version in the statefile before allowing soup to continue
+  if [[ -f "$es_required_version_statefile" ]]; then
+    # required so verification script should have already been created
+    if [[ ! -f "$es_verification_script" ]]; then
+        create_intermediate_upgrade_verification_script $es_verification_script
    fi

-    for statefile in "${es_required_version_statefile_base}"-*; do
-        [[ -f $statefile ]] || continue
-
-        local es_required_version_statefile_value=$(cat "$statefile")
-
-        if [[ "$es_required_version_statefile_value" == "$target_es_version" ]]; then
-            echo "Intermediate upgrade to ES $target_es_version is in progress. Skipping Elasticsearch version compatibility check."
-            is_active_intermediate_upgrade=0
-            continue
-        fi
-
-        # use sort to check if es_required_statefile_value is < the current es_version.
-        if [[ "$(printf '%s\n' $es_required_version_statefile_value $es_version | sort -V | head -n1)" == "$es_required_version_statefile_value" ]]; then
-            rm -f "$statefile"
-            continue
-        fi
-
-        if [[ ! -f "$es_verification_script" ]]; then
-            create_intermediate_upgrade_verification_script "$es_verification_script"
-        fi
-
-        echo -e "\n##############################################################################################################################\n"
-        echo "A previously required intermediate Elasticsearch upgrade was detected. Verifying that all Searchnodes/Heavynodes have successfully upgraded Elasticsearch to $es_required_version_statefile_value before proceeding with soup to avoid potential data loss!"
-        timeout --foreground 4000 bash "$es_verification_script" "$es_required_version_statefile_value" "$statefile"
-        if [[ $? -ne 0 ]]; then
-            echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
-
-            echo "A previous required intermediate Elasticsearch upgrade to $es_required_version_statefile_value has yet to successfully complete across the grid. Please allow time for all Searchnodes/Heavynodes to have upgraded Elasticsearch to $es_required_version_statefile_value before running soup again to avoid potential data loss!"
-
-            echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
-
-            exit 161
-        fi
-        echo -e "\n##############################################################################################################################\n"
-    done
-
-    # if current soup is an intermediate upgrade we can skip the upgrade map check below
-    if [[ $is_active_intermediate_upgrade -eq 0 ]]; then
-        return 0
-    fi
-
-    if [[ " ${es_upgrade_map[$es_version]} " =~ " $target_es_version " || "$es_version" == "$target_es_version" ]]; then
-        # supported upgrade
-        return 0
-    else
-        compatible_versions=${es_upgrade_map[$es_version]}
-        if [[ -z "$compatible_versions" ]]; then
-            # If current ES version is not explicitly defined in the upgrade map, we know they have an intermediate upgrade to do.
-            # We default to the lowest ES version defined in es_to_so_version as $first_es_required_version
-            local first_es_required_version=$(printf '%s\n' "${!es_to_so_version[@]}" | sort -V | head -n1)
-            next_step_so_version=${es_to_so_version[$first_es_required_version]}
-            required_es_upgrade_version="$first_es_required_version"
-        else
-            next_step_so_version=${es_to_so_version[${compatible_versions##* }]}
-            required_es_upgrade_version="${compatible_versions##* }"
-        fi
-        echo -e "\n##############################################################################################################################\n"
-        echo -e "You are currently running Security Onion $INSTALLEDVERSION. You will need to update to version $next_step_so_version before updating to $(cat $UPDATE_DIR/VERSION).\n"
-
-        es_required_version_statefile="${es_required_version_statefile_base}-${required_es_upgrade_version}"
-        echo "$required_es_upgrade_version" > "$es_required_version_statefile"
-
-        # We expect to upgrade to the latest compatiable minor version of ES
-        create_intermediate_upgrade_verification_script "$es_verification_script"
-
-        if [[ $is_airgap -eq 0 ]]; then
-            run_airgap_intermediate_upgrade
-        else
-            if [[ ! -z $ISOLOC ]]; then
-                originally_requested_iso_location="$ISOLOC"
-            fi
-            # Make sure ISOLOC is not set. Network installs that used soup -f would have ISOLOC set.
-            unset ISOLOC
-
-            run_network_intermediate_upgrade
-	    fi
-    fi
-
-}
-
-run_airgap_intermediate_upgrade() {
-    local originally_requested_so_version=$(cat $UPDATE_DIR/VERSION)
-    # preserve ISOLOC value, so we can try to use it post intermediate upgrade
-    local originally_requested_iso_location="$ISOLOC"
-
-    # make sure a fresh ISO gets mounted
-    unmount_update
-
-    echo "You can download the $next_step_so_version ISO image from https://download.securityonion.net/file/securityonion/securityonion-$next_step_so_version.iso"
-    echo -e "\nIf you have the next ISO / USB ready, enter the path now eg. /dev/sdd, /home/onion/securityonion-$next_step_so_version.iso:"
-
-    while [[ -z "$next_iso_location" ]] || [[ ! -f "$next_iso_location" && ! -b "$next_iso_location" ]]; do
-        # List removable devices if any are present
-        local removable_devices=$(lsblk -no PATH,SIZE,TYPE,MOUNTPOINTS,RM | awk '$NF==1')
-        if [[ -n "$removable_devices" ]]; then
-            echo "PATH        SIZE    TYPE    MOUNTPOINTS    RM"
-            echo "$removable_devices"
-        fi
-
-        read -rp "Device/ISO Path (or 'exit' to quit): " next_iso_location
-        if [[ "${next_iso_location,,}" == "exit" ]]; then
-            echo "Exiting soup. Before reattempting to upgrade to $originally_requested_so_version, please first upgrade to $next_step_so_version to ensure Elasticsearch can properly update through the required versions."
-
-            exit 160
-        fi
-
-        if [[ ! -f "$next_iso_location" && ! -b "$next_iso_location" ]]; then
-            echo "$next_iso_location is not a valid file or block device."
-            next_iso_location=""
-        fi
-    done
-
-    echo "Using $next_iso_location for required intermediary upgrade."
-    exec bash <<EOF
-        ISOLOC=$next_iso_location soup -y && \
-        ISOLOC=$next_iso_location soup -y && \
-
-        echo -e "\n##############################################################################################################################\n" && \
-        echo -e "Verifying Elasticsearch was successfully upgraded to $required_es_upgrade_version across the grid. This part can take a while as Searchnodes/Heavynodes sync up with the Manager! \n\nOnce verification completes the next soup will begin automatically. If verification takes longer than 1 hour it will stop waiting and your grid will remain at $next_step_so_version. Allowing for all Searchnodes/Heavynodes to upgrade Elasticsearch to the required version on their own time.\n" && \
-
-        timeout --foreground 4000 bash /tmp/so_intermediate_upgrade_verification.sh $required_es_upgrade_version $es_required_version_statefile && \
-
-        echo -e "\n##############################################################################################################################\n" && \
-
-        # automatically start the next soup if the original ISO isn't using the same block device we just used
-        if [[ -n "$originally_requested_iso_location" ]] && [[ "$originally_requested_iso_location" != "$next_iso_location" ]]; then
-            umount /tmp/soagupdate
-            ISOLOC=$originally_requested_iso_location soup -y && \
-            ISOLOC=$originally_requested_iso_location soup -y
-        else
-            echo "Could not automatically start next soup to $originally_requested_so_version. Soup will now exit here at $(cat /etc/soversion)" && \
-
-            exit 170
-        fi
-
-        echo -e "\n##############################################################################################################################\n"
-EOF
-}
-
-run_network_intermediate_upgrade() {
-    # preserve BRANCH value if set originally
-    if [[ -n "$BRANCH" ]]; then
-        local originally_requested_so_branch="$BRANCH"
-    else
-        local originally_requested_so_branch="2.4/main"
-    fi
-
-    echo "Starting automated intermediate upgrade to $next_step_so_version."
-    echo "After completion, the system will automatically attempt to upgrade to the latest version."
+    local es_required_version_statefile_value=$(cat $es_required_version_statefile)
    echo -e "\n##############################################################################################################################\n"
-    exec bash << EOF
-        BRANCH=$next_step_so_version soup -y && \
-        BRANCH=$next_step_so_version soup -y && \
+    echo "A previously required intermediate Elasticsearch upgrade was detected. Verifying that all Searchnodes/Heavynodes have successfully upgraded Elasticsearch to $es_required_version_statefile_value before proceeding with soup to avoid potential data loss!"
+    # create script using version in statefile
+    timeout --foreground 4000 bash "$es_verification_script" "$es_required_version_statefile_value" "$es_required_version_statefile"
+    if [[ $? -ne 0 ]]; then
+        echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"

-        echo -e "\n##############################################################################################################################\n" && \
-        echo -e "Verifying Elasticsearch was successfully upgraded to $required_es_upgrade_version across the grid. This part can take a while as Searchnodes/Heavynodes sync up with the Manager! \n\nOnce verification completes the next soup will begin automatically. If verification takes longer than 1 hour it will stop waiting and your grid will remain at $next_step_so_version. Allowing for all Searchnodes/Heavynodes to upgrade Elasticsearch to the required version on their own time.\n" && \
+        echo "A previous required intermediate Elasticsearch upgrade to $es_required_version_statefile_value has yet to successfully complete across the grid. Please allow time for all Searchnodes/Heavynodes to have upgraded Elasticsearch to $es_required_version_statefile_value before running soup again to avoid potential data loss!"

-        timeout --foreground 4000 bash /tmp/so_intermediate_upgrade_verification.sh $required_es_upgrade_version $es_required_version_statefile && \
+        echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
+        exit 161
+    fi
+    echo -e "\n##############################################################################################################################\n"
+  fi
+
+  if [[ " ${es_upgrade_map[$es_version]} " =~ " $target_es_version " || "$es_version" == "$target_es_version" ]]; then
+    # supported upgrade
+    return 0
+  else
+    compatible_versions=${es_upgrade_map[$es_version]}
+    if [[ -z "$compatible_versions" ]]; then
+      # If current ES version is not explicitly defined in the upgrade map, we know they have an intermediate upgrade to do.
+        # We default to the lowest ES version defined in es_to_so_version as $first_es_required_version
+      local first_es_required_version=$(printf '%s\n' "${!es_to_so_version[@]}" | sort -V | head -n1)
+      next_step_so_version=${es_to_so_version[$first_es_required_version]}
+      required_es_upgrade_version="$first_es_required_version"
+    else
+      next_step_so_version=${es_to_so_version[${compatible_versions##* }]}
+      required_es_upgrade_version="${compatible_versions##* }"
+    fi
+      echo -e "\n##############################################################################################################################\n"
+      echo -e "You are currently running Security Onion $INSTALLEDVERSION. You will need to update to version $next_step_so_version before updating to $(cat $UPDATE_DIR/VERSION).\n"
+
+      echo "$required_es_upgrade_version" > "$es_required_version_statefile"
+
+    # We expect to upgrade to the latest compatiable minor version of ES
+    create_intermediate_upgrade_verification_script $es_verification_script
+
+    if [[ $is_airgap -eq 0 ]]; then
+      echo "You can download the $next_step_so_version ISO image from https://download.securityonion.net/file/securityonion/securityonion-$next_step_so_version.iso"
+      echo "*** Once you have updated to $next_step_so_version, you can then run soup again to update to $(cat $UPDATE_DIR/VERSION). ***"
+      echo -e "\n##############################################################################################################################\n"
+      exit 160
+    else
+      # preserve BRANCH value if set originally
+      if [[ -n "$BRANCH" ]]; then
+        local originally_requested_so_version="$BRANCH"
+      else
+        local originally_requested_so_version="2.4/main"
+      fi
+
+      echo "Starting automated intermediate upgrade to $next_step_so_version."
+      echo "After completion, the system will automatically attempt to upgrade to the latest version."
+      echo -e "\n##############################################################################################################################\n"
+      exec bash -c "BRANCH=$next_step_so_version soup -y && BRANCH=$next_step_so_version soup -y && \
+       echo -e \"\n##############################################################################################################################\n\" && \
+       echo -e \"Verifying Elasticsearch was successfully upgraded to $required_es_upgrade_version across the grid. This part can take a while as Searchnodes/Heavynodes sync up with the Manager! \n\nOnce verification completes the next soup will begin automatically. If verification takes longer than 1 hour it will stop waiting and your grid will remain at $next_step_so_version. Allowing for all Searchnodes/Heavynodes to upgrade Elasticsearch to the required version on their own time.\n\" \
+       && timeout --foreground 4000 bash /tmp/so_intermediate_upgrade_verification.sh $required_es_upgrade_version $es_required_version_statefile && \
+       echo -e \"\n##############################################################################################################################\n\" \
+       && BRANCH=$originally_requested_so_version soup -y && BRANCH=$originally_requested_so_version soup -y"
+    fi
+  fi

-        echo -e "\n##############################################################################################################################\n" && \
-        if [[ -n "$originally_requested_iso_location" ]]; then
-            # nonairgap soup that used -f originally, runs intermediate upgrade using network + BRANCH, later coming back to the original ISO for the last soup
-            ISOLOC=$originally_requested_iso_location soup -y && \
-            ISOLOC=$originally_requested_iso_location soup -y
-        else
-            BRANCH=$originally_requested_so_branch soup -y && \
-            BRANCH=$originally_requested_so_branch soup -y
-        fi
-        echo -e "\n##############################################################################################################################\n"
-EOF
 }

 create_intermediate_upgrade_verification_script() {
@@ -2128,7 +2019,6 @@ main() {
  echo "Verifying we have the latest soup script."
  verify_latest_update_script

-  echo "Verifying Elasticsearch version compatibility before upgrading."
  verify_es_version_compatibility

  echo "Let's see if we need to update Security Onion."
--- a/salt/manager/tools/sbin/soupto3
+++ b/salt/manager/tools/sbin/soupto3
@@ -0,0 +1,184 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
+. /usr/sbin/so-common
+
+UPDATE_URL=https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/refs/heads/3/main/VERSION
+
+# Check if already running version 3
+CURRENT_VERSION=$(cat /etc/soversion 2>/dev/null)
+if [[ "$CURRENT_VERSION" =~ ^3\. ]]; then
+    echo ""
+    echo "========================================================================="
+    echo " Already Running Security Onion 3"
+    echo "========================================================================="
+    echo ""
+    echo " This system is already running Security Onion $CURRENT_VERSION."
+    echo " Use 'soup' to update within the 3.x release line."
+    echo ""
+    exit 0
+fi
+
+echo ""
+echo "Checking PCAP settings."
+echo ""
+
+# Check pcapengine setting - must be SURICATA before upgrading to version 3
+PCAP_ENGINE=$(lookup_pillar "pcapengine")
+
+PCAP_DELETED=false
+
+prompt_delete_pcap() {
+    read -rp " Would you like to delete all remaining Stenographer PCAP data? (y/N): " DELETE_PCAP
+    if [[ "$DELETE_PCAP" =~ ^[Yy]$ ]]; then
+        echo ""
+        echo " WARNING: This will permanently delete all Stenographer PCAP data"
+        echo " on all nodes. This action cannot be undone."
+        echo ""
+        read -rp " Are you sure? (y/N): " CONFIRM_DELETE
+        if [[ "$CONFIRM_DELETE" =~ ^[Yy]$ ]]; then
+            echo ""
+            echo " Deleting Stenographer PCAP data on all nodes..."
+            salt '*' cmd.run "rm -rf /nsm/pcap/* && rm -rf /nsm/pcapindex/*"
+            echo " Done."
+            PCAP_DELETED=true
+        else
+            echo ""
+            echo " Delete cancelled."
+        fi
+    fi
+}
+
+pcapengine_not_changed() {
+    echo ""
+    echo " PCAP engine must be set to SURICATA before upgrading to Security Onion 3."
+    echo " You can change this in SOC by navigating to:"
+    echo "   Configuration -> global -> pcapengine"
+}
+
+prompt_change_engine() {
+    local current_engine=$1
+    echo ""
+    read -rp " Would you like to change the PCAP engine to SURICATA now? (y/N): " CHANGE_ENGINE
+    if [[ "$CHANGE_ENGINE" =~ ^[Yy]$ ]]; then
+        if [[ "$PCAP_DELETED" != "true" ]]; then
+            echo ""
+            echo " WARNING: Stenographer PCAP data was not deleted. If you proceed,"
+            echo " this data will no longer be accessible through SOC and will never"
+            echo " be automatically deleted. You will need to manually remove it later."
+            echo ""
+            read -rp " Continue with changing pcapengine to SURICATA? (y/N): " CONFIRM_CHANGE
+            if [[ ! "$CONFIRM_CHANGE" =~ ^[Yy]$ ]]; then
+                pcapengine_not_changed
+                return 1
+            fi
+        fi
+        echo ""
+        echo " Updating PCAP engine to SURICATA..."
+        so-yaml.py replace /opt/so/saltstack/local/pillar/global/soc_global.sls global.pcapengine SURICATA
+        echo " Done."
+        return 0
+    else
+        pcapengine_not_changed
+        return 1
+    fi
+}
+
+case "$PCAP_ENGINE" in
+    SURICATA)
+        echo "PCAP engine settings OK."
+        ;;
+    TRANSITION|STENO)
+        echo ""
+        echo "========================================================================="
+        echo " PCAP Engine Check Failed"
+        echo "========================================================================="
+        echo ""
+        echo " Your PCAP engine is currently set to $PCAP_ENGINE."
+        echo ""
+        echo " Before upgrading to Security Onion 3, Stenographer PCAP data must be"
+        echo " removed and the PCAP engine must be set to SURICATA."
+        echo ""
+        echo " To check remaining Stenographer PCAP usage, run:"
+        echo "   salt '*' cmd.run 'du -sh /nsm/pcap'"
+        echo ""
+
+        prompt_delete_pcap
+        if ! prompt_change_engine "$PCAP_ENGINE"; then
+            echo ""
+            exit 1
+        fi
+        ;;
+    *)
+        echo ""
+        echo "========================================================================="
+        echo " PCAP Engine Check Failed"
+        echo "========================================================================="
+        echo ""
+        echo " Unable to determine the PCAP engine setting (got: '$PCAP_ENGINE')."
+        echo " Please ensure the PCAP engine is set to SURICATA."
+        echo " In SOC, navigate to Configuration -> global -> pcapengine"
+        echo " and change the value to SURICATA."
+        echo ""
+        exit 1
+        ;;
+esac
+
+echo ""
+echo "Checking Versions."
+echo ""
+
+# Check if Security Onion 3 has been released
+VERSION=$(curl -sSf "$UPDATE_URL" 2>/dev/null)
+
+if [[ -z "$VERSION" ]]; then
+    echo ""
+    echo "========================================================================="
+    echo " Unable to Check Version"
+    echo "========================================================================="
+    echo ""
+    echo " Could not retrieve version information from:"
+    echo "   $UPDATE_URL"
+    echo ""
+    echo " Please check your network connection and try again."
+    echo ""
+    exit 1
+fi
+
+if [[ "$VERSION" == "UNRELEASED" ]]; then
+    echo ""
+    echo "========================================================================="
+    echo " Security Onion 3 Not Available"
+    echo "========================================================================="
+    echo ""
+    echo " Security Onion 3 has not been released yet."
+    echo ""
+    echo " Please check back later or visit https://securityonion.net for updates."
+    echo ""
+    exit 1
+fi
+
+# Validate version format (e.g., 3.0.2)
+if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+    echo ""
+    echo "========================================================================="
+    echo " Invalid Version"
+    echo "========================================================================="
+    echo ""
+    echo " Received unexpected version format: '$VERSION'"
+    echo ""
+    echo " Please check back later or visit https://securityonion.net for updates."
+    echo ""
+    exit 1
+fi
+
+echo "Security Onion 3 ($VERSION) is available. Upgrading..."
+echo ""
+
+# All checks passed - proceed with upgrade
+BRANCH=3/main soup
--- a/salt/soc/defaults.yaml
+++ b/salt/soc/defaults.yaml
@@ -1494,19 +1494,10 @@ soc:
                branch: main
                folder: securityonion-normalized
        assistant:
+          apiUrl: https://onionai.securityonion.net
+          healthTimeoutSeconds: 3
          systemPromptAddendum: ""
          systemPromptAddendumMaxLength: 50000
-          adapters:
-            - name: SOAI
-              protocol: securityonion_ai_cloud
-              apiUrl: https://onionai.securityonion.net
-              healthTimeoutSeconds: 5
-            - name: Gemini
-              protocol: gemini
-              apiKey: ""
-              serviceAccountJSON: ""
-              serviceAccountLocation: ""
-              healthTimeoutSeconds: 5
        salt:
          queueDir: /opt/sensoroni/queue
          timeoutMs: 45000
@@ -2668,7 +2659,6 @@ soc:
              contextLimitLarge: 1000000
              lowBalanceColorAlert: 500000
              enabled: true
-              adapter: SOAI
            - id: qwen-235b
              displayName: QWEN 235B ($)
              origin: China
@@ -2676,4 +2666,3 @@ soc:
              contextLimitLarge: 256000
              lowBalanceColorAlert: 500000
              enabled: true
-              adapter: SOAI
--- a/salt/soc/soc_soc.yaml
+++ b/salt/soc/soc_soc.yaml
@@ -650,6 +650,13 @@ soc:
                label: Folder
            airgap: *pbRepos
        assistant:
+          apiUrl:
+            description: The URL of the AI gateway.
+            global: True
+          healthTimeoutSeconds:
+            description: Timeout in seconds for the Onion AI health check.
+            global: True
+            advanced: True
          systemPromptAddendum:
            description: Additional context to provide to the AI assistant about this SOC deployment. This can include information about your environment, policies, or any other relevant details that can help the AI provide more accurate and tailored assistance. Long prompts may be shortened.
            global: True
@@ -659,40 +666,6 @@ soc:
            description: Maximum length of the system prompt addendum. Longer prompts will be truncated.
            global: True
            advanced: True
-          adapters:
-            description: Configuration for AI adapters used by the Onion AI assistant.
-            global: True
-            advanced: True
-            forcedType: "[]{}"
-            syntax: json
-            uiElements:
-            - field: name
-              label: Adapter Name
-              regex: "^(?!.*@).+$"
-              regexFailureMessage: Adapter name cannot contain the '@' character
-              required: True
-            - field: protocol
-              label: Protocol
-              required: True
-              options:
-                - securityonion_ai_cloud
-                - gemini
-            - field: apiUrl
-              label: API URL
-              required: False
-            - field: apiKey
-              label: API Key
-              required: False
-            - field: serviceAccountJSON
-              label: Service Account JSON
-              required: False
-              multiline: True
-            - field: serviceAccountLocation
-              label: Service Account Location
-              required: False
-            - field: healthTimeoutSeconds
-              label: Health Timeout Seconds
-              required: False
      client:
        assistant:
          enabled:
@@ -731,8 +704,6 @@ soc:
            - field: id
              label: Model ID
              required: True
-              regex: "^(?!.*@).+$"
-              regexFailureMessage: Model ID cannot contain the '@' character
            - field: displayName
              label: Display Name
              required: True
--- a/salt/telegraf/config.sls
+++ b/salt/telegraf/config.sls
@@ -7,7 +7,6 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'telegraf/map.jinja' import TELEGRAFMERGED %}
-{%   from 'logstash/map.jinja' import LOGSTASH_MERGED %}

 # add Telegraf to monitor all the things
 tgraflogdir:
@@ -90,7 +89,6 @@ tgrafconf:
    - defaults:
        GLOBALS: {{ GLOBALS }}
        TELEGRAFMERGED: {{ TELEGRAFMERGED }}
-        LOGSTASH_MERGED: {{ LOGSTASH_MERGED }}

 # this file will be read by telegraf to send node details (management interface, monitor interface, etc)
 # into influx
--- a/salt/telegraf/etc/telegraf.conf
+++ b/salt/telegraf/etc/telegraf.conf
@@ -7,7 +7,7 @@
 {%- set UNIQUEID = salt['pillar.get']('sensor:uniqueid', '') %}
 {%- set ZEEK_ENABLED = salt['pillar.get']('zeek:enabled', True) %}
 {%- set MDENGINE = GLOBALS.md_engine %}
-{%- set LOGSTASH_ENABLED = LOGSTASH_MERGED.enabled %}
+{%- set LOGSTASH_ENABLED = salt['pillar.get']('logstash:enabled', False) %}
 # Global tags can be specified here in key="value" format.
 [global_tags]
  role = "{{ GLOBALS.role.split('-') | last }}"
--- a/setup/so-setup
+++ b/setup/so-setup
@@ -828,7 +828,6 @@ if ! [[ -f $install_opt_file ]]; then
 		fi
 		checkin_at_boot
 		set_initial_firewall_access
-        initialize_elasticsearch_indices "so-case so-casehistory so-assistant-session so-assistant-chat"
 		# run a final highstate before enabling scheduled highstates. 
 		# this will ensure so-elasticsearch-ilm-policy-load and so-elasticsearch-templates-load have a chance to run after elasticfleet is setup
 		info "Running final highstate for setup"
Author	SHA1	Message	Date
Mike Reeves	be35b59b8c	Update echo messages for PCAP engine clarity	2026-02-24 10:04:26 -05:00
Mike Reeves	7170289a5e	Continue upgrade after pcapengine is changed to SURICATA Instead of exiting and requiring the user to rerun the script after changing pcapengine to SURICATA, let the script continue to the version check and upgrade.	2026-02-23 11:35:32 -05:00
Mike Reeves	ca040044bb	Use so-yaml to update pcapengine pillar and fix file path Replace fragile sed with so-yaml.py replace for proper YAML handling. Also correct the pillar file path from soc_soc.sls to soc_global.sls.	2026-02-23 11:16:30 -05:00
Mike Reeves	f17e2961ed	Add PCAP orphan warning and require SURICATA before upgrade - Warn users that undeleted Stenographer PCAP data will be inaccessible and never automatically cleaned up if they switch to SURICATA without deleting it first - Require pcapengine to be set to SURICATA before allowing upgrade, with clear messaging when the user declines to change it	2026-02-23 11:05:30 -05:00
Mike Reeves	bbc7668786	Add version check, PCAP cleanup prompts, and SOC config references to soupto3 - Skip upgrade if already running Security Onion 3.x.x - Add interactive prompts to delete Stenographer PCAP data (with double confirmation) and change pcapengine to SURICATA - Direct users to SOC Configuration UI instead of editing pillar files directly - Consolidate TRANSITION and STENO cases to reduce repeated code	2026-02-23 10:49:54 -05:00
Mike Reeves	1888f9e757	Soup to 3	2026-02-23 10:07:16 -05:00